diff --git a/csv-validator-cmd/README.md b/csv-validator-cmd/README.md index c852c2de..747bdf29 100644 --- a/csv-validator-cmd/README.md +++ b/csv-validator-cmd/README.md @@ -32,6 +32,8 @@ Usage: validate [options] The path to the CSV Schema file to use for validation --disable-utf8-validation Disable UTF-8 validation for CSV files + --skip-file-checks + Skip integrity, checksum and file existence checks --show-progress Show progress --help diff --git a/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala b/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala index 915c8d2a..6ba9511b 100644 --- a/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala +++ b/csv-validator-cmd/src/main/scala/uk/gov/nationalarchives/csv/validator/cmd/CsvValidatorCmdApp.scala @@ -51,7 +51,8 @@ object CsvValidatorCmdApp extends App { csvSchemaPath: Path = Paths.get("."), csvSchemaEncoding: Charset = CsvValidator.DEFAULT_ENCODING, disableUtf8Validation:Boolean = false, - progressCallback: Option[ProgressCallback] = None) + progressCallback: Option[ProgressCallback] = None, + skipFileChecks: Boolean = false) def run(args: Array[String]): ExitStatus = { @@ -69,6 +70,7 @@ object CsvValidatorCmdApp extends App { opt[Charset]('x', "csv-encoding").optional().action { (x,c) => c.copy(csvEncoding = x) } text("Defines the charset encoding used in the CSV file") opt[Charset]('y', "csv-schema-encoding").optional().action { (x,c) => c.copy(csvSchemaEncoding = x) }.text("Defines the charset encoding used in the CSV Schema file") opt[Unit]("disable-utf8-validation").optional().action {(_, c) => c.copy(disableUtf8Validation = true)}.text("Disable UTF-8 validation for CSV files.") + opt[Unit]("skip-file-checks").optional().action {(_, c) => c.copy(progressCallback = Some(commandLineProgressCallback()))}.text("Skip integrity, checksum and file existence checks") opt[Unit]("show-progress").optional().action {(_, c) => c.copy(progressCallback = Some(commandLineProgressCallback()))}.text("Show progress") arg[Path]("").validate { x => if(Files.exists(x) && Files.isReadable(x)) success else failure(s"Cannot access CSV file: ${x.toString}") }.action { (x,c) => c.copy(csvPath = x) }.text("The path to the CSV file to validate") arg[Path]("").validate { x => if(Files.exists(x) && Files.isReadable(x)) success else failure(s"Cannot access CSV Schema file: ${x.toString}") }.action { (x,c) => c.copy(csvSchemaPath = x) }.text("The path to the CSV Schema file to use for validation") @@ -84,7 +86,8 @@ object CsvValidatorCmdApp extends App { config.substitutePaths, config.caseSensitivePaths, config.traceParser, - config.progressCallback + config.progressCallback, + config.skipFileChecks ) } getOrElse { //arguments are bad, usage message will have been displayed @@ -141,7 +144,7 @@ object CsvValidatorCmdApp extends App { } def getColumnFromCsv(csvFile: TextFile, csvSchemaFile: TextFile, columnName: String): List[String] = Try { - val validator = createValidator(true, Nil, false, false) + val validator = createValidator(true, Nil, false, false, false) val csv = validator.loadCsvFile(csvFile, csvSchemaFile) csv.headOption.map(_.indexOf("identifier")).map { identifierIdx => csv.tail.map(arr => arr(identifierIdx)) @@ -157,9 +160,10 @@ object CsvValidatorCmdApp extends App { enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: Option[ProgressCallback], + skipFileChecks: Boolean, onRow: ValidatedNel[FailMessage, Any] => Unit = rowCallback ): ExitStatus = { - val validator = createValidator(failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace) + val validator = createValidator(failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, skipFileChecks) validator.parseSchema(schemaFile) match { case Validated.Invalid(errors) => (prettyPrint(errors), SystemExitCodes.InvalidSchema) case Validated.Valid(schema) => diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala index 57479497..4f57e5bb 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidator.scala @@ -30,11 +30,11 @@ object CsvValidator { type PathTo = String type SubstitutePath = (PathFrom, PathTo) - def createValidator(failFast: Boolean, pathSubstitutionsList: List[SubstitutePath], enforceCaseSensitivePathChecksSwitch: Boolean, traceSwitch: Boolean) = { + def createValidator(failFast: Boolean, pathSubstitutionsList: List[SubstitutePath], enforceCaseSensitivePathChecksSwitch: Boolean, traceSwitch: Boolean, skipFileChecksSwitch: Boolean) = { if(failFast) { - new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = pathSubstitutionsList; val enforceCaseSensitivePathChecks = enforceCaseSensitivePathChecksSwitch; val trace = traceSwitch } + new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = pathSubstitutionsList; val enforceCaseSensitivePathChecks = enforceCaseSensitivePathChecksSwitch; val trace = traceSwitch; val skipFileChecks = skipFileChecksSwitch} } else { - new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = pathSubstitutionsList; val enforceCaseSensitivePathChecks = enforceCaseSensitivePathChecksSwitch; val trace = traceSwitch } + new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = pathSubstitutionsList; val enforceCaseSensitivePathChecks = enforceCaseSensitivePathChecksSwitch; val trace = traceSwitch; val skipFileChecks = skipFileChecksSwitch } } } } diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala index 52087729..206625dc 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaParser.scala @@ -49,7 +49,7 @@ with TraceableParsers { */ val enforceCaseSensitivePathChecks: Boolean - + val skipFileChecks: Boolean lazy val versionHeader: PackratParser[String] = "VersionDecl" ::= ("version" ~> versionLiteral ) @@ -146,6 +146,7 @@ with TraceableParsers { val ecspc = enforceCaseSensitivePathChecks val ps = pathSubstitutions val t = trace + val sfc = skipFileChecks SchemaValidator.versionValid(version).map(Failure(_, next)).getOrElse { version match { @@ -153,6 +154,7 @@ with TraceableParsers { val parser1_2 = new SchemaParser1_2 {override val enforceCaseSensitivePathChecks: Boolean = ecspc override val pathSubstitutions: List[(String, String)] = ps override val trace: Boolean = t + override val skipFileChecks: Boolean = sfc } parser1_2.parseVersionAware(reader) match { @@ -165,6 +167,7 @@ with TraceableParsers { val parser1_1 = new SchemaParser1_1 {override val enforceCaseSensitivePathChecks: Boolean = ecspc override val pathSubstitutions: List[(String, String)] = ps override val trace: Boolean = t + override val skipFileChecks: Boolean = sfc } parser1_1.parseVersionAware(reader) match { @@ -177,6 +180,7 @@ with TraceableParsers { val parser1_0 = new SchemaParser1_0 {override val enforceCaseSensitivePathChecks: Boolean = ecspc override val pathSubstitutions: List[(String, String)] = ps override val trace: Boolean = t + override val skipFileChecks: Boolean = sfc } parser1_0.parseVersionAware(reader) match { diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/Rule.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/Rule.scala index 551947f1..7c7f5e34 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/Rule.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/Rule.scala @@ -106,17 +106,21 @@ case class RegExpRule(regex: String) extends Rule("regex") { } //TODO note the use of `Seq(rootPath): _*` when extending Rule, this is to workaround this bug https://issues.scala-lang.org/browse/SI-7436. This pattern is repeated below! -case class FileExistsRule(pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean, rootPath: ArgProvider = Literal(None)) extends Rule("fileExists", Seq(rootPath): _*) { +case class FileExistsRule(pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean, rootPath: ArgProvider = Literal(None), skipFileChecks: Boolean = false) extends Rule("fileExists", Seq(rootPath): _*) { override def valid(filePath: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None) = { + if(skipFileChecks) { + true + } else { + val ruleValue = rootPath.referenceValue(columnIndex, row, schema) - val ruleValue = rootPath.referenceValue(columnIndex, row, schema) - - val fileExists = ruleValue match { - case Some(rp) => new FileSystem(rp, filePath, pathSubstitutions).exists(enforceCaseSensitivePathChecks) - case None => new FileSystem(filePath, pathSubstitutions).exists(enforceCaseSensitivePathChecks) + ruleValue match { + case Some(rp) => new FileSystem(rp, filePath, pathSubstitutions).exists(enforceCaseSensitivePathChecks) + case None => new FileSystem(filePath, pathSubstitutions).exists(enforceCaseSensitivePathChecks) + } } - fileExists + + } override def toError = s"""$ruleName""" + (if (rootPath == Literal(None)) "" else s"""(${rootPath.toError})""") @@ -317,18 +321,21 @@ case class UniqueMultiRule(columns: List[ColumnReference]) extends Rule("unique( } } -case class ChecksumRule(rootPath: ArgProvider, file: ArgProvider, algorithm: String, pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean = false) extends Rule("checksum", Seq(rootPath, file): _*) with FileWildcardSearch[String] { +case class ChecksumRule(rootPath: ArgProvider, file: ArgProvider, algorithm: String, pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean = false, skipFileChecks: Boolean = false) extends Rule("checksum", Seq(rootPath, file): _*) with FileWildcardSearch[String] { - def this(file: ArgProvider, algorithm: String, pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean) = this(Literal(None), file, algorithm, pathSubstitutions, enforceCaseSensitivePathChecks) - def this(file: ArgProvider, algorithm: String, enforceCaseSensitivePathChecks: Boolean) = this(Literal(None), file, algorithm, List.empty[(String,String)], enforceCaseSensitivePathChecks) + def this(file: ArgProvider, algorithm: String, pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean) = this(Literal(None), file, algorithm, pathSubstitutions, enforceCaseSensitivePathChecks, false) + def this(file: ArgProvider, algorithm: String, enforceCaseSensitivePathChecks: Boolean, skipFileChecks: Boolean) = this(Literal(None), file, algorithm, List.empty[(String,String)], enforceCaseSensitivePathChecks, false) override def evaluate(columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean] = None): RuleValidation[Any] = { val columnDefinition = schema.columnDefinitions(columnIndex) - - search(filename(columnIndex, row, schema)) match { - case Validated.Valid(hexValue: String) if hexValue == cellValue(columnIndex,row,schema) => true.validNel[String] - case Validated.Valid(hexValue: String) => s"""$toError file "${TypedPath(filename(columnIndex, row, schema)._1 + filename(columnIndex, row, schema)._2).toPlatform}" checksum match fails for line: ${row.lineNumber}, column: ${columnDefinition.id}, ${toValueError(row,columnIndex)}. Computed checksum value:"${hexValue}"""".invalidNel[Any] - case Validated.Invalid(errMsg) => s"$toError ${errMsg.head} for line: ${row.lineNumber}, column: ${columnDefinition.id}, ${toValueError(row,columnIndex)}".invalidNel[Any] + if(skipFileChecks) { + Validated.Valid("") + } else { + search(filename(columnIndex, row, schema)) match { + case Validated.Valid(hexValue: String) if hexValue == cellValue(columnIndex, row, schema) => true.validNel[String] + case Validated.Valid(hexValue: String) => s"""$toError file "${TypedPath(filename(columnIndex, row, schema)._1 + filename(columnIndex, row, schema)._2).toPlatform}" checksum match fails for line: ${row.lineNumber}, column: ${columnDefinition.id}, ${toValueError(row, columnIndex)}. Computed checksum value:"${hexValue}"""".invalidNel[Any] + case Validated.Invalid(errMsg) => s"$toError ${errMsg.head} for line: ${row.lineNumber}, column: ${columnDefinition.id}, ${toValueError(row, columnIndex)}".invalidNel[Any] + } } } diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParser.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParser.scala index 260ee4a4..e90cd788 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParser.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParser.scala @@ -481,9 +481,9 @@ trait SchemaParser extends BaseSchemaParser { */ lazy val fileExistsExpr: PackratParser[FileExistsRule] = "FileExistsExpr" ::= ("fileExists" ~> opt("(" ~> stringProvider <~ ")")).withFailureMessage("Invalid fileExists rule") ^^ { case None => - FileExistsRule(pathSubstitutions, enforceCaseSensitivePathChecks) + FileExistsRule(pathSubstitutions, enforceCaseSensitivePathChecks, skipFileChecks = skipFileChecks) case Some(s) => - FileExistsRule(pathSubstitutions, enforceCaseSensitivePathChecks, s) + FileExistsRule(pathSubstitutions, enforceCaseSensitivePathChecks, s, skipFileChecks = skipFileChecks) } @@ -492,7 +492,7 @@ trait SchemaParser extends BaseSchemaParser { */ lazy val checksumExpr = "ChecksumExpr" ::= ("checksum(" ~> fileExpr <~ ",") ~ stringLiteral <~ ")" ^^ { case files ~ algorithm => - ChecksumRule(files._1.getOrElse(Literal(None)), files._2, algorithm, pathSubstitutions, enforceCaseSensitivePathChecks) + ChecksumRule(files._1.getOrElse(Literal(None)), files._2, algorithm, pathSubstitutions, enforceCaseSensitivePathChecks, skipFileChecks) } /** diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/Rule.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/Rule.scala index 497b108f..84ec9bff 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/Rule.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/Rule.scala @@ -66,7 +66,7 @@ case class SwitchRule(elseRules: Option[List[Rule]], cases:(Rule, List[Rule])*) -case class IntegrityCheckRule(pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean, rootPath: ArgProvider = Literal(None), topLevelFolder: String = "content", includeFolder: Boolean = false) extends Rule("integrityCheck", Seq(rootPath): _*) { +case class IntegrityCheckRule(pathSubstitutions: List[(String,String)], enforceCaseSensitivePathChecks: Boolean, rootPath: ArgProvider = Literal(None), topLevelFolder: String = "content", includeFolder: Boolean = false, skipFileChecks: Boolean = false) extends Rule("integrityCheck", Seq(rootPath): _*) { //TODO introduce state, not very functional var filesMap = Map[String, Set[Path]]() @@ -83,8 +83,10 @@ case class IntegrityCheckRule(pathSubstitutions: List[(String,String)], enforceC } override def valid(filePath: String, columnDefinition: ColumnDefinition, columnIndex: Int, row: Row, schema: Schema, mayBeLast: Option[Boolean]): Boolean = { - - if (!filePath.isEmpty){ + if (skipFileChecks) { + true + } + else if (!filePath.isEmpty){ val ruleValue = rootPath.referenceValue(columnIndex, row, schema) val filePathS = if (FILE_SEPARATOR == WINDOWS_FILE_SEPARATOR) diff --git a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaParser.scala b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaParser.scala index 2561ba8d..c2b0d8af 100644 --- a/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaParser.scala +++ b/csv-validator-core/src/main/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/SchemaParser.scala @@ -78,9 +78,9 @@ trait SchemaParser extends SchemaParser1_0 { lazy val integrityCheckExpr: PackratParser[IntegrityCheckRule] = "IntegrityCheckExpr" ::= ("integrityCheck" ~> "(" ~> opt(stringProvider <~ ",") ~ opt(stringLiteral <~ ",") ~ stringLiteral <~ ")" ).withFailureMessage("Invalid integrityCheck rule") ^^ { case rp ~ topLevelFolder ~ includeFolder if (includeFolder == "includeFolder") => - IntegrityCheckRule(pathSubstitutions, enforceCaseSensitivePathChecks, rp.getOrElse(Literal(None)), topLevelFolder.getOrElse("content"), true) + IntegrityCheckRule(pathSubstitutions, enforceCaseSensitivePathChecks, rp.getOrElse(Literal(None)), topLevelFolder.getOrElse("content"), true, skipFileChecks) case rp ~ topLevelFolder ~ includeFolder if (includeFolder == "excludeFolder") => - IntegrityCheckRule(pathSubstitutions, enforceCaseSensitivePathChecks, rp.getOrElse(Literal(None)), topLevelFolder.getOrElse("content"), false) + IntegrityCheckRule(pathSubstitutions, enforceCaseSensitivePathChecks, rp.getOrElse(Literal(None)), topLevelFolder.getOrElse("content"), false, skipFileChecks) } diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorAcceptanceSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorAcceptanceSpec.scala index 23d03fae..b6284137 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorAcceptanceSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorAcceptanceSpec.scala @@ -31,6 +31,7 @@ class MetaDataValidatorAcceptanceSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = false val trace = false + val skipFileChecks = false def validateR(csv: io.Reader, schema: Schema): this.type#MetaDataValidation[Any] = validate(csv, schema, None) } @@ -39,6 +40,7 @@ class MetaDataValidatorAcceptanceSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = true val trace = false + val skipFileChecks = false } import v.{validate, validateR, parseSchema} @@ -406,7 +408,7 @@ class MetaDataValidatorAcceptanceSpec extends Specification with TestResources { } "Validate fail fast" should { - val app = new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false } + val app = new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } "only report first error for invalid @TotalColumns" in { app.validate(TextFile(Paths.get(base).resolve("totalColumnsFailMetaData.csv")), parse(base + "/totalColumnsSchema.csvs"), None) must beLike { diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBigFileSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBigFileSpec.scala index a3295701..660f5a38 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBigFileSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBigFileSpec.scala @@ -25,14 +25,14 @@ class MetaDataValidatorBigFileSpec extends Specification with TestResources { "Big file" should { "succeed with no stack overflow for all errors" in { - val v = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false } + val v = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } def parse(filePath: String): Schema = v.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) v.validate(TextFile(Paths.get(base).resolve("bigMetaData.csv")), parse(base + "/bigSchema.csvs"), None) must beLike { case Validated.Valid(_) => ok } } "succeed with no stack overflow for fail fast" in { - val v = new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false } + val v = new CsvValidator with FailFastMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } def parse(filePath: String): Schema = v.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) v.validate(TextFile(Paths.get(base).resolve("bigMetaData.csv")), parse(base + "/bigSchema.csvs"), None) must beLike { case Validated.Valid(_) => ok } diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBusinessAcceptanceSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBusinessAcceptanceSpec.scala index 7b64bb98..e97977ef 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBusinessAcceptanceSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorBusinessAcceptanceSpec.scala @@ -22,7 +22,7 @@ class MetaDataValidatorBusinessAcceptanceSpec extends Specification with TestRes val base = resourcePath("acceptance/dp") - val v: CsvValidator = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false } + val v: CsvValidator = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } import v.{validate, parseSchema} def parse(filePath: String): Schema = parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorChecksumSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorChecksumSpec.scala index 482d1762..90b40370 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorChecksumSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorChecksumSpec.scala @@ -29,6 +29,7 @@ class MetaDataValidatorChecksumSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = false val trace = false + val skipFileChecks = false override def parse(reader: Reader): ParseResult[Schema] = super.parse(reader) match { case s @ Success(schema: Schema, _) => s diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorFileCountSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorFileCountSpec.scala index b8a08b44..db9705c4 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorFileCountSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorFileCountSpec.scala @@ -26,6 +26,7 @@ class MetaDataValidatorFileCountSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = false val trace = false + val skipFileChecks = false override def parse(reader: Reader): ParseResult[Schema] = super.parse(reader) match { case s@Success(schema: Schema, _) => s case NoSuccess(message, next) => throw new RuntimeException(message) diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorIntegrityCheckSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorIntegrityCheckSpec.scala index 8b16b263..c25651d4 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorIntegrityCheckSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorIntegrityCheckSpec.scala @@ -20,10 +20,11 @@ import java.nio.file.Paths @RunWith(classOf[JUnitRunner]) class MetaDataValidatorIntegrityCheckSpec extends Specification with TestResources { - def buildValidator(substitutionPath: List[(String,String)]) : CsvValidator = new CsvValidator with AllErrorsMetaDataValidator { + def buildValidator(substitutionPath: List[(String,String)], skipFileChecksFlag: Boolean = false) : CsvValidator = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = substitutionPath val enforceCaseSensitivePathChecks = false val trace = false + val skipFileChecks = skipFileChecksFlag } def parse(filePath: String, validator: CsvValidator): Schema = validator.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) @@ -66,6 +67,15 @@ class MetaDataValidatorIntegrityCheckSpec extends Specification with TestResourc // message.toString must contain("file2 are not listed in ") } + "succeed for metadatafile missing files if skipFileChecks is true" in { + + val substitutionPaths = List(("file:///T:/WORK/RF_5/", headerPath)) + val validator = buildValidator(substitutionPaths, true) + val result = validator.validate(TextFile(Paths.get(headerPath).resolve("integrityCheckMetaData-missing-files.csv")), parse(headerPath + "/integrityCheckSchema.csvs", validator), None) + + result.isValid mustEqual true + } + "fail for metadatafile missing files - header" in { val substitutionPaths = List(("file:///T:/WORK/RF_5/",headerPath)) diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorSpec.scala index da4b20d4..6ff7e833 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/MetaDataValidatorSpec.scala @@ -27,6 +27,7 @@ class MetaDataValidatorSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = false val trace = false + val skipFileChecks = false override def parse(reader: Reader): ParseResult[Schema] = { super.parse(reader) match { case s @ Success(schema: Schema, _) => s diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/NotEmptyBugTest.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/NotEmptyBugTest.scala index 2fef087c..420de9d6 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/NotEmptyBugTest.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/NotEmptyBugTest.scala @@ -28,7 +28,8 @@ class NotEmptyBugTest extends Specification with TestResources { val pathSubstitutions = List[(String,String)]() val enforceCaseSensitivePathChecks = false val trace = false - + val skipFileChecks = false + override def parse(reader: Reader): ParseResult[Schema] = { super.parse(reader) match { case s@Success(schema: Schema, _) => s diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorFileEncodingSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorFileEncodingSpec.scala index 6eb5d713..b621505f 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorFileEncodingSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorFileEncodingSpec.scala @@ -26,7 +26,7 @@ class CsvValidatorFileEncodingSpec extends Specification with TestResources { "Validation" should { - val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false } + val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } def parse(filePath: String): Schema = app.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) "fail for non UTF-8 file" in { diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala index d865deb4..f48e7116 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/api/CsvValidatorSpec.scala @@ -23,7 +23,7 @@ import java.nio.file.Paths class CsvValidatorSpec extends Specification with TestResources { "Parsing schema" should { - val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false } + val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[SubstitutePath](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } "report position on parse fail" in { @@ -45,7 +45,7 @@ class CsvValidatorSpec extends Specification with TestResources { } "Validation" should { - val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false } + val app = new CsvValidator with AllErrorsMetaDataValidator { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } def parse(filePath: String): Schema = app.parseSchema(TextFile(Paths.get(filePath))) fold (f => throw new IllegalArgumentException(f.toString()), s => s) diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaSpecBase.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaSpecBase.scala index 23e2b26f..c3246090 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaSpecBase.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/SchemaSpecBase.scala @@ -14,7 +14,7 @@ import uk.gov.nationalarchives.csv.validator.schema.v1_0.NotEmptyRule trait SchemaSpecBase extends Specification { - object TestSchemaParser extends SchemaParser { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false } + object TestSchemaParser extends SchemaParser { val pathSubstitutions = List[(String,String)](); val enforceCaseSensitivePathChecks = false; val trace = false; val skipFileChecks = false } def buildSchema1_0(globalDirective: GlobalDirective*)(columnDefinition: ColumnDefinition*) = Schema(globalDirective.toList, columnDefinition.toList, "1.0") diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/ChecksumRuleSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/ChecksumRuleSpec.scala index 4171300b..a2cc7baa 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/ChecksumRuleSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/ChecksumRuleSpec.scala @@ -28,16 +28,24 @@ class ChecksumRuleSpec extends Specification with TestResources { "Checksum" should { "fail when calculated algorithm does not match given string value" in { - val checksumRule = new ChecksumRule(Literal(Some(checksumPath)), "MD5", false) + val checksumRule = new ChecksumRule(Literal(Some(checksumPath)), "MD5", false, false) checksumRule.evaluate(0, Row(List(Cell("699d61aff25f16a5560372e610da91ab")), 1), Schema(List(TotalColumns(1), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { case Validated.Invalid(m) => m.toList mustEqual List("""checksum(file("""" + checksumPath + """"), "MD5") file """" + checksumPath + """" checksum match fails for line: 1, column: column1, value: "699d61aff25f16a5560372e610da91ab". Computed checksum value:"232762380299115da6995e4c4ac22fa2"""") } } - "succeed when calculated algorithm does match given string value" in { - val checksumRule = new ChecksumRule(Literal(Some(checksumPath)), "MD5", false) + "succeed when calculated algorithm does not match given string value and skipFileChecks is true" in { + val checksumRule = new ChecksumRule(Literal(Some(checksumPath)), "MD5", false, true) + + checksumRule.evaluate(0, Row(List(Cell("699d61aff25f16a5560372e610da91ab")), 1), Schema(List(TotalColumns(1), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must beLike { + case Validated.Invalid(m) => m.toList mustEqual List("""checksum(file("""" + checksumPath + """"), "MD5") file """" + checksumPath + """" checksum match fails for line: 1, column: column1, value: "699d61aff25f16a5560372e610da91ab". Computed checksum value:"232762380299115da6995e4c4ac22fa2"""") + } + } + "succeed when calculated algorithm does match given string value" in { + val checksumRule = new ChecksumRule(Literal(Some(checksumPath)), "MD5", false, false) + checksumRule.evaluate(0, Row(List(Cell("232762380299115da6995e4c4ac22fa2")), 1), Schema(List(TotalColumns(1), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) } @@ -68,7 +76,7 @@ class ChecksumRuleSpec extends Specification with TestResources { val pathSubstitutions = List[(String,String)]( ("bob", relBasePath + FILE_SEPARATOR) ) - + val checksumRule = new ChecksumRule(Literal(Some("""bob/uk/gov/nationalarchives/csv/validator/schema/v1_0/checksum.csvs""")), "MD5", pathSubstitutions, false) checksumRule.evaluate(0, Row(List(Cell("232762380299115da6995e4c4ac22fa2")), 1), Schema(List(TotalColumns(1), NoHeader()), List(ColumnDefinition(NamedColumnIdentifier("column1"))))) mustEqual Validated.Valid(true) } diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/FileExistsRuleSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/FileExistsRuleSpec.scala index ec64c5ef..3cd15831 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/FileExistsRuleSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/FileExistsRuleSpec.scala @@ -43,6 +43,10 @@ class FileExistsRuleSpec extends Specification with TestResources { } } + "succeed with a non-existent file if skipFileChecks is true" in { + FileExistsRule(emptyPathSubstitutions, false, skipFileChecks = true).evaluate(0, Row(List(Cell("some/non/existent/file")), 1), Schema(globalDirsOne, List(ColumnDefinition(NamedColumnIdentifier("column1"))))) must be_==(Validated.Valid(true)) + } + "fail for empty file path" in { FileExistsRule(emptyPathSubstitutions, false).evaluate(1, Row(List(Cell("abc"), Cell("")), 2), Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2"))))) must beLike { case Validated.Invalid(messages) => messages.head mustEqual "fileExists fails for line: 2, column: column2, value: \"\"" diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserRulesSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserRulesSpec.scala index feeb57b1..196d7716 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserRulesSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_0/SchemaParserRulesSpec.scala @@ -77,7 +77,7 @@ class SchemaParserRulesSpec extends SchemaSpecBase { @totalColumns 1 Name: fileExists""" - parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("Name"), List(FileExistsRule(emptyPathSubs, false, Literal(None))), _)), _), _) => ok } + parse(new StringReader(schema)) must beLike { case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("Name"), List(FileExistsRule(emptyPathSubs, false, Literal(None), false)), _)), _), _) => ok } } // "fail for file exists rule with empty ()" in { @@ -94,7 +94,7 @@ class SchemaParserRulesSpec extends SchemaSpecBase { Name: fileExists("some/root/path")""" parse(new StringReader(schema)) must beLike { - case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("Name"), List(FileExistsRule(emptyPathSubs, false, Literal(Some(rootPath)))), _)), _), _) => { + case Success(Schema(_, List(ColumnDefinition(NamedColumnIdentifier("Name"), List(FileExistsRule(emptyPathSubs, false, Literal(Some(rootPath)), false)), _)), _), _) => { rootPath mustEqual "some/root/path" } } diff --git a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/IntegrityCheckRuleSpec.scala b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/IntegrityCheckRuleSpec.scala index d3390ee1..49b7b621 100644 --- a/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/IntegrityCheckRuleSpec.scala +++ b/csv-validator-core/src/test/scala/uk/gov/nationalarchives/csv/validator/schema/v1_1/IntegrityCheckRuleSpec.scala @@ -70,6 +70,15 @@ class IntegrityCheckRuleSpec extends Specification with TestResources { } } + "succeed for empty file path if skipFileChecks is true" in { + val integrityCheckRule = IntegrityCheckRule(emptyPathSubstitutions, false, skipFileChecks = true) + val schema: Schema = Schema(globalDirsTwo, List(ColumnDefinition(NamedColumnIdentifier("column1")), ColumnDefinition(NamedColumnIdentifier("column2")))) + + integrityCheckRule.evaluate(1, Row(List(Cell("abc"), Cell(relIntegrityCheckForRulePath)), 1), schema, Some(true)) mustEqual Validated.Valid(true) + + integrityCheckRule.evaluate(1, Row(List(Cell("abc"), Cell("")), 2), schema, Some(false)) mustEqual Validated.Valid(true) + } + "succeed for file that exists with no root file path" in { val integrityCheckRule = IntegrityCheckRule(emptyPathSubstitutions,false) diff --git a/csv-validator-java-api/src/main/scala/uk/gov/nationalarchives/csv/validator/api/java/CsvValidatorJavaBridge.scala b/csv-validator-java-api/src/main/scala/uk/gov/nationalarchives/csv/validator/api/java/CsvValidatorJavaBridge.scala index 100a7a7c..58addc52 100644 --- a/csv-validator-java-api/src/main/scala/uk/gov/nationalarchives/csv/validator/api/java/CsvValidatorJavaBridge.scala +++ b/csv-validator-java-api/src/main/scala/uk/gov/nationalarchives/csv/validator/api/java/CsvValidatorJavaBridge.scala @@ -29,27 +29,27 @@ object CsvValidatorJavaBridge { @deprecated def validate(csvFile: String, csvEncoding: Charset, csvSchemaFile: String, csvSchemaEncoding: Charset, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean): JList[FailMessage] = - validate(csvFile, csvEncoding, true, csvSchemaFile, csvSchemaEncoding, false, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, None) + validate(csvFile, csvEncoding, true, csvSchemaFile, csvSchemaEncoding, false, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, None, false) @deprecated def validate(csvFile: String, csvEncoding: Charset, csvSchemaFile: String, csvSchemaEncoding: Charset, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: ProgressCallback): JList[FailMessage] = { val sProgressCallback = new SProgressCallback { override def update(complete: this.type#Percentage) = progress.update(complete) } - validate(csvFile, csvEncoding, true, csvSchemaFile, csvSchemaEncoding, false, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, Some(sProgressCallback)) + validate(csvFile, csvEncoding, true, csvSchemaFile, csvSchemaEncoding, false, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, Some(sProgressCallback), false) } def validate(csvFile: String, csvEncoding: Charset, validateCsvEncoding: Boolean, csvSchemaFile: String, csvSchemaEncoding: Charset, validateCsvSchemaEncoding: Boolean, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean): JList[FailMessage] = - validate(csvFile, csvEncoding, validateCsvEncoding, csvSchemaFile, csvSchemaEncoding, validateCsvSchemaEncoding, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, None) + validate(csvFile, csvEncoding, validateCsvEncoding, csvSchemaFile, csvSchemaEncoding, validateCsvSchemaEncoding, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, None, false) def validate(csvFile: String, csvEncoding: Charset, validateCsvEncoding: Boolean, csvSchemaFile: String, csvSchemaEncoding: Charset, validateCsvSchemaEncoding: Boolean, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: ProgressCallback): JList[FailMessage] = { val sProgressCallback = new SProgressCallback { override def update(complete: this.type#Percentage) = progress.update(complete) } - validate(csvFile, csvEncoding, validateCsvEncoding, csvSchemaFile, csvSchemaEncoding, validateCsvSchemaEncoding, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, Some(sProgressCallback)) + validate(csvFile, csvEncoding, validateCsvEncoding, csvSchemaFile, csvSchemaEncoding, validateCsvSchemaEncoding, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, Some(sProgressCallback), false) } - private def validate(csvFile: String, csvEncoding: Charset, validateCsvEncoding: Boolean, csvSchemaFile: String, csvSchemaEncoding: Charset, validateCsvSchemaEncoding: Boolean, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: Option[SProgressCallback]): JList[FailMessage] = { + private def validate(csvFile: String, csvEncoding: Charset, validateCsvEncoding: Boolean, csvSchemaFile: String, csvSchemaEncoding: Charset, validateCsvSchemaEncoding: Boolean, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: Option[SProgressCallback], skipFileChecks: Boolean): JList[FailMessage] = { import scala.jdk.CollectionConverters._ @@ -63,7 +63,7 @@ object CsvValidatorJavaBridge { errors.map{ asJavaMessage(_) }.toList.asJava case Validated.Valid(_) => - val validator = createValidator(failFast, pathSubs, enforceCaseSensitivePathChecks, trace) + val validator = createValidator(failFast, pathSubs, enforceCaseSensitivePathChecks, trace, skipFileChecks) validator.parseSchema(csvSchemaTextFile) match { case Validated.Invalid(errors) => @@ -88,13 +88,13 @@ object CsvValidatorJavaBridge { validate(csvData, csvSchema, failFast, pathSubstitutionsList, enforceCaseSensitivePathChecks, trace, Some(sProgressCallback)) } - private def validate(csvData: JReader, csvSchema: JReader, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: Option[SProgressCallback]): JList[FailMessage] = { + private def validate(csvData: JReader, csvSchema: JReader, failFast: Boolean, pathSubstitutionsList: JList[Substitution], enforceCaseSensitivePathChecks: Boolean, trace: Boolean, progress: Option[SProgressCallback], skipFileChecks: Boolean = false): JList[FailMessage] = { import scala.jdk.CollectionConverters._ val pathSubs: List[(String,String)] = pathSubstitutionsList.asScala.map( x => (x.getFrom, x.getTo)).toList - - val validator = createValidator(failFast, pathSubs, enforceCaseSensitivePathChecks, trace) + + val validator = createValidator(failFast, pathSubs, enforceCaseSensitivePathChecks, trace, skipFileChecks) validator.parseSchema(csvSchema) match { case Validated.Invalid(errors) => diff --git a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala index 61965083..2066f503 100644 --- a/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala +++ b/csv-validator-ui/src/main/scala/uk/gov/nationalarchives/csv/validator/ui/CsvValidatorUi.scala @@ -135,7 +135,7 @@ object CsvValidatorUi extends SimpleSwingApplication { } } - private def validate(csvFilePath: String, csvEncoding: Charset, csvSchemaFilePath: String, csvSchemaEncoding: Charset, failOnFirstError: Boolean, pathSubstitutions: List[(String, String)], enforceCaseSensitivePathChecks: Boolean, progress: Option[ProgressCallback], validateEncoding: Boolean)(output: String => Unit) : Unit = { + private def validate(csvFilePath: String, csvEncoding: Charset, csvSchemaFilePath: String, csvSchemaEncoding: Charset, failOnFirstError: Boolean, pathSubstitutions: List[(String, String)], enforceCaseSensitivePathChecks: Boolean, progress: Option[ProgressCallback], validateEncoding: Boolean, skipFileChecks: Boolean, outputTextSuffix: String)(output: String => Unit) : Unit = { def toConsole(msg: String): Unit = Swing.onEDT { output(msg) @@ -169,11 +169,12 @@ object CsvValidatorUi extends SimpleSwingApplication { enforceCaseSensitivePathChecks, trace = false, progress, + skipFileChecks, rowCallback ) cliResult._2 match { - case SystemExitCodes.ValidCsv => toConsole("PASS") + case SystemExitCodes.ValidCsv => toConsole(s"PASS$outputTextSuffix") case _ => toConsole(cliResult._1) } } @@ -338,6 +339,7 @@ object CsvValidatorUi extends SimpleSwingApplication { scrollPane.viewportView = txtArReport private val btnValidate = new Button("Validate") + private val btnValidateMetadataOnly = new Button("Validate Metadata Only") private val progressBar = new ProgressBar() progressBar.visible = false @@ -366,35 +368,46 @@ object CsvValidatorUi extends SimpleSwingApplication { } - btnValidate.reactions += onClick(displayWait( - suspendUi = { - btnValidate.enabled = false - this.peer.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)) - this.progressBar.value = 0 - this.progressBar.visible = true - - }, - action = { - txtArReport.text = "" - CsvValidatorUi.this.validate( - txtCsvFile.getText, - settingsPanel.csvEncoding, - txtCsvSchemaFile.getText, - settingsPanel.csvSchemaEncoding, - settingsPanel.failOnFirstError, - settingsPanel.pathSubstitutions, - settingsPanel.enforceCaseSensitivePathChecks, - Some(progress), - settingsPanel.validateUtf8 - ) - }, - output = outputToReport, - resumeUi = { - btnValidate.enabled = true - this.peer.setCursor(Cursor.getDefaultCursor) - //this.progressBar.visible = false - } - )) + btnValidate.reactions += validateOnClick(false) + btnValidateMetadataOnly.reactions += validateOnClick(true) + + private def validateOnClick(skipFileChecks: Boolean) = { + val suffix = if(skipFileChecks) " (Metadata Only)" else "" + onClick(displayWait( + suspendUi = { + btnValidate.enabled = false + btnValidateMetadataOnly.enabled = false + this.peer.setCursor(Cursor.getPredefinedCursor(Cursor.WAIT_CURSOR)) + this.progressBar.value = 0 + this.progressBar.visible = true + + }, + action = { + txtArReport.text = "" + CsvValidatorUi.this.validate( + txtCsvFile.getText, + settingsPanel.csvEncoding, + txtCsvSchemaFile.getText, + settingsPanel.csvSchemaEncoding, + settingsPanel.failOnFirstError, + settingsPanel.pathSubstitutions, + settingsPanel.enforceCaseSensitivePathChecks, + Some(progress), + settingsPanel.validateUtf8, + skipFileChecks, + suffix + ) + }, + output = outputToReport, + resumeUi = { + btnValidate.enabled = true + btnValidateMetadataOnly.enabled = true + btnValidate.peer.setCursor(Cursor.getDefaultCursor) + btnValidateMetadataOnly.peer.setCursor(Cursor.getDefaultCursor) + //this.progressBar.visible = false + } + )) + } private val separator2 = new Separator @@ -434,6 +447,7 @@ object CsvValidatorUi extends SimpleSwingApplication { layout.row.center.fill.add(settingsPanel) layout.row.center.fill.add(btnValidate) + layout.row.center.fill.add(btnValidateMetadataOnly) layout.row.center.fill.add(progressBar) layout.row.center.fill.add(separator1)