Add XML format normalization; add XML unit tests.

harvard-lts · Apr 9, 2018 · f98f8b6 · f98f8b6
1 parent 4f2c3ab
commit f98f8b6
Show file tree

Hide file tree

Showing 7 changed files with 267 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -14,3 +14,4 @@ TAGS
 /build/
 /fits-tests.log
 /javadoc/
+/target/
diff --git a/testfiles/output/test.xml_XmlUnitExpectedOutput.xml b/testfiles/output/test.xml_XmlUnitExpectedOutput.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<fits xmlns="http://hul.harvard.edu/ois/xml/ns/fits/fits_output" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hul.harvard.edu/ois/xml/ns/fits/fits_output http://hul.harvard.edu/ois/xml/xsd/fits/fits_output.xsd" version="1.2.1" timestamp="4/9/18 4:45 PM">
+  <identification>
+    <identity format="Extensible Markup Language" mimetype="text/xml" toolname="FITS" toolversion="1.2.1">
+      <tool toolname="Droid" toolversion="6.3" />
+      <tool toolname="Jhove" toolversion="1.20" />
+      <tool toolname="file utility" toolversion="5.31" />
+      <tool toolname="Exiftool" toolversion="10.00" />
+      <tool toolname="NLNZ Metadata Extractor" toolversion="3.6GA" />
+      <tool toolname="OIS XML Metadata" toolversion="0.2" />
+      <tool toolname="ffident" toolversion="0.2" />
+      <tool toolname="Tika" toolversion="1.10" />
+      <version toolname="Droid" toolversion="6.3">1.0</version>
+      <externalIdentifier toolname="Droid" toolversion="6.3" type="puid">fmt/101</externalIdentifier>
+    </identity>
+  </identification>
+  <fileinfo>
+    <size toolname="Jhove" toolversion="1.20">3219</size>
+    <filepath toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">/Users/dan179/git/git-daveneiman/fits/testfiles/test.xml</filepath>
+    <filename toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">test.xml</filename>
+    <md5checksum toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">43ebf426e75119483bbea6729ba567e9</md5checksum>
+    <fslastmodified toolname="OIS File Information" toolversion="0.2" status="SINGLE_RESULT">1523306483000</fslastmodified>
+  </fileinfo>
+  <filestatus>
+    <well-formed toolname="Jhove" toolversion="1.20" status="SINGLE_RESULT">true</well-formed>
+    <valid toolname="Jhove" toolversion="1.20" status="SINGLE_RESULT">true</valid>
+  </filestatus>
+  <metadata>
+    <text>
+      <charset toolname="Jhove" toolversion="1.20">utf-8</charset>
+      <markupBasis toolname="Jhove" toolversion="1.20">XML</markupBasis>
+      <markupBasisVersion toolname="Jhove" toolversion="1.20">1.0</markupBasisVersion>
+      <markupLanguage toolname="OIS XML Metadata" toolversion="0.2" status="SINGLE_RESULT">http://www.loc.gov/standards/mets/mets.xsd</markupLanguage>
+    </text>
+  </metadata>
+  <statistics fitsExecutionTime="1430">
+    <tool toolname="MediaInfo" toolversion="0.7.75" status="did not run" />
+    <tool toolname="OIS Audio Information" toolversion="0.1" status="did not run" />
+    <tool toolname="ADL Tool" toolversion="0.1" status="did not run" />
+    <tool toolname="VTT Tool" toolversion="0.1" status="did not run" />
+    <tool toolname="Droid" toolversion="6.3" executionTime="270" />
+    <tool toolname="Jhove" toolversion="1.20" executionTime="1386" />
+    <tool toolname="file utility" toolversion="5.31" executionTime="613" />
+    <tool toolname="Exiftool" toolversion="10.00" executionTime="614" />
+    <tool toolname="NLNZ Metadata Extractor" toolversion="3.6GA" executionTime="590" />
+    <tool toolname="OIS File Information" toolversion="0.2" executionTime="204" />
+    <tool toolname="OIS XML Metadata" toolversion="0.2" executionTime="261" />
+    <tool toolname="ffident" toolversion="0.2" executionTime="586" />
+    <tool toolname="Tika" toolversion="1.10" executionTime="438" />
+  </statistics>
+</fits>
+
diff --git a/testfiles/test.xml b/testfiles/test.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="utf-8" standalone="no"?>
+<mets LABEL="Manks, Dorothy St. John. Massachusetts Horticultural Society library :a classification scheme for horticulture. Boston, Mass. : The Society, 1951." TYPE="PAGEDOBJECT" PROFILE="HUL" xmlns="http://www.loc.gov/METS/" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:pds="http://hul.harvard.edu/ois/xml/ns/pds" xsi:schemaLocation="http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/mets.xsd http://hul.harvard.edu/ois/xml/ns/pds http://hul.harvard.edu/ois/xml/xsd/pds/pds.xsd">
+ <dmdSec ID="C0" GROUPID="CITATION">
+  <mdWrap MDTYPE="MODS">
+   <xmlData>
+    <mods:mods xmlns:mods="http://www.loc.gov/mods/v3" xsi:schemaLocation="http://www.loc.gov/mods/v3 http://www.loc.gov/standards/mods/v3/mods-3-0.xsd" version="3.0">
+     <mods:name>
+      <mods:displayForm>Manks, Dorothy St. John</mods:displayForm>
+     </mods:name>
+     <mods:titleInfo>
+      <mods:title>Massachusetts Horticultural Society library :a classification scheme for horticulture</mods:title>
+     </mods:titleInfo>
+     <mods:identifier type="hollis">123</mods:identifier>
+    </mods:mods>
+   </xmlData>
+  </mdWrap>
+ </dmdSec>
+ <dmdSec ID="C1" GROUPID="CITATION">
+  <mdWrap MDTYPE="OTHER" OTHERMDTYPE="HULPDS">
+   <xmlData>
+    <pds:pds/>
+   </xmlData>
+  </mdWrap>
+ </dmdSec>
+ <fileSec>
+  <fileGrp ID="text-plain">
+   <file ID="Fkeyed_text-321808_page_1.txt" MIMETYPE="text/plain">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662232"/>
+   </file>
+   <file ID="Fkeyed_text-321809_page_2.txt" MIMETYPE="text/plain">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662233"/>
+   </file>
+   <file ID="Fkeyed_text-321811_page_3.txt" MIMETYPE="text/plain">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662234"/>
+   </file>
+  </fileGrp>
+  <fileGrp ID="image-tiff">
+   <file ID="Fdeliverable-168798_page_1.tif" MIMETYPE="image/tiff">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662229"/>
+   </file>
+   <file ID="Fdeliverable-168799_page_2.tif" MIMETYPE="image/tiff">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662230"/>
+   </file>
+   <file ID="Fdeliverable-168800_page_3.tif" MIMETYPE="image/tiff">
+    <FLocat LOCTYPE="OTHER" OTHERLOCTYPE="OracleID" xlink:type="simple" xlink:href="6662231"/>
+   </file>
+  </fileGrp>
+ </fileSec>
+ <structMap>
+  <div LABEL="Citation Label8" DMDID="C0 C1" TYPE="CITATION">
+   <div LABEL="First Section" TYPE="INTERMEDIATE">
+    <div ORDER="1" ORDERLABEL="1" LABEL="page 1" TYPE="PAGE">
+     <fptr FILEID="Fdeliverable-168798_page_1.tif"/>
+     <fptr FILEID="Fkeyed_text-321808_page_1.txt"/>
+    </div>
+    <div ORDER="2" ORDERLABEL="2" LABEL="page 2" TYPE="PAGE">
+     <fptr FILEID="Fdeliverable-168799_page_2.tif"/>
+     <fptr FILEID="Fkeyed_text-321809_page_2.txt"/>
+    </div>
+    <div ORDER="3" ORDERLABEL="3" LABEL="page 3" TYPE="PAGE">
+     <fptr FILEID="Fdeliverable-168800_page_3.tif"/>
+     <fptr FILEID="Fkeyed_text-321811_page_3.txt"/>
+    </div>
+   </div>
+  </div>
+ </structMap>
+</mets>
diff --git a/tests/edu/harvard/hul/ois/fits/junit/XmlFileTest.java b/tests/edu/harvard/hul/ois/fits/junit/XmlFileTest.java
@@ -0,0 +1,66 @@
+/* 
+ * Copyright 2016 Harvard University Library
+ * 
+ * This file is part of FITS (File Information Tool Set).
+ * 
+ * FITS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * FITS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with FITS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package edu.harvard.hul.ois.fits.junit;
+
+import java.io.File;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import edu.harvard.hul.ois.fits.Fits;
+import edu.harvard.hul.ois.fits.FitsOutput;
+import edu.harvard.hul.ois.fits.tests.AbstractXmlUnitTest;
+
+/**
+ * These generated FITS output with tool output on the given files files.
+ * 
+ * @author dan179
+ */
+public class XmlFileTest extends AbstractXmlUnitTest {
+
+	/*
+	 *  Only one Fits instance is needed to run tests.
+	 *  This also speeds up the tests.
+	 */
+	private static Fits fits;
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		// Set up FITS for entire class.
+		File fitsConfigFile = new File("testfiles/properties/fits-full-with-tool-output.xml");
+		fits = new Fits(null, fitsConfigFile);
+	}
+
+	@AfterClass
+	public static void afterClass() {
+		fits = null;
+	}
+
+	@Test
+	public void testXmlFile() throws Exception {
+
+    	String inputFilename = "test.xml";
+    	File input = new File("testfiles/" + inputFilename);
+    	FitsOutput fitsOut = fits.examine(input);
+    	fitsOut.addStandardCombinedFormat();
+    	fitsOut.saveToDisk("test-generated-output/" + inputFilename + "_Output.xml");
+	}
+
+}
diff --git a/tests/edu/harvard/hul/ois/fits/junit/XmlFileXmlUnitTest.java b/tests/edu/harvard/hul/ois/fits/junit/XmlFileXmlUnitTest.java
@@ -0,0 +1,80 @@
+/* 
+ * Copyright 2016 Harvard University Library
+ * 
+ * This file is part of FITS (File Information Tool Set).
+ * 
+ * FITS is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * FITS is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ * 
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with FITS.  If not, see <http://www.gnu.org/licenses/>.
+ */
+package edu.harvard.hul.ois.fits.junit;
+
+import java.io.File;
+import java.util.Scanner;
+
+import org.jdom.output.Format;
+import org.jdom.output.XMLOutputter;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import edu.harvard.hul.ois.fits.Fits;
+import edu.harvard.hul.ois.fits.FitsOutput;
+import edu.harvard.hul.ois.fits.tests.AbstractXmlUnitTest;
+
+/**
+ * These generated FITS output with tool output on the given files files.
+ * These tests should be run with &lt;display-tool-output&gt;false&lt;/display-tool-output&gt; in fits.xml.
+ * 
+ * @author dan179
+ */
+public class XmlFileXmlUnitTest extends AbstractXmlUnitTest {
+
+	/*
+	 *  Only one Fits instance is needed to run tests.
+	 *  This also speeds up the tests.
+	 */
+	private static Fits fits;
+
+	@BeforeClass
+	public static void beforeClass() throws Exception {
+		// Set up FITS for entire class.
+		fits = new Fits();
+	}
+
+	@AfterClass
+	public static void afterClass() {
+		fits = null;
+	}
+
+	@Test
+	public void testXmlFile() throws Exception {
+
+    	String inputFilename = "test.xml";
+    	File input = new File("testfiles/" + inputFilename);
+    	FitsOutput fitsOut = fits.examine(input);
+    	fitsOut.saveToDisk("test-generated-output/" + inputFilename + ACTUAL_OUTPUT_FILE_SUFFIX);
+
+		XMLOutputter serializer = new XMLOutputter(Format.getPrettyFormat());
+		String actualXmlStr = serializer.outputString(fitsOut.getFitsXml());
+
+		// Read in the expected XML file
+		Scanner scan = new Scanner(new File(
+				"testfiles/output/" + inputFilename + EXPECTED_OUTPUT_FILE_SUFFIX));
+		String expectedXmlStr = scan.
+				useDelimiter("\\Z").next();
+		scan.close();
+
+		testActualAgainstExpected(actualXmlStr, expectedXmlStr, inputFilename);
+	}
+
+}
diff --git a/tests/edu/harvard/hul/ois/fits/junit/ZipXmlTest.java b/tests/edu/harvard/hul/ois/fits/junit/ZipXmlTest.java
@@ -30,7 +30,6 @@
 
 /**
  * These generated FITS output with tool output on the given files files.
- * These tests should be run with &lt;display-tool-output&gt;false&lt;/display-tool-output&gt; in fits.xml.
  * 
  * @author dan179
  */

diff --git a/xml/fileutility/fileutility_to_fits.xslt b/xml/fileutility/fileutility_to_fits.xslt
@@ -217,7 +217,7 @@
 						</xsl:analyze-string>					
 					</xsl:when>						
 					<!--  XML -->
-					<xsl:when test="$mime='application/xml'">
+					<xsl:when test="$mime='application/xml' or $mime='text/xml'">
 						<!-- test if format contains: XML VERSION* SOMETHING,* ELSE* -->
 						<!-- whereas: * is optional --> 
 						<!-- version <= file-5.24 example: XML document text -->
-Original file line number
+Diff line change
@@ Expand Up / @@ -30,7 +30,6 @@ @@
     /**
      * These generated FITS output with tool output on the given files files.
-     * These tests should be run with &lt;display-tool-output&gt;false&lt;/display-tool-output&gt; in fits.xml.
      *
      * @author dan179
      */
@@ Expand Down @@