Merge pull request #240 from kb-dk/logProblemsFixed

Use slf4j globally to fix log problems. Log4j2 in warc-indexer, log4j in hadoop modules
ukwa · Dec 7, 2020 · a166803 · a166803
2 parents 0a15781 + 9276f6a
commit a166803
Show file tree

Hide file tree

Showing 101 changed files with 727 additions and 822 deletions.
diff --git a/digipres-tika/pom.xml b/digipres-tika/pom.xml
@@ -55,6 +55,15 @@
       </plugins>
    </build>
    <dependencies>
+
+      <dependency>
+         <!--slf4j simple for tests-->
+         <groupId>org.slf4j</groupId>
+         <artifactId>slf4j-simple</artifactId>
+         <version>1.7.30</version>
+         <scope>test</scope>
+      </dependency>
+
       <dependency>
          <groupId>com.typesafe</groupId>
          <artifactId>config</artifactId>
@@ -75,10 +84,6 @@
          <artifactId>tika-parsers</artifactId>
          <version>${tika.version}</version>
          <exclusions>
-            <exclusion>
-               <artifactId>slf4j-api</artifactId>
-               <groupId>org.slf4j</groupId>
-            </exclusion>
             <exclusion>
                <groupId>org.apache.cxf</groupId>
                <artifactId>cxf-rt-transports-http</artifactId>
@@ -123,16 +128,7 @@
          <type>jar</type>
          <scope>test</scope>
       </dependency>
-      <dependency>
-         <groupId>org.apache.logging.log4j</groupId>
-         <artifactId>log4j-1.2-api</artifactId>
-         <version>2.13.1</version>
-      </dependency>
-      <dependency>
-         <groupId>org.apache.logging.log4j</groupId>
-         <artifactId>log4j-core</artifactId>
-         <version>2.13.2</version>
-      </dependency>
+
       <dependency>
          <groupId>com.github.stephenc.java-iso-tools</groupId>
          <artifactId>loopy-core</artifactId>
@@ -145,10 +141,6 @@
          <artifactId>nanite-core</artifactId>
          <version>1.4.0-97</version>
          <exclusions>
-            <exclusion>
-               <artifactId>slf4j-api</artifactId>
-               <groupId>org.slf4j</groupId>
-            </exclusion>
             <exclusion>
                <groupId>org.apache.cxf</groupId>
                <artifactId>cxf-rt-bindings-http</artifactId>
@@ -185,10 +177,6 @@
                <groupId>org.springframework</groupId>
                <artifactId>spring-beans</artifactId>
             </exclusion>
-            <exclusion>
-               <groupId>org.slf4j</groupId>
-               <artifactId>slf4j-log4j12</artifactId>
-            </exclusion>
          </exclusions>
       </dependency>
       <dependency>
@@ -201,10 +189,6 @@
          <artifactId>webarchive-commons</artifactId>
          <version>${webarchive.commons.version}</version>
          <exclusions>
-            <exclusion>
-               <artifactId>slf4j-api</artifactId>
-               <groupId>org.slf4j</groupId>
-            </exclusion>
             <exclusion>
                <artifactId>hadoop-core</artifactId>
                <groupId>org.apache.hadoop</groupId>
@@ -232,12 +216,7 @@
          <artifactId>guava</artifactId>
          <version>18.0</version>
       </dependency>
-      <dependency>
-         <groupId>org.slf4j</groupId>
-         <artifactId>slf4j-simple</artifactId>
-         <version>1.7.4</version>
-         <scope>test</scope>
-      </dependency>
+
       <dependency>
          <groupId>org.netpreserve.openwayback</groupId>
          <artifactId>openwayback-core</artifactId>

diff --git a/digipres-tika/src/main/java/uk/bl/wa/tika/PreservationParser.java b/digipres-tika/src/main/java/uk/bl/wa/tika/PreservationParser.java
@@ -30,14 +30,15 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import org.apache.log4j.Logger;
 import org.apache.tika.exception.TikaException;
 import org.apache.tika.extractor.EmbeddedDocumentExtractor;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.Parser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
 
@@ -50,7 +51,7 @@
  *
  */
 public class PreservationParser extends AutoDetectParser {
-    private static Logger log = Logger.getLogger(PreservationParser.class.getName());
+    private static Logger log = LoggerFactory.getLogger(PreservationParser.class.getName());
 
     public static final String EXT_MIME_TYPE = "Extended-MIME-Type";
 

diff --git a/digipres-tika/src/main/java/uk/bl/wa/tika/TikaDeepIdentifier.java b/digipres-tika/src/main/java/uk/bl/wa/tika/TikaDeepIdentifier.java
@@ -33,7 +33,6 @@
 import java.io.InterruptedIOException;
 
 import org.apache.commons.io.IOUtils;
-import org.apache.log4j.Logger;
 import org.apache.tika.detect.CompositeDetector;
 import org.apache.tika.detect.Detector;
 import org.apache.tika.io.TikaInputStream;
@@ -42,6 +41,8 @@
 import org.apache.tika.parser.AutoDetectParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.sax.WriteOutContentHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.xml.sax.ContentHandler;
 
 
@@ -52,7 +53,7 @@
  */
 public class TikaDeepIdentifier {
 
-    private static Logger log = Logger.getLogger(TikaDeepIdentifier.class.getName());
+    private static Logger log = LoggerFactory.getLogger(TikaDeepIdentifier.class.getName());
 
     private static int MAX_BUF = 1024*1024;
 

diff --git a/digipres-tika/src/main/java/uk/bl/wa/tika/detect/HighlightJSDetector.java b/digipres-tika/src/main/java/uk/bl/wa/tika/detect/HighlightJSDetector.java
@@ -26,13 +26,14 @@
 import javax.script.ScriptEngineManager;
 import javax.script.ScriptException;
 
-import org.apache.log4j.Logger;
 import org.apache.tika.detect.Detector;
 import org.apache.tika.io.CloseShieldInputStream;
 import org.apache.tika.metadata.Metadata;
 import org.apache.tika.metadata.Property;
 import org.apache.tika.mime.MediaType;
 import org.apache.tika.mime.MediaTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
@@ -43,7 +44,7 @@
 import java.util.HashMap;
 
 public class HighlightJSDetector implements Detector {
-    private static Logger log = Logger.getLogger(HighlightJSDetector.class.getName());
+    private static Logger log = LoggerFactory.getLogger(HighlightJSDetector.class.getName());
 
     /**
      * 

diff --git a/digipres-tika/src/main/java/uk/bl/wa/tika/parser/pdf/pdfbox/PDFParser.java b/digipres-tika/src/main/java/uk/bl/wa/tika/parser/pdf/pdfbox/PDFParser.java
@@ -51,7 +51,6 @@
 
 import org.apache.jempbox.xmp.XMPMetadata;
 import org.apache.jempbox.xmp.XMPSchemaPDF;
-import org.apache.log4j.Logger;
 import org.apache.pdfbox.cos.COSArray;
 import org.apache.pdfbox.cos.COSBase;
 import org.apache.pdfbox.cos.COSDictionary;
@@ -71,6 +70,8 @@
 import org.apache.tika.parser.AbstractParser;
 import org.apache.tika.parser.ParseContext;
 import org.apache.tika.parser.PasswordProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.xml.sax.ContentHandler;
 import org.xml.sax.SAXException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -87,7 +88,7 @@
  */
 public class PDFParser extends AbstractParser {
 
-    private static Logger log = Logger.getLogger(PDFParser.class);
+    private static Logger log = LoggerFactory.getLogger(PDFParser.class);
 
     /** Serial version UID */
     private static final long serialVersionUID = -752276948656079347L;

diff --git a/pom.xml b/pom.xml
@@ -167,6 +167,12 @@
    </build>
 
    <repositories>
+      <repository>
+         <id>version99</id>
+         <!-- highly available repository serving empty artifacts -->
+         <url>http://version99.qos.ch/</url>
+      </repository>
+
       <repository>
          <id>maven-restlet</id>
          <name>Public online Restlet repository</name>
@@ -205,4 +211,32 @@
          <url>https://repository.apache.org/snapshots/</url>
       </repository>
    </repositories>
+
+   <dependencies>
+      <!--        Logging-->
+      <dependency>
+         <groupId>org.slf4j</groupId>
+         <artifactId>slf4j-api</artifactId>
+         <version>${slf4j.version.override}</version>
+      </dependency>
+      <dependency>
+         <groupId>org.slf4j</groupId>
+         <artifactId>jcl-over-slf4j</artifactId>
+         <version>${slf4j.version.override}</version>
+      </dependency>
+   </dependencies>
+   <dependencyManagement>
+      <dependencies>
+         <dependency><!--Per default replace commons logging with empty jar, for all dependencies-->
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>99-empty</version>
+         </dependency>
+         <dependency> <!--Ignore log4j dependencies. We only include log4j if we actually want to-->
+            <groupId>log4j</groupId>
+            <artifactId>log4j</artifactId>
+            <version>99-empty</version>
+         </dependency>
+      </dependencies>
+   </dependencyManagement>
 </project>
diff --git a/warc-hadoop-crawler-utils/src/main/java/uk/bl/wa/hadoop/recrawl/PersistLogMapper.java b/warc-hadoop-crawler-utils/src/main/java/uk/bl/wa/hadoop/recrawl/PersistLogMapper.java
@@ -22,8 +22,8 @@
 import org.apache.commons.codec.binary.Base64;
 import org.apache.commons.httpclient.HttpParser;
 import org.apache.commons.lang.SerializationUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.mapred.MapReduceBase;
 import org.apache.hadoop.mapred.Mapper;
@@ -51,7 +51,7 @@
 @SuppressWarnings({ "unchecked", "deprecation" })
 public class PersistLogMapper extends MapReduceBase implements
     Mapper<Text, WritableArchiveRecord, Text, Text> {
-    private static final Log LOGGER = LogFactory.getLog(PersistLogMapper.class);
+    private static final Logger LOGGER = LoggerFactory.getLogger(PersistLogMapper.class);
 
     ArchiveRecordHeader responseHeader = null;
     CrawlURI curi = null;

diff --git a/warc-hadoop-indexer/pom.xml b/warc-hadoop-indexer/pom.xml
@@ -19,7 +19,7 @@
 		<plugins>
 			<plugin>
 				<artifactId>maven-assembly-plugin</artifactId>
-				<version>3.1.0</version>
+				<version>3.3.0</version>
 				<configuration>
 					<descriptors>
 						<descriptor>src/main/assembly/hadoop-job.xml</descriptor>
@@ -46,6 +46,18 @@
 		</plugins>
 	</build>
 	<dependencies>
+
+		<dependency>
+			<groupId>org.slf4j</groupId>
+			<artifactId>slf4j-log4j12</artifactId>
+			<version>${slf4j.version.override}</version>
+		</dependency>
+		<dependency>
+			<groupId>log4j</groupId>
+			<artifactId>log4j</artifactId>
+			<version>1.2.17</version>
+		</dependency>
+
         <dependency>
             <groupId>uk.bl.wa.discovery</groupId>
             <artifactId>warc-hadoop-recordreaders</artifactId>
@@ -64,66 +76,17 @@
 			<version>3.2.0-SNAPSHOT</version>
 			<exclusions>
 				<exclusion>
-					<artifactId>logback-classic</artifactId>
-					<groupId>ch.qos.logback</groupId>
-				</exclusion>
-				<exclusion>
-					<artifactId>logback-core</artifactId>
-					<groupId>ch.qos.logback</groupId>
-				</exclusion>
-				<exclusion>
-				   <groupId>org.slf4j</groupId>
-				   <artifactId>jcl-over-slf4j</artifactId>
-				</exclusion>
-				<exclusion>
-				   <groupId>org.slf4j</groupId>
-				   <artifactId>slf4j-api</artifactId>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>*</artifactId>
 				</exclusion>
-				<exclusion>
-				   <groupId>org.slf4j</groupId>
-				   <artifactId>jul-to-slf4j</artifactId>
-				</exclusion>
-				<exclusion>
-				   <groupId>org.slf4j</groupId>
-				   <artifactId>slf4j-log4j12</artifactId>
-				</exclusion>
-               <exclusion>
-                  <groupId>commons-logging</groupId>
-                  <artifactId>commons-logging</artifactId>
-               </exclusion>
-			</exclusions>
-		</dependency>
-		<dependency>
-			<groupId>org.slf4j</groupId>
-			<artifactId>slf4j-log4j12</artifactId>
-			<version>${slf4j.version.override}</version>
-			<exclusions>
-			   <exclusion>
-			      <groupId>org.slf4j</groupId>
-			      <artifactId>slf4j-api</artifactId>
-			   </exclusion>
 			</exclusions>
 		</dependency>
 		<dependency>
 		   <groupId>org.apache.solr</groupId>
 		   <artifactId>solr-solrj</artifactId>
 		   <version>${solr.version}</version>
-		   <exclusions>
-		      <exclusion>
-		         <groupId>org.slf4j</groupId>
-		         <artifactId>slf4j-api</artifactId>
-		      </exclusion>
-		      <exclusion>
-		         <groupId>org.slf4j</groupId>
-		         <artifactId>jcl-over-slf4j</artifactId>
-		      </exclusion>
-		   </exclusions>
-		</dependency>
-		<dependency>
-		   <groupId>org.slf4j</groupId>
-		   <artifactId>slf4j-api</artifactId>
-		   <version>${slf4j.version.override}</version>
 		</dependency>
+
       <dependency>
          <groupId>org.apache.hadoop</groupId>
          <artifactId>hadoop-core</artifactId>
@@ -135,12 +98,6 @@
          <artifactId>hadoop-test</artifactId>
          <version>${hadoop.version}</version>
          <scope>test</scope>
-         <exclusions>
-            <exclusion>
-               <artifactId>slf4j-api</artifactId>
-               <groupId>org.slf4j</groupId>
-            </exclusion>
-         </exclusions>
       </dependency>
       <dependency>
         <groupId>org.apache.mrunit</groupId>

diff --git a/warc-hadoop-indexer/src/main/java/uk/bl/wa/apache/solr/hadoop/WebArchiveIndexerMapper.java b/warc-hadoop-indexer/src/main/java/uk/bl/wa/apache/solr/hadoop/WebArchiveIndexerMapper.java
@@ -30,8 +30,8 @@
 //import java.security.NoSuchAlgorithmException;
 //import java.util.Properties;
 //
-//import org.apache.commons.logging.Log;
-//import org.apache.commons.logging.LogFactory;
+//import org.slf4j.Logger;
+//import org.slf4j.LoggerFactory;
 //import org.apache.hadoop.io.Text;
 //import org.apache.log4j.PropertyConfigurator;
 //import org.archive.io.ArchiveRecordHeader;
@@ -51,8 +51,8 @@
 // */
 //public class WebArchiveIndexerMapper extends
 //        SolrMapper<Text, WritableArchiveRecord> {
-//    private static final Log LOG = LogFactory
-//            .getLog(WebArchiveIndexerMapper.class);
+//    private static final Logger LOG = LoggerFactory
+//            .getLogger(WebArchiveIndexerMapper.class);
 //
 //    private WARCIndexer windex;
 //