diff --git a/NOTICE b/NOTICE index bf5ec6d3d..58b02727f 100644 --- a/NOTICE +++ b/NOTICE @@ -43,6 +43,7 @@ This project includes: CDI APIs under Apache License, Version 2.0 Common Eclipse Runtime under Eclipse Public License - v 1.0 Commons Codec under The Apache Software License, Version 2.0 + Commons Compiler under New BSD License Commons IO under The Apache Software License, Version 2.0 Commons Logging under The Apache Software License, Version 2.0 Core Runtime under Eclipse Public License - v 1.0 @@ -57,7 +58,7 @@ This project includes: Fluent API for Apache HttpClient under Apache License Guava: Google Core Libraries for Java under The Apache Software License, Version 2.0 Hamcrest Core under New BSD License - janino.wso2:janino under Apache License, Version 2.0 + Janino under New BSD License javax.inject under The Apache Software License, Version 2.0 JCL 1.1.1 implemented over SLF4J under MIT License JSR-250 Common Annotations for the JavaTM Platform under COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 diff --git a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/codeIt.xml b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/codeIt.xml index 2f9c34ebb..b796777d9 100644 --- a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/codeIt.xml +++ b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/codeIt.xml @@ -438,7 +438,7 @@ public PepperMapper createPepperMapper(SElementId sElementId){ In case you are wondering, yes this sounds a bit strange, since each file ending which is not contained in the second list won't be imported by default. But there is an option to set this to import each file, no matter - wjat's the ending. + what's the ending. Now let's show some code for adapting. The following snippet is placed into the method 'isReadyToStart()', but even could be located inside the constructor: @@ -454,7 +454,7 @@ public PepperMapper createPepperMapper(SElementId sElementId){ getSDocumentEndings().add(ENDING_LEAF_FOLDER); ... } - In general the paramter of the method 'In general the parameter of the method 'getSDocumentEndings()' is just a String, but there are some predefined endings you can use. The two lines marked as option 1, will add the endings 'xml' and 'tab' to the list of file endings to be imported. That means, that diff --git a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/quickStart.xml b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/quickStart.xml index d7fec216a..f39b0ee20 100644 --- a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/quickStart.xml +++ b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/quickStart.xml @@ -18,7 +18,7 @@ Download Eclipse Eclipse is available in several flavors e.g. for web developers, - mobile developers, C++ developers etc.. We recommend the Eclipse + mobile developers, C++ developers etc. We recommend the Eclipse Modeling Tools, since you might want to create or use a model for the format you want to support. In this documentation, we used Eclipse kepler, version 4.3.1 (see: http://www.graphviz.org/) into a svg, png or another graphical format. - . + + Replace the entry with an absolute path to that file. @@ -373,8 +374,8 @@ public class MyImporter extends PepperImporterImpl Sometimes it is necessary to include libraries, which are not accessible via a Maven repository and therefore can not be resolved by Maven. In that case we recommend, to create a 'lib' folder in the project directory and to copy all the - libraries you need into it. Unfortunately, you have register them twice, first - for Maven and second for OSGi. + libraries you need into it. Unfortunately, you have to register them twice, + first for Maven and second for OSGi. To register such a library to Maven, you need to install them to your local Maven repository. You can do that with: diff --git a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/testing.xml b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/testing.xml index 46e33b54f..050ba9ca0 100644 --- a/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/testing.xml +++ b/pepper-doc/src/main/docbkx/pepper_modulesDevelopersGuide/testing.xml @@ -77,8 +77,8 @@ protected void setUp() throws Exception { //X SCorpus objects assertEquals(X, getFixture().getSaltProject() .getSCorpusGraphs().getSCorpora().size()); -}More - samples for tests can be found in the sample project in the classes +} + More samples for tests can be found in the sample project in the classes SampleImporterTest, SampleManipulatorTest and SampleExporterTest. @@ -88,5 +88,16 @@ protected void setUp() throws Exception { the layers, you want to use. For more information, take a look into the Salt quick user's guide (see ). + The class de.hu_berlin.german.korpling.saltnpepper.pepper.testFramework.PepperTestUtil + provides some very helpful methods like getSrcResources() to get a reference to the + resources folder (.../src/main/resources) or the method getTestResources to get a + reference to the test resource folder (.../src/test/resources). + When you are implementing an importer and an exporter you may want to make sure that + no data is lost, you can orchestrate the im- and the exporter and compare the input + files with the output files. In that case implementing the classes PepperImporterTest + and PepperExporterTest want do the job, because they assume that you are testing a + single module. To load multiple modules in the Pepper test environment, use the method + PepperUtil.start(Collection<PepperModule> fixtures) instead and pass your im- and + your exporter. diff --git a/pepper-framework/NOTICE b/pepper-framework/NOTICE index 587ed6636..2675dea49 100644 --- a/pepper-framework/NOTICE +++ b/pepper-framework/NOTICE @@ -21,6 +21,7 @@ This project includes: Apache Commons Lang under The Apache Software License, Version 2.0 carrot-osgi-anno-scr-core under The BSD License carrot-osgi-anno-scr-make under The BSD License + Commons IO under The Apache Software License, Version 2.0 Guava: Google Core Libraries for Java under The Apache Software License, Version 2.0 Hamcrest Core under New BSD License JUnit under Common Public License Version 1.0 diff --git a/pepper-framework/pepper-testEnvironment.launch b/pepper-framework/pepper-testEnvironment.launch index 7e417a9b4..986de3f3e 100644 --- a/pepper-framework/pepper-testEnvironment.launch +++ b/pepper-framework/pepper-testEnvironment.launch @@ -15,12 +15,8 @@ -<<<<<<< .mine - -======= ->>>>>>> .r8615 @@ -238,11 +234,7 @@ -<<<<<<< .mine - -======= ->>>>>>> .r8615 diff --git a/pepper-framework/pom.xml b/pepper-framework/pom.xml index 030167176..d1411ccf7 100644 --- a/pepper-framework/pom.xml +++ b/pepper-framework/pom.xml @@ -6,7 +6,7 @@ de.hu_berlin.german.korpling.saltnpepper pepper - 2.1.0 + 2.1.1-SNAPSHOT ../pom.xml diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/CorpusDesc.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/CorpusDesc.java index da666ed5c..1642038d6 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/CorpusDesc.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/CorpusDesc.java @@ -20,9 +20,10 @@ import org.eclipse.emf.common.util.URI; /** - * This class realizes a description of a corpus to be imported or exported. The description consists of a - * path addressing the location of the corpus and a format description, in which the corpus currently is or - * is supposed to be persist in. + * This class realizes a description of a corpus to be imported or exported. The + * description consists of a path addressing the location of the corpus and a + * format description, in which the corpus currently is or is supposed to be + * persist in. * * @author Florian Zipser */ @@ -33,64 +34,77 @@ public class CorpusDesc { public CorpusDesc() { super(); } + /** format description belonging to this corpus */ - protected FormatDesc formatDesc= null; + protected FormatDesc formatDesc = null; + /** - * Returns a set format description. - * If no {@link FormatDesc} object was set, a new one is created. + * Returns a set format description. If no {@link FormatDesc} object was + * set, a new one is created. + * * @return a format description */ public FormatDesc getFormatDesc() { - if (formatDesc== null){ + if (formatDesc == null) { synchronized (this) { - formatDesc= new FormatDesc(); + formatDesc = new FormatDesc(); } } return formatDesc; } + /** * Sets the format description for this corpus description. - * @param formatDesc format description object + * + * @param formatDesc + * format description object * @return this */ public CorpusDesc setFormatDesc(FormatDesc formatDesc) { this.formatDesc = formatDesc; - return(this); + return (this); } - - /** location of corpus**/ - protected URI corpusPath= null; + + /** location of corpus **/ + protected URI corpusPath = null; + /** * Returns the path of where to store or from where to load this corpus. + * * @return location of corpus */ public URI getCorpusPath() { return corpusPath; } + /** * Sets the path of where to store or from where to load this corpus. - * @param corpusPath location of corpus + * + * @param corpusPath + * location of corpus * @param this object */ public CorpusDesc setCorpusPath(URI corpusPath) { this.corpusPath = corpusPath; - return(this); + return (this); } + /** - * Returns a string representation of this object. - * Note: This representation cannot be used for serialization/deserialization purposes. + * Returns a string representation of this object. Note: This + * representation cannot be used for serialization/deserialization + * purposes. */ @Override - public String toString(){ - StringBuilder str= new StringBuilder(); + public String toString() { + StringBuilder str = new StringBuilder(); str.append(getCorpusPath()); - if (getFormatDesc()!= null){ + if (getFormatDesc() != null) { str.append("("); str.append(getFormatDesc().getFormatName()); str.append(", "); str.append(getFormatDesc().getFormatVersion()); str.append(")"); } - return(str.toString()); + return (str.toString()); } } diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/FormatDesc.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/FormatDesc.java index 0f2004913..8e57b8c04 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/FormatDesc.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/FormatDesc.java @@ -106,11 +106,8 @@ public FormatDesc setFormatReference(URI formatReference) { } /** - * Two objects are equal: - *
    - *
  • if they are the same object
  • - *
  • if the passed object is of type {@link FormatDesc} and both {@link FormatDesc#formatName} and both {@link FormatDesc#formatName} are the same
  • - *
+ * Compares a {@link CorpusDesc} object with this object. Both objects are equal, if and only if + * the have the same format name and format version. The comparison is case insensitive. */ @Override public boolean equals(Object obj){ @@ -120,9 +117,9 @@ public boolean equals(Object obj){ if (obj!= null){ if (obj instanceof FormatDesc){ if ( (getFormatName()!= null)&& - (getFormatName().equals(((FormatDesc) obj).getFormatName()))&& + (getFormatName().equalsIgnoreCase(((FormatDesc) obj).getFormatName()))&& (getFormatVersion()!= null)&& - (getFormatVersion().equals(((FormatDesc) obj).getFormatVersion()))){ + (getFormatVersion().equalsIgnoreCase(((FormatDesc) obj).getFormatVersion()))){ return(true); }else{ return(false); diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperConfiguration.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperConfiguration.java index 6a962e345..d3a620f19 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperConfiguration.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperConfiguration.java @@ -107,6 +107,11 @@ public class PepperConfiguration extends Properties { * terminates. */ public static final String PROP_KEEP_TEMP_DOCS = PROP_PREFIX + ".keepTempDocs"; + /** + * Name of the property to determine Property to determine whether the status report should contain a progress for each module + * or just the global progress. + */ + public static final String PROP_DETAILED_STATUS_REPORT = PROP_PREFIX + ".detailedStatusReport"; /** * name of the flag to determine a workspace for pepper, where all jobs are * stored by default. @@ -148,6 +153,7 @@ public PepperConfiguration() { put(PROP_CALL_GC_AFTER_DOCUMENT, Boolean.TRUE); put(PROP_MEMORY_POLICY, MEMORY_POLICY.MODERATE); put(PROP_MAX_AMOUNT_OF_SDOCUMENTS, 10); + put(PROP_DETAILED_STATUS_REPORT, true); } /** @@ -362,4 +368,12 @@ public Integer getReportInterval() { String interval = getProperty(PROP_REPORT_INTERVAL, new Integer(1000).toString()); return (Integer.valueOf(interval)); } + /** + * Property to determine whether the status report should contain a progress for each module or just the global progress. + * @return + */ + public Boolean getDetaialedStatReport() { + String callGC = getProperty(PROP_DETAILED_STATUS_REPORT); + return (Boolean.valueOf(callGC)); + } } diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperJob.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperJob.java index 49a4b735f..345b8e195 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperJob.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperJob.java @@ -17,12 +17,15 @@ */ package de.hu_berlin.german.korpling.saltnpepper.pepper.common; +import java.io.File; +import java.io.IOException; import java.util.List; import java.util.Vector; import org.eclipse.emf.common.util.URI; import de.hu_berlin.german.korpling.saltnpepper.pepper.exceptions.WorkflowException; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.ModuleController; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.SaltProject; public abstract class PepperJob { @@ -35,6 +38,38 @@ public abstract class PepperJob { public String getId() { return id; } + + /** The base directory is either the directory from which the Pepper workflow description was loaded or the directory from + * which Pepper was started. **/ + private URI baseDir= null; + + /** + * Sets the base directory for this {@link ModuleController}. The base directory is either + * the directory from which the Pepper workflow description was loaded or the directory from + * which Pepper was started. + * @param baseDir base directory + */ + public void setBaseDir(URI baseDir){ + this.baseDir= baseDir; + } + + /** + * Returns the base directory for this {@link ModuleController}. The base directory is either + * the directory from which the Pepper workflow description was loaded or the directory from + * which Pepper was started. + * @return base directory + */ + public URI getBaseDir(){ + if (baseDir== null){ + try { + baseDir= URI.createFileURI(new File("./").getCanonicalPath()); + } catch (IOException e) { + baseDir= URI.createFileURI(new File("./").getAbsolutePath()); + } + } + return(baseDir); + } + /** status of job **/ protected JOB_STATUS status= JOB_STATUS.NOT_STARTED; /** @@ -89,21 +124,6 @@ public void addStepDesc(StepDesc stepDesc){ public StepDesc createStepDesc(){ return(new StepDesc()); } -// /** -// * Creates a {@link StepDesc} object an returns it. Further the {@link StepDesc} type {@link StepDesc}s -// * module type is set to the passed one. -// * @param moduleType -// * @return -// */ -// public StepDesc createStepDesc(MODULE_TYPE moduleType){ -// if (moduleType== null){ -// throw new PepperException("Cannot create a step desc object when passed module type is empty."); -// } -// StepDesc stepDesc= createStepDesc(); -// stepDesc.setModuleType(moduleType); -// addStepDesc(stepDesc); -// return(stepDesc); -// } /** * Starts the conversion of this job. diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperUtil.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperUtil.java index f2cb7d644..59f2d48b4 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperUtil.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/PepperUtil.java @@ -534,9 +534,11 @@ public static class PepperJobReporter extends Thread { * the interval in which the status is printed */ public PepperJobReporter(PepperJob pepperJob, int interval) { - if (pepperJob == null) + if (pepperJob == null){ throw new PepperException("Cannot observe Pepper job, because it was null."); + } this.pepperJob = pepperJob; + this.interval= interval; } /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/ModuleControllerImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/ModuleControllerImpl.java index ce6938dcb..8763d129d 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/ModuleControllerImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/ModuleControllerImpl.java @@ -17,12 +17,17 @@ */ package de.hu_berlin.german.korpling.saltnpepper.pepper.core; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.StandardCopyOption; import java.util.HashSet; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.locks.ReentrantLock; +import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,8 +39,12 @@ import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.ModuleController; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperImporter; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModule; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperties; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleException; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpus; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SElementId; /** @@ -217,8 +226,7 @@ public synchronized Future importCorpusStructure(SCorpusGraph sCorpusGraph){ } if (!getBusyLock().tryLock()){ throw new PepperInActionException("Cannot start importing corpus structure, since this module controller currently imports a corpus structure."); - } - else{ + }else{ this.sCorpusGraph= sCorpusGraph; ExecutorService executor = Executors.newSingleThreadExecutor(); Runnable task = new Runnable() { @@ -247,26 +255,32 @@ public void run() { * request all {@link DocumentController} object waiting in the incoming {@link DocumentBus}. */ public synchronized Future processDocumentStructures(){ - if (getPepperModule()== null) + if (getPepperModule()== null){ throw new PepperFWException("Cannot start imort corpus structure, because the contained Pepper module is null."); - if (!getBusyLock().tryLock()) + } + if (!getBusyLock().tryLock()){ throw new PepperInActionException("Cannot start importing corpus structure, since this module controller currently imports a corpus structure."); - else{ + }else{ ExecutorService executor = Executors.newSingleThreadExecutor(); Runnable task = new Runnable() { public void run() { + //calls before() to do some work before everything is processed when set in customization property + before(); getPepperModule().start(); if (getControllList().size()!= 0){ throw new PepperModuleException(getPepperModule(), "Some documents are still in the processing queue by module '"+getPepperModule().getName()+"' and neither set to '"+DOCUMENT_STATUS.COMPLETED+"', '"+DOCUMENT_STATUS.DELETED+"' or '"+DOCUMENT_STATUS.FAILED+"'. Remaining documents are: "+getControllList()); } getOutputDocumentBus().finish(getPepperModule().getModuleController().getId()); mLogger.debug("[{}] completed processing of documents and corpora. ", ((getPepperModule()!= null)?getPepperModule().getName():" EMPTY ")); + //calls after() to do some work after everything was processed when set in customization property + after(); } }; - if (!getBusyLock().tryLock()) + if (!getBusyLock().tryLock()){ throw new PepperInActionException("cannot import document structure, because module controller '"+getId()+"' currently is busy with another process."); + } getBusyLock().lock(); Future future= null; try{ @@ -277,6 +291,103 @@ public void run() { return(future); } } + + /** {@inheritDoc PepperModule#before(SElementId)} */ + private void before() throws PepperModuleException { + + } + + /** {@inheritDoc PepperModule#after(SElementId)} */ + private void after() throws PepperModuleException { + if (getPepperModule().getProperties().getProperty(PepperModuleProperties.PROP_AFTER_COPY_RES) != null) { + // copies resources as files from source to target + + String resString = (String) getPepperModule().getProperties().getProperty(PepperModuleProperties.PROP_AFTER_COPY_RES).getValue(); + copyResources(resString); + } + } + + /** + * Reads customization property {@link PepperModuleProperties#PROP_AFTER_COPY_RES} and copies the listed + * resources to the named target folder. + */ + protected void copyResources(String resString){ + if ((resString != null) && (!resString.isEmpty())) { + String[] resources = resString.split(";"); + if (resources.length > 0) { + for (String resource : resources) { + resource = resource.trim(); + String[] parts = resource.split("->"); + if (parts.length == 2) { + String sourceStr= parts[0]; + String targetStr= parts[1]; + sourceStr= sourceStr.trim(); + targetStr= targetStr.trim(); + + //check if source and target is given + boolean copyOk= true; + if ( (sourceStr== null)|| + (sourceStr.isEmpty())){ + logger.warn("Cannot copy resources for '"+getPepperModule().getName()+"' because no source file was given in property value '"+resource+"'. "); + copyOk= false; + } + if ( (targetStr== null)|| + (targetStr.isEmpty())){ + logger.warn("Cannot copy resources for '"+getPepperModule().getName()+"' because no target file was given in property value '"+resource+"'. "); + copyOk= false; + } + if (copyOk){ + File source= new File(sourceStr); + File target= new File(targetStr); + + // in case of source or target aren't absolute resolve them against current Job's base directory + String baseDir= getJob().getBaseDir().toFileString(); + if (!baseDir.endsWith("/")){ + baseDir= baseDir+ "/"; + } + if (!source.isAbsolute()){ + source= new File(baseDir + sourceStr); + } + if (!source.exists()){ + logger.warn("Cannot copy resources for '"+getPepperModule().getName()+"' because source does not exist '"+source.getAbsolutePath()+"'. Check the property value '"+resource+"'. "); + }else{ + //only copy if source exists + + if (!target.isAbsolute()){ + target= new File(baseDir + targetStr); + } + if (!target.exists()){ + target.mkdirs(); + } + try { + if (source.isDirectory()){ + targetStr= target.getAbsolutePath(); + if (!targetStr.endsWith("/")){ + targetStr= targetStr+"/"; + } + target= new File(targetStr+source.getName()); + FileUtils.copyDirectory(source, target); + logger.trace("Copied resource from '"+source.getAbsolutePath()+"' to '"+target.getAbsolutePath()+"'."); + }else{ + targetStr= target.getCanonicalPath(); + if (!targetStr.endsWith("/")){ + targetStr= targetStr+ "/"; + } + target= new File(targetStr+source.getName()); + FileUtils.copyFile(source, target); + logger.trace("Copied resource from '"+source.getAbsolutePath()+"' to '"+target.getAbsolutePath()+"'."); + } + } catch (IOException e) { + logger.warn("Cannot copy resources for '"+getPepperModule().getName()+"' because of '"+e.getMessage()+"'. Check the property value '"+resource+"'. "); + } + } + } + } + } + } + } + } + /** A lock determining, whether this object currently is busy with importing corpus structure or importing document structure. **/ protected ReentrantLock busyLock= null; /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/PepperJobImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/PepperJobImpl.java index 77b46b41f..a07130d71 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/PepperJobImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/core/PepperJobImpl.java @@ -26,6 +26,7 @@ import java.io.Reader; import java.text.DecimalFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -43,6 +44,7 @@ import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.emf.common.util.URI; @@ -69,7 +71,6 @@ import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperExporter; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperImporter; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModule; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperties; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperty; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleException; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleXMLResourceException; @@ -773,7 +774,7 @@ public String getStatusReport() { distance = globalId.length(); } } - // distance is distance plus 4??? plus length of string sleep + // distance is distance plus 4??? plus length of string 'sleep' distance = distance + 4 + sleep.length() + DOCUMENT_STATUS.IN_PROGRESS.toString().length(); StringBuilder docInfo = null; for (DocumentController docController : getDocumentControllers()) { @@ -797,10 +798,13 @@ public String getStatusReport() { retVal.append(new DecimalFormat("###.##").format(progressOverAll / numOfDocuments * 100) + "%"); } retVal.append("\n"); - retVal.append(detailedStr.toString()); + retVal.append("processing time:\t"); + retVal.append(DurationFormatUtils.formatDurationHMS(getProcessingTime())); + retVal.append("\n"); + if (getConfiguration().getDetaialedStatReport()){ + retVal.append(detailedStr.toString()); + } } - - retVal.append("-------------------------------------------------------------------------"); retVal.append("\n"); return (retVal.toString()); @@ -813,19 +817,34 @@ public String getStatusReport() { * Checks for each {@link PepperModule} in all steps, if it is ready to * start, via calling {@link PepperModule#isReadyToStart()}. * - * @return false, if one of the {@link PepperModule} objects returned false. + * @return a list of steps whose modules are not ready to start */ - protected synchronized Boolean checkReadyToStart() { - Boolean retVal = true; + protected synchronized Collection>> checkReadyToStart() { + ArrayList>> retVal = new ArrayList<>(); for (Step step : getAllSteps()) { if (!step.getModuleController().getPepperModule().isReadyToStart()) { - retVal = false; + Pair> stepReason= new ImmutablePair>(step, step.getModuleController().getPepperModule().getStartProblems()); + retVal.add(stepReason); logger.error("Cannot run pepper job '" + getId() + "', because one of the involved modules '" + step.getModuleController().getPepperModule().getFingerprint() + "' is not ready to run."); } } return (retVal); } + /** Stores the time when this job was started **/ + private Long startTime= 0l; + /** Returns the time when this job was started **/ + private Long getStartTime(){ + return startTime; + } + /** + * Returns the amount of time the job already took. + * @return time in milli seconds + */ + public Long getProcessingTime(){ + return System.currentTimeMillis()- startTime; + } + /** * Specifies if this job currently runs a conversion. If this is the case, * some other operations, like adding {@link Step}s cannot be done @@ -851,13 +870,23 @@ public void convert() { } inProgress.lock(); try { + startTime= System.currentTimeMillis(); status = JOB_STATUS.IN_PROGRESS; if (!isWired) { wire(); } if (!isReadyToStart) { - if (!checkReadyToStart()) { - throw new PepperException("Cannot run Pepper job '" + getId() + "', because one of the involved job is not ready to run."); + Collection>> notReadyModules= checkReadyToStart(); + if (notReadyModules.size() != 0) { + StringBuilder str= new StringBuilder(); + for (Pair> problems: notReadyModules){ + str.append("["); + str.append(problems.getLeft()); + str.append(": "); + str.append(problems.getRight()); + str.append("], "); + } + throw new PepperException("Cannot run Pepper job '" + getId() + "', because at least one of the involved job is not ready to run: '"+str.toString()+"'. "); } } if (!isImportedCorpusStructure){ @@ -1230,7 +1259,9 @@ private void save_module(XMLStreamWriter xml, StepDesc step) throws XMLStreamExc public void load(URI uri) { if (uri.isFile()) { File wdFile = new File(uri.toFileString()); - + //set folder containing workflow description as base dir + setBaseDir(uri.trimSegments(1)); + SAXParser parser; XMLReader xmlReader; SAXParserFactory factory = SAXParserFactory.newInstance(); @@ -1243,9 +1274,9 @@ public void load(URI uri) { xmlReader = parser.getXMLReader(); xmlReader.setContentHandler(contentHandler); } catch (ParserConfigurationException e) { - throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "'.", e); + throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "': "+e.getMessage()+". ", e); } catch (Exception e) { - throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "'.", e); + throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "': "+e.getMessage()+". ", e); } try { InputStream inputStream = new FileInputStream(wdFile); @@ -1260,13 +1291,14 @@ public void load(URI uri) { xmlReader.setContentHandler(contentHandler); xmlReader.parse(wdFile.getAbsolutePath()); } catch (Exception e1) { - throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "'.", e1); + throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file '" + wdFile.getAbsolutePath() + "': "+e1.getMessage()+". ", e1); } } catch (Exception e) { - if (e instanceof PepperModuleException) + if (e instanceof PepperModuleException){ throw (PepperModuleException) e; - else - throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file'" + wdFile + "', because of a nested exception. ", e); + }else{ + throw new PepperModuleXMLResourceException("Cannot load Pepper workflow description file'" + wdFile + "', because of a nested exception: "+e.getMessage()+". ", e); + } } } else{ throw new UnsupportedOperationException("Currently Pepper can only load workflow description from local files."); diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/ModuleController.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/ModuleController.java index 98decd3ac..d6156d0d2 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/ModuleController.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/ModuleController.java @@ -19,6 +19,8 @@ import java.util.concurrent.Future; +import org.eclipse.emf.common.util.URI; + import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperJob; import de.hu_berlin.german.korpling.saltnpepper.pepper.core.DocumentBus; import de.hu_berlin.german.korpling.saltnpepper.pepper.core.ModuleControllerImpl; @@ -65,7 +67,7 @@ public interface ModuleController { * @param job new {@link PepperJobImpl} object */ public void setJob_basic(PepperJobImpl job); - + /** * The {@link DocumentBus} object working as input for this {@link ModuleControllerImpl}. All documents on bus * will be processed and set to {@link #outputDocumentBus} diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperMapperController.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperMapperController.java index 98916f0ea..9c8d30d28 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperMapperController.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperMapperController.java @@ -114,25 +114,25 @@ public interface PepperMapperController extends Runnable{ * @param pepperModule containing {@link PepperModule} object */ public void setPepperModule(PepperModule pepperModule); - /** - * Invokes processings, before the mapping was started. This could be helpful, for instance to make some preparations - * for the mapping. To trigger this pre processing for a specific Pepper module a set of customization properties is - * available. Customization properties triggering a pre processing starts with {@value PepperModuleProperties#PREFIX_PEPPER_BEFORE}. - * This method is called by the method {@link #map()}, before {@link PepperMapper#mapSDocument()} was called. - * @param sElementId id of either {@link SDocument} or {@link SCorpus} object to be prepared - * @throws PepperModuleException - */ - public void before(SElementId sElementId) throws PepperModuleException; - - /** - * Invokes processings, after the mapping is done. This could be helpful, for instance to make some processing - * after the mapping e.g. adding all created nodes and relations to a layer. - * To trigger this post processing for a specific Pepper module a set of customization properties is - * available. Customization properties triggering a post processing starts with {@value PepperModuleProperties#PREFIX_PEPPER_AFTER}. - * This method is called by the method {@link #map()}, after {@link PepperMapper#mapSDocument()} was called. - * @param sElementId id of either {@link SDocument} or {@link SCorpus} object to be post processed - * @throws PepperModuleException - */ - public void after(SElementId sElementId) throws PepperModuleException; +// /** +// * Invokes processings, before the mapping was started. This could be helpful, for instance to make some preparations +// * for the mapping. To trigger this pre processing for a specific Pepper module a set of customization properties is +// * available. Customization properties triggering a pre processing starts with {@value PepperModuleProperties#PREFIX_PEPPER_BEFORE}. +// * This method is called by the method {@link #map()}, before {@link PepperMapper#mapSDocument()} was called. +// * @param sElementId id of either {@link SDocument} or {@link SCorpus} object to be prepared +// * @throws PepperModuleException +// */ +// public void before(SElementId sElementId) throws PepperModuleException; +// +// /** +// * Invokes processings, after the mapping is done. This could be helpful, for instance to make some processing +// * after the mapping e.g. adding all created nodes and relations to a layer. +// * To trigger this post processing for a specific Pepper module a set of customization properties is +// * available. Customization properties triggering a post processing starts with {@value PepperModuleProperties#PREFIX_PEPPER_AFTER}. +// * This method is called by the method {@link #map()}, after {@link PepperMapper#mapSDocument()} was called. +// * @param sElementId id of either {@link SDocument} or {@link SCorpus} object to be post processed +// * @throws PepperModuleException +// */ +// public void after(SElementId sElementId) throws PepperModuleException; } diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModule.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModule.java index 881b1593c..c172f5d3b 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModule.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModule.java @@ -17,6 +17,7 @@ */ package de.hu_berlin.german.korpling.saltnpepper.pepper.modules; +import java.util.Collection; import java.util.List; import org.eclipse.emf.common.util.URI; @@ -229,6 +230,13 @@ public interface PepperModule{ */ void setSymbolicName(String value); + /** + * If {@link #isReadyToStart()} has returned false, this method returns a list of reasons + * why this module is not ready to start. + * @return a list describing the reasons, or an empty list if there were no problems + */ + public Collection getStartProblems(); + /** * This method is called by the pepper framework after initializing this object and directly before start processing. * Initializing means setting properties {@link PepperModuleProperties}, setting temporary files, resources etc. . @@ -242,7 +250,8 @@ public interface PepperModule{ *
  • if the {@link MODULE_TYPE} is not null
  • *
  • if the name is not null
  • * - * When overriding this method, please call super.isReadyToStart() first. + * When overriding this method, please call super.isReadyToStart() first and in case a + * problem occured add it to the list {@link #getStartProblems()}. * @return false, {@link PepperModule} instance is not ready for any reason, true, else. */ public boolean isReadyToStart() throws PepperModuleNotReadyException; @@ -348,4 +357,38 @@ public interface PepperModule{ * @param controller The object which is done with its job */ public void done(PepperMapperController controller); + + /** + * Invokes processings, before the mapping was started. This could be + * helpful, for instance to make some preparations for the mapping. To + * trigger this pre processing for a specific Pepper module a set of + * customization properties is available. Customization properties + * triggering a pre processing starts with + * {@value PepperModuleProperties#PREFIX_PEPPER_BEFORE}. This method is + * called by the method {@link #map()}, before + * {@link PepperMapper#mapSDocument()} was called. + * + * @param sElementId + * id of either {@link SDocument} or {@link SCorpus} object to be + * prepared + * @throws PepperModuleException + */ + public void before(SElementId sElementId) throws PepperModuleException; + + /** + * Invokes processings, after the mapping is done. This could be helpful, + * for instance to make some processing after the mapping e.g. adding all + * created nodes and relations to a layer. To trigger this post processing + * for a specific Pepper module a set of customization properties is + * available. Customization properties triggering a post processing starts + * with {@value PepperModuleProperties#PREFIX_PEPPER_AFTER}. This method is + * called by the method {@link #map()}, after + * {@link PepperMapper#mapSDocument()} was called. + * + * @param sElementId + * id of either {@link SDocument} or {@link SCorpus} object to be + * post processed + * @throws PepperModuleException + */ + public void after(SElementId sElementId) throws PepperModuleException; } diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModuleProperties.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModuleProperties.java index a358729df..fc48aa6cf 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModuleProperties.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/PepperModuleProperties.java @@ -86,6 +86,15 @@ public class PepperModuleProperties implements Serializable { * object. */ public static final String PROP_AFTER_ADD_SLAYER = PREFIX_PEPPER_AFTER + "addSLayer"; + + /** + * Copies one or more source files to one or more target files after processing. This is very helpful, + * in case of customizations should be done in target format. If you use relative paths, the are anchored + * to either the location of the workflow description file or where Pepper was started. + * Syntax is: SOURCE_FILE -> TARGET_FILE (; SOURCE_FILE -> TARGET_FILE)* + * + */ + public static final String PROP_AFTER_COPY_RES = PREFIX_PEPPER_AFTER + "copyRes"; /** * Consumes a semicolon separated list of names for {@link SLayer} objects. * For each list element, one {@link SLayer} is created and added to all @@ -93,6 +102,14 @@ public class PepperModuleProperties implements Serializable { * object. */ public static final String PROP_BEFORE_ADD_SLAYER = PREFIX_PEPPER_BEFORE + "addSLayer"; + /** + * Reads meta data for corpora and subcorpora in a very simple attribute-value format like:
    + * a=b
    + * c=d
    + * To enable the reading of meta data set this property to the file ending of the metadata file. + * For instance in case of the file is named data.meta: {@value #PROP_BEFORE_READ_META}=meta + */ + public static final String PROP_BEFORE_READ_META= PREFIX_PEPPER_BEFORE + "readMeta"; /** * Creates instance of {@link PepperModuleProperties} and initializes it @@ -103,7 +120,9 @@ public class PepperModuleProperties implements Serializable { */ public PepperModuleProperties() { addProperty(new PepperModuleProperty(PROP_BEFORE_ADD_SLAYER, String.class, "Consumes a semicolon separated list of names for {@link SLayer} objects. For each list element, one layer is created and added to all nodes and relations of a document-structure before the mapping was processed.")); + addProperty(new PepperModuleProperty(PROP_BEFORE_READ_META, String.class, "Reads meta data for corpora and subcorpora in a very simple attribute-value format like: a=b. To enable the reading of meta data set this property to the file ending of the metadata file. For instance in case of the file is named data.meta: {@value #PROP_BEFORE_READ_META}=meta")); addProperty(new PepperModuleProperty(PROP_AFTER_ADD_SLAYER, String.class, "Consumes a semicolon separated list of names for {@link SLayer} objects. For each list element, one layer is created and added to all nodes and relations of a document-structure after the mapping was processed.")); + addProperty(new PepperModuleProperty(PROP_AFTER_COPY_RES, String.class, "Copies one or more source files to one or more target files after processing. This is very helpful, in case of customizations should be done in target format. If you use relative paths, the are anchored to either the location of the workflow description file or where Pepper was started. The syntax is as follows: SOURCE_FILE -> TARGET_FILE (; SOURCE_FILE -> TARGET_FILE)*.")); } /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperExporterImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperExporterImpl.java index 5456cffa8..51f4dddaf 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperExporterImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperExporterImpl.java @@ -39,6 +39,7 @@ import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SElementId; +import java.util.LinkedList; /** * This is an abstract implementation of {@link PepperExporter}. This class @@ -52,10 +53,10 @@ public abstract class PepperExporterImpl extends PepperModuleImpl implements PepperExporter { /** * Creates a {@link PepperModule} of type {@link MODULE_TYPE#EXPORTER}. The - * name of this module is set to "MyExporter". - *
    + * name of this module is set to "MyExporter".
    * We recommend to use the constructor - * {@link PepperExporterImpl#PepperExporterImpl(String)} and pass a proper name. + * {@link PepperExporterImpl#PepperExporterImpl(String)} and pass a proper + * name. */ protected PepperExporterImpl() { super("MyExporter"); @@ -193,7 +194,7 @@ public synchronized Map getSElementId2ResourceTable() { public void exportCorpusStructure() { if ((getExportMode() != null) && (!getExportMode().equals(EXPORT_MODE.NO_EXPORT))) { if (this.getSaltProject() != null) { - Collection corpGraphs = Collections.synchronizedList(this.getSaltProject().getSCorpusGraphs()); + Collection corpGraphs = new LinkedList<>(this.getSaltProject().getSCorpusGraphs()); for (SCorpusGraph sCorpusGraph : corpGraphs) { if (sCorpusGraph == null) { logger.warn("An empty SDocumentGraph is in list of SaltProject. This might be a bug of pepper framework."); @@ -219,16 +220,17 @@ public void exportCorpusStructure() { for (String segment : sDocument.getSElementPath().segments()) { resourceURI = resourceURI.appendSegment(segment); } - resourceURI= resourceURI.appendFileExtension(ending); + resourceURI = resourceURI.appendFileExtension(ending); getSElementId2ResourceTable().put(sDocument.getSElementId(), resourceURI); - - //in case of folders in hierarchie does not exist, create them - String fileName= resourceURI.toFileString(); - if (fileName== null){ + + // in case of folders in hierarchie does not + // exist, create them + String fileName = resourceURI.toFileString(); + if (fileName == null) { resourceURI.toString(); } - File outFile= new File(fileName); - if (!outFile.getParentFile().exists()){ + File outFile = new File(fileName); + if (!outFile.getParentFile().exists()) { outFile.getParentFile().mkdirs(); } } diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperImporterImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperImporterImpl.java index 833fba6e7..489137264 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperImporterImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperImporterImpl.java @@ -129,31 +129,31 @@ public void setCorpusDesc(CorpusDesc newCorpusDefinition) { * {@inheritDoc PepperImporter#readFirstLines(URI, int)} */ @Override - public String readFirstLines(final URI corpusPath, final int lines){ - String retVal= null; - if (corpusPath!= null){ - File importPath= new File(corpusPath.toFileString()); - try(BufferedReader br = new BufferedReader(new FileReader(importPath))) { - StringBuilder sb = new StringBuilder(); - String line = br.readLine(); - int i= 0; - while (line != null) { - sb.append(line); - sb.append(System.lineSeparator()); - line = br.readLine(); - i++; - if (i>= lines){ - break; - } - } - retVal = sb.toString(); - } catch (IOException e) { - return(null); + public String readFirstLines(final URI corpusPath, final int lines) { + String retVal = null; + if (corpusPath != null) { + File importPath = new File(corpusPath.toFileString()); + try (BufferedReader br = new BufferedReader(new FileReader(importPath))) { + StringBuilder sb = new StringBuilder(); + String line = br.readLine(); + int i = 0; + while (line != null) { + sb.append(line); + sb.append(System.lineSeparator()); + line = br.readLine(); + i++; + if (i >= lines) { + break; + } + } + retVal = sb.toString(); + } catch (IOException e) { + return (null); } } - return(retVal); + return (retVal); } - + /** * Stores {@link SElementId} objects corresponding to either a * {@link SDocument} or a {@link SCorpus} object, which has been created @@ -216,27 +216,27 @@ public void importCorpusStructure(SCorpusGraph corpusGraph) throws PepperModuleE if ((this.getCorpusDesc().getCorpusPath().toFileString().endsWith("/")) || (this.getCorpusDesc().getCorpusPath().toFileString().endsWith("\\"))) { this.getCorpusDesc().setCorpusPath(this.getCorpusDesc().getCorpusPath().trimSegments(1)); } - Boolean containsDocuments= importCorpusStructureRec(this.getCorpusDesc().getCorpusPath(), null); - if (logger.isDebugEnabled()){ - if (getSElementId2ResourceTable().size()> 0){ - StringBuilder str= new StringBuilder(); + Boolean containsDocuments = importCorpusStructureRec(this.getCorpusDesc().getCorpusPath(), null); + if (logger.isDebugEnabled()) { + if (getSElementId2ResourceTable().size() > 0) { + StringBuilder str = new StringBuilder(); str.append("["); str.append(getName()); str.append("]"); str.append(" import corpora and documents: \n"); - for (URI uri: getSElementId2ResourceTable().values()){ + for (URI uri : getSElementId2ResourceTable().values()) { str.append("\t"); str.append(uri); str.append("\n"); - } + } logger.debug(str.toString()); } } - if (getSElementId2ResourceTable().size()== 0){ - logger.warn("[{}] No corpora and documents fount to import in '{}'. ", getName() ,this.getCorpusDesc().getCorpusPath()); + if (getSElementId2ResourceTable().size() == 0) { + logger.warn("[{}] No corpora and documents fount to import in '{}'. ", getName(), this.getCorpusDesc().getCorpusPath()); } - if (!containsDocuments){ - logger.warn("[{}] No documents fount to import in '{}'. ", getName() ,this.getCorpusDesc().getCorpusPath()); + if (!containsDocuments) { + logger.warn("[{}] No documents fount to import in '{}'. ", getName(), this.getCorpusDesc().getCorpusPath()); } } @@ -258,8 +258,8 @@ public void importCorpusStructure(SCorpusGraph corpusGraph) throws PepperModuleE * @throws IOException */ protected Boolean importCorpusStructureRec(URI currURI, SCorpus parent) { - Boolean retVal= false; - + Boolean retVal = false; + // set name for corpus graph if ((this.getSCorpusGraph().getSName() == null) || (this.getSCorpusGraph().getSName().isEmpty())) { this.getSCorpusGraph().setSName(currURI.lastSegment()); @@ -278,9 +278,10 @@ protected Boolean importCorpusStructureRec(URI currURI, SCorpus parent) { if (currFile.isDirectory()) { for (File file : currFile.listFiles()) { try { - //if retval is true or returned value is true set retVal to true - Boolean containsDocuments= importCorpusStructureRec(URI.createFileURI(file.getCanonicalPath()), sCorpus); - retVal=(retVal || containsDocuments); + // if retval is true or returned value is true + // set retVal to true + Boolean containsDocuments = importCorpusStructureRec(URI.createFileURI(file.getCanonicalPath()), sCorpus); + retVal = (retVal || containsDocuments); } catch (IOException e) { throw new PepperModuleException("Cannot import corpus structure, because cannot create a URI out of file '" + file + "'. ", e); } @@ -288,7 +289,7 @@ protected Boolean importCorpusStructureRec(URI currURI, SCorpus parent) { } }// resource is a SCorpus else if (STYPE_NAME.SDOCUMENT.equals(type)) { - retVal= true; + retVal = true; // resource is a SDocument if (parent == null) { // if there is no corpus given, create one with name of @@ -310,7 +311,7 @@ else if (STYPE_NAME.SDOCUMENT.equals(type)) { }// resource is a SDocument }// do not ignore resource }// if file is not part of ignore list - return(retVal); + return (retVal); } /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperMapperControllerImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperMapperControllerImpl.java index ca9a6330a..bb09f381d 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperMapperControllerImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperMapperControllerImpl.java @@ -19,6 +19,9 @@ import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import de.hu_berlin.german.korpling.saltnpepper.pepper.common.DOCUMENT_STATUS; import de.hu_berlin.german.korpling.saltnpepper.pepper.core.DocumentControllerImpl; import de.hu_berlin.german.korpling.saltnpepper.pepper.exceptions.NotInitializedException; @@ -29,15 +32,10 @@ import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperMapper; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperMapperController; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModule; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperties; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleException; -import de.hu_berlin.german.korpling.saltnpepper.salt.SaltFactory; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpus; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SElementId; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SLayer; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SNode; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SRelation; /** * The class {@link PepperMapperControllerImpl} is a communicator class between @@ -54,6 +52,8 @@ */ public class PepperMapperControllerImpl extends Thread implements PepperMapperController { + protected Logger logger = LoggerFactory.getLogger("Pepper"); + /** * Initializes this object and sets its {@link ThreadGroup} and the name of * the thread. @@ -76,8 +76,9 @@ public PepperMapperControllerImpl(ThreadGroup threadGroup, String threadName) { @Override public void setPepperMapper(PepperMapper pepperMapper) { this.pepperMapper = pepperMapper; - if (this.pepperMapper != null) + if (this.pepperMapper != null){ mappingSubjects = pepperMapper.getMappingSubjects(); + } } /** @@ -137,9 +138,9 @@ public SElementId getSElementId() { */ @Override public void setSElementId(SElementId sElementId) { - if (sElementId == null) + if (sElementId == null){ throw new PepperModuleException(getPepperMapper(), "Cannot set an empty sElementId."); - + } MappingSubject subj = null; if (getMappingSubjects().size() < 1) { subj = new MappingSubject(); @@ -178,6 +179,7 @@ else if (progress != null) */ @Override public void run() { + //if an exception is thrown, it will be stored to do some clean up first and throw it it afterwards. PepperException origException= null; try { this.map(); @@ -228,17 +230,19 @@ public void map() { progress = 0d; mappingResult = this.getPepperMapper().mapSCorpus(); progress = 1d; - } else if (this.getPepperMapper().getSDocument() != null){ + } else if (this.getPepperMapper().getSDocument() != null) { + //real document mapping //preprocessing for (MappingSubject subj: getMappingSubjects()){ - before(subj.getSElementId()); + getPepperModule().before(subj.getSElementId()); } //real document mapping mappingResult = this.getPepperMapper().mapSDocument(); //postprocessing for (MappingSubject subj: getMappingSubjects()){ - after(subj.getSElementId()); + getPepperModule().after(subj.getSElementId()); } + }else{ throw new NotInitializedException("Cannot start mapper, because neither the SDocument nor the SCorpus value is set."); } @@ -249,76 +253,7 @@ public void map() { } } - /** {@inheritDoc PepperModule#before(SElementId)} */ - @Override - public void before(SElementId sElementId) throws PepperModuleException { - if (getPepperMapper().getProperties()!= null){ - if ( (sElementId!= null)&& - (sElementId.getSIdentifiableElement() != null)){ - if (sElementId.getSIdentifiableElement() instanceof SDocument){ - SDocument sDoc= (SDocument) sElementId.getSIdentifiableElement(); - - //add layers - String layers= (String)getPepperMapper().getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_ADD_SLAYER).getValue(); - addSLayers(sDoc, layers); - }else if (sElementId.getSIdentifiableElement() instanceof SCorpus){ - - } - } - } - } - - /** {@inheritDoc PepperModule#after(SElementId)} */ - @Override - public void after(SElementId sElementId) throws PepperModuleException { - if (getPepperMapper().getProperties()!= null){ - if ( (sElementId!= null)&& - (sElementId.getSIdentifiableElement() != null)){ - if (sElementId.getSIdentifiableElement() instanceof SDocument){ - SDocument sDoc= (SDocument) sElementId.getSIdentifiableElement(); - //add layers - String layers= (String)getPepperMapper().getProperties().getProperty(PepperModuleProperties.PROP_AFTER_ADD_SLAYER).getValue(); - addSLayers(sDoc, layers); - - }else if (sElementId.getSIdentifiableElement() instanceof SCorpus){ - - } - } - } - } - - private void addSLayers(SDocument sDoc, String layers){ - if ( (layers!= null)&& - (!layers.isEmpty())){ - String[] layerArray= layers.split(";"); - if (layerArray.length> 0){ - for (String layer: layerArray){ - layer= layer.trim(); - //create SLayer and add to document-structure - List sLayers= sDoc.getSDocumentGraph().getSLayerByName(layer); - SLayer sLayer= null; - if ( (sLayers!= null)&& - (sLayers.size() > 0)){ - sLayer= sLayers.get(0); - } - if (sLayer== null){ - sLayer= SaltFactory.eINSTANCE.createSLayer(); - sLayer.setSName(layer); - sDoc.getSDocumentGraph().addSLayer(sLayer); - } - //add all nodes to new layer - for (SNode sNode: sDoc.getSDocumentGraph().getSNodes()){ - sNode.getSLayers().add(sLayer); - } - //add all relations to new layer - for (SRelation sRel: sDoc.getSDocumentGraph().getSRelations()){ - sRel.getSLayers().add(sLayer); - } - } - } - } - } - + /** {@link PepperModule} containing this object **/ protected PepperModule pepperModule = null; @@ -328,6 +263,9 @@ private void addSLayers(SDocument sDoc, String layers){ @Override public void setPepperModule(PepperModule pepperModule) { this.pepperModule = pepperModule; + if (this.pepperModule!= null){ + logger= LoggerFactory.getLogger(getPepperModule().getName()); + } } /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperModuleImpl.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperModuleImpl.java index 9a82beb28..ee89695e2 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperModuleImpl.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/PepperModuleImpl.java @@ -18,7 +18,10 @@ package de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl; import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.ConcurrentModificationException; @@ -26,6 +29,7 @@ import java.util.Hashtable; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Vector; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; @@ -60,6 +64,12 @@ import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SElementId; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SIdentifiableElement; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SLayer; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SMetaAnnotatableElement; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SNode; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SRelation; +import java.util.LinkedList; /** * TODO make docu @@ -70,24 +80,27 @@ public class PepperModuleImpl implements PepperModule, UncaughtExceptionHandler protected Logger logger = LoggerFactory.getLogger("Pepper"); /** - * Creates a {@link PepperModule} object, which is either a {@link MODULE_TYPE#IMPORTER}, a {@link MODULE_TYPE#MANIPULATOR} or - * a {@link MODULE_TYPE#EXPORTER}. The name of this module is set to "MyModule". - *
    - * We recommend to use the constructor {@link PepperModuleImpl#PepperModuleImpl(String)} and pass a proper name. + * Creates a {@link PepperModule} object, which is either a + * {@link MODULE_TYPE#IMPORTER}, a {@link MODULE_TYPE#MANIPULATOR} or a + * {@link MODULE_TYPE#EXPORTER}. The name of this module is set to + * "MyModule".
    + * We recommend to use the constructor + * {@link PepperModuleImpl#PepperModuleImpl(String)} and pass a proper name. */ protected PepperModuleImpl() { this("MyModule"); } - + /** - * Creates a {@link PepperModule} object, which is either a {@link MODULE_TYPE#IMPORTER}, a {@link MODULE_TYPE#MANIPULATOR} or - * a {@link MODULE_TYPE#EXPORTER}. The passed name is set as the modules name. + * Creates a {@link PepperModule} object, which is either a + * {@link MODULE_TYPE#IMPORTER}, a {@link MODULE_TYPE#MANIPULATOR} or a + * {@link MODULE_TYPE#EXPORTER}. The passed name is set as the modules name. */ protected PepperModuleImpl(String name) { setName(name); - logger= LoggerFactory.getLogger(name); + logger = LoggerFactory.getLogger(name); getFingerprint(); - if (getProperties()== null){ + if (getProperties() == null) { setProperties(new PepperModuleProperties()); } } @@ -133,8 +146,8 @@ protected void setName(String name) { getFingerprint().setName(name); } } - - /** + + /** * {@inheritDoc PepperModule#getVersion()} */ @Override @@ -194,7 +207,7 @@ public void setSupplierContact(URI supplierContact) { getFingerprint().setSupplierContact(supplierContact); } - /** + /** * {@inheritDoc PepperModule#getProperties()} */ @Override @@ -209,7 +222,7 @@ public PepperModuleProperties getProperties() { public void setProperties(PepperModuleProperties properties) { getFingerprint().setProperties(properties); } - + /** * TODO make docu */ @@ -332,31 +345,47 @@ protected void activate(ComponentContext componentContext) { this.componentContext = componentContext; if ((componentContext != null) && (componentContext.getBundleContext() != null) && (componentContext.getBundleContext().getBundle() != null)) { this.setSymbolicName(componentContext.getBundleContext().getBundle().getSymbolicName()); - this.setVersion(componentContext.getBundleContext().getBundle().getVersion().toString()); + this.setVersion(componentContext.getBundleContext().getBundle().getVersion().toString()); } } + /** a list containing reasons why this module is not ready to start **/ + private Collection startProblems= new ArrayList(); + /** + * {@inheritDoc PepperModule#getStartProblems()} + */ + @Override + public Collection getStartProblems(){ + return(startProblems); + } /** * {@inheritDoc PepperModule#isReadyToStart()} */ + @Override public boolean isReadyToStart() throws PepperModuleNotReadyException { Boolean retVal = true; if (getResources() == null) { + startProblems.add("No resource is given for module."); retVal = false; } else { File resourceFile = new File(getResources().toFileString()); if (!resourceFile.exists()) { + startProblems.add("Given resource file '"+resourceFile.getAbsolutePath()+"' does not exist."); retVal = false; } } if (getModuleType() == null) { + startProblems.add("No module-type is set for module."); retVal = false; } if (getName() == null) { + startProblems.add("No name is set for module."); retVal = false; } return (retVal); } + + /** * the controller object, which acts as bridge between Pepper framework and @@ -515,7 +544,7 @@ protected Map getDocumentId2DC() { */ @Override public void start() throws PepperModuleException { - if (getSaltProject() == null){ + if (getSaltProject() == null) { throw new PepperFWException("No salt project was set in module '" + getName() + ", " + getVersion() + "'."); } // creating new thread group for mapper threads @@ -533,7 +562,7 @@ public void start() throws PepperModuleException { getDocumentId2DC().put(SaltFactory.eINSTANCE.getGlobalId(sElementId), documentController); // call for using push-method try { - //start mapping + // start mapping start(sElementId); } catch (Exception e) { if (this.isStartOverridden) { @@ -641,7 +670,7 @@ public void done(PepperMapperController controller) { } } } - + /** * This method is called by method {@link #start()}, if the method was not * overridden by the current class. If this is not the case, this method @@ -660,31 +689,31 @@ public void start(SElementId sElementId) throws PepperModuleException { // copy all corpora into finite list corporaToEnd = new Vector(); - List corpGraphs = Collections.synchronizedList(this.getSaltProject().getSCorpusGraphs()); + // copy the list before iterating over it (the original one might get modified) + List corpGraphs = new LinkedList<>(this.getSaltProject().getSCorpusGraphs()); for (SCorpusGraph sCorpusGraph : corpGraphs) { if (sCorpusGraph != null) { - if (MODULE_TYPE.IMPORTER.equals(getModuleType())){ - boolean belongsToSetCorpusGraph= false; - if ( (sCorpusGraph.getSElementId()!= null)&& - (((PepperImporter)this).getSCorpusGraph()!= null)&& - (((PepperImporter)this).getSCorpusGraph().getSElementId()!= null)){ - if (sCorpusGraph.getSElementId().equals(((PepperImporter)this).getSCorpusGraph().getSElementId())){ - belongsToSetCorpusGraph= true; + if (MODULE_TYPE.IMPORTER.equals(getModuleType())) { + boolean belongsToSetCorpusGraph = false; + if ((sCorpusGraph.getSElementId() != null) && (((PepperImporter) this).getSCorpusGraph() != null) && (((PepperImporter) this).getSCorpusGraph().getSElementId() != null)) { + if (sCorpusGraph.getSElementId().equals(((PepperImporter) this).getSCorpusGraph().getSElementId())) { + belongsToSetCorpusGraph = true; } - }else{ - if (sCorpusGraph.equals(((PepperImporter)this).getSCorpusGraph())){ - belongsToSetCorpusGraph= true; + } else { + if (sCorpusGraph.equals(((PepperImporter) this).getSCorpusGraph())) { + belongsToSetCorpusGraph = true; } } - //in case of module is an importer, only import corpora from set corpus graph - if (belongsToSetCorpusGraph){ + // in case of module is an importer, only import corpora + // from set corpus graph + if (belongsToSetCorpusGraph) { for (SCorpus sCorpus : sCorpusGraph.getSCorpora()) { corporaToEnd.add(sCorpus); } } - }else{ - //if module is not an importer, process all corpora - + } else { + // if module is not an importer, process all corpora + for (SCorpus sCorpus : sCorpusGraph.getSCorpora()) { corporaToEnd.add(sCorpus); } @@ -698,7 +727,7 @@ public void start(SElementId sElementId) throws PepperModuleException { PepperMapperController controller = new PepperMapperControllerImpl(mapperThreadGroup, this.getName() + "_mapper(" + sElementId.getSId() + ")"); String id = sElementId.getSId(); - if (sElementId.getSIdentifiableElement() instanceof SDocument){ + if (sElementId.getSIdentifiableElement() instanceof SDocument) { id = SaltFactory.eINSTANCE.getGlobalId(sElementId); } this.getMapperControllers().put(id, controller); @@ -707,9 +736,9 @@ public void start(SElementId sElementId) throws PepperModuleException { PepperMapper mapper = this.createPepperMapper(sElementId); mapper.setProperties(this.getProperties()); - + if (this instanceof PepperImporter) { - if (mapper.getResourceURI()== null){ + if (mapper.getResourceURI() == null) { URI resource = ((PepperImporter) this).getSElementId2ResourceTable().get(sElementId); mapper.setResourceURI(resource); } @@ -726,7 +755,7 @@ public void start(SElementId sElementId) throws PepperModuleException { if (this.isMultithreaded()) { controller.start(); } else - controller.map(); + controller.run(); } } @@ -737,6 +766,194 @@ public PepperMapper createPepperMapper(SElementId sElementId) { throw new NotInitializedException("Cannot start mapping, because the method createPepperMapper() of module '" + this.getName() + "' has not been overridden. Please check that first."); } + /** + * Invokes processings, before the mapping was started. This could be + * helpful, for instance to make some preparations for the mapping. To + * trigger this pre processing for a specific Pepper module a set of + * customization properties is available. Customization properties + * triggering a pre processing starts with + * {@value PepperModuleProperties#PREFIX_PEPPER_BEFORE}. This method is + * called by the method {@link #map()}, before + * {@link PepperMapper#mapSDocument()} was called. + * + * @param sElementId + * id of either {@link SDocument} or {@link SCorpus} object to be + * prepared + * @throws PepperModuleException + */ + @Override + public void before(SElementId sElementId) throws PepperModuleException { + if (getProperties() != null) { + if (getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_ADD_SLAYER) != null) { + // add slayers after processing + + if ((sElementId != null) && (sElementId.getSIdentifiableElement() != null)) { + if (sElementId.getSIdentifiableElement() instanceof SDocument) { + SDocument sDoc = (SDocument) sElementId.getSIdentifiableElement(); + + // add layers + String layers = (String) getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_ADD_SLAYER).getValue(); + addSLayers(sDoc, layers); + } else if (sElementId.getSIdentifiableElement() instanceof SCorpus) { + + } + } + } + if ((getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_READ_META) != null) && (getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_READ_META).getValue() != null)) { + // read meta data + + readMeta(sElementId); + } + } + } + + /** + * Invokes processings, after the mapping is done. This could be helpful, + * for instance to make some processing after the mapping e.g. adding all + * created nodes and relations to a layer. To trigger this post processing + * for a specific Pepper module a set of customization properties is + * available. Customization properties triggering a post processing starts + * with {@value PepperModuleProperties#PREFIX_PEPPER_AFTER}. This method is + * called by the method {@link #map()}, after + * {@link PepperMapper#mapSDocument()} was called. + * + * @param sElementId + * id of either {@link SDocument} or {@link SCorpus} object to be + * post processed + * @throws PepperModuleException + */ + @Override + public void after(SElementId sElementId) throws PepperModuleException { + if (getProperties() != null) { + if ((sElementId != null) && (sElementId.getSIdentifiableElement() != null)) { + if (sElementId.getSIdentifiableElement() instanceof SDocument) { + SDocument sDoc = (SDocument) sElementId.getSIdentifiableElement(); + if (getProperties().getProperty(PepperModuleProperties.PROP_AFTER_ADD_SLAYER) != null) { + // add slayers after processing + String layers = (String) getProperties().getProperty(PepperModuleProperties.PROP_AFTER_ADD_SLAYER).getValue(); + addSLayers(sDoc, layers); + } + } else if (sElementId.getSIdentifiableElement() instanceof SCorpus) { + + } + } + } + } + + // **************************************************************************************** + // *** functions for before() and after() + + /** + * Adds the passed layer to all nodes and objects in the passed + * {@link SDocument}. + * + * @param sDoc + * @param layers + */ + public void addSLayers(SDocument sDoc, String layers) { + if ((layers != null) && (!layers.isEmpty())) { + String[] layerArray = layers.split(";"); + if (layerArray.length > 0) { + for (String layer : layerArray) { + layer = layer.trim(); + // create SLayer and add to document-structure + List sLayers = sDoc.getSDocumentGraph().getSLayerByName(layer); + SLayer sLayer = null; + if ((sLayers != null) && (sLayers.size() > 0)) { + sLayer = sLayers.get(0); + } + if (sLayer == null) { + sLayer = SaltFactory.eINSTANCE.createSLayer(); + sLayer.setSName(layer); + sDoc.getSDocumentGraph().addSLayer(sLayer); + } + // add all nodes to new layer + for (SNode sNode : sDoc.getSDocumentGraph().getSNodes()) { + sNode.getSLayers().add(sLayer); + } + // add all relations to new layer + for (SRelation sRel : sDoc.getSDocumentGraph().getSRelations()) { + sRel.getSLayers().add(sLayer); + } + } + } + } + } + + /** + * Loads meta data form a meta data file and adds them to the object + * corresponding to the passed {@link SElementId}. The meta data file is + * localized in the directory in case of the URI corresponding to passed + * {@link SElementId} is a directory or (in case the corresponding URI + * addresses a file) in the same directory as the resource corresponding to + * the passed {@link SElementId}. The meta data file must have the ending + * passed in {@link PepperModuleProperties#PROP_BEFORE_READ_META}. + * + * @param sElementId + * identifying the current object + */ + public void readMeta(SElementId sElementId) { + if (this instanceof PepperImporter) { + URI resourceURI = ((PepperImporter) this).getSElementId2ResourceTable().get(sElementId); + Object endingObj = getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_READ_META).getValue(); + if (endingObj != null) { + String ending = endingObj.toString().trim(); + if (resourceURI != null) { + File resource = new File(resourceURI.toFileString()); + File metaFile = null; + if (resource.isDirectory()) { + // resource is directory, search for meta data file + // (all files having customized ending) + File[] files = resource.listFiles(); + if (files != null) { + for (File file : resource.listFiles()) { + if (file.getName().equalsIgnoreCase(sElementId.getSElementPath().lastSegment() + "." + ending)) { + metaFile = file; + break; + } + } + } + } else { + // resource is a file, search for meta data file + // (file having the same name as current corpus or + // document and having customized ending) + + String[] parts = resource.getName().split("[.]"); + if (parts != null) { + String currEnding = parts[parts.length - 1]; + resource.getAbsolutePath().lastIndexOf("."); + metaFile = new File(resource.getAbsolutePath().substring(0, resource.getAbsolutePath().lastIndexOf(".")) + "." + ending); + if (!metaFile.exists()) { + metaFile = null; + } + } + } + if (metaFile != null) { + Properties props = new Properties(); + try { + props.load(new FileInputStream(metaFile)); + } catch (IOException e) { + logger.warn("Tried to load meta data file '" + metaFile.getAbsolutePath() + "', but a problem occured: " + e.getMessage() + ". ", e); + } + for (Object key : props.keySet()) { + SIdentifiableElement container = sElementId.getSIdentifiableElement(); + if ((container != null) && (container instanceof SMetaAnnotatableElement)) { + if (!((SMetaAnnotatableElement) container).hasLabel(key.toString())) { + ((SMetaAnnotatableElement) container).createSMetaAnnotation(null, key.toString(), props.getProperty(key.toString())); + } else { + logger.warn("Cannot add meta annotation '" + key.toString() + "', because it already exist on object '" + sElementId.getSId() + "' please check file '" + metaFile.getAbsolutePath() + "'. "); + } + } + } + } + } + } + } + } + + // *** functions for before() and after() + // **************************************************************************************** + /** * A list of all corpora, which should be called in method {@link #end()}. * The interim storage necessary because of @@ -752,10 +969,10 @@ public PepperMapper createPepperMapper(SElementId sElementId) { @Override public void end() throws PepperModuleException { logger.trace("[{}] start processing corpus structure (manipulating or exporting). ", getName()); - if (getSaltProject() == null){ + if (getSaltProject() == null) { throw new PepperModuleException(this, "Error in method end() salt project was empty."); } - if (getSaltProject().getSCorpusGraphs() == null){ + if (getSaltProject().getSCorpusGraphs() == null) { throw new PepperModuleException(this, "Error in method end() corpus graphs of salt project were empty."); } if (corporaToEnd != null) { @@ -773,7 +990,9 @@ public void end() throws PepperModuleException { */ @Override public void uncaughtException(Thread t, Throwable e) { - logger.error("An exception was thrown by the mapper threads '"+t+"'. ", e); + //TODO this is a workaround because of a bug in slf4j. Currently errors are not passable to slf4j. Therefore just the error message is passed, and because this is quite unuseful, the stacktrace is also printed. + logger.error("An exception was thrown by the mapper threads '" + t + "'. ", e.getMessage()); + e.printStackTrace(); if (logger instanceof NOPLogger) { e.printStackTrace(); } @@ -784,19 +1003,20 @@ public void uncaughtException(Thread t, Throwable e) { */ @Override public Double getProgress(String globalId) { - if (globalId == null) + if (globalId == null){ throw new PepperFWException("Cannot return the progress for an empty sDocumentId."); - + } PepperMapperController controller = this.getMapperControllers().get(globalId); // outcommented for downwards compatibility to modules implemented with // < pepper 1.1.6 // if (controller== null) // throw new // PepperFWException("Cannot return the progress for sDocumentId '"+sDocumentId+"', because no mapper controller exists. This might be a bug."); - if (controller != null) + if (controller != null){ return (controller.getProgress()); - else + }else{ return (null); + } } /** diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperModuleTest.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperModuleTest.java index 301072540..9fcff0b01 100644 --- a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperModuleTest.java +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperModuleTest.java @@ -23,34 +23,17 @@ import java.io.File; import java.io.IOException; -import java.util.List; -import java.util.Vector; +import java.util.ArrayList; +import java.util.Collection; import org.eclipse.emf.common.util.URI; import org.junit.Test; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.CorpusDesc; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.FormatDesc; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.MEMORY_POLICY; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.MODULE_TYPE; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperConfiguration; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperJob; -import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperUtil; import de.hu_berlin.german.korpling.saltnpepper.pepper.core.ModuleControllerImpl; -import de.hu_berlin.german.korpling.saltnpepper.pepper.core.PepperImpl; -import de.hu_berlin.german.korpling.saltnpepper.pepper.core.PepperJobImpl; -import de.hu_berlin.german.korpling.saltnpepper.pepper.core.Step; import de.hu_berlin.german.korpling.saltnpepper.pepper.exceptions.PepperTestException; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.DocumentController; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperExporter; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperImporter; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperManipulator; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModule; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.doNothing.DoNothingExporter; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.doNothing.DoNothingImporter; -import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleTestException; import de.hu_berlin.german.korpling.saltnpepper.pepper.util.FileComparator; -import de.hu_berlin.german.korpling.saltnpepper.salt.SaltFactory; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.SaltCommonFactory; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; @@ -104,9 +87,9 @@ public void initilaize(){ } /** - * Name of the directory where tests for Pepper and Pepper modules can be stored. + * {@inheritDoc PepperTestUtil#TMP_TEST_DIR} */ - public static String TMP_TEST_DIR="pepper-test"; + public static String TMP_TEST_DIR=PepperTestUtil.TMP_TEST_DIR; /** * Returns a {@link File} object pointing to a temporary path, where the caller can store temporary files. @@ -130,38 +113,24 @@ public File getTempPath(String testDirectory){ } /** - * Returns a {@link File} object pointing to a temporary path, where the caller can store temporary files. - * The temporary path is located in the temporary directory provided by the underlying os. The resulting - * directory is located in TEMP_PATH_BY_OS/{@value #TMP_TEST_DIR}/testDirectory. - * @param testDirectory last part of the temporary path - * @return a file object locating to a temporary folder, where files can be stored temporarily + * {@inheritDoc PepperTestUtil#getTempPath_static(String)} */ public static File getTempPath_static(String testDirectory){ - if ( (testDirectory== null)|| - (testDirectory.isEmpty())){ - throw new PepperModuleTestException("Cannot return a temporary directory, since the given last part is empty."); - } - File retVal= null; - retVal= PepperUtil.getTempTestFile(TMP_TEST_DIR+"/"+testDirectory); - return(retVal); + return(PepperTestUtil.getTempPath_static(testDirectory)); } /** - * Returns a default test folder, where to find resources for tests. When using the default - * maven structure, this folder is located at 'src/test/resources/'. - * @return a folder where to find test resources + * {@inheritDoc PepperTestUtil#getTestResources()} */ public static String getTestResources(){ - return("src/test/resources/"); + return(PepperTestUtil.getTestResources()); } /** - * Returns a default test folder, where to find resources for tests. When using the default - * maven structure, this folder is located at 'src/test/resources/'. - * @return a folder where to find test resources + * {@inheritDoc PepperTestUtil#getSrcResources()} */ public static String getSrcResources(){ - return("src/main/resources/"); + return(PepperTestUtil.getSrcResources()); } /** @@ -182,112 +151,12 @@ public static String getSrcResources(){ * * */ - public void start() - { - if (this.getFixture()== null) - throw new PepperModuleTestException("Cannot start Pepper module, because the fixture is not set."); - - if (this.getFixture().getSaltProject()== null) - this.getFixture().setSaltProject(SaltFactory.eINSTANCE.createSaltProject()); - - File tmpFolder= getTempPath("pepperModuleTest"); - - PepperImpl pepper= new PepperImpl(); - PepperConfiguration conf= new PepperConfiguration(); - conf.setProperty(PepperConfiguration.PROP_MEMORY_POLICY, MEMORY_POLICY.MODERATE.toString()); - pepper.setConfiguration(conf); - PepperJob job= pepper.getJob(pepper.createJob()); - - if (!(job instanceof PepperJobImpl)) - throw new PepperModuleTestException("Cannot start Pepper module test, because '"+PepperJob.class+"' is not of type '"+PepperJobImpl.class+"'. "); - - ((PepperJobImpl)job).setSaltProject(getFixture().getSaltProject()); - - CorpusDesc corpusDesc; - FormatDesc formatDesc; - - Step fixtureStep= null; - fixtureStep= new Step("fixture_step"); - fixtureStep.setModuleType(getFixture().getModuleType()); - fixtureStep.setName(getFixture().getName()); - fixtureStep.setVersion(getFixture().getVersion()); - if (getFixture() instanceof PepperImporter){ - fixtureStep.setCorpusDesc(((PepperImporter)getFixture()).getCorpusDesc()); - }else if (getFixture() instanceof PepperExporter){ - fixtureStep.setCorpusDesc(((PepperExporter)getFixture()).getCorpusDesc()); - } - fixtureStep.setPepperModule(getFixture()); - ((PepperJobImpl)job).addStep(fixtureStep); - - URI dummyResourceURI= URI.createFileURI(new File(System.getProperty("java.io.tmpdir")).getAbsolutePath()); - - //define export step - Step exportStep= new Step("doNothing_export_step"); - exportStep.setModuleType(MODULE_TYPE.EXPORTER); - exportStep.setName(DoNothingImporter.MODULE_NAME); - PepperExporter exporter= new DoNothingExporter(); - exporter.setResources(dummyResourceURI); - exportStep.setPepperModule(exporter); - - corpusDesc= new CorpusDesc(); - corpusDesc.setCorpusPath(URI.createFileURI(tmpFolder.getAbsolutePath())); - formatDesc= new FormatDesc(); - formatDesc.setFormatName(DoNothingImporter.FORMAT_NAME); - formatDesc.setFormatVersion(DoNothingImporter.FORMAT_VERSION); - corpusDesc.setFormatDesc(formatDesc); - exportStep.setCorpusDesc(corpusDesc); - exporter.setCorpusDesc(corpusDesc); - //define export step - - //define import step - List importSteps= new Vector(); - for (SCorpusGraph sCorpusgraph: getFixture().getSaltProject().getSCorpusGraphs()){ - Step importStep= new Step("doNothing_import_step"); - importStep.setModuleType(MODULE_TYPE.IMPORTER); - importStep.setName(DoNothingImporter.MODULE_NAME); - PepperImporter importer= new DoNothingImporter(); - importer.setResources(dummyResourceURI); - importStep.setPepperModule(importer); - - corpusDesc= new CorpusDesc(); - corpusDesc.setCorpusPath(URI.createFileURI(tmpFolder.getAbsolutePath())); - formatDesc= new FormatDesc(); - formatDesc.setFormatName(DoNothingImporter.FORMAT_NAME); - formatDesc.setFormatVersion(DoNothingImporter.FORMAT_VERSION); - corpusDesc.setFormatDesc(formatDesc); - importer.setCorpusDesc(corpusDesc); - importSteps.add(importStep); - } - //define import step - if (getFixture() instanceof PepperImporter){ - ((PepperJobImpl)job).addStep(exportStep); - }else if (getFixture() instanceof PepperManipulator){ - if ( (importSteps.size()== 0)&& - (getFixture().getSaltProject().getSCorpusGraphs().size()== 0)){ - throw new PepperModuleTestException(getFixture(), "Please add either an importer to workflow or create a filled SaltProject to be manipulated. "); - } - for (Step importStep: importSteps){ - ((PepperJobImpl)job).addStep(importStep); - } - ((PepperJobImpl)job).addStep(exportStep); - }else if (getFixture() instanceof PepperExporter){ - if ( (importSteps.size()== 0)&& - (getFixture().getSaltProject()== null)){ - throw new PepperModuleTestException(getFixture(), "Please add either an importer to workflow or create a filled SaltProject to be exported. "); - } - for (Step importStep: importSteps){ - ((PepperJobImpl)job).addStep(importStep); - } - }else{ - throw new PepperModuleTestException(getFixture(), "Cannot run test, because given fixture '"+getFixture()+"' was neither of type '"+PepperImporter.class+"', '"+PepperManipulator.class+"' nor '"+PepperExporter.class+"'. "); - } - - job.convert(); - - for (DocumentController controller: ((PepperJobImpl)job).getDocumentControllers()){ - controller.awake(); - } - } + public void start(){ + Collection fixtures= new ArrayList(); + fixtures.add(getFixture()); + PepperTestUtil.start(fixtures); + } + @Test public void testSetGetCorpusGraph() { diff --git a/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperTestUtil.java b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperTestUtil.java new file mode 100644 index 000000000..50c8b1b4a --- /dev/null +++ b/pepper-framework/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/testFramework/PepperTestUtil.java @@ -0,0 +1,264 @@ +/** + * Copyright 2009 Humboldt-Universität zu Berlin, INRIA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + */ +package de.hu_berlin.german.korpling.saltnpepper.pepper.testFramework; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; + +import org.eclipse.emf.common.util.URI; + +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.CorpusDesc; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.FormatDesc; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.MEMORY_POLICY; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.MODULE_TYPE; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperConfiguration; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperJob; +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperUtil; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.ModuleControllerImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.PepperImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.PepperJobImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.Step; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.DocumentController; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperExporter; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperImporter; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperManipulator; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModule; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.doNothing.DoNothingExporter; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.doNothing.DoNothingImporter; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleTestException; +import de.hu_berlin.german.korpling.saltnpepper.salt.SaltFactory; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.SaltProject; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; + +/** + * This class offers some static fields and methods for testing {@link PepperModule} objects. + * @author florian + * + */ +public class PepperTestUtil { + /** + * Name of the directory where tests for Pepper and Pepper modules can be stored. + */ + public static String TMP_TEST_DIR="pepper-test"; + + /** + * Returns a {@link File} object pointing to a temporary path, where the caller can store temporary files. + * The temporary path is located in the temporary directory provided by the underlying os. The resulting + * directory is located in TEMP_PATH_BY_OS/{@value #TMP_TEST_DIR}/testDirectory. + * @param testDirectory last part of the temporary path + * @return a file object locating to a temporary folder, where files can be stored temporarily + */ + public static File getTempPath_static(String testDirectory){ + if ( (testDirectory== null)|| + (testDirectory.isEmpty())){ + throw new PepperModuleTestException("Cannot return a temporary directory, since the given last part is empty."); + } + File retVal= null; + retVal= PepperUtil.getTempTestFile(TMP_TEST_DIR+"/"+testDirectory); + return(retVal); + } + + /** + * Returns a default test folder, where to find resources for tests. When using the default + * maven structure, this folder is located at 'src/test/resources/'. + * @return a folder where to find test resources + */ + public static String getSrcResources(){ + return("src/main/resources/"); + } + /** + * Returns a default test folder, where to find resources for tests. When using the default + * maven structure, this folder is located at 'src/test/resources/'. + * @return a folder where to find test resources + */ + public static String getTestResources(){ + return("src/test/resources/"); + } + /** + * Creates an alibi {@link Step} for the method {@link #start(Collection)}. + * + * @param createAlibiImporter + * @return + */ + private static Step createAlibiStep(boolean createAlibiImporter) { + URI dummyResourceURI = URI.createFileURI(new File(System.getProperty("java.io.tmpdir")).getAbsolutePath()); + FormatDesc formatDesc; + + // set tmp folder + File tmpFolder = PepperModuleTest.getTempPath_static("pepperModuleTest"); + + Step step = null; + CorpusDesc corpusDesc = new CorpusDesc(); + corpusDesc.setCorpusPath(URI.createFileURI(tmpFolder.getAbsolutePath())); + formatDesc = new FormatDesc(); + formatDesc.setFormatName(DoNothingImporter.FORMAT_NAME); + formatDesc.setFormatVersion(DoNothingImporter.FORMAT_VERSION); + corpusDesc.setFormatDesc(formatDesc); + + if (createAlibiImporter) { + step = new Step("doNothing_import_step"); + PepperImporter importer = new DoNothingImporter(); + importer.setResources(dummyResourceURI); + importer.setCorpusDesc(corpusDesc); + step.setPepperModule(importer); + step.setModuleType(MODULE_TYPE.IMPORTER); + } else { + step = new Step("doNothing_export_step"); + PepperExporter exporter = new DoNothingExporter(); + exporter.setCorpusDesc(corpusDesc); + exporter.setResources(dummyResourceURI); + step.setPepperModule(exporter); + step.setModuleType(MODULE_TYPE.EXPORTER); + } + step.setCorpusDesc(corpusDesc); + step.setName(DoNothingImporter.MODULE_NAME); + + return (step); + } + + /** + * This methods starts the processing of Pepper in the development + * environment for a set of Pepper modules. This enables the test of + * multiple modules at once, for instance the import and export of data to + * check no data is lost. In case of the fixture is {@link PepperImporter}, + * first the method + * {@link PepperImporter#importCorpusStructure(SCorpusGraph)} is called. For + * all kinds of fixture, the method + * {@link PepperModule#start(de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SElementId)} + * is called for each {@link SDocument} object contained in the variable + * {@link PepperModule#getSaltProject()}. This method will wait, until each + * {@link ModuleControllerImpl} return having finished the process.
    + * To create a test using this method do the following:
    + *
      + *
    • Create {@link CorpusDefinition} and add it to this object and set its + * {@link FormatDefinition} and corpus path
    • + *
    • Create a {@link SCorpusGraph} object as the one to be filled and add + * it with + * + *
      +	 * this.getFixture().getSaltProject().getSCorpusGraphs().add(importedCorpusGraph);
      +	 * this.getFixture().importCorpusStructure(importedCorpusGraph);
      +	 * 
      + * + *
    • + *
    + */ + public static void start(Collection fixtures) { + if (fixtures == null) { + throw new PepperModuleTestException("Cannot start Pepper modules, because the list of fixtures is not set."); + } + // + Collection importers = new ArrayList(); + Collection manipulators = new ArrayList(); + Collection exporters = new ArrayList(); + + /** + * set the salt project for all modules, if it is already set, check + * that it is equal in all modules. Extract all importers, manipulators + * and exporters + */ + SaltProject saltProject = null; + int i = 1; + for (PepperModule fixture : fixtures) { + if (i == 1) { + saltProject = fixture.getSaltProject(); + } else if (saltProject != fixture.getSaltProject()) { + throw new PepperModuleTestException("Cannot run test because the SaltProject objects are not equal for all Pepper modules. "); + } + i++; + + //if module does not have a resource, set a default one + fixture.setResources(URI.createFileURI("src/main/resources")); + + // fill importers manipulators and exporters collection + if (fixture instanceof PepperImporter) { + importers.add((PepperImporter) fixture); + } else if (fixture instanceof PepperManipulator) { + manipulators.add((PepperManipulator) fixture); + } else if (fixture instanceof PepperExporter) { + exporters.add((PepperExporter) fixture); + } + } + if (saltProject == null) { + saltProject = SaltFactory.eINSTANCE.createSaltProject(); + for (PepperModule fixture : fixtures) { + fixture.setSaltProject(saltProject); + } + } + + // Create a Pepper object + PepperImpl pepper = new PepperImpl(); + PepperConfiguration conf = new PepperConfiguration(); + conf.setProperty(PepperConfiguration.PROP_MEMORY_POLICY, MEMORY_POLICY.MODERATE.toString()); + pepper.setConfiguration(conf); + + // create a Pepper job object + PepperJob job = pepper.getJob(pepper.createJob()); + if (!(job instanceof PepperJobImpl)) { + throw new PepperModuleTestException("Cannot start Pepper module test, because '" + PepperJob.class + "' is not of type '" + PepperJobImpl.class + "'. "); + } else { + ((PepperJobImpl) job).setSaltProject(saltProject); + } + + /** Create a step for each fixture. **/ + for (PepperModule fixture : fixtures) { + Step fixtureStep = null; + fixtureStep = new Step("fixture_step"); + fixtureStep.setModuleType(fixture.getModuleType()); + fixtureStep.setName(fixture.getName()); + fixtureStep.setVersion(fixture.getVersion()); + if (fixture instanceof PepperImporter) { + fixtureStep.setCorpusDesc(((PepperImporter) fixture).getCorpusDesc()); + } else if (fixture instanceof PepperExporter) { + fixtureStep.setCorpusDesc(((PepperExporter) fixture).getCorpusDesc()); + } + fixtureStep.setPepperModule(fixture); + ((PepperJobImpl) job).addStep(fixtureStep); + } + + /** Create and add alibi steps **/ + if ((importers.size() == 0) || (importers.size() != saltProject.getSCorpusGraphs().size())) { + for (SCorpusGraph cGraph : saltProject.getSCorpusGraphs()) { + boolean isAssociated = false; + for (PepperModule fixture : fixtures) { + if (fixture.getSCorpusGraph() == cGraph) { + isAssociated = true; + break; + } + } + if (!isAssociated) { + Step alibiStep= createAlibiStep(true); + ((PepperJobImpl) job).addStep(alibiStep); + } + } + } + if (exporters.size() == 0) { + Step alibiStep= createAlibiStep(false); + ((PepperJobImpl) job).addStep(alibiStep); + } + + // start the conversion + job.convert(); + + for (DocumentController controller : ((PepperJobImpl) job).getDocumentControllers()) { + controller.awake(); + } + } +} diff --git a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/tests/PepperUtilTest.java b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/tests/PepperUtilTest.java index 772756fcd..e0824569f 100644 --- a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/tests/PepperUtilTest.java +++ b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/common/tests/PepperUtilTest.java @@ -28,6 +28,11 @@ import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperUtil; +/** + * This class is a test class to tes {@link PepperUtil}. + * @author florian + * + */ public class PepperUtilTest { @Before diff --git a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/ModuleControllerImplTest.java b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/ModuleControllerImplTest.java new file mode 100644 index 000000000..78dcc5f6f --- /dev/null +++ b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/ModuleControllerImplTest.java @@ -0,0 +1,70 @@ +/** + * Copyright 2009 Humboldt-Universität zu Berlin, INRIA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + */ +package de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.tests; + +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.apache.commons.io.FileUtils; +import org.eclipse.emf.common.util.URI; +import org.junit.Before; +import org.junit.Test; + +import de.hu_berlin.german.korpling.saltnpepper.pepper.common.PepperUtil; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.ModuleControllerImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.core.PepperJobImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.PepperManipulatorImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.testFramework.PepperModuleTest; + +public class ModuleControllerImplTest extends ModuleControllerImpl { + + public ModuleControllerImplTest() { + super("1"); + } + + @Before + public void setUp() throws Exception { + // setFixture(this); + this.setJob_basic(new PepperJobImpl("1")); + this.setPepperModule_basic(new PepperManipulatorImpl() { + }); + } + + @Test + public void testCopyRes() throws IOException { + File fromPath = new File(PepperModuleTest.getTestResources() + "/copyRes/"); + File toPath = PepperUtil.getTempTestFile("to"); + + FileUtils.deleteDirectory(toPath); + + this.getJob().setBaseDir(URI.createFileURI(fromPath.getCanonicalPath())); + String prop = "file1.txt -> " + toPath + "; file2.txt-> " + toPath.getAbsolutePath() + "; ./folder-> " + toPath; + this.copyResources(prop); + + File file1 = new File(toPath.getCanonicalPath() + "/file1.txt"); + File file2 = new File(toPath.getCanonicalPath() + "/file2.txt"); + File file3 = new File(toPath.getCanonicalPath() + "/folder/file3.txt"); + + assertTrue(file1.getCanonicalPath() + " does not exist", file1.exists()); + assertTrue(file2.getCanonicalPath() + " does not exist", file2.exists()); + assertTrue(file3.getCanonicalPath() + " does not exist", file3.exists()); + } + +} diff --git a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperMapperControllerImplTest.java b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperMapperControllerImplTest.java index 7e42b2fbc..99f015ffc 100644 --- a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperMapperControllerImplTest.java +++ b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperMapperControllerImplTest.java @@ -17,22 +17,16 @@ */ package de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.tests; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import org.junit.Before; import org.junit.Test; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperMapperController; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperties; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.exceptions.PepperModuleException; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.PepperMapperControllerImpl; import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.PepperMapperImpl; -import de.hu_berlin.german.korpling.saltnpepper.salt.SaltFactory; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SLayer; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SNode; -import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SRelation; -import de.hu_berlin.german.korpling.saltnpepper.salt.samples.SampleGenerator; public class PepperMapperControllerImplTest { @@ -54,25 +48,12 @@ public void setUp() throws Exception { } @Test - public void test_PropAddSLayer() { - SDocument sDoc= SaltFactory.eINSTANCE.createSDocument(); - SampleGenerator.createSDocumentStructure(sDoc); - int layersBefore = sDoc.getSDocumentGraph().getSLayers().size(); - getFixture().getPepperMapper().getProperties().setPropertyValue(PepperModuleProperties.PROP_AFTER_ADD_SLAYER, "layer1; layer2"); - sDoc.setSElementId(SaltFactory.eINSTANCE.createSElementId()); - getFixture().after(sDoc.getSElementId()); - - assertEquals(layersBefore+2, sDoc.getSDocumentGraph().getSLayers().size()); - SLayer layer1= sDoc.getSDocumentGraph().getSLayers().get(layersBefore); - SLayer layer2= sDoc.getSDocumentGraph().getSLayers().get(layersBefore+1); - for (SNode sNode: sDoc.getSDocumentGraph().getSNodes()){ - assertTrue(sNode.getSLayers().contains(layer1)); - assertTrue(sNode.getSLayers().contains(layer2)); - } - for (SRelation sRel: sDoc.getSDocumentGraph().getSRelations()){ - assertTrue(sRel.getSLayers().contains(layer1)); - assertTrue(sRel.getSLayers().contains(layer2)); + public void testSetSElementId(){ + try{ + getFixture().setSElementId(null); + fail(); + }catch(PepperModuleException e){ + // do nothing } } - } diff --git a/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperModuleImplTest.java b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperModuleImplTest.java new file mode 100644 index 000000000..bb4aba757 --- /dev/null +++ b/pepper-framework/src/test/java/de/hu_berlin/german/korpling/saltnpepper/pepper/modules/impl/tests/PepperModuleImplTest.java @@ -0,0 +1,136 @@ +/** + * Copyright 2009 Humboldt-Universität zu Berlin, INRIA. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * + */ +package de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.tests; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.File; +import java.io.IOException; + +import org.eclipse.emf.common.util.URI; +import org.junit.Before; +import org.junit.Test; + +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.PepperModuleProperties; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.PepperImporterImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.modules.impl.PepperModuleImpl; +import de.hu_berlin.german.korpling.saltnpepper.pepper.testFramework.PepperModuleTest; +import de.hu_berlin.german.korpling.saltnpepper.salt.SaltFactory; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpus; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SCorpusGraph; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCommon.sCorpusStructure.SDocument; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SLayer; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SNode; +import de.hu_berlin.german.korpling.saltnpepper.salt.saltCore.SRelation; +import de.hu_berlin.german.korpling.saltnpepper.salt.samples.SampleGenerator; + +public class PepperModuleImplTest extends PepperImporterImpl{ + + private PepperModuleImpl fixture = null; + + public PepperModuleImpl getFixture() { + return fixture; + } + + public void setFixture(PepperModuleImpl fixture) { + this.fixture = fixture; + } + + @Before + public void setUp() throws Exception { + setFixture(this); + getFixture().setProperties(new PepperModuleProperties()); + } + + @Test + public void testPropAddSLayer() { + SDocument sDoc = SaltFactory.eINSTANCE.createSDocument(); + SampleGenerator.createSDocumentStructure(sDoc); + int layersBefore = sDoc.getSDocumentGraph().getSLayers().size(); + getFixture().getProperties().setPropertyValue(PepperModuleProperties.PROP_AFTER_ADD_SLAYER, "layer1; layer2"); + sDoc.setSElementId(SaltFactory.eINSTANCE.createSElementId()); + getFixture().after(sDoc.getSElementId()); + + assertEquals(layersBefore + 2, sDoc.getSDocumentGraph().getSLayers().size()); + SLayer layer1 = sDoc.getSDocumentGraph().getSLayers().get(layersBefore); + SLayer layer2 = sDoc.getSDocumentGraph().getSLayers().get(layersBefore + 1); + for (SNode sNode : sDoc.getSDocumentGraph().getSNodes()) { + assertTrue(sNode.getSLayers().contains(layer1)); + assertTrue(sNode.getSLayers().contains(layer2)); + } + for (SRelation sRel : sDoc.getSDocumentGraph().getSRelations()) { + assertTrue(sRel.getSLayers().contains(layer1)); + assertTrue(sRel.getSLayers().contains(layer2)); + } + } + + @Test + public void testPropReadMeta() throws IOException{ + File corpusPath = new File(PepperModuleTest.getTestResources() + "/readMeta/"); + URI corpusURI= URI.createFileURI(corpusPath.getCanonicalPath()); + SCorpusGraph graph= SaltFactory.eINSTANCE.createSCorpusGraph(); + SCorpus corpus= graph.createSCorpus(null, "corpus"); + SCorpus subCorpus= graph.createSCorpus(corpus, "subcorpus"); + SDocument document= graph.createSDocument(subCorpus, "document"); + + this.getSElementId2ResourceTable().put(corpus.getSElementId(), corpusURI.appendSegment("corpus")); + this.getSElementId2ResourceTable().put(subCorpus.getSElementId(), corpusURI.appendSegment("corpus").appendSegment("subcorpus")); + this.getSElementId2ResourceTable().put(document.getSElementId(), corpusURI.appendSegment("corpus").appendSegment("subcorpus").appendSegment("document.txt")); + + getFixture().getProperties().getProperty(PepperModuleProperties.PROP_BEFORE_READ_META).setValueString("meta"); + + getFixture().readMeta(corpus.getSElementId()); + assertEquals(2, corpus.getSMetaAnnotations().size()); + assertEquals("b", corpus.getSMetaAnnotation("a").getSValue()); + assertEquals("d", corpus.getSMetaAnnotation("c").getSValue()); + + getFixture().readMeta(subCorpus.getSElementId()); + assertEquals(2, subCorpus.getSMetaAnnotations().size()); + assertEquals("2", subCorpus.getSMetaAnnotation("1").getSValue()); + assertEquals("4", subCorpus.getSMetaAnnotation("3").getSValue()); + + getFixture().readMeta(document.getSElementId()); + assertEquals(2, document.getSMetaAnnotations().size()); + assertEquals("Bart", document.getSMetaAnnotation("name").getSValue()); + assertEquals("Springfield", document.getSMetaAnnotation("place").getSValue()); + } + + @Test + public void test_PropAddSLayer() { + SDocument sDoc = SaltFactory.eINSTANCE.createSDocument(); + SampleGenerator.createSDocumentStructure(sDoc); + int layersBefore = sDoc.getSDocumentGraph().getSLayers().size(); + getFixture().getProperties().setPropertyValue(PepperModuleProperties.PROP_AFTER_ADD_SLAYER, "layer1; layer2"); + sDoc.setSElementId(SaltFactory.eINSTANCE.createSElementId()); + + getFixture().after(sDoc.getSElementId()); + + assertEquals(layersBefore + 2, sDoc.getSDocumentGraph().getSLayers().size()); + SLayer layer1 = sDoc.getSDocumentGraph().getSLayers().get(layersBefore); + SLayer layer2 = sDoc.getSDocumentGraph().getSLayers().get(layersBefore + 1); + for (SNode sNode : sDoc.getSDocumentGraph().getSNodes()) { + assertTrue(sNode.getSLayers().contains(layer1)); + assertTrue(sNode.getSLayers().contains(layer2)); + } + for (SRelation sRel : sDoc.getSDocumentGraph().getSRelations()) { + assertTrue(sRel.getSLayers().contains(layer1)); + assertTrue(sRel.getSLayers().contains(layer2)); + } + } +} diff --git a/pepper-framework/src/test/resources/copyRes/file1.txt b/pepper-framework/src/test/resources/copyRes/file1.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/pepper-framework/src/test/resources/copyRes/file1.txt @@ -0,0 +1 @@ +test diff --git a/pepper-framework/src/test/resources/copyRes/file2.txt b/pepper-framework/src/test/resources/copyRes/file2.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/pepper-framework/src/test/resources/copyRes/file2.txt @@ -0,0 +1 @@ +test diff --git a/pepper-framework/src/test/resources/copyRes/folder/file3.txt b/pepper-framework/src/test/resources/copyRes/folder/file3.txt new file mode 100644 index 000000000..9daeafb98 --- /dev/null +++ b/pepper-framework/src/test/resources/copyRes/folder/file3.txt @@ -0,0 +1 @@ +test diff --git a/pepper-framework/src/test/resources/readMeta/corpus/corpus.meta b/pepper-framework/src/test/resources/readMeta/corpus/corpus.meta new file mode 100644 index 000000000..a69ba90a4 --- /dev/null +++ b/pepper-framework/src/test/resources/readMeta/corpus/corpus.meta @@ -0,0 +1,2 @@ +a=b +c=d diff --git a/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.meta b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.meta new file mode 100644 index 000000000..7bfc92481 --- /dev/null +++ b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.meta @@ -0,0 +1,2 @@ +name=Bart +place=Springfield diff --git a/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.txt b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.txt new file mode 100644 index 000000000..261e13c40 --- /dev/null +++ b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/document.txt @@ -0,0 +1 @@ +This is just a placeholder. diff --git a/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/subcorpus.meta b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/subcorpus.meta new file mode 100644 index 000000000..c14d15f27 --- /dev/null +++ b/pepper-framework/src/test/resources/readMeta/corpus/subcorpus/subcorpus.meta @@ -0,0 +1,2 @@ +1=2 +3=4 diff --git a/pepper-lib/NOTICE b/pepper-lib/NOTICE index 5d8ba12f8..2a126cdbe 100755 --- a/pepper-lib/NOTICE +++ b/pepper-lib/NOTICE @@ -43,6 +43,7 @@ This project includes: CDI APIs under Apache License, Version 2.0 Common Eclipse Runtime under Eclipse Public License - v 1.0 Commons Codec under The Apache Software License, Version 2.0 + Commons Compiler under New BSD License Commons IO under The Apache Software License, Version 2.0 Commons Logging under The Apache Software License, Version 2.0 Core Runtime under Eclipse Public License - v 1.0 @@ -56,7 +57,7 @@ This project includes: Fluent API for Apache HttpClient under Apache License Guava: Google Core Libraries for Java under The Apache Software License, Version 2.0 Hamcrest Core under New BSD License - janino.wso2 under Apache License, Version 2.0 + Janino under New BSD License javax.inject under The Apache Software License, Version 2.0 JCL 1.1.1 implemented over SLF4J under MIT License JSR-250 Common Annotations for the JavaTM Platform under COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0 diff --git a/pepper-lib/conf/.gitignore b/pepper-lib/conf/.gitignore index d746c6c2c..24f9cb615 100644 --- a/pepper-lib/conf/.gitignore +++ b/pepper-lib/conf/.gitignore @@ -1,3 +1,4 @@ /pepper-test.properties /logback-test.xml /dep/ +*.properties~ diff --git a/pepper-lib/conf/modules.xml b/pepper-lib/conf/modules.xml index 51b4220b9..af18a8599 100644 --- a/pepper-lib/conf/modules.xml +++ b/pepper-lib/conf/modules.xml @@ -46,6 +46,9 @@ pepperModules-RelANNISModules + + pepperModules-ANNISModules + pepperModules-RidgesModules diff --git a/pepper-lib/conf/pepper.properties b/pepper-lib/conf/pepper.properties index 887b9136a..fcd91e6e7 100644 --- a/pepper-lib/conf/pepper.properties +++ b/pepper-lib/conf/pepper.properties @@ -19,7 +19,12 @@ pepper.maxAmountOfProcessedSDocuments=4 pepper.removeSDocumentAfterProcessing=true ########## -# Sets the path, where to store temporary files, which are necessary for the conversion. The deafult is set to the systems temprorary folder. +# Determines if an SDocument-object shall be removed after it was processed by all PepperModules +########## +pepper.gcAfterDocumentSleep=false + +########## +# Sets the path, where to store temporary files, which are necessary for the conversion. The default is set to the systems temporary folder. ########## #pepper.temporaries= @@ -37,3 +42,13 @@ pepper.removeSDocumentAfterProcessing=true # Determines the width of the console and can be set to 80 or 120. By default the value is set to 80 if the system is windows based and 120 otherwise. ########## pepper.console.width=120 + +########## +#Determines whether the status report should contain a progress for each module or just the global progress +########## +pepper.detailedStatusReport=true + +########## +#Determines the time interval of the convert status report +########## +pepper.reportInterval= 10000 diff --git a/pepper-lib/nb-configuration.xml b/pepper-lib/nb-configuration.xml index f70e86bc8..4ece09552 100644 --- a/pepper-lib/nb-configuration.xml +++ b/pepper-lib/nb-configuration.xml @@ -32,6 +32,6 @@ Any value defined here will override the pom.xml file value but is only applicab false 80 none - none + all diff --git a/pepper-lib/nbactions-convert.xml b/pepper-lib/nbactions-convert.xml index fe3a62682..a7cb55ff7 100644 --- a/pepper-lib/nbactions-convert.xml +++ b/pepper-lib/nbactions-convert.xml @@ -10,7 +10,7 @@ org.codehaus.mojo:exec-maven-plugin:1.2.1:exec - -Dlogback.configurationFile=./conf/logback.xml -Dpepper.home=/home/thomas/opt/SaltNPepper_2014.12.09-SNAPSHOT/ -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/tiger2/paula2relANNIS.pepperParams + -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/test2/merge.pepper java /home/thomas/saltnpepper/pepper/pepper-lib/ @@ -25,7 +25,7 @@ org.codehaus.mojo:exec-maven-plugin:1.2.1:exec - -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Dlogback.configurationFile=./conf/logback.xml -Dpepper.home=/home/thomas/opt/SaltNPepper_2014.12.09-SNAPSHOT/ -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/tiger2/paula2relANNIS.pepperParams + -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/test2/merge.pepper java true /home/thomas/saltnpepper/pepper/pepper-lib/ @@ -41,7 +41,7 @@ org.codehaus.mojo:exec-maven-plugin:1.2.1:exec - -Dlogback.configurationFile=./conf/logback.xml -Dpepper.home=/home/thomas/opt/SaltNPepper_2014.12.09-SNAPSHOT/ -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/tiger2/paula2relANNIS.pepperParams + -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter /home/thomas/korpora/relannis4/test2/merge.pepper java /home/thomas/saltnpepper/pepper/pepper-lib/ diff --git a/pepper-lib/nbactions.xml b/pepper-lib/nbactions.xml new file mode 100644 index 000000000..dd68ca861 --- /dev/null +++ b/pepper-lib/nbactions.xml @@ -0,0 +1,49 @@ + + + + run + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2.1:exec + + + -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter + java + /home/thomas/saltnpepper/pepper/pepper-lib + + + + debug + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2.1:exec + + + -Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter + java + true + /home/thomas/saltnpepper/pepper/pepper-lib + + + + profile + + jar + + + process-classes + org.codehaus.mojo:exec-maven-plugin:1.2.1:exec + + + -Dlogback.configurationFile=./conf/logback.xml -classpath %classpath de.hu_berlin.german.korpling.saltnpepper.pepper.cli.PepperStarter + java + /home/thomas/saltnpepper/pepper/pepper-lib + + + diff --git a/pepper-lib/pepperConversion.launch b/pepper-lib/pepperConversion.launch new file mode 100644 index 000000000..734c355af --- /dev/null +++ b/pepper-lib/pepperConversion.launch @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pepper-lib/pepperInteractive.launch b/pepper-lib/pepperInteractive.launch new file mode 100644 index 000000000..db702d4c4 --- /dev/null +++ b/pepper-lib/pepperInteractive.launch @@ -0,0 +1,27 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pepper-lib/pepperStart.sh b/pepper-lib/pepperStart.sh old mode 100644 new mode 100755 diff --git a/pepper-lib/pom.xml b/pepper-lib/pom.xml index b36b4ff29..a39eb2f9a 100644 --- a/pepper-lib/pom.xml +++ b/pepper-lib/pom.xml @@ -6,21 +6,16 @@ de.hu_berlin.german.korpling.saltnpepper pepper - 2.1.0 + 2.1.1-SNAPSHOT ../pom.xml - - WSO2repo - WSO2repo - http://maven.wso2.org/nexus/content/groups/wso2-public/ - - - Jabylon - Jabylon - http://www.jabylon.org/maven/ - - + + Jabylon + Jabylon + http://www.jabylon.org/maven/ + + scm:svn:https://korpling.german.hu-berlin.de/svn/saltnpepper/pepper/trunk/pepper-starter/ https://korpling.german.hu-berlin.de/svn/saltnpepper/pepper/pepper-starter/ @@ -58,9 +53,9 @@ 2.4 - janino.wso2 + org.codehaus.janino janino - 2.5.15.wso2v1 + 2.7.8 org.eclipse.aether @@ -155,6 +150,7 @@ plugins/** lib/** + conf/dep/** false diff --git a/pepper-lib/src/assemble/assemblies.xml b/pepper-lib/src/assemble/assemblies.xml index d6bb78d45..548abb97f 100644 --- a/pepper-lib/src/assemble/assemblies.xml +++ b/pepper-lib/src/assemble/assemblies.xml @@ -24,17 +24,26 @@ ${basedir}/plugins/ ${basedir}/lib/ - - ${basedir}/pepperStart.* + + ${basedir}/pepperStart.bat LICENSE NOTICE readme.txt + + 0644 - - + + ${project.groupId}.${project.artifactId}_${project.version} + + + ${basedir}/pepperStart.sh + + 0755 + + ${project.groupId}.${project.artifactId}_${project.version} diff --git a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/ConvertWizzardConsole.java b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/ConvertWizzardConsole.java index bbfc18d6d..eaf9c3f85 100644 --- a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/ConvertWizzardConsole.java +++ b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/ConvertWizzardConsole.java @@ -85,14 +85,15 @@ public class ConvertWizzardConsole { private static final String PROMPT = "wizzard"; private static final String MSG_IM = "\tPlease enter the number or the name of the importer you want to use. "; - private static final String MSG_IMPORT_CORPUS = "\tPlease enter (a further) path to corpus you want to import or press enter to skip. The current path is:'"+new File("").getAbsolutePath()+"'. "; + private static final String MSG_IMPORT_CORPUS = "\tPlease enter a (further) path to corpus you want to import or press enter to skip. When you use a relative path make the relative to:'"+new File("").getAbsolutePath()+"/'. "; private static final String MSG_PROP = "\tTo use a customization property, please enter it's number or name, the '=' and a value (e.g. 'name=value', or 'number=value'). To skip the customiazation, press enter. "; private static final String MSG_MAN = "\tIf you want to use a manipulator, please enter it's number or name, or press enter to skip. "; private static final String MSG_NO_PROPS = "\tNo customization properties available."; private static final String MSG_NO_VALID_MODULE = "\tSorry could not match the input, please enter the number or the name of the module again. "; private static final String MSG_NO_VALID_PROP = "\tSorry could not match the input, please enter the number or the name of the property followed by '=' and the value again. "; private static final String MSG_EX = "\tPlease enter the number or the name of the exporter you want to use. "; - private static final String MSG_EX_CORPUS = "\tPlease enter (a further) path to which you want to export the corpus. "; + private static final String MSG_EX_CORPUS = "\tPlease enter a (further) path to which you want to export the corpus or press enter to skip. When you use a relative path make the relative to:'"+new File("").getAbsolutePath()+"/'. "; + private static final String MSG_ABORTED = "Creating of Pepper workflow aborted by user's input. "; /** Determines if debug mode is on or off **/ @@ -311,7 +312,7 @@ public boolean importPhase(PepperJob pepperJob) { int state = 0; String input = null; StepDesc stepDesc = null; - out.println(MSG_IMPORT_CORPUS); + out.println(MSG_IMPORT_CORPUS.replace("(further) ", "")); // a map containing each registered module and a corresponding number, // to make selection easier (key= number, value= module desc) Map number2Module = null; @@ -351,6 +352,7 @@ public boolean importPhase(PepperJob pepperJob) { (!path.endsWith("/"))){ path= path + "/"; } + out.println("import corpus from: "+ path); stepDesc.getCorpusDesc().setCorpusPath(URI.createFileURI(path)); if ((number2Module == null) || (name2Module == null)) { @@ -526,7 +528,7 @@ public boolean exportPhase(PepperJob pepperJob) { int state = 0; String input = null; StepDesc stepDesc = null; - out.println(MSG_EX_CORPUS); + out.println(MSG_EX_CORPUS.replace("(further) ", "")); // a map containing each registered module and a corresponding number, // to make selection easier (key= number, value= module desc) Map number2Module = null; @@ -564,6 +566,7 @@ public boolean exportPhase(PepperJob pepperJob) { if (!path.endsWith("/")){ path= path + "/"; } + out.println("export corpus to: "+ path); stepDesc = pepperJob.createStepDesc(); stepDesc.setModuleType(MODULE_TYPE.EXPORTER); stepDesc.getCorpusDesc().setCorpusPath(URI.createFileURI(path)); diff --git a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/PepperStarter.java b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/PepperStarter.java index ae11b924c..3ed17e820 100644 --- a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/PepperStarter.java +++ b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/cli/PepperStarter.java @@ -17,6 +17,8 @@ */ package de.hu_berlin.german.korpling.saltnpepper.pepper.cli; +import ch.qos.logback.classic.LoggerContext; + import java.io.BufferedReader; import java.io.File; import java.io.IOException; @@ -43,8 +45,10 @@ import javax.xml.transform.stream.StreamResult; import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.Pair; import org.eclipse.emf.common.util.URI; +import org.osgi.framework.Bundle; import org.osgi.framework.BundleException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -139,6 +143,8 @@ public void setPepperConfiguration(PepperStarterConfiguration pepperConf) { } public enum COMMAND { + PRINT_DEPS("dependencies", "deps", "Bundle id or GROUP_ID::ARTIFACT_ID::VERSION::MAVEN_REPOSITORY_URL or plugin names split by space; parameter all prints dependencies for all plugins", "displays all dependencies of the specified component"), + // UPDATE("update", "u", "module name or location", "Updates the pepper module(s). Parameter \"all\" updates all modules listed in modules.xml."), // LIST_ALL("list", "l", null, "A table with information about all available Pepper modules."), @@ -153,7 +159,7 @@ public enum COMMAND { // EXIT("exit", "e", null, "Exits Pepper."), // - CONVERT("convert", "c", "workflow file", "Loads the passed 'workflow-file' and starts the conversion."), + CONVERT("convert", "c", "workflow file", "If no workflow file is passed, Pepper opens a conversion wizzard, which help you through the definition of a workflow proecess. If a 'worklow file' is passed, this file is load and the described workflow will be started."), // OSGI("osgi", "o", null, "Opens a console to access the underlying OSGi environment, if OSGi is used."), INSTALL_START("install_start", "is", "module path", "Installs the Pepper module located at 'module path' and starts it."), // UPDATE("update", "up", "module path", @@ -252,7 +258,7 @@ public String list() { if (isDebug) { e.printStackTrace(); } - retVal.append("Cannot not display any Pepper module. Calling " + COMMAND.START_OSGI.getName() + " might solve the problem. "); + retVal.append("Cannot display any Pepper module. Call " + COMMAND.START_OSGI.getName() + " might solve the problem. "); return (retVal.toString()); } number2module= new HashMap(moduleDescs.size()); @@ -278,7 +284,7 @@ public String list(String moduleName) { if (isDebug) { e.printStackTrace(); } - retVal.append("Cannot not display any Pepper module."); + retVal.append("Cannot display any Pepper module."); return (retVal.toString()); } Integer numOfModule = null; @@ -324,7 +330,8 @@ public String list(String moduleName) { map[1][0] = "version:"; map[1][1] = moduleDesc.getVersion(); map[2][0] = "supplier:"; - map[2][1] = moduleDesc.getSupplierContact().toString(); + map[2][1] = moduleDesc.getSupplierContact() == null ? + "" : moduleDesc.getSupplierContact().toString(); map[3][0] = "description:"; map[3][1] = moduleDesc.getDesc(); @@ -476,10 +483,10 @@ public void convert(String workFlowFile) { try { pepperJob.convert(); timestamp = System.currentTimeMillis() - timestamp; - output.println("conversion ended successfully, required time: " + (timestamp / 1000) + " s"); + output.println("conversion ended successfully, required time: " + DurationFormatUtils.formatDurationHMS(timestamp) + " s"); } catch (Exception e) { timestamp = System.currentTimeMillis() - timestamp; - output.println("CONVERSION ENDED WITH ERRORS, REQUIRED TIME: " + (timestamp / 1000) + " s"); + output.println("CONVERSION ENDED WITH ERRORS, REQUIRED TIME: " + DurationFormatUtils.formatDurationHMS(timestamp) + " s"); output.println(PepperUtil.breakString(" ", e.getMessage() + " (" + e.getClass().getSimpleName() + ")")); output.println("full stack trace:"); e.printStackTrace(output); @@ -795,7 +802,7 @@ private boolean write2ConfigFile(String groupId, String artifactId, String repos } catch (ParserConfigurationException | SAXException | IOException | FactoryConfigurationError | TransformerFactoryConfigurationError | TransformerException e) { logger.error("Could not read module table."); - logger.trace(" ", e);// TODO okay? + logger.trace(" ", e); return false; } return true; @@ -977,6 +984,8 @@ public void runInteractive() { output.println(debug()); } else if ((COMMAND.UPDATE.getName().equalsIgnoreCase(command)) || (COMMAND.UPDATE.getAbbreviation().equalsIgnoreCase(command))) { output.println(update(params)); + } else if (COMMAND.PRINT_DEPS.getName().equalsIgnoreCase(command) || COMMAND.PRINT_DEPS.getAbbreviation().equalsIgnoreCase(command)) { + output.print(printDependencies(params)); } else { output.println("Type 'help' for help."); } @@ -985,11 +994,70 @@ public void runInteractive() { if (!isDebug) { output.println("For more details enter '" + COMMAND.DEBUG.getName() + "' and redo last action. "); } else { - logger.error(" ", e.getMessage()); + String msg= e.getMessage(); + if ( (msg!= null)&& + (!msg.isEmpty())){ + logger.error(" ", e.getMessage()); + }else{ + e.printStackTrace(); + } } } } } + + /** This method reads the dependency command's parameters, calls pepper to + * invoke the dependency collection and returns the results. + * @param params -- command line parameters + * @return dependency tree print */ + private String printDependencies(List params) { + PepperOSGiConnector connector = (PepperOSGiConnector)getPepper(); + Map> moduleTable = null; + try { + moduleTable = getModuleTable(); + } catch (ParserConfigurationException | SAXException | IOException e) { + logger.error("Could not parse module table."); + return "No operation performed."; + } + + StringBuilder retVal = new StringBuilder(); + String groupId = null; + String version = null; + for (String param : params) { + if ("all".equalsIgnoreCase(param)){ + retVal.delete(0, retVal.length()); + for (String module : moduleTable.keySet()){ + groupId = moduleTable.get(module).getLeft(); + Bundle bundle = connector.getBundle(groupId, module, null); + version = bundle==null? null : bundle.getVersion().toString().replace(".SNAPSHOT", "-SNAPSHOT"); + if (version==null) { + logger.info(module.concat(" not installed. Collecting dependencies for newest version.")); + } + retVal.append(connector.printDependencies(moduleTable.get(module).getLeft(), module, version, moduleTable.get(module).getRight())).append(System.lineSeparator()).append(System.lineSeparator()); + } + return retVal.toString(); + } + else if (param.contains("::")){ + String[] coords = params.get(0).split("::"); + if (coords.length==4){ + retVal.append(connector.printDependencies(coords[0], coords[1], coords[2], coords[3])).append(System.lineSeparator()); + } + } + else if (moduleTable.keySet().contains(param)){ + groupId = moduleTable.get(param).getLeft(); + Bundle bundle = connector.getBundle(groupId, param, null); + version = bundle==null? null : bundle.getVersion().toString().replace(".SNAPSHOT", "-SNAPSHOT"); + if (version==null) { + logger.info(param.concat(" not installed. Collecting dependencies for newest version.")); + } + retVal.append(connector.printDependencies(moduleTable.get(param).getLeft(), param, version, moduleTable.get(param).getRight())).append(System.lineSeparator()); + } + else if (param.matches("#?[0-9]+")){ + retVal.append(connector.printDependencies(param.replace("#", ""))).append(System.lineSeparator()); + } + } + return retVal.toString(); + } // REMOVED THIS BECAUSE OF DEPENDENCY TO CONCRETE LOGGING FRAMEWORK IS // CONTRA SLF4J IDEA. @@ -1014,6 +1082,14 @@ public static void main(String[] args) throws Exception { PepperConnector pepper = null; boolean endedWithErrors = false; + if(LoggerFactory.getILoggerFactory() instanceof LoggerContext) { + LoggerContext lc = (LoggerContext) LoggerFactory.getILoggerFactory(); + // in our OSGI environment package data extraction + // results in strange errors when log() is called with an + // exception as argument + lc.setPackagingDataEnabled(false); + } + try { PepperStarterConfiguration pepperProps = new PepperStarterConfiguration(); pepperProps.load(); @@ -1058,7 +1134,7 @@ public static void main(String[] args) throws Exception { for (int i = 1; i < args.length; i++) { params.add(args[i]); } - logger.info(starter.update(params)); + logger.info(starter.update(params)); } else if (("-p".equalsIgnoreCase(args[0])) || ("-w".equalsIgnoreCase(args[0])) || (args[0] != null)) { String workFlowFile = null; if (("-p".equalsIgnoreCase(args[0])) || ("-w".equalsIgnoreCase(args[0]))) { diff --git a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/connectors/impl/MavenAccessor.java b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/connectors/impl/MavenAccessor.java index 96596dd71..c7a29ee95 100644 --- a/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/connectors/impl/MavenAccessor.java +++ b/pepper-lib/src/main/java/de/hu_berlin/german/korpling/saltnpepper/pepper/connectors/impl/MavenAccessor.java @@ -27,6 +27,8 @@ import java.text.DecimalFormatSymbols; import java.util.ArrayList; import java.util.Collections; +import java.util.Dictionary; +import java.util.Enumeration; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; @@ -166,9 +168,12 @@ private static enum STATUS{ /** delimiter for artifact strings */ private static final String DELIMITER = ":"; + /** path to temporary repository */ + private final String PATH_LOCAL_REPO; + /* MAVEN UTILS */ /** maven/aether utility */ - RepositorySystem system = null; + RepositorySystem mvnSystem = null; /** this Map contains all repos already used in this pepper session, key is url, value is repo */ HashMap repos = null; /** maven/aether utility used to build Objects of class {@link RemoteRepository}. */ @@ -186,12 +191,13 @@ public MavenAccessor(PepperOSGiConnector pepperOSGiConnector){ locator.addService( RepositoryConnectorFactory.class, BasicRepositoryConnectorFactory.class ); locator.addService( TransporterFactory.class, FileTransporterFactory.class ); locator.addService( TransporterFactory.class, HttpTransporterFactory.class ); - system = locator.getService( RepositorySystem.class ); + mvnSystem = locator.getService( RepositorySystem.class ); } repoBuilder = new RemoteRepository.Builder("", "default", ""); repos = new HashMap(); forbiddenFruits = new HashSet(); parentDependencies = new HashMap>(); + PATH_LOCAL_REPO = pepperOSGiConnector.getPepperStarterConfiguration().getTempPath()+"/local-repo/"; init(); initDependencies(); } @@ -242,11 +248,15 @@ private boolean initDependencies(){ collectRequest.addRepository(repos.get(CENTRAL_REPO)); collectRequest.setRootArtifact(pepArt); try { - CollectResult collectResult = system.collectDependencies( session, collectRequest ); + CollectResult collectResult = mvnSystem.collectDependencies( session, collectRequest ); List allDeps = getAllDependencies(collectResult.getRoot(), false); parentDependencies.put(frameworkVersion.replace("-SNAPSHOT", ""), allDeps); - for (Dependency dependency : allDeps){ - forbiddenFruits.add(dependency.getArtifact().toString()+DELIMITER+STATUS.FINAL); + Bundle bundle = null; + STATUS status = null; + for (Dependency dependency : allDeps){ + bundle = pepperOSGiConnector.getBundle(dependency.getArtifact().getGroupId(), dependency.getArtifact().getArtifactId(), null); + status = bundle==null || bundle.getHeaders().get("Bundle-SymbolicName").contains("singleton:=true")? STATUS.FINAL : STATUS.OVERRIDABLE; + forbiddenFruits.add(dependency.getArtifact().toString().concat(DELIMITER).concat(status.toString()).concat(DELIMITER).concat(bundle==null?"":bundle.getSymbolicName())); } write2Blacklist(); collectResult = null; @@ -262,8 +272,8 @@ private boolean initDependencies(){ private DefaultRepositorySystemSession getNewSession(){ DefaultRepositorySystemSession session = new DefaultRepositorySystemSession(); - LocalRepository localRepo = new LocalRepository( pepperOSGiConnector.getPepperStarterConfiguration().getTempPath()+"/local-repo/" ); - LocalRepositoryManager repoManager = system.newLocalRepositoryManager( session, localRepo ); + LocalRepository localRepo = new LocalRepository( PATH_LOCAL_REPO ); + LocalRepositoryManager repoManager = mvnSystem.newLocalRepositoryManager( session, localRepo ); session.setLocalRepositoryManager( repoManager ); session.setRepositoryListener(repoListener); session.setTransferListener(transferListener); @@ -354,7 +364,7 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b VersionRangeRequest rangeRequest = new VersionRangeRequest(); rangeRequest.addRepository(repo); rangeRequest.setArtifact(artifact); - VersionRangeResult rangeResult = system.resolveVersionRange(session, rangeRequest); + VersionRangeResult rangeResult = mvnSystem.resolveVersionRange(session, rangeRequest); rangeRequest.setArtifact( artifact ); /* utils needed for request */ @@ -382,7 +392,7 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b update = newestVersion.compareTo(installedVersion) > 0; artifactRequest.setArtifact(artifact); try{ - artifactResult = system.resolveArtifact(session, artifactRequest); + artifactResult = mvnSystem.resolveArtifact(session, artifactRequest); artifact = artifactResult.getArtifact(); srcExists = update && artifact.getFile().exists(); file = artifact.getFile(); @@ -416,7 +426,7 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b collectRequest.setRoot( new Dependency( artifact, "" ) ); collectRequest.addRepository(repos.get(CENTRAL_REPO)); collectRequest.addRepository(repo); - CollectResult collectResult = system.collectDependencies( session, collectRequest ); + CollectResult collectResult = mvnSystem.collectDependencies( session, collectRequest ); List allDependencies = getAllDependencies(collectResult.getRoot(), true); /* we have to remove the dependencies of pepperParent from the dependency list, since they are (sometimes) @@ -431,36 +441,29 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b if (parentVersion==null){ logger.warn(artifactId+": Could not perform update: pepper-parent version could not be determined."); return false; - } + } + Version max = isCompatiblePlugin(parentVersion); + if (!ignoreFrameworkVersion && max!=null){ + logger.info( + (new StringBuilder()) + .append("No update was performed because of a version incompatibility according to pepper-framework: ") + .append(newLine).append(artifactId).append(" only supported up to ").append(max.toString()).append(", but ").append(pepperOSGiConnector.getFrameworkVersion()).append(" is installed!") + .append(newLine).append("You can make pepper ignore this by using \"update").append(isSnapshot? " snapshot ":" ").append("iv ") + .append(artifactId).append("\"").toString()); + return false; + } allDependencies = cleanDependencies(allDependencies, session, parentVersion); Bundle bundle = null; Dependency dependency = null; //in the following we ignore the first dependency (i=0), because it is the module itself for (int i=1; i itDeps = null; Artifact installArtifact = null; for (int i=installArtifacts.size()-1; i>=0; i--){ try { @@ -479,20 +478,10 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b logger.info("installing: "+installArtifact); bundle = pepperOSGiConnector.installAndCopy(installArtifact.getFile().toURI()); if (i!=0){//the module itself must not be put on the blacklist - itDeps = forbiddenFruits.iterator(); - nxt = itDeps.next(); - while (nxt!=null){ - next = nxt.split(DELIMITER); - if (next[0].equals(installArtifact.getGroupId()) && next[1].equals(installArtifact.getArtifactId())){ - forbiddenFruits.remove(nxt); - logger.debug("Removed dependency from blacklist: "+nxt); - nxt = null; - } else { - nxt = itDeps.hasNext()? itDeps.next() : null; - } - } - forbiddenFruits.add(installArtifact.toString()+DELIMITER+STATUS.OVERRIDABLE); - logger.debug("Put dependency on blacklist: "+installArtifact.toString()); + putOnBlacklist(installArtifact); + }else if (installedBundle!=null){ + pepperOSGiConnector.remove(installedBundle.getSymbolicName()); + logger.info("Successfully removed version ".concat(installedBundle.getVersion().toString()).concat(" of ").concat(artifactId)); } if (bundle!=null){ bundle.start(); @@ -533,71 +522,71 @@ public boolean update(String groupId, String artifactId, String repositoryUrl, b return update; } + private Version isCompatiblePlugin(String pluginFrameworkVersion){ + VersionScheme vScheme = new GenericVersionScheme(); + Version frameworkVersion; + try { + frameworkVersion = vScheme.parseVersion(pepperOSGiConnector.getFrameworkVersion().replace(".SNAPSHOT", "").replace("-SNAPSHOT", "")); + Version depParentVersion = vScheme.parseVersion(pluginFrameworkVersion.replace("-SNAPSHOT", "")); + int m = 1+Integer.parseInt(depParentVersion.toString().split("\\.")[0]); + Version maxVersion = vScheme.parseVersion(m+".0.0"); + if (!(depParentVersion.compareTo(frameworkVersion)<=0 && frameworkVersion.compareTo(maxVersion)<0 + && !(frameworkVersion.equals(depParentVersion) && pepperOSGiConnector.getFrameworkVersion().contains("SNAPSHOT") && !pluginFrameworkVersion.contains("SNAPSHOT")))){ + return maxVersion; + } + } catch (InvalidVersionSpecificationException e) { + logger.error("Could not compare required framework version to running framework. Update will not be performed."); + } + return null; + } + /** * This method returns all dependencies as list. * Elementary dependencies and their daughters are skipped. */ private List getAllDependencies(DependencyNode startNode, boolean skipFramework){ - if ("provided".equalsIgnoreCase(startNode.getDependency().getScope())){ - forbiddenFruits.add(startNode.getDependency().getArtifact().toString()+DELIMITER+STATUS.OVERRIDABLE); - return Collections.emptyList(); - } List retVal = new ArrayList(); retVal.add(startNode.getDependency()); for (DependencyNode node : startNode.getChildren()){ - if ((!skipFramework || !node.getDependency().getArtifact().getArtifactId().contains("salt-")) && !dependencyAlreadyInstalled(node.getArtifact().toString())) { - retVal.addAll( getAllDependencies(node, skipFramework) ); - } + boolean isFramework = ARTIFACT_ID_PEPPER_FRAMEWORK.equals(node.getArtifact().getArtifactId()); + boolean isSalt = node.getArtifact().getArtifactId().contains("salt-"); + if ((isFramework&&!skipFramework)||(!isFramework&&!isSalt)) { + String blackListLine = getBlackListString(node.getArtifact()); + if (blackListLine!=null && blackListLine.split(DELIMITER)[4].equals(STATUS.FINAL.toString())){//dependency already installed AND singleton + //do nothing at the Moment (TODO-> maybe implement a version range check, that enables an exchange of singletons) + } + else{//dependency not installed yet or not singleton + if ("provided".equalsIgnoreCase(startNode.getDependency().getScope())){ + putOnBlacklist(node.getArtifact()); + return Collections.emptyList(); + }else{ + retVal.addAll(getAllDependencies(node, skipFramework)); + } + } + } + else if (skipFramework&&isFramework){//we need this for checking compatibility + retVal.add(node.getDependency()); + } } return retVal; } - /** - * Checks, if the given coords belong to a dependency that's already - * installed - * @param artifactString - * @return - */ - private boolean dependencyAlreadyInstalled(String artifactString){ - String[] coords = artifactString.split(DELIMITER); - String[] testCoords = null; - for (String dependencyString : forbiddenFruits){ - testCoords = dependencyString.split(DELIMITER); - if (STATUS.OVERRIDABLE.equals(testCoords[4]) && - coords[1].equals(testCoords[1]) && /*artifactId*/ - coords[0].equals(testCoords[0]) /*groupId*/ - ){ - /* check version */ - VersionScheme vScheme = new GenericVersionScheme(); - try { - Version version = vScheme.parseVersion(coords[3]); - Version testVersion = vScheme.parseVersion(testCoords[3]); - if (version.compareTo(testVersion)<=0){ - return true; - } - else { - /* find and delete dependency */ - String bundleName = pepperOSGiConnector.getBundleNameByDependency(coords[0], coords[1]); - if (bundleName!=null){ - try { - pepperOSGiConnector.remove(bundleName); - logger.info("removed dependency "+coords[1]+". Newer version is about to be installed."); - return false; - } catch (BundleException | IOException e) { - logger.warn("Could not delete dependency "+coords[1]+", so its older version remains."); - return true; - } - } - } - } catch (InvalidVersionSpecificationException e) { - logger.warn("Could not compare versions of dependency "+coords[1]+", so it will be dropped."); - } - return true; - } else if (STATUS.FINAL.equals(testCoords[4])){ - return true; + private void putOnBlacklist(Artifact artifact){ + if (getBlackListString(artifact)==null){//for safety reasons (future use of this method, etc) we do the check + Bundle bundle = pepperOSGiConnector.getBundle(artifact.getGroupId(), artifact.getArtifactId(), null); + STATUS status = bundle==null || !pepperOSGiConnector.isSingleton(bundle)? STATUS.OVERRIDABLE : STATUS.FINAL; + forbiddenFruits.add(artifact.toString().concat(DELIMITER).concat(status.toString()).concat(DELIMITER).concat(bundle==null?"":bundle.getSymbolicName())); + logger.debug("Put dependency on blacklist: ".concat(artifact.toString())); + } + } + private String getBlackListString(Artifact artifact){ + String as = artifact.toString().substring(0, artifact.toString().lastIndexOf(DELIMITER.charAt(0))); + for (String artifactString : forbiddenFruits){ + if (artifactString.startsWith(as)){ + return artifactString; } } - return false; + return null; } /** @@ -645,7 +634,6 @@ public String getBlacklist(){ * @return */ private List cleanDependencies(List dependencies, RepositorySystemSession session, String parentVersion){ - Dependency pepperFramework = null; try { final List parentDeps; List checkList = parentDependencies.get(parentVersion.replace("-SNAPSHOT", "")); @@ -655,7 +643,7 @@ private List cleanDependencies(List dependencies, Reposi collectRequest.addRepository(repos.get(CENTRAL_REPO)); collectRequest.addRepository(repos.get(KORPLING_MAVEN_REPO)); CollectResult collectResult; - collectResult = system.collectDependencies( session, collectRequest ); + collectResult = mvnSystem.collectDependencies( session, collectRequest ); parentDeps = getAllDependencies(collectResult.getRoot(), false); parentDependencies.put(parentVersion.replace("-SNAPSHOT", ""), parentDeps); }else{ @@ -669,32 +657,44 @@ private List cleanDependencies(List dependencies, Reposi itDeps = null; int j=0; List newDeps = new ArrayList(); - pepperFramework = next; + STATUS status = null; for (int i=0; i retVal = new ArrayList(); - retVal.add(pepperFramework); return retVal; } + private STATUS getDependencyStatus(String dependencyString){ + dependencyString = dependencyString.substring(0, dependencyString.lastIndexOf(':')); + for (String fruit : forbiddenFruits){ + if (fruit.startsWith(dependencyString)){ + if (fruit.split(DELIMITER)[4].equals(STATUS.FINAL.toString())){ + return STATUS.FINAL; + } + return STATUS.OVERRIDABLE; + } + } + return null; + } + /** * This method builds a RemoteRepository for diverse maven/aether purposes. * @param id @@ -707,6 +707,86 @@ private RemoteRepository buildRepo(String id, String url){ return repoBuilder.build(); } + /** This method starts invokes the computation of the dependency tree. If no version is + * provided, the highest version in the specified maven repository is used. If no repository + * is provided, maven central and the korpling maven repository are used for trial. */ + public String printDependencies(String groupId, String artifactId, String version, String repositoryUrl){ + /* repositories */ + RemoteRepository repo = null; + if (repositoryUrl==null){ + repo = repos.get(KORPLING_MAVEN_REPO); + if (repo==null){ + repo = buildRepo("korpling", KORPLING_MAVEN_REPO); + repos.put(KORPLING_MAVEN_REPO, repo); + } + } else { + repo = repos.get(repositoryUrl); + if (repo==null){ + repo = buildRepo("repository", repositoryUrl); + repos.put(repositoryUrl, repo); + } + } + /* version range resolution and dependency collection */ + DefaultRepositorySystemSession session = getNewSession(); + Artifact artifact = new DefaultArtifact(groupId, artifactId, "pom", version==null? "[0,)" : version); + if (version==null){ + VersionRangeRequest request = new VersionRangeRequest(); + request.setArtifact(artifact); + if (repositoryUrl==null){request.addRepository(repos.get(CENTRAL_REPO));} + request.addRepository(repo); + try { + VersionRangeResult rangeResult = mvnSystem.resolveVersionRange(session, request); + version = rangeResult.getHighestVersion().toString(); + artifact.setVersion(version); + } catch (VersionRangeResolutionException e) { + logger.error("Could not determine newest version."); + return null; + } + } + CollectRequest collectRequest = new CollectRequest(); + collectRequest.setRoot( new Dependency( artifact, "" ) ); + if (repositoryUrl==null){collectRequest.addRepository(repos.get(CENTRAL_REPO));} + collectRequest.addRepository(repo); + CollectResult collectResult; + try { + collectResult = mvnSystem.collectDependencies( session, collectRequest ); + return getDependencyPrint(collectResult.getRoot(), 0); + } catch (DependencyCollectionException e) { + logger.error("Could not print dependencies for ".concat(artifactId).concat(".")); + } + return null; + } + + /** this method recursively computes */ + private String getDependencyPrint(DependencyNode startNode, int depth){ + String d = ""; + for (int i=0; i installBundles(URI pluginPath, List dropinPath File[] fileLocations = new File(dropinLocation.getPath()) .listFiles((FilenameFilter) new SuffixFileFilter(".jar")); - for (File bundleJar : fileLocations) { - // check if file is file-object - if (bundleJar.isFile() && bundleJar.canRead()) { - // check if file is file jar - URI bundleURI = bundleJar.toURI(); - Bundle bundle = install(bundleURI); - if (bundle != null) { - bundles.add(bundle); - logger.debug("\t\tinstalling bundle: " + bundle. - getSymbolicName() + "-" + bundle.getVersion()); - - // set system property for bundle pathes - if (osgiBundlesProp == null) { - osgiBundlesProp = new StringBuilder(); + if (fileLocations!= null){ + for (File bundleJar : fileLocations) { + // check if file is file-object + if (bundleJar.isFile() && bundleJar.canRead()) { + // check if file is file jar + URI bundleURI = bundleJar.toURI(); + Bundle bundle = install(bundleURI); + if (bundle != null) { + bundles.add(bundle); + logger.debug("\t\tinstalling bundle: " + bundle.getSymbolicName() + "-" + bundle.getVersion()); + + // set system property for bundle pathes + if (osgiBundlesProp == null) { + osgiBundlesProp = new StringBuilder(); + } + osgiBundlesProp.append("reference:"); + osgiBundlesProp.append(bundleURI); + osgiBundlesProp.append(","); } - osgiBundlesProp.append("reference:"); - osgiBundlesProp.append(bundleURI); - osgiBundlesProp.append(","); } } } @@ -696,18 +697,8 @@ public Collection selfTest() { * This method checks the pepperModules in the modules.xml for updates * and triggers the installation process if a newer version is available */ - public boolean update(String groupId, String artifactId, String repositoryUrl, boolean isSnapshot, boolean ignoreFrameworkVersion){ - - /* checking, if a correlating bundle already exist, which would mean, the module is already installed */ - Bundle installedBundle = null; - List bundles = new ArrayList(bundleIdMap.values()); - for (int i=0; i de.hu_berlin.german.korpling.saltnpepper pepper - 2.1.0 + 2.1.1-SNAPSHOT ../pom.xml 4.0.0 @@ -33,7 +33,7 @@ serviceComponents.xml - 2.1.0 + 2.1.1-SNAPSHOT @@ -97,13 +97,13 @@ com.carrotgarden.osgi carrot-osgi-anno-scr-core ${carrot-osgi-anno-scr-core.version} - compile + provided com.carrotgarden.osgi carrot-osgi-anno-scr-make ${carrot-osgi-anno-scr-make.version} - compile + provided diff --git a/pepper-sampleModules/NOTICE b/pepper-sampleModules/NOTICE index 2640018bf..64c140d13 100644 --- a/pepper-sampleModules/NOTICE +++ b/pepper-sampleModules/NOTICE @@ -21,6 +21,7 @@ This project includes: Apache Commons Lang under The Apache Software License, Version 2.0 carrot-osgi-anno-scr-core under The BSD License carrot-osgi-anno-scr-make under The BSD License + Commons IO under The Apache Software License, Version 2.0 de.hu_berlin.german.korpling.saltnpepper.pepperModules-SampleModules under The Apache Software License, Version 2.0 Guava: Google Core Libraries for Java under The Apache Software License, Version 2.0 Hamcrest Core under New BSD License diff --git a/pepper-sampleModules/pom.xml b/pepper-sampleModules/pom.xml index 928c1636b..7fd50fe62 100644 --- a/pepper-sampleModules/pom.xml +++ b/pepper-sampleModules/pom.xml @@ -36,7 +36,7 @@ de.hu_berlin.german.korpling.saltnpepper pepper-parentModule - 2.1.0 + 2.1.1-SNAPSHOT ../pepper-parentModule/pom.xml This project provides three sample modules for Pepper. It provides an importer, an exporter and a manipulator. The project can be used as a container for creating new PepperModules. One just have to override the areas marked with TODO. diff --git a/pom.xml b/pom.xml index 91b82d0bd..57192f51c 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 de.hu_berlin.german.korpling.saltnpepper pepper - 2.1.0 + 2.1.1-SNAPSHOT pom pepper-framework @@ -64,8 +64,7 @@ ${basedir}/target/classes/OSGI-INF/service-component - - 2.1.0 + 2.1.1-SNAPSHOT 3.3.2 1.7.5 4.11 @@ -74,9 +73,10 @@ 3.9.1-v20130814-1242 2.0.2 2.0.2 - 2.7.6 + 2.7.8 1.0.2.v20150114 3.4.0.v20140312-2051 + 2.4 2.5.3 @@ -133,11 +133,18 @@ org.osgi org.osgi.core ${org.osgi.core.version} + provided org.osgi org.osgi.compendium ${org.osgi.compendium.version} + provided + + + commons-io + commons-io + ${commons.io.version} @@ -149,11 +156,13 @@ com.carrotgarden.osgi carrot-osgi-anno-scr-core ${carrot-osgi-anno-scr-core.version} + provided com.carrotgarden.osgi carrot-osgi-anno-scr-make ${carrot-osgi-anno-scr-make.version} + provided @@ -341,6 +350,21 @@ + + + org.apache.maven.plugins + maven-dependency-plugin + 2.10 + + + + + + org.apache.maven.plugins + maven-surefire-plugin + 2.18.1 + + ${project.groupId}.${project.artifactId}_${project.version}