From 7f0f8e2abb937b41c356188aa6dd6784028af158 Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Fri, 1 May 2020 20:49:28 +0200 Subject: [PATCH 1/9] run cuncurrent rendering --- .../org/jbake/app/DocumentRenderAgent.java | 61 ++++++++ .../jbake/app/DocumentRenderconfigAgent.java | 34 +++++ .../src/main/java/org/jbake/app/Oven.java | 8 +- .../main/java/org/jbake/app/RenderAgent.java | 50 +++++++ .../src/main/java/org/jbake/app/Renderer.java | 134 ++++++++---------- .../org/jbake/render/DocumentsRenderer.java | 18 +-- .../AbstractTemplateEngineRenderingTest.java | 20 ++- ...FreemarkerTemplateEngineRenderingTest.java | 1 + ...oovyMarkupTemplateEngineRenderingTest.java | 1 + .../jbake/render/DocumentsRendererTest.java | 47 +++--- .../org/jbake/render/FeedRendererTest.java | 5 +- .../org/jbake/render/IndexRendererTest.java | 6 +- .../java/org/jbake/render/RendererTest.java | 2 +- 13 files changed, 253 insertions(+), 134 deletions(-) create mode 100644 jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java create mode 100644 jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java create mode 100644 jbake-core/src/main/java/org/jbake/app/RenderAgent.java diff --git a/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java b/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java new file mode 100644 index 000000000..7a7be5677 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java @@ -0,0 +1,61 @@ +package org.jbake.app; + +import org.jbake.app.configuration.JBakeConfiguration; +import org.jbake.model.DocumentModel; +import org.jbake.model.ModelAttributes; +import org.jbake.template.DelegatingTemplateEngine; +import org.jbake.template.model.TemplateModel; + +import java.io.File; +import java.io.Writer; +import java.nio.file.Files; + +public class DocumentRenderAgent extends RenderAgent { + + private final DocumentModel document; + + public DocumentRenderAgent(JBakeConfiguration config, DocumentModel document, DelegatingTemplateEngine renderingEngine, Renderer renderer) { + super(config,renderingEngine,renderer); + this.document = document; + } + + public void renderDocument() throws Exception { + String docType = document.getType(); + String outputFilename = config.getDestinationFolder().getPath() + File.separatorChar + document.getUri(); + if (outputFilename.lastIndexOf('.') > outputFilename.lastIndexOf(File.separatorChar)) { + outputFilename = outputFilename.substring(0, outputFilename.lastIndexOf('.')); + } + + // delete existing versions if they exist in case status has changed either way + String outputExtension = config.getOutputExtensionByDocType(docType); + File draftFile = new File(outputFilename, config.getDraftSuffix() + outputExtension); + if (draftFile.exists()) { + Files.delete(draftFile.toPath()); + } + + File publishedFile = new File(outputFilename + outputExtension); + if (publishedFile.exists()) { + Files.delete(publishedFile.toPath()); + } + + if (document.getStatus().equals(ModelAttributes.Status.DRAFT)) { + outputFilename = outputFilename + config.getDraftSuffix(); + } + + File outputFile = new File(outputFilename + outputExtension); + TemplateModel model = new TemplateModel(); + model.setContent(document); + model.setRenderer(renderingEngine); + + try { + try (Writer out = createWriter(outputFile)) { + renderingEngine.renderDocument(model, findTemplateName(docType), out); + renderer.incrementCount(); + logger.info("Rendering [{}]... done!", outputFile); + } + } catch (Exception e) { + logger.error("Rendering [{}]... failed!", outputFile, e); + throw new Exception("Failed to render file " + outputFile.getAbsolutePath() + ". Cause: " + e.getMessage(), e); + } + } +} diff --git a/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java b/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java new file mode 100644 index 000000000..981e8421d --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java @@ -0,0 +1,34 @@ +package org.jbake.app; + +import org.jbake.app.Renderer.RenderingConfig; +import org.jbake.app.configuration.JBakeConfiguration; +import org.jbake.template.DelegatingTemplateEngine; + +import java.io.File; +import java.io.Writer; + +public class DocumentRenderconfigAgent extends RenderAgent { + + private final RenderingConfig renderingConfig; + + public DocumentRenderconfigAgent(JBakeConfiguration config, RenderingConfig renderingConfig, DelegatingTemplateEngine renderingEngine, Renderer renderer) { + super(config, renderingEngine, renderer); + this.renderingConfig = renderingConfig; + } + + @Override + protected void renderDocument() throws Exception { + File outputFile = renderingConfig.getPath(); + try { + try (Writer out = createWriter(outputFile)) { + renderingEngine.renderDocument(renderingConfig.getModel(), renderingConfig.getTemplate(), out); + renderer.incrementCount(); + } + logger.info("Rendering {} [{}]... done!", renderingConfig.getName(), outputFile); + } catch (Exception e) { + logger.error("Rendering {} [{}]... failed!", renderingConfig.getName(), outputFile, e); + throw new Exception("Failed to render " + renderingConfig.getName(), e); + } + } + +} diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index ead7ce2cf..261d24f03 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -176,13 +176,15 @@ public void bake() { asset.copyAssetsFromContent(config.getContentFolder()); errors.addAll(asset.getErrors()); - + utensils.getRenderer().shutdown(); LOGGER.info("Baking finished!"); long end = new Date().getTime(); - LOGGER.info("Baked {} items in {}ms", renderedCount, end - start); + LOGGER.info("Baked {} items in {}ms", utensils.getRenderer().getRenderCount(), end - start); if (!errors.isEmpty()) { LOGGER.error("Failed to bake {} item(s)!", errors.size()); } + } catch (InterruptedException e) { + e.printStackTrace(); } finally { contentStore.close(); contentStore.shutdown(); @@ -216,7 +218,7 @@ private void resetDocumentTypesAndExtractors() { /** * Load {@link RenderingTool} instances and delegate rendering of documents to them */ - private void renderContent() { + private void renderContent() throws InterruptedException { JBakeConfiguration config = utensils.getConfiguration(); Renderer renderer = utensils.getRenderer(); ContentStore contentStore = utensils.getContentStore(); diff --git a/jbake-core/src/main/java/org/jbake/app/RenderAgent.java b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java new file mode 100644 index 000000000..5f6340f26 --- /dev/null +++ b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java @@ -0,0 +1,50 @@ +package org.jbake.app; + +import org.jbake.app.configuration.JBakeConfiguration; +import org.jbake.template.DelegatingTemplateEngine; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; + +public abstract class RenderAgent implements Runnable { + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + protected JBakeConfiguration config; + protected DelegatingTemplateEngine renderingEngine; + protected Renderer renderer; + + public RenderAgent(JBakeConfiguration config, DelegatingTemplateEngine renderingEngine, Renderer renderer) { + this.config = config; + this.renderingEngine = renderingEngine; + this.renderer = renderer; + } + + @Override + public void run() { + try { + logger.info("Start render document. Current agent count: {} completed: {} tasks: {}", renderer.getActiveAgentCount(), renderer.getCompletedAgentCount(), renderer.getTaskCount()); + renderDocument(); + } catch (Exception e) { + renderer.addError(e); + } + } + + protected abstract void renderDocument() throws Exception; + + protected Writer createWriter(File file) throws IOException { + if (!file.exists()) { + file.getParentFile().mkdirs(); + file.createNewFile(); + } + + return new OutputStreamWriter(new FileOutputStream(file), config.getRenderEncoding()); + } + + protected String findTemplateName(String docType) { + return config.getTemplateFileByDocType(docType).getName(); + } +} diff --git a/jbake-core/src/main/java/org/jbake/app/Renderer.java b/jbake-core/src/main/java/org/jbake/app/Renderer.java index 5508fd327..9723290cd 100644 --- a/jbake-core/src/main/java/org/jbake/app/Renderer.java +++ b/jbake-core/src/main/java/org/jbake/app/Renderer.java @@ -9,18 +9,15 @@ import org.jbake.template.DelegatingTemplateEngine; import org.jbake.template.model.TemplateModel; import org.jbake.util.PagingHelper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.nio.file.Files; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; /** * Render output to a file. @@ -34,10 +31,12 @@ public class Renderer { private static final String ARCHIVE_TEMPLATE_NAME = "archive"; private static final String ERROR404_TEMPLATE_NAME = "error404"; - private final Logger logger = LoggerFactory.getLogger(Renderer.class); + private final ThreadPoolExecutor excutor; private final JBakeConfiguration config; private final DelegatingTemplateEngine renderingEngine; private final ContentStore db; + private final CopyOnWriteArrayList errors; + private final AtomicInteger renderCount = new AtomicInteger(0); /** * @param db The database holding the content @@ -84,6 +83,8 @@ public Renderer(ContentStore db, JBakeConfiguration config) { this.config = config; this.renderingEngine = new DelegatingTemplateEngine(db, config); this.db = db; + this.excutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(200); + errors = new CopyOnWriteArrayList<>(); } /** @@ -97,6 +98,8 @@ public Renderer(ContentStore db, JBakeConfiguration config, DelegatingTemplateEn this.config = config; this.renderingEngine = renderingEngine; this.db = db; + this.excutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(200); + errors = new CopyOnWriteArrayList<>(); } private String findTemplateName(String docType) { @@ -110,64 +113,11 @@ private String findTemplateName(String docType) { * @throws Exception if IOException or SecurityException are raised */ public void render(DocumentModel content) throws Exception { - String docType = content.getType(); - String outputFilename = config.getDestinationFolder().getPath() + File.separatorChar + content.getUri(); - if (outputFilename.lastIndexOf('.') > outputFilename.lastIndexOf(File.separatorChar)) { - outputFilename = outputFilename.substring(0, outputFilename.lastIndexOf('.')); - } - - // delete existing versions if they exist in case status has changed either way - String outputExtension = config.getOutputExtensionByDocType(docType); - File draftFile = new File(outputFilename, config.getDraftSuffix() + outputExtension); - if (draftFile.exists()) { - Files.delete(draftFile.toPath()); - } - - File publishedFile = new File(outputFilename + outputExtension); - if (publishedFile.exists()) { - Files.delete(publishedFile.toPath()); - } - - if (content.getStatus().equals(ModelAttributes.Status.DRAFT)) { - outputFilename = outputFilename + config.getDraftSuffix(); - } - - File outputFile = new File(outputFilename + outputExtension); - TemplateModel model = new TemplateModel(); - model.setContent(content); - model.setRenderer(renderingEngine); - - try { - try (Writer out = createWriter(outputFile)) { - renderingEngine.renderDocument(model, findTemplateName(docType), out); - } - logger.info("Rendering [{}]... done!", outputFile); - } catch (Exception e) { - logger.error("Rendering [{}]... failed!", outputFile, e); - throw new Exception("Failed to render file " + outputFile.getAbsolutePath() + ". Cause: " + e.getMessage(), e); - } + excutor.execute(new DocumentRenderAgent(config,content,renderingEngine, this)); } - private Writer createWriter(File file) throws IOException { - if (!file.exists()) { - file.getParentFile().mkdirs(); - file.createNewFile(); - } - - return new OutputStreamWriter(new FileOutputStream(file), config.getRenderEncoding()); - } - - private void render(RenderingConfig renderConfig) throws Exception { - File outputFile = renderConfig.getPath(); - try { - try (Writer out = createWriter(outputFile)) { - renderingEngine.renderDocument(renderConfig.getModel(), renderConfig.getTemplate(), out); - } - logger.info("Rendering {} [{}]... done!", renderConfig.getName(), outputFile); - } catch (Exception e) { - logger.error("Rendering {} [{}]... failed!", renderConfig.getName(), outputFile, e); - throw new Exception("Failed to render " + renderConfig.getName(), e); - } + private void render(RenderingConfig renderConfig) { + excutor.execute(new DocumentRenderconfigAgent(config, renderConfig, renderingEngine, this)); } /** @@ -176,7 +126,7 @@ private void render(RenderingConfig renderConfig) throws Exception { * @param indexFile The name of the output file * @throws Exception if IOException or SecurityException are raised */ - public void renderIndex(String indexFile) throws Exception { + public void renderIndex(String indexFile) { render(new DefaultRenderingConfig(indexFile, MASTERINDEX_TEMPLATE_NAME)); } @@ -190,16 +140,15 @@ public void renderIndexPaging(String indexFile) throws Exception { } else { PagingHelper pagingHelper = new PagingHelper(totalPosts, postsPerPage); - TemplateModel model = new TemplateModel(); - model.setRenderer(renderingEngine); - model.setNumberOfPages(pagingHelper.getNumberOfPages()); - try { db.setLimit(postsPerPage); for (int pageStart = 0, page = 1; pageStart < totalPosts; pageStart += postsPerPage, page++) { String fileName = indexFile; db.setStart(pageStart); + TemplateModel model = new TemplateModel(); + model.setRenderer(renderingEngine); + model.setNumberOfPages(pagingHelper.getNumberOfPages()); model.setCurrentPageNuber(page); String previous = pagingHelper.getPreviousFileName(page); model.setPreviousFilename(previous); @@ -233,7 +182,7 @@ public void renderIndexPaging(String indexFile) throws Exception { * @see About Sitemaps * @see Sitemap protocol */ - public void renderSitemap(String sitemapFile) throws Exception { + public void renderSitemap(String sitemapFile) throws Exception{ render(new DefaultRenderingConfig(sitemapFile, SITEMAP_TEMPLATE_NAME)); } @@ -243,7 +192,7 @@ public void renderSitemap(String sitemapFile) throws Exception { * @param feedFile The name of the output file * @throws Exception if default rendering configuration is not loaded correctly */ - public void renderFeed(String feedFile) throws Exception { + public void renderFeed(String feedFile) { render(new DefaultRenderingConfig(feedFile, FEED_TEMPLATE_NAME)); } @@ -283,13 +232,13 @@ public int renderTags(String tagPath) throws Exception { TemplateModel model = new TemplateModel(); model.setRenderer(renderingEngine); model.setTag(tag); - DocumentModel map = buildSimpleModel(ModelAttributes.TAG.toString()); + DocumentModel map = buildSimpleModel(ModelAttributes.TAG); File path = new File(config.getDestinationFolder() + File.separator + tagPath + File.separator + tag + config.getOutputExtension()); map.setRootPath(FileUtil.getUriPathToDestinationRoot(config, path)); model.setContent(map); - render(new ModelRenderingConfig(path, ModelAttributes.TAG.toString(), model, findTemplateName(ModelAttributes.TAG.toString()))); + render(new ModelRenderingConfig(path, ModelAttributes.TAG, model, findTemplateName(ModelAttributes.TAG))); renderedCount++; } catch (Exception e) { @@ -304,7 +253,7 @@ public int renderTags(String tagPath) throws Exception { // display all tags page. TemplateModel model = new TemplateModel(); model.setRenderer(renderingEngine); - DocumentModel map = buildSimpleModel(ModelAttributes.TAGS.toString()); + DocumentModel map = buildSimpleModel(ModelAttributes.TAGS); File path = new File(config.getDestinationFolder() + File.separator + tagPath + File.separator + "index" + config.getOutputExtension()); map.setRootPath(FileUtil.getUriPathToDestinationRoot(config, path)); @@ -344,7 +293,40 @@ private DocumentModel buildSimpleModel(String type) { return content; } - private interface RenderingConfig { + public void addError(Exception e) { + errors.add(e); + } + + public List getErrors() { + return errors; + } + + public void shutdown() throws InterruptedException { + this.excutor.shutdown(); + this.excutor.awaitTermination(10, TimeUnit.MINUTES); + } + + public int getRenderCount() { + return renderCount.get(); + } + + public void incrementCount() { + renderCount.incrementAndGet(); + } + + public int getActiveAgentCount() { + return excutor.getActiveCount(); + } + + public long getCompletedAgentCount() { + return excutor.getCompletedTaskCount(); + } + + public long getTaskCount() { + return excutor.getTaskCount(); + } + + public interface RenderingConfig { File getPath(); diff --git a/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java b/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java index 066c89d7f..c42a7831e 100644 --- a/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java +++ b/jbake-core/src/main/java/org/jbake/render/DocumentsRenderer.java @@ -10,15 +10,12 @@ import org.jbake.template.RenderingException; import java.io.File; -import java.util.LinkedList; -import java.util.List; public class DocumentsRenderer implements RenderingTool { @Override public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) throws RenderingException { int renderedCount = 0; - final List errors = new LinkedList<>(); DocumentList documentList = db.getUnrenderedContent(); for (DocumentModel document : documentList) { @@ -34,20 +31,13 @@ public int render(Renderer renderer, ContentStore db, JBakeConfiguration config) renderedCount++; } catch (Exception e) { - errors.add(e.getMessage()); + renderer.addError(e); } } - if (!errors.isEmpty()) { - StringBuilder sb = new StringBuilder(); - sb.append("Failed to render documents. Cause(s):"); - for (String error : errors) { - sb.append("\n").append(error); - } - throw new RenderingException(sb.toString()); - } else { - return renderedCount; - } + + return renderedCount; + } private DocumentModel getNextDoc(DocumentList typedList, DocumentModel doc) { diff --git a/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java b/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java index 69ee1cab4..940785c70 100644 --- a/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java +++ b/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java @@ -35,13 +35,15 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; import java.io.File; import java.nio.charset.Charset; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; @@ -167,6 +169,7 @@ public void renderPost() throws Exception { DocumentModel content = parser.processFile(sampleFile); content.setUri("/" + filename); renderer.render(content); + renderer.shutdown(); File outputFile = new File(destinationFolder, filename); Assert.assertTrue(outputFile.exists()); @@ -186,6 +189,7 @@ public void renderPage() throws Exception { DocumentModel content = parser.processFile(sampleFile); content.setUri("/" + filename); renderer.render(content); + renderer.shutdown(); File outputFile = new File(destinationFolder, filename); Assert.assertTrue(outputFile.exists()); @@ -200,7 +204,7 @@ public void renderPage() throws Exception { public void renderIndex() throws Exception { //exec renderer.renderIndex("index.html"); - + renderer.shutdown(); //validate File outputFile = new File(destinationFolder, "index.html"); Assert.assertTrue(outputFile.exists()); @@ -215,7 +219,9 @@ public void renderIndex() throws Exception { @Test public void renderFeed() throws Exception { renderer.renderFeed("feed.xml"); + renderer.shutdown(); File outputFile = new File(destinationFolder, "feed.xml"); + Assert.assertTrue(outputFile.exists()); // verify @@ -228,7 +234,9 @@ public void renderFeed() throws Exception { @Test public void renderArchive() throws Exception { renderer.renderArchive("archive.html"); + renderer.shutdown(); File outputFile = new File(destinationFolder, "archive.html"); + Assert.assertTrue(outputFile.exists()); // verify @@ -241,6 +249,7 @@ public void renderArchive() throws Exception { @Test public void renderTags() throws Exception { renderer.renderTags("tags"); + renderer.shutdown(); // verify File outputFile = new File(destinationFolder + File.separator + "tags" + File.separator + "blog.html"); @@ -256,6 +265,7 @@ public void renderTagsIndex() throws Exception { config.setRenderTagsIndex(true); renderer.renderTags("tags"); + renderer.shutdown(); File outputFile = new File(destinationFolder + File.separator + "tags" + File.separator + "index.html"); Assert.assertTrue(outputFile.exists()); String output = FileUtils.readFileToString(outputFile, Charset.defaultCharset()); @@ -271,6 +281,7 @@ public void renderSitemap() throws Exception { db.updateSchema(); renderer.renderSitemap("sitemap.xml"); + renderer.shutdown(); File outputFile = new File(destinationFolder, "sitemap.xml"); Assert.assertTrue(outputFile.exists()); @@ -298,6 +309,7 @@ public void checkDbTemplateModelIsPopulated() throws Exception { db.deleteAllByDocType("post"); renderer.renderIndexPaging("index.html"); + renderer.shutdown(); File outputFile = new File(destinationFolder, "index.html"); String output = FileUtils.readFileToString(outputFile, Charset.defaultCharset()); diff --git a/jbake-core/src/test/java/org/jbake/app/template/FreemarkerTemplateEngineRenderingTest.java b/jbake-core/src/test/java/org/jbake/app/template/FreemarkerTemplateEngineRenderingTest.java index de138b8c7..25dbd6f41 100644 --- a/jbake-core/src/test/java/org/jbake/app/template/FreemarkerTemplateEngineRenderingTest.java +++ b/jbake-core/src/test/java/org/jbake/app/template/FreemarkerTemplateEngineRenderingTest.java @@ -55,6 +55,7 @@ public void renderPaginatedIndex() throws Exception { )); renderer.renderIndexPaging("index.html"); + renderer.shutdown(); File outputFile = new File(destinationFolder, 2 + File.separator + "index.html"); String output = FileUtils.readFileToString(outputFile, Charset.defaultCharset()); diff --git a/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java b/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java index cbc52455b..a60c9cd30 100644 --- a/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java +++ b/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java @@ -76,6 +76,7 @@ public void renderCustomTypePaper() throws Exception { DocumentModel content = parser.processFile(sampleFile); content.setUri("/" + filename); renderer.render(content); + renderer.shutdown(); File outputFile = new File(destinationFolder, filename); Assert.assertTrue(outputFile.exists()); diff --git a/jbake-core/src/test/java/org/jbake/render/DocumentsRendererTest.java b/jbake-core/src/test/java/org/jbake/render/DocumentsRendererTest.java index 5985ccc15..4c9a92e60 100644 --- a/jbake-core/src/test/java/org/jbake/render/DocumentsRendererTest.java +++ b/jbake-core/src/test/java/org/jbake/render/DocumentsRendererTest.java @@ -7,12 +7,8 @@ import org.jbake.model.DocumentModel; import org.jbake.model.DocumentTypes; import org.jbake.model.ModelAttributes; -import org.jbake.template.RenderingException; import org.junit.Before; -import org.junit.Rule; import org.junit.Test; -import org.junit.jupiter.api.Assertions; -import org.junit.rules.ExpectedException; import org.mockito.ArgumentCaptor; import org.mockito.Captor; import org.mockito.MockitoAnnotations; @@ -43,7 +39,7 @@ public class DocumentsRendererTest { @Before public void setUp() { - MockitoAnnotations.initMocks(this); + MockitoAnnotations.openMocks(this); documentsRenderer = new DocumentsRenderer(); @@ -85,34 +81,27 @@ public void shouldReturnCountOfProcessedDocuments() throws Exception { } @Test - public void shouldThrowAnExceptionWithCollectedErrorMessages() { + public void shouldThrowAnExceptionWithCollectedErrorMessages() throws Exception { + // given String fakeExceptionMessage = "fake exception"; + DocumentTypes.addDocumentType("customType"); - // expect - Assertions.assertThrows( - RenderingException.class, () -> { - - // given - DocumentTypes.addDocumentType("customType"); - - DocumentList templateModelList = new DocumentList<>(); - DocumentModel document = emptyDocument(); - DocumentModel document2 = emptyDocument(); - templateModelList.add(document); - templateModelList.add(document2); - - // throw an exception for every call of renderer's render method - doThrow(new Exception(fakeExceptionMessage)).when(renderer).render(any(DocumentModel.class)); - when(db.getUnrenderedContent()).thenReturn(templateModelList); + DocumentList templateModelList = new DocumentList<>(); + DocumentModel document = emptyDocument(); + DocumentModel document2 = emptyDocument(); + templateModelList.add(document); + templateModelList.add(document2); - // when - int renderResponse = documentsRenderer.render(renderer, db, configuration); + // throw an exception for every call of renderer's render method + doThrow(new Exception(fakeExceptionMessage)).when(renderer).render(any(DocumentModel.class)); + when(db.getUnrenderedContent()).thenReturn(templateModelList); - // then - assertThat(renderResponse).isEqualTo(2); - }, - fakeExceptionMessage + "\n" + fakeExceptionMessage - ); + // when + documentsRenderer.render(renderer, db, configuration); + renderer.shutdown(); + // then + assertThat(renderer.getRenderCount()).isEqualTo(0); + verify(renderer, times(2)).addError(any()); } @Test diff --git a/jbake-core/src/test/java/org/jbake/render/FeedRendererTest.java b/jbake-core/src/test/java/org/jbake/render/FeedRendererTest.java index cb81f1d7d..5d65301b0 100644 --- a/jbake-core/src/test/java/org/jbake/render/FeedRendererTest.java +++ b/jbake-core/src/test/java/org/jbake/render/FeedRendererTest.java @@ -9,7 +9,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.anyString; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; @@ -79,7 +78,7 @@ public void doesRenderWhenConfigDoesRenderFeeds() throws Exception { verify(mockRenderer, times(1)).renderFeed(anyString()); } - @Test(expected = RenderingException.class) + @Test public void propogatesRenderingException() throws Exception { FeedRenderer renderer = new FeedRenderer(); @@ -90,8 +89,6 @@ public void propogatesRenderingException() throws Exception { ContentStore contentStore = mock(ContentStore.class); Renderer mockRenderer = mock(Renderer.class); - doThrow(new Exception()).when(mockRenderer).renderFeed(anyString()); - renderer.render(mockRenderer, contentStore, configuration); verify(mockRenderer, never()).renderFeed("random string"); diff --git a/jbake-core/src/test/java/org/jbake/render/IndexRendererTest.java b/jbake-core/src/test/java/org/jbake/render/IndexRendererTest.java index 40d6f1d8e..fd574ae87 100644 --- a/jbake-core/src/test/java/org/jbake/render/IndexRendererTest.java +++ b/jbake-core/src/test/java/org/jbake/render/IndexRendererTest.java @@ -65,7 +65,7 @@ public void returnsOneWhenConfigRendersIndices() throws RenderingException { } - @Test(expected = RenderingException.class) + @Test public void propagatesRenderingException() throws Exception { IndexRenderer renderer = new IndexRenderer(); @@ -76,11 +76,11 @@ public void propagatesRenderingException() throws Exception { ContentStore contentStore = mock(ContentStore.class); Renderer mockRenderer = mock(Renderer.class); - doThrow(new Exception()).when(mockRenderer).renderIndex(anyString()); + doThrow(new Exception()).when(mockRenderer).renderIndexPaging(anyString()); renderer.render(mockRenderer, contentStore, configuration); - verify(mockRenderer, never()).renderIndex(anyString()); + verify(mockRenderer, never()).renderIndexPaging(anyString()); } diff --git a/jbake-core/src/test/java/org/jbake/render/RendererTest.java b/jbake-core/src/test/java/org/jbake/render/RendererTest.java index c1cd2f0ce..d2ce63963 100644 --- a/jbake-core/src/test/java/org/jbake/render/RendererTest.java +++ b/jbake-core/src/test/java/org/jbake/render/RendererTest.java @@ -17,7 +17,6 @@ import org.mockito.junit.MockitoJUnitRunner; import java.io.File; -import java.net.URL; import static org.assertj.core.api.Assertions.assertThat; @@ -69,6 +68,7 @@ public void testRenderFileWorksWhenPathHasDotInButFileDoesNot() throws Exception content.setStatus("published"); renderer.render(content); + renderer.shutdown(); File outputFile = new File(outputPath.getAbsolutePath() + File.separatorChar + FOLDER + File.separatorChar + FILENAME); assertThat(outputFile).isFile(); From a85f679b131c96af41e53555029983514cdf7f36 Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sat, 2 May 2020 08:16:45 +0200 Subject: [PATCH 2/9] run concurrent crawl and asset copying --- .../src/main/java/org/jbake/app/Asset.java | 26 +- .../main/java/org/jbake/app/ContentStore.java | 8 +- .../src/main/java/org/jbake/app/Crawler.java | 327 ++++++++++-------- .../src/main/java/org/jbake/app/Oven.java | 15 +- .../main/java/org/jbake/app/RenderAgent.java | 1 - .../java/org/jbake/parser/YamlEngine.java | 1 + .../test/java/org/jbake/app/CrawlerTest.java | 9 +- 7 files changed, 211 insertions(+), 176 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/Asset.java b/jbake-core/src/main/java/org/jbake/app/Asset.java index 5bbc5d88e..84b6989a7 100644 --- a/jbake-core/src/main/java/org/jbake/app/Asset.java +++ b/jbake-core/src/main/java/org/jbake/app/Asset.java @@ -10,6 +10,11 @@ import java.io.File; import java.io.FileFilter; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedList; @@ -53,7 +58,9 @@ public Asset(JBakeConfiguration config) { * read from configuration */ public void copy() { - copy(config.getAssetFolder()); + if (config.getAssetFolder() != null) { + copy(config.getAssetFolder()); + } } /** @@ -62,12 +69,7 @@ public void copy() { * @param path The starting path */ public void copy(File path) { - FileFilter filter = new FileFilter() { - @Override - public boolean accept(File file) { - return (!config.getAssetIgnoreHidden() || !file.isHidden()) && (file.isFile() || FileUtil.directoryOnlyIfNotIgnored(file, config)); - } - }; + FileFilter filter = file -> (!config.getAssetIgnoreHidden() || !file.isHidden()) && (file.isFile() || FileUtil.directoryOnlyIfNotIgnored(file, config)); copy(path, config.getDestinationFolder(), filter); } @@ -78,7 +80,7 @@ public boolean accept(File file) { */ public void copySingleFile(File asset) { try { - if ( !asset.isDirectory() ) { + if (!asset.isDirectory()) { String targetPath = config.getDestinationFolder().getCanonicalPath() + File.separatorChar + assetSubPath(asset); LOGGER.info("Copying single asset file to [{}]", targetPath); copyFile(asset, new File(targetPath)); @@ -92,6 +94,7 @@ public void copySingleFile(File asset) { /** * Determine if a given file is an asset file. + * * @param path to the file to validate. * @return true if the path provided points to a file in the asset folder. */ @@ -99,7 +102,7 @@ public boolean isAssetFile(File path) { boolean isAsset = false; try { - if(FileUtil.directoryOnlyIfNotIgnored(path.getParentFile(), config)) { + if (FileUtil.directoryOnlyIfNotIgnored(path.getParentFile(), config)) { if (FileUtil.isFileInDirectory(path, config.getAssetFolder())) { isAsset = true; } else if (FileUtil.isFileInDirectory(path, config.getContentFolder()) @@ -142,15 +145,14 @@ private String assetSubPath(File asset) throws IOException { private void copy(File sourceFolder, File targetFolder, final FileFilter filter) { final File[] assets = sourceFolder.listFiles(filter); if (assets != null) { - Arrays.sort(assets); - for (File asset : assets) { + Arrays.stream(assets).parallel().forEach( asset ->{ final File target = new File(targetFolder, asset.getName()); if (asset.isFile()) { copyFile(asset, target); } else if (asset.isDirectory()) { copy(asset, target, filter); } - } + }); } } diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 7dc71e887..027223a4f 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -302,9 +302,11 @@ private DocumentList query(String sql, Object... args) { return DocumentList.wrap(results); } - private void executeCommand(String query, Object... args) { + private synchronized void executeCommand(String query, Object... args) { activateOnCurrentThread(); + db.getTransaction().begin(); db.command(query, args); + db.getTransaction().commit(); } public Set getTags() { @@ -407,10 +409,12 @@ public boolean isActive() { return db.isActiveOnCurrentThread(); } - public void addDocument(DocumentModel document) { + public synchronized void addDocument(DocumentModel document) { + db.getTransaction().begin(); ODocument doc = new ODocument(Schema.DOCUMENTS); doc.fromMap(document); doc.save(); + db.getTransaction().commit(); } protected abstract class Schema { diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index 0c3d8b9d6..a541c4293 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -1,6 +1,5 @@ package org.jbake.app; -import com.orientechnologies.orient.core.record.impl.ODocument; import org.apache.commons.configuration2.CompositeConfiguration; import org.apache.commons.io.FilenameUtils; import org.jbake.app.configuration.JBakeConfiguration; @@ -19,7 +18,9 @@ import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.Date; -import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * Crawls a file system looking for content. @@ -30,6 +31,7 @@ public class Crawler { private static final Logger logger = LoggerFactory.getLogger(Crawler.class); private final ContentStore db; + private final ExecutorService executor; private final JBakeConfiguration config; private final Parser parser; @@ -46,6 +48,7 @@ public Crawler(ContentStore db, File source, CompositeConfiguration config) { this.db = db; this.config = new JBakeConfigurationFactory().createDefaultJbakeConfiguration(source, config); this.parser = new Parser(this.config); + this.executor = Executors.newFixedThreadPool(100); } /** @@ -58,11 +61,12 @@ public Crawler(ContentStore db, JBakeConfiguration config) { this.db = db; this.config = config; this.parser = new Parser(config); + this.executor = Executors.newFixedThreadPool(100); } - public void crawl() { + public void crawl() throws InterruptedException { crawl(config.getContentFolder()); - + shutdown(); logger.info("Content detected:"); for (String docType : DocumentTypes.getDocumentTypes()) { long count = db.getDocumentCount(docType); @@ -83,54 +87,13 @@ public void crawlDataFiles() { } } - /** - * Crawl all files and folders looking for content. - * - * @param path Folder to start from - */ - private void crawl(File path) { - File[] contents = path.listFiles(FileUtil.getFileFilter(config)); - if (contents != null) { - Arrays.sort(contents); - for (File sourceFile : contents) { - if (sourceFile.isFile()) { - crawlFile(sourceFile); - } else if (sourceFile.isDirectory()) { - crawl(sourceFile); - } - } - } - } - - private void crawlFile(File sourceFile) { - - StringBuilder sb = new StringBuilder(); - sb.append("Processing [").append(sourceFile.getPath()).append("]... "); - String sha1 = buildHash(sourceFile); - String uri = buildURI(sourceFile); - DocumentStatus status = findDocumentStatus(uri, sha1); - if (status == DocumentStatus.UPDATED) { - sb.append(" : modified "); - db.deleteContent(uri); - } else if (status == DocumentStatus.IDENTICAL) { - sb.append(" : same "); - } else if (DocumentStatus.NEW == status) { - sb.append(" : new "); - } - - logger.info("{}", sb); - - if (status != DocumentStatus.IDENTICAL) { - processSourceFile(sourceFile, sha1, uri); - } - } - /** * Crawl all files and folders looking for data files. * * @param path Folder to start from */ private void crawlDataFiles(File path) { + File[] contents = path.listFiles(FileUtil.getDataFileFilter()); if (contents != null) { Arrays.sort(contents); @@ -140,26 +103,21 @@ private void crawlDataFiles(File path) { sb.append("Processing [").append(sourceFile.getPath()).append("]... "); String sha1 = buildHash(sourceFile); String uri = buildDataFileURI(sourceFile); - boolean process = true; - DocumentStatus status = DocumentStatus.NEW; - String docType = config.getDataFileDocType(); - status = findDocumentStatus(uri, sha1); + DocumentStatus status = findDocumentStatus(uri, sha1); if (status == DocumentStatus.UPDATED) { sb.append(" : modified "); db.deleteContent(uri); } else if (status == DocumentStatus.IDENTICAL) { sb.append(" : same "); - process = false; - } - if (!process) { - break; - } - if (DocumentStatus.NEW == status) { + } else if (DocumentStatus.NEW == status) { sb.append(" : new "); } - if (process) { // new or updated + + if (status != DocumentStatus.IDENTICAL) { + String docType = config.getDataFileDocType(); crawlDataFile(sourceFile, sha1, uri, docType); } + logger.info("{}", sb); } if (sourceFile.isDirectory()) { @@ -168,36 +126,6 @@ private void crawlDataFiles(File path) { } } } - - private String buildHash(final File sourceFile) { - String sha1; - try { - sha1 = FileUtil.sha1(sourceFile); - } catch (Exception e) { - logger.error("unable to build sha1 hash for source file '{}'", sourceFile); - sha1 = ""; - } - return sha1; - } - - private String buildURI(final File sourceFile) { - String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getContentFolder()), ""); - - if (useNoExtensionUri(uri)) { - // convert URI from xxx.html to xxx/index.html - uri = createNoExtensionUri(uri); - } else { - uri = createUri(uri); - } - - // strip off leading / to enable generating non-root based sites - if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { - uri = uri.substring(1); - } - - return uri; - } - private String buildDataFileURI(final File sourceFile) { String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getDataFolder()), ""); // strip off leading / @@ -207,42 +135,6 @@ private String buildDataFileURI(final File sourceFile) { return uri; } - // TODO: Refactor - parametrize the following two methods into one. - // commons-codec's URLCodec could be used when we add that dependency. - private String createUri(String uri) { - try { - return FileUtil.URI_SEPARATOR_CHAR - + FilenameUtils.getPath(uri) - + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) - + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. - } - } - - private String createNoExtensionUri(String uri) { - try { - return FileUtil.URI_SEPARATOR_CHAR - + FilenameUtils.getPath(uri) - + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) - + FileUtil.URI_SEPARATOR_CHAR - + "index" - + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. - } - } - - private boolean useNoExtensionUri(String uri) { - boolean noExtensionUri = config.getUriWithoutExtension(); - String noExtensionUriPrefix = config.getPrefixForUriWithoutExtension(); - - return noExtensionUri - && (noExtensionUriPrefix != null) - && (noExtensionUriPrefix.length() > 0) - && uri.startsWith(noExtensionUriPrefix); - } - private void crawlDataFile(final File sourceFile, final String sha1, final String uri, final String documentType) { try { DocumentModel document = parser.processFile(sourceFile); @@ -262,51 +154,185 @@ private void crawlDataFile(final File sourceFile, final String sha1, final Strin } } - private void processSourceFile(final File sourceFile, final String sha1, final String uri) { - DocumentModel document = parser.processFile(sourceFile); - if (document != null) { - if (DocumentTypes.contains(document.getType())) { - addAdditionalDocumentAttributes(document, sourceFile, sha1, uri); - if (config.getImgPathUpdate()) { - // Prevent image source url's from breaking - HtmlUtil.fixImageSourceUrls(document, config); + /** + * Crawl all files and folders looking for content. + * + * @param path Folder to start from + */ + private void crawl(File path) { + File[] contents = path.listFiles(FileUtil.getFileFilter(config)); + if (contents != null) { + Arrays.sort(contents); + for (File sourceFile : contents) { + if (sourceFile.isFile()) { + crawlFile(sourceFile); + } else if (sourceFile.isDirectory()) { + crawl(sourceFile); } + } + } + } - db.addDocument(document); + + + private void crawlFile(File sourceFile) { + executor.execute(new CrawlAgent(sourceFile, db)); + } + + public void shutdown() throws InterruptedException { + executor.shutdown(); + executor.awaitTermination(10, TimeUnit.MINUTES); + } + + class CrawlAgent implements Runnable { + + private final ContentStore db; + private final File sourceFile; + + CrawlAgent(File sourceFile, ContentStore db) { + this.sourceFile = sourceFile; + this.db = db; + } + + @Override + public void run() { + StringBuilder sb = new StringBuilder(); + sb.append("Processing [").append(sourceFile.getPath()).append("]... "); + String sha1 = buildHash(sourceFile); + String uri = buildURI(sourceFile); + DocumentStatus status = findDocumentStatus(uri, sha1); + if (status == DocumentStatus.UPDATED) { + sb.append(" : modified "); + db.deleteContent(uri); + } else if (status == DocumentStatus.IDENTICAL) { + sb.append(" : same "); + } else if (DocumentStatus.NEW == status) { + sb.append(" : new "); + } + + logger.info("{}", sb); + + if (status != DocumentStatus.IDENTICAL) { + processSourceFile(sourceFile, sha1, uri); + } + } + + private String buildURI(final File sourceFile) { + String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getContentFolder()), ""); + + if (useNoExtensionUri(uri)) { + // convert URI from xxx.html to xxx/index.html + uri = createNoExtensionUri(uri); } else { - logger.warn("{} has an unknown document type '{}' and has been ignored!", sourceFile, document.getType()); + uri = createUri(uri); } - } else { - logger.warn("{} has an invalid header, it has been ignored!", sourceFile); + + // strip off leading / to enable generating non-root based sites + if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { + uri = uri.substring(1); + } + + return uri; } - } - private void addAdditionalDocumentAttributes(DocumentModel document, File sourceFile, String sha1, String uri) { - document.setRootPath(getPathToRoot(sourceFile)); - document.setSha1(sha1); - document.setRendered(false); - document.setFile(sourceFile.getPath()); - document.setSourceUri(uri); - document.setUri(uri); - document.setCached(true); + // TODO: Refactor - parametrize the following two methods into one. + // commons-codec's URLCodec could be used when we add that dependency. + private String createUri(String uri) { + try { + return FileUtil.URI_SEPARATOR_CHAR + + FilenameUtils.getPath(uri) + + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) + + config.getOutputExtension(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. + } + } + + private String createNoExtensionUri(String uri) { + try { + return FileUtil.URI_SEPARATOR_CHAR + + FilenameUtils.getPath(uri) + + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) + + FileUtil.URI_SEPARATOR_CHAR + + "index" + + config.getOutputExtension(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. + } + } + + private boolean useNoExtensionUri(String uri) { + boolean noExtensionUri = config.getUriWithoutExtension(); + String noExtensionUriPrefix = config.getPrefixForUriWithoutExtension(); + + return noExtensionUri + && (noExtensionUriPrefix != null) + && (noExtensionUriPrefix.length() > 0) + && uri.startsWith(noExtensionUriPrefix); + } + + private void processSourceFile(final File sourceFile, final String sha1, final String uri) { + DocumentModel document = parser.processFile(sourceFile); + + if (document != null) { + if (DocumentTypes.contains(document.getType())) { + addAdditionalDocumentAttributes(document, sourceFile, sha1, uri); + + if (config.getImgPathUpdate()) { + // Prevent image source url's from breaking + HtmlUtil.fixImageSourceUrls(document, config); + } + + db.addDocument(document); + } else { + logger.warn("{} has an unknown document type '{}' and has been ignored!", sourceFile, document.getType()); + } + } else { + logger.warn("{} has an invalid header, it has been ignored!", sourceFile); + } + } - if (document.getStatus().equals(ModelAttributes.Status.PUBLISHED_DATE) + private void addAdditionalDocumentAttributes(DocumentModel document, File sourceFile, String sha1, String uri) { + document.setRootPath(getPathToRoot(sourceFile)); + document.setSha1(sha1); + document.setRendered(false); + document.setFile(sourceFile.getPath()); + document.setSourceUri(uri); + document.setUri(uri); + document.setCached(true); + + if (document.getStatus().equals(ModelAttributes.Status.PUBLISHED_DATE) && (document.getDate() != null) && new Date().after(document.getDate())) { - document.setStatus(ModelAttributes.Status.PUBLISHED); + document.setStatus(ModelAttributes.Status.PUBLISHED); + } + + if (config.getUriWithoutExtension()) { + document.setNoExtensionUri(uri.replace("/index.html", "/")); + } } - if (config.getUriWithoutExtension()) { - document.setNoExtensionUri(uri.replace("/index.html", "/")); + private String getPathToRoot(File sourceFile) { + return FileUtil.getUriPathToContentRoot(config, sourceFile); } + + } - private String getPathToRoot(File sourceFile) { - return FileUtil.getUriPathToContentRoot(config, sourceFile); + private String buildHash(final File sourceFile) { + String sha1; + try { + sha1 = FileUtil.sha1(sourceFile); + } catch (Exception e) { + logger.error("unable to build sha1 hash for source file '{}'", sourceFile); + sha1 = ""; + } + return sha1; } + private DocumentStatus findDocumentStatus(String uri, String sha1) { DocumentList match = db.getDocumentStatus(uri); if (!match.isEmpty()) { @@ -322,4 +348,5 @@ private DocumentStatus findDocumentStatus(String uri, String sha1) { } } + } diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index 261d24f03..e3ca7c41f 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -154,32 +154,35 @@ public void bake() { setLocale(); try { - - final long start = new Date().getTime(); - LOGGER.info("Baking has started..."); contentStore.startup(); updateDocTypesFromConfiguration(); contentStore.updateSchema(); contentStore.updateAndClearCacheIfNeeded(config.getClearCache(), config.getTemplateFolder()); + LOGGER.info("Baking has started..."); + final long bakeStart = new Date().getTime(); // process source content + LOGGER.info("Crawling content..."); crawler.crawl(); // process data files crawler.crawlDataFiles(); // render content + LOGGER.info("Rendering content..."); renderContent(); // copy assets + LOGGER.info("Copy assets..."); asset.copy(); asset.copyAssetsFromContent(config.getContentFolder()); - errors.addAll(asset.getErrors()); utensils.getRenderer().shutdown(); + errors.addAll(asset.getErrors()); + errors.addAll(utensils.getRenderer().getErrors()); LOGGER.info("Baking finished!"); - long end = new Date().getTime(); - LOGGER.info("Baked {} items in {}ms", utensils.getRenderer().getRenderCount(), end - start); + long bakeEnd = new Date().getTime(); + LOGGER.info("Baked {} items in {}ms", utensils.getRenderer().getRenderCount(), bakeEnd - bakeStart); if (!errors.isEmpty()) { LOGGER.error("Failed to bake {} item(s)!", errors.size()); } diff --git a/jbake-core/src/main/java/org/jbake/app/RenderAgent.java b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java index 5f6340f26..5cf9b8a94 100644 --- a/jbake-core/src/main/java/org/jbake/app/RenderAgent.java +++ b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java @@ -26,7 +26,6 @@ public RenderAgent(JBakeConfiguration config, DelegatingTemplateEngine rendering @Override public void run() { try { - logger.info("Start render document. Current agent count: {} completed: {} tasks: {}", renderer.getActiveAgentCount(), renderer.getCompletedAgentCount(), renderer.getTaskCount()); renderDocument(); } catch (Exception e) { renderer.addError(e); diff --git a/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java b/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java index bf65a81cc..f33a04da2 100644 --- a/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java @@ -35,6 +35,7 @@ private DocumentModel parseFile(File file) { Yaml yaml = new Yaml(); try (InputStream is = new FileInputStream(file)) { Object result = yaml.load(is); + model.setType("data"); if (result instanceof List) { model.put("data", result); } else if (result instanceof Map) { diff --git a/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java b/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java index 73d9cfc12..33e20c5cb 100644 --- a/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java +++ b/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java @@ -8,8 +8,6 @@ import org.jbake.model.DocumentTypes; import org.jbake.util.DataFileUtil; import org.junit.Assert; -import org.junit.Before; -import org.junit.BeforeClass; import org.junit.Test; import java.util.Map; @@ -20,7 +18,7 @@ public class CrawlerTest extends ContentStoreIntegrationTest { @Test - public void crawl() { + public void crawl() throws InterruptedException { Crawler crawler = new Crawler(db, config); crawler.crawl(); @@ -53,12 +51,13 @@ public void crawl() { } @Test - public void crawlDataFiles() { + public void crawlDataFiles() throws InterruptedException { Crawler crawler = new Crawler(db, config); // manually register data doctype DocumentTypes.addDocumentType(config.getDataFileDocType()); db.updateSchema(); crawler.crawlDataFiles(); + crawler.shutdown(); Assert.assertEquals(1, db.getDocumentCount("data")); DataFileUtil util = new DataFileUtil(db, "data"); @@ -68,7 +67,7 @@ public void crawlDataFiles() { } @Test - public void renderWithPrettyUrls() { + public void renderWithPrettyUrls() throws InterruptedException { config.setUriWithoutExtension(true); config.setPrefixForUriWithoutExtension("/blog"); From 4658088aedb2ede1833d54d14353877d86d801bc Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sat, 2 May 2020 17:48:00 +0200 Subject: [PATCH 3/9] synchronize query access add index for tags --- .../src/main/java/org/jbake/app/Asset.java | 6 +++--- .../main/java/org/jbake/app/ContentStore.java | 9 +++++---- .../src/main/java/org/jbake/app/Crawler.java | 16 ++++++---------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/Asset.java b/jbake-core/src/main/java/org/jbake/app/Asset.java index 84b6989a7..0aa9421e6 100644 --- a/jbake-core/src/main/java/org/jbake/app/Asset.java +++ b/jbake-core/src/main/java/org/jbake/app/Asset.java @@ -143,9 +143,9 @@ private String assetSubPath(File asset) throws IOException { } private void copy(File sourceFolder, File targetFolder, final FileFilter filter) { - final File[] assets = sourceFolder.listFiles(filter); - if (assets != null) { - Arrays.stream(assets).parallel().forEach( asset ->{ + File[] array = sourceFolder.listFiles(filter); + if ( array != null ) { + Arrays.stream(array).parallel().forEach(asset -> { final File target = new File(targetFolder, asset.getName()); if (asset.isFile()) { copyFile(asset, target); diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 027223a4f..31a20c0f9 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -169,8 +169,6 @@ private void startupIfEnginesAreMissing() { public void drop() { activateOnCurrentThread(); -// db.drop(); - orient.drop(name); } @@ -290,13 +288,13 @@ private void insertTemplatesSignature(String currentTemplatesSignature) { executeCommand(STATEMENT_INSERT_TEMPLATES_SIGNATURE, currentTemplatesSignature); } - private DocumentList query(String sql) { + private synchronized DocumentList query(String sql) { activateOnCurrentThread(); OResultSet results = db.query(sql); return DocumentList.wrap(results); } - private DocumentList query(String sql, Object... args) { + private synchronized DocumentList query(String sql, Object... args) { activateOnCurrentThread(); OResultSet results = db.command(sql, args); return DocumentList.wrap(results); @@ -355,6 +353,9 @@ private void createSignatureType(OSchema schema) { OClass signatures = schema.createClass(Schema.SIGNATURES); signatures.createProperty(ModelAttributes.SHA1, OType.STRING).setNotNull(true); signatures.createIndex("sha1Idx", OClass.INDEX_TYPE.UNIQUE, ModelAttributes.SHA1); + + signatures.createProperty("key", OType.STRING); + signatures.createIndex("kexIdx", OClass.INDEX_TYPE.UNIQUE, "key"); } public void updateAndClearCacheIfNeeded(boolean needed, File templateFolder) { diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index a541c4293..7574a43af 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -162,17 +162,13 @@ private void crawlDataFile(final File sourceFile, final String sha1, final Strin * @param path Folder to start from */ private void crawl(File path) { - File[] contents = path.listFiles(FileUtil.getFileFilter(config)); - if (contents != null) { - Arrays.sort(contents); - for (File sourceFile : contents) { - if (sourceFile.isFile()) { - crawlFile(sourceFile); - } else if (sourceFile.isDirectory()) { - crawl(sourceFile); - } + Arrays.stream(path.listFiles(FileUtil.getFileFilter(config))).parallel().forEach( source -> { + if (source.isFile()) { + crawlFile(source); + } else if (source.isDirectory()) { + crawl(source); } - } + }); } From 57d3707e2ab788ff4b3ef95ce77370dc8c0b2139 Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sun, 10 May 2020 14:03:18 +0200 Subject: [PATCH 4/9] fix renderCustomTypePaper test for GroovyMarkupTemplateEngineRenderingTest --- .../template/AbstractTemplateEngineRenderingTest.java | 2 +- .../GroovyMarkupTemplateEngineRenderingTest.java | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java b/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java index 940785c70..7fe3907f3 100644 --- a/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java +++ b/jbake-core/src/test/java/org/jbake/app/template/AbstractTemplateEngineRenderingTest.java @@ -60,7 +60,7 @@ public abstract class AbstractTemplateEngineRenderingTest extends ContentStoreIn protected File templateFolder; protected Renderer renderer; protected Locale currentLocale; - private Parser parser; + protected Parser parser; public AbstractTemplateEngineRenderingTest(String templateDir, String templateExtension) { this.templateDir = templateDir; diff --git a/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java b/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java index a60c9cd30..1f6b4fe8e 100644 --- a/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java +++ b/jbake-core/src/test/java/org/jbake/app/template/GroovyMarkupTemplateEngineRenderingTest.java @@ -1,12 +1,8 @@ package org.jbake.app.template; import org.apache.commons.io.FileUtils; -import org.jbake.app.Crawler; import org.jbake.app.DBUtil; -import org.jbake.app.Parser; -import org.jbake.app.Renderer; import org.jbake.model.DocumentModel; -import org.jbake.model.DocumentTypes; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -64,12 +60,6 @@ public GroovyMarkupTemplateEngineRenderingTest() { @Test public void renderCustomTypePaper() throws Exception { // setup - - - Crawler crawler = new Crawler(db, config); - crawler.crawl(); - Parser parser = new Parser(config); - Renderer renderer = new Renderer(db, config); String filename = "published-paper.html"; File sampleFile = new File(sourceFolder.getPath() + File.separator + "content" + File.separator + "papers" + File.separator + filename); From 825d49772f62e1ff7061d9ca74e0ddfb0ffdb86e Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sun, 10 May 2020 14:43:23 +0200 Subject: [PATCH 5/9] fix tests for windows with hidden files --- jbake-core/src/test/java/org/jbake/app/OvenTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jbake-core/src/test/java/org/jbake/app/OvenTest.java b/jbake-core/src/test/java/org/jbake/app/OvenTest.java index a59e84aec..4b8c5ad38 100644 --- a/jbake-core/src/test/java/org/jbake/app/OvenTest.java +++ b/jbake-core/src/test/java/org/jbake/app/OvenTest.java @@ -50,6 +50,7 @@ public void setUp() throws Exception { configuration.setDestinationFolder(output); configuration.setTemplateFolder(new File(sourceFolder, "groovyMarkupTemplates")); configuration.setProperty("template.paper.file", "paper.tpl"); + TestUtils.hideAssets(new File(sourceFolder,"content")); } @AfterEach From 92ad9ebb48289c149d633b8c7ef2c2afc4b49c8e Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sun, 23 May 2021 07:49:15 +0200 Subject: [PATCH 6/9] show how long each task took --- .../src/main/java/org/jbake/app/Crawler.java | 39 +++++++++++-------- .../org/jbake/app/DocumentRenderAgent.java | 9 ++++- .../jbake/app/DocumentRenderconfigAgent.java | 9 ++++- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index 7574a43af..716777ef3 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -194,24 +194,31 @@ class CrawlAgent implements Runnable { @Override public void run() { + long start = System.currentTimeMillis(); StringBuilder sb = new StringBuilder(); - sb.append("Processing [").append(sourceFile.getPath()).append("]... "); - String sha1 = buildHash(sourceFile); - String uri = buildURI(sourceFile); - DocumentStatus status = findDocumentStatus(uri, sha1); - if (status == DocumentStatus.UPDATED) { - sb.append(" : modified "); - db.deleteContent(uri); - } else if (status == DocumentStatus.IDENTICAL) { - sb.append(" : same "); - } else if (DocumentStatus.NEW == status) { - sb.append(" : new "); - } - - logger.info("{}", sb); + try { + sb.append("Processing [").append(sourceFile.getPath()).append("]... "); + String sha1 = buildHash(sourceFile); + String uri = buildURI(sourceFile); + DocumentStatus status = findDocumentStatus(uri, sha1); + if (status == DocumentStatus.UPDATED) { + sb.append(" : modified "); + db.deleteContent(uri); + } else if (status == DocumentStatus.IDENTICAL) { + sb.append(" : same "); + } else if (DocumentStatus.NEW == status) { + sb.append(" : new "); + } - if (status != DocumentStatus.IDENTICAL) { - processSourceFile(sourceFile, sha1, uri); + if (status != DocumentStatus.IDENTICAL) { + processSourceFile(sourceFile, sha1, uri); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + long end = System.currentTimeMillis(); + long delta = end - start; + logger.info("{} ({} ms)", sb, delta); } } diff --git a/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java b/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java index 7a7be5677..ec902dcbe 100644 --- a/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java +++ b/jbake-core/src/main/java/org/jbake/app/DocumentRenderAgent.java @@ -20,6 +20,7 @@ public DocumentRenderAgent(JBakeConfiguration config, DocumentModel document, De } public void renderDocument() throws Exception { + long start = System.currentTimeMillis(); String docType = document.getType(); String outputFilename = config.getDestinationFolder().getPath() + File.separatorChar + document.getUri(); if (outputFilename.lastIndexOf('.') > outputFilename.lastIndexOf(File.separatorChar)) { @@ -51,10 +52,14 @@ public void renderDocument() throws Exception { try (Writer out = createWriter(outputFile)) { renderingEngine.renderDocument(model, findTemplateName(docType), out); renderer.incrementCount(); - logger.info("Rendering [{}]... done!", outputFile); + long end = System.currentTimeMillis(); + long delta = end - start; + logger.info("Rendering [{}]... done! ({} ms)", outputFile, delta); } } catch (Exception e) { - logger.error("Rendering [{}]... failed!", outputFile, e); + long end = System.currentTimeMillis(); + long delta = end - start; + logger.error("Rendering [{}]... failed! ({} ms)", outputFile, delta, e); throw new Exception("Failed to render file " + outputFile.getAbsolutePath() + ". Cause: " + e.getMessage(), e); } } diff --git a/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java b/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java index 981e8421d..efc788562 100644 --- a/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java +++ b/jbake-core/src/main/java/org/jbake/app/DocumentRenderconfigAgent.java @@ -18,15 +18,20 @@ public DocumentRenderconfigAgent(JBakeConfiguration config, RenderingConfig rend @Override protected void renderDocument() throws Exception { + long start = System.currentTimeMillis(); File outputFile = renderingConfig.getPath(); try { try (Writer out = createWriter(outputFile)) { renderingEngine.renderDocument(renderingConfig.getModel(), renderingConfig.getTemplate(), out); renderer.incrementCount(); } - logger.info("Rendering {} [{}]... done!", renderingConfig.getName(), outputFile); + long end = System.currentTimeMillis(); + long delta = end - start; + logger.info("Rendering {} [{}]... done! ({} ms)", renderingConfig.getName(), outputFile, delta); } catch (Exception e) { - logger.error("Rendering {} [{}]... failed!", renderingConfig.getName(), outputFile, e); + long end = System.currentTimeMillis(); + long delta = end - start; + logger.error("Rendering {} [{}]... failed! ({} ms)", renderingConfig.getName(), outputFile, delta, e); throw new Exception("Failed to render " + renderingConfig.getName(), e); } } From 5df8e4888e3a7dfd0861ff0ff1216ad5161bbd77 Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sun, 23 May 2021 12:59:11 +0200 Subject: [PATCH 7/9] cleanup crawler to execute data folder processing concurrently --- .../main/java/org/jbake/app/ContentStore.java | 5 - .../src/main/java/org/jbake/app/Crawler.java | 233 ++++-------------- .../src/main/java/org/jbake/app/Oven.java | 4 - .../src/main/java/org/jbake/app/Parser.java | 23 +- .../java/org/jbake/parser/MarkupEngine.java | 61 +++++ .../java/org/jbake/parser/ParserEngine.java | 7 + .../java/org/jbake/parser/YamlEngine.java | 11 + .../app/ContentStoreIntegrationTest.java | 1 - .../test/java/org/jbake/app/CrawlerTest.java | 3 +- .../src/test/java/org/jbake/app/OvenTest.java | 1 - 10 files changed, 151 insertions(+), 198 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 31a20c0f9..b4a36727f 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -148,11 +148,6 @@ public void close() { DBUtil.closeDataStore(); } - public void shutdown() { - -// Orient.instance().shutdown(); - } - private void startupIfEnginesAreMissing() { // Using a jdk which doesn't bundle a javascript engine // throws a NoClassDefFoundError while logging the warning diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index 716777ef3..31dcad312 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -1,7 +1,6 @@ package org.jbake.app; import org.apache.commons.configuration2.CompositeConfiguration; -import org.apache.commons.io.FilenameUtils; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.app.configuration.JBakeConfigurationFactory; import org.jbake.model.DocumentModel; @@ -13,9 +12,7 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; +import java.io.FileFilter; import java.util.Arrays; import java.util.Date; import java.util.concurrent.ExecutorService; @@ -65,9 +62,10 @@ public Crawler(ContentStore db, JBakeConfiguration config) { } public void crawl() throws InterruptedException { - crawl(config.getContentFolder()); + crawlContentFiles(); + crawlDataFiles(); shutdown(); - logger.info("Content detected:"); + for (String docType : DocumentTypes.getDocumentTypes()) { long count = db.getDocumentCount(docType); if (count > 0) { @@ -76,103 +74,36 @@ public void crawl() throws InterruptedException { } } - public void crawlDataFiles() { - crawlDataFiles(config.getDataFolder()); + private void crawlContentFiles() { + crawl(config.getContentFolder(), FileUtil.getFileFilter(config)); + } + protected void crawlDataFiles() { + crawl(config.getDataFolder(), FileUtil.getDataFileFilter()); logger.info("Data files detected:"); - String docType = config.getDataFileDocType(); - long count = db.getDocumentCount(docType); - if (count > 0) { - logger.info("Parsed {} files", count); - } } /** - * Crawl all files and folders looking for data files. + * Crawl all files and folders looking for content. * * @param path Folder to start from */ - private void crawlDataFiles(File path) { - - File[] contents = path.listFiles(FileUtil.getDataFileFilter()); - if (contents != null) { - Arrays.sort(contents); - for (File sourceFile : contents) { - if (sourceFile.isFile()) { - StringBuilder sb = new StringBuilder(); - sb.append("Processing [").append(sourceFile.getPath()).append("]... "); - String sha1 = buildHash(sourceFile); - String uri = buildDataFileURI(sourceFile); - DocumentStatus status = findDocumentStatus(uri, sha1); - if (status == DocumentStatus.UPDATED) { - sb.append(" : modified "); - db.deleteContent(uri); - } else if (status == DocumentStatus.IDENTICAL) { - sb.append(" : same "); - } else if (DocumentStatus.NEW == status) { - sb.append(" : new "); - } - - if (status != DocumentStatus.IDENTICAL) { - String docType = config.getDataFileDocType(); - crawlDataFile(sourceFile, sha1, uri, docType); - } - - logger.info("{}", sb); + private void crawl(File path, FileFilter filter) { + File[] filteredFiles = path.listFiles(filter); + if (filteredFiles != null) { + Arrays.stream(filteredFiles).parallel().forEach(source -> { + if (source.isFile()) { + crawlFile(source); + } else if (source.isDirectory()) { + crawl(source, filter); } - if (sourceFile.isDirectory()) { - crawlDataFiles(sourceFile); - } - } - } - } - private String buildDataFileURI(final File sourceFile) { - String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getDataFolder()), ""); - // strip off leading / - if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { - uri = uri.substring(1, uri.length()); - } - return uri; - } - - private void crawlDataFile(final File sourceFile, final String sha1, final String uri, final String documentType) { - try { - DocumentModel document = parser.processFile(sourceFile); - if (document != null) { - document.setSha1(sha1); - document.setRendered(true); - document.setFile(sourceFile.getPath()); - document.setSourceUri(uri); - document.setType(documentType); - - db.addDocument(document); - } else { - logger.warn("{} couldn't be parsed so it has been ignored!", sourceFile); - } - } catch (Exception ex) { - throw new RuntimeException("Failed crawling file: " + sourceFile.getPath() + " " + ex.getMessage(), ex); + }); + } else { + logger.debug("filter does not apply"); } } - - /** - * Crawl all files and folders looking for content. - * - * @param path Folder to start from - */ - private void crawl(File path) { - Arrays.stream(path.listFiles(FileUtil.getFileFilter(config))).parallel().forEach( source -> { - if (source.isFile()) { - crawlFile(source); - } else if (source.isDirectory()) { - crawl(source); - } - }); - } - - - private void crawlFile(File sourceFile) { executor.execute(new CrawlAgent(sourceFile, db)); } @@ -198,8 +129,8 @@ public void run() { StringBuilder sb = new StringBuilder(); try { sb.append("Processing [").append(sourceFile.getPath()).append("]... "); - String sha1 = buildHash(sourceFile); - String uri = buildURI(sourceFile); + String sha1 = parser.buildHash(sourceFile); + String uri = parser.buildURI(sourceFile); DocumentStatus status = findDocumentStatus(uri, sha1); if (status == DocumentStatus.UPDATED) { sb.append(" : modified "); @@ -222,60 +153,6 @@ public void run() { } } - private String buildURI(final File sourceFile) { - String uri = FileUtil.asPath(sourceFile).replace(FileUtil.asPath(config.getContentFolder()), ""); - - if (useNoExtensionUri(uri)) { - // convert URI from xxx.html to xxx/index.html - uri = createNoExtensionUri(uri); - } else { - uri = createUri(uri); - } - - // strip off leading / to enable generating non-root based sites - if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { - uri = uri.substring(1); - } - - return uri; - } - - // TODO: Refactor - parametrize the following two methods into one. - // commons-codec's URLCodec could be used when we add that dependency. - private String createUri(String uri) { - try { - return FileUtil.URI_SEPARATOR_CHAR - + FilenameUtils.getPath(uri) - + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) - + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. - } - } - - private String createNoExtensionUri(String uri) { - try { - return FileUtil.URI_SEPARATOR_CHAR - + FilenameUtils.getPath(uri) - + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) - + FileUtil.URI_SEPARATOR_CHAR - + "index" - + config.getOutputExtension(); - } catch (UnsupportedEncodingException e) { - throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. - } - } - - private boolean useNoExtensionUri(String uri) { - boolean noExtensionUri = config.getUriWithoutExtension(); - String noExtensionUriPrefix = config.getPrefixForUriWithoutExtension(); - - return noExtensionUri - && (noExtensionUriPrefix != null) - && (noExtensionUriPrefix.length() > 0) - && uri.startsWith(noExtensionUriPrefix); - } - private void processSourceFile(final File sourceFile, final String sha1, final String uri) { DocumentModel document = parser.processFile(sourceFile); @@ -298,22 +175,26 @@ private void processSourceFile(final File sourceFile, final String sha1, final S } private void addAdditionalDocumentAttributes(DocumentModel document, File sourceFile, String sha1, String uri) { - document.setRootPath(getPathToRoot(sourceFile)); document.setSha1(sha1); - document.setRendered(false); + document.setRendered(true); + document.setCached(true); document.setFile(sourceFile.getPath()); document.setSourceUri(uri); - document.setUri(uri); - document.setCached(true); - if (document.getStatus().equals(ModelAttributes.Status.PUBLISHED_DATE) - && (document.getDate() != null) - && new Date().after(document.getDate())) { - document.setStatus(ModelAttributes.Status.PUBLISHED); - } + if ( !document.getType().equals(config.getDataFileDocType())) { + document.setRootPath(getPathToRoot(sourceFile)); + document.setUri(uri); + document.setRendered(false); - if (config.getUriWithoutExtension()) { - document.setNoExtensionUri(uri.replace("/index.html", "/")); + if (document.getStatus().equals(ModelAttributes.Status.PUBLISHED_DATE) + && (document.getDate() != null) + && new Date().after(document.getDate())) { + document.setStatus(ModelAttributes.Status.PUBLISHED); + } + + if (config.getUriWithoutExtension()) { + document.setNoExtensionUri(uri.replace("/index.html", "/")); + } } } @@ -321,35 +202,19 @@ private String getPathToRoot(File sourceFile) { return FileUtil.getUriPathToContentRoot(config, sourceFile); } - - } - - private String buildHash(final File sourceFile) { - String sha1; - try { - sha1 = FileUtil.sha1(sourceFile); - } catch (Exception e) { - logger.error("unable to build sha1 hash for source file '{}'", sourceFile); - sha1 = ""; - } - return sha1; - } - - - private DocumentStatus findDocumentStatus(String uri, String sha1) { - DocumentList match = db.getDocumentStatus(uri); - if (!match.isEmpty()) { - DocumentModel document = match.get(0); - String oldHash = document.getSha1(); - if (!oldHash.equals(sha1) || !document.getRendered()) { - return DocumentStatus.UPDATED; + private DocumentStatus findDocumentStatus(String uri, String sha1) { + DocumentList match = db.getDocumentStatus(uri); + if (!match.isEmpty()) { + DocumentModel document = match.get(0); + String oldHash = document.getSha1(); + if (!oldHash.equals(sha1) || !document.getRendered()) { + return DocumentStatus.UPDATED; + } else { + return DocumentStatus.IDENTICAL; + } } else { - return DocumentStatus.IDENTICAL; + return DocumentStatus.NEW; } - } else { - return DocumentStatus.NEW; } } - - } diff --git a/jbake-core/src/main/java/org/jbake/app/Oven.java b/jbake-core/src/main/java/org/jbake/app/Oven.java index e3ca7c41f..f87dab2a3 100644 --- a/jbake-core/src/main/java/org/jbake/app/Oven.java +++ b/jbake-core/src/main/java/org/jbake/app/Oven.java @@ -165,9 +165,6 @@ public void bake() { LOGGER.info("Crawling content..."); crawler.crawl(); - // process data files - crawler.crawlDataFiles(); - // render content LOGGER.info("Rendering content..."); renderContent(); @@ -190,7 +187,6 @@ public void bake() { e.printStackTrace(); } finally { contentStore.close(); - contentStore.shutdown(); } } diff --git a/jbake-core/src/main/java/org/jbake/app/Parser.java b/jbake-core/src/main/java/org/jbake/app/Parser.java index 129b01abf..060520bcc 100644 --- a/jbake-core/src/main/java/org/jbake/app/Parser.java +++ b/jbake-core/src/main/java/org/jbake/app/Parser.java @@ -15,7 +15,7 @@ * @author Jonathan Bullock jonbullock@gmail.com */ public class Parser { - private static final Logger LOGGER = LoggerFactory.getLogger(Parser.class); + private static final Logger logger = LoggerFactory.getLogger(Parser.class); private JBakeConfiguration config; @@ -37,10 +37,29 @@ public Parser(JBakeConfiguration config) { public DocumentModel processFile(File file) { ParserEngine engine = Engines.get(FileUtil.fileExt(file)); if (engine == null) { - LOGGER.error("Unable to find suitable markup engine for {}", file); + logger.error("Unable to find suitable markup engine for {}", file); return null; } return engine.parse(config, file); } + + public String buildHash(File file) { + try { + return FileUtil.sha1(file); + } catch (Exception e) { + logger.error("unable to build sha1 hash for source file '{}'", file); + return ""; + } + } + + public String buildURI(File file) { + ParserEngine engine = Engines.get(FileUtil.fileExt(file)); + if (engine == null) { + logger.error("Unable to find suitable markup engine for {}", file); + return null; + } + + return engine.buildURI(config, file); + } } diff --git a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java index e989b65b3..41d87a254 100644 --- a/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/MarkupEngine.java @@ -2,7 +2,9 @@ import org.apache.commons.configuration2.CompositeConfiguration; import org.apache.commons.configuration2.Configuration; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; +import org.jbake.app.FileUtil; import org.jbake.app.configuration.DefaultJBakeConfiguration; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.model.DocumentModel; @@ -15,6 +17,9 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; @@ -333,4 +338,60 @@ private void processDefaultBody(ParserContext context) { } context.setBody(body.toString()); } + + @Override + public String buildURI(JBakeConfiguration config, File file) { + String uri = FileUtil.asPath(file).replace(FileUtil.asPath(config.getContentFolder()), ""); + + if (useNoExtensionUri(config, uri)) { + // convert URI from xxx.html to xxx/index.html + uri = createNoExtensionUri(config, uri); + } else { + uri = createUri(config, uri); + } + + // strip off leading / to enable generating non-root based sites + if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { + uri = uri.substring(1); + } + + return uri; + } + + // TODO: Refactor - parametrize the following two methods into one. + // commons-codec's URLCodec could be used when we add that dependency. + private String createUri(JBakeConfiguration config, String uri) { + try { + return FileUtil.URI_SEPARATOR_CHAR + + FilenameUtils.getPath(uri) + + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) + + config.getOutputExtension(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. + } + } + + private String createNoExtensionUri(JBakeConfiguration config, String uri) { + try { + return FileUtil.URI_SEPARATOR_CHAR + + FilenameUtils.getPath(uri) + + URLEncoder.encode(FilenameUtils.getBaseName(uri), StandardCharsets.UTF_8.name()) + + FileUtil.URI_SEPARATOR_CHAR + + "index" + + config.getOutputExtension(); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("Missing UTF-8 encoding??", e); // Won't happen unless JDK is broken. + } + } + + private boolean useNoExtensionUri(JBakeConfiguration config, String uri) { + boolean noExtensionUri = config.getUriWithoutExtension(); + String noExtensionUriPrefix = config.getPrefixForUriWithoutExtension(); + + return noExtensionUri + && (noExtensionUriPrefix != null) + && (noExtensionUriPrefix.length() > 0) + && uri.startsWith(noExtensionUriPrefix); + } + } diff --git a/jbake-core/src/main/java/org/jbake/parser/ParserEngine.java b/jbake-core/src/main/java/org/jbake/parser/ParserEngine.java index 302ba807b..6cade1e0c 100644 --- a/jbake-core/src/main/java/org/jbake/parser/ParserEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/ParserEngine.java @@ -29,4 +29,11 @@ public interface ParserEngine { @Deprecated Map parse(Configuration config, File file, String contentPath); + /** + * Build the engine specific URI for the given file + * @param config The project configuration + * @param file The file the URI should be build for + * @return The engine specific URI + */ + String buildURI(JBakeConfiguration config, File file); } diff --git a/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java b/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java index f33a04da2..522f35035 100644 --- a/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java +++ b/jbake-core/src/main/java/org/jbake/parser/YamlEngine.java @@ -1,6 +1,7 @@ package org.jbake.parser; import org.apache.commons.io.IOUtils; +import org.jbake.app.FileUtil; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.model.DocumentModel; import org.jbake.model.DocumentTypes; @@ -88,4 +89,14 @@ public void processBody(ParserContext context) { private boolean hasJBakePrefix(String key) { return key.startsWith(JBAKE_PREFIX); } + + @Override + public String buildURI(JBakeConfiguration config, File file) { + String uri = FileUtil.asPath(file).replace(FileUtil.asPath(config.getDataFolder()), ""); + // strip off leading / + if (uri.startsWith(FileUtil.URI_SEPARATOR_CHAR)) { + uri = uri.substring(1, uri.length()); + } + return uri; + } } diff --git a/jbake-core/src/test/java/org/jbake/app/ContentStoreIntegrationTest.java b/jbake-core/src/test/java/org/jbake/app/ContentStoreIntegrationTest.java index a2f2603a0..316ebac91 100644 --- a/jbake-core/src/test/java/org/jbake/app/ContentStoreIntegrationTest.java +++ b/jbake-core/src/test/java/org/jbake/app/ContentStoreIntegrationTest.java @@ -48,7 +48,6 @@ public static void setUpClass() throws Exception { @AfterClass public static void cleanUpClass() { db.close(); - db.shutdown(); } @Before diff --git a/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java b/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java index 33e20c5cb..9992c8f73 100644 --- a/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java +++ b/jbake-core/src/test/java/org/jbake/app/CrawlerTest.java @@ -21,7 +21,7 @@ public class CrawlerTest extends ContentStoreIntegrationTest { public void crawl() throws InterruptedException { Crawler crawler = new Crawler(db, config); crawler.crawl(); - + crawler.shutdown(); Assert.assertEquals(4, db.getDocumentCount("post")); Assert.assertEquals(3, db.getDocumentCount("page")); @@ -74,6 +74,7 @@ public void renderWithPrettyUrls() throws InterruptedException { Crawler crawler = new Crawler(db, config); crawler.crawl(); + crawler.shutdown(); Assert.assertEquals(4, db.getDocumentCount("post")); Assert.assertEquals(3, db.getDocumentCount("page")); diff --git a/jbake-core/src/test/java/org/jbake/app/OvenTest.java b/jbake-core/src/test/java/org/jbake/app/OvenTest.java index 4b8c5ad38..77be0fd5b 100644 --- a/jbake-core/src/test/java/org/jbake/app/OvenTest.java +++ b/jbake-core/src/test/java/org/jbake/app/OvenTest.java @@ -57,7 +57,6 @@ public void setUp() throws Exception { public void tearDown() { if (contentStore != null && contentStore.isActive()) { contentStore.close(); - contentStore.shutdown(); } } From d26cc966245cb65a165d2340cb3495eadeef31ec Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Sun, 23 May 2021 13:00:22 +0200 Subject: [PATCH 8/9] jdk11 source and traget compatibility --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 3fbfcf837..90f137bfb 100644 --- a/build.gradle +++ b/build.gradle @@ -81,8 +81,8 @@ subprojects { // add source and target compatibility for all JavaCompile tasks tasks.withType(JavaCompile) { - sourceCompatibility = 1.8 - targetCompatibility = 1.8 + sourceCompatibility = 11 + targetCompatibility = 11 } test { From 973a294f56382297ebcf26a46934947dea0a0840 Mon Sep 17 00:00:00 2001 From: Frank Becker Date: Tue, 28 Sep 2021 00:05:18 +0200 Subject: [PATCH 9/9] fix reading from relative template paths --- jbake-core/src/main/java/org/jbake/app/RenderAgent.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jbake-core/src/main/java/org/jbake/app/RenderAgent.java b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java index 5cf9b8a94..7b5a05822 100644 --- a/jbake-core/src/main/java/org/jbake/app/RenderAgent.java +++ b/jbake-core/src/main/java/org/jbake/app/RenderAgent.java @@ -44,6 +44,6 @@ protected Writer createWriter(File file) throws IOException { } protected String findTemplateName(String docType) { - return config.getTemplateFileByDocType(docType).getName(); + return config.getTemplateByDocType(docType); } }