From 5054cf01001496782ad2c6cf7856c6504af6ad71 Mon Sep 17 00:00:00 2001 From: lvca Date: Mon, 10 Jan 2022 13:44:55 -0500 Subject: [PATCH 1/3] Replaced OrientDB with ArcadeDB --- .gitignore | 1 + gradle.properties | 2 +- jbake-core/build.gradle | 2 +- .../main/java/org/jbake/app/ContentStore.java | 143 ++++++------------ .../src/main/java/org/jbake/app/Crawler.java | 12 +- .../src/main/java/org/jbake/app/DBUtil.java | 12 +- .../main/java/org/jbake/app/DocumentList.java | 10 +- .../jbake/template/JadeTemplateEngine.java | 15 +- .../template/ThymeleafTemplateEngine.java | 11 +- .../java/org/jbake/FakeDocumentBuilder.java | 7 +- .../java/org/jbake/app/ContentStoreTest.java | 4 +- .../java/org/jbake/app/PaginationTest.java | 3 +- 12 files changed, 82 insertions(+), 140 deletions(-) diff --git a/.gitignore b/.gitignore index 82ab65eb3..4abf6fcd2 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ build/ .gradle .gradletasknamecache +.DS_Store diff --git a/gradle.properties b/gradle.properties index cb81cde7c..6ef933e0f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -24,7 +24,7 @@ jade4jVersion = 1.3.2 jsoupVersion = 1.14.3 jgitVersion = 6.0.0.202111291000-r logbackVersion = 1.2.10 -orientDbVersion = 3.0.41 +arcadeDbVersion = 21.12.1 pebbleVersion = 3.1.5 slf4jVersion = 1.7.32 snakeYamlVersion = 1.30 diff --git a/jbake-core/build.gradle b/jbake-core/build.gradle index a64a6f07c..c43df6352 100644 --- a/jbake-core/build.gradle +++ b/jbake-core/build.gradle @@ -37,7 +37,7 @@ dependencies { implementation("com.googlecode.json-simple:json-simple:$jsonSimpleVersion") { exclude group: "junit", module: "junit" } - implementation "com.orientechnologies:orientdb-core:$orientDbVersion" + implementation "com.arcadedb:arcadedb-engine:$arcadeDbVersion" api "org.asciidoctor:asciidoctorj:$asciidoctorjVersion", optional api "org.codehaus.groovy:groovy:$groovyVersion", optional api "org.codehaus.groovy:groovy-templates:$groovyVersion", optional diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 7dc71e887..9a8803ea2 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -23,29 +23,20 @@ */ package org.jbake.app; -import com.orientechnologies.common.log.OLogManager; -import com.orientechnologies.orient.core.Orient; -import com.orientechnologies.orient.core.db.ODatabaseSession; -import com.orientechnologies.orient.core.db.ODatabaseType; -import com.orientechnologies.orient.core.db.OrientDB; -import com.orientechnologies.orient.core.db.OrientDBConfig; -import com.orientechnologies.orient.core.metadata.schema.OClass; -import com.orientechnologies.orient.core.metadata.schema.OSchema; -import com.orientechnologies.orient.core.metadata.schema.OType; -import com.orientechnologies.orient.core.record.impl.ODocument; -import com.orientechnologies.orient.core.sql.executor.OResultSet; -import com.orientechnologies.orient.core.sql.query.OSQLSynchQuery; -import org.jbake.launcher.SystemExit; +import com.arcadedb.database.Database; +import com.arcadedb.database.DatabaseFactory; +import com.arcadedb.database.MutableDocument; +import com.arcadedb.query.sql.executor.ResultSet; +import com.arcadedb.schema.DocumentType; +import com.arcadedb.schema.Type; import org.jbake.model.DocumentModel; import org.jbake.model.DocumentTypes; import org.jbake.model.ModelAttributes; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; +import java.io.*; +import java.util.*; /** * @author jdlee @@ -73,11 +64,11 @@ public class ContentStore { private final String type; private final String name; - private ODatabaseSession db; + private Database db; - private long start = -1; - private long limit = -1; - private OrientDB orient; + private long start = -1; + private long limit = -1; + private DatabaseFactory factory; public ContentStore(final String type, String name) { this.type = type; @@ -86,20 +77,14 @@ public ContentStore(final String type, String name) { public void startup() { - startupIfEnginesAreMissing(); + factory = new DatabaseFactory(name); - if (type.equalsIgnoreCase(ODatabaseType.PLOCAL.name())) { - orient = new OrientDB(type + ":" + name, OrientDBConfig.defaultConfig()); - } else { - orient = new OrientDB(type + ":", OrientDBConfig.defaultConfig()); - } - - orient.createIfNotExists(name, ODatabaseType.valueOf(type.toUpperCase())); - - db = orient.open(name, "admin", "admin"); - - activateOnCurrentThread(); + if( !factory.exists() ) + db = factory.create(); + else + db = factory.open(); + db.setAutoTransaction(true); updateSchema(); } @@ -126,64 +111,37 @@ public void resetPagination() { public final void updateSchema() { - OSchema schema = db.getMetadata().getSchema(); + com.arcadedb.schema.Schema schema = db.getSchema(); - if (!schema.existsClass(Schema.DOCUMENTS)) { + if (!schema.existsType(Schema.DOCUMENTS)) { createDocType(schema); } - if (!schema.existsClass(Schema.SIGNATURES)) { + if (!schema.existsType(Schema.SIGNATURES)) { createSignatureType(schema); } } public void close() { if (db != null) { - activateOnCurrentThread(); db.close(); } - if (orient != null) { - orient.close(); + if (factory != null) { + factory.close(); } DBUtil.closeDataStore(); } public void shutdown() { - -// Orient.instance().shutdown(); - } - - private void startupIfEnginesAreMissing() { - // Using a jdk which doesn't bundle a javascript engine - // throws a NoClassDefFoundError while logging the warning - // see https://github.com/orientechnologies/orientdb/issues/5855 - OLogManager.instance().setWarnEnabled(false); - - // If an instance of Orient was previously shutdown all engines are removed. - // We need to startup Orient again. - if (Orient.instance().getEngines().isEmpty()) { - Orient.instance().startup(); - } - OLogManager.instance().setWarnEnabled(true); + close(); } public void drop() { - activateOnCurrentThread(); -// db.drop(); - - orient.drop(name); - } - - private void activateOnCurrentThread() { - if (db != null) { - db.activateOnCurrentThread(); - } else { - System.out.println("db is null on activate"); - } + if( db != null) + db.drop(); } public long getDocumentCount(String docType) { - activateOnCurrentThread(); String statement = String.format(STATEMENT_GET_DOCUMENT_COUNT_BY_TYPE, docType); return (long) query(statement).get(0).get("count"); } @@ -291,20 +249,17 @@ private void insertTemplatesSignature(String currentTemplatesSignature) { } private DocumentList query(String sql) { - activateOnCurrentThread(); - OResultSet results = db.query(sql); + ResultSet results = db.query("sql", sql); return DocumentList.wrap(results); } private DocumentList query(String sql, Object... args) { - activateOnCurrentThread(); - OResultSet results = db.command(sql, args); + ResultSet results = db.command("sql", sql, args); return DocumentList.wrap(results); } private void executeCommand(String query, Object... args) { - activateOnCurrentThread(); - db.command(query, args); + db.command("sql", query, args); } public Set getTags() { @@ -330,29 +285,29 @@ public Set getAllTags() { return result; } - private void createDocType(final OSchema schema) { + private void createDocType(final com.arcadedb.schema.Schema schema) { logger.debug("Create document class"); - OClass page = schema.createClass(Schema.DOCUMENTS); - page.createProperty(ModelAttributes.SHA1, OType.STRING).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "sha1Index", OClass.INDEX_TYPE.NOTUNIQUE, ModelAttributes.SHA1); - page.createProperty(ModelAttributes.SOURCE_URI, OType.STRING).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "sourceUriIndex", OClass.INDEX_TYPE.UNIQUE, ModelAttributes.SOURCE_URI); - page.createProperty(ModelAttributes.CACHED, OType.BOOLEAN).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "cachedIndex", OClass.INDEX_TYPE.NOTUNIQUE, ModelAttributes.CACHED); - page.createProperty(ModelAttributes.RENDERED, OType.BOOLEAN).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "renderedIndex", OClass.INDEX_TYPE.NOTUNIQUE, ModelAttributes.RENDERED); - page.createProperty(ModelAttributes.STATUS, OType.STRING).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "statusIndex", OClass.INDEX_TYPE.NOTUNIQUE, ModelAttributes.STATUS); - page.createProperty(ModelAttributes.TYPE, OType.STRING).setNotNull(true); - page.createIndex(Schema.DOCUMENTS + "typeIndex", OClass.INDEX_TYPE.NOTUNIQUE, ModelAttributes.TYPE); + DocumentType page = schema.createDocumentType(Schema.DOCUMENTS); + page.createProperty(ModelAttributes.SHA1, Type.STRING); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE,false, ModelAttributes.SHA1); + page.createProperty(ModelAttributes.SOURCE_URI, Type.STRING); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, true, ModelAttributes.SOURCE_URI); + page.createProperty(ModelAttributes.CACHED, Type.BOOLEAN); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, false, ModelAttributes.CACHED); + page.createProperty(ModelAttributes.RENDERED, Type.BOOLEAN); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, false, ModelAttributes.RENDERED); + page.createProperty(ModelAttributes.STATUS, Type.STRING); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, false, ModelAttributes.STATUS); + page.createProperty(ModelAttributes.TYPE, Type.STRING); + page.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, false, ModelAttributes.TYPE); } - private void createSignatureType(OSchema schema) { - OClass signatures = schema.createClass(Schema.SIGNATURES); - signatures.createProperty(ModelAttributes.SHA1, OType.STRING).setNotNull(true); - signatures.createIndex("sha1Idx", OClass.INDEX_TYPE.UNIQUE, ModelAttributes.SHA1); + private void createSignatureType(com.arcadedb.schema.Schema schema) { + DocumentType signatures = schema.createDocumentType(Schema.SIGNATURES); + signatures.createProperty(ModelAttributes.SHA1, Type.STRING); + signatures.createTypeIndex(com.arcadedb.schema.Schema.INDEX_TYPE.LSM_TREE, true, ModelAttributes.SHA1); } public void updateAndClearCacheIfNeeded(boolean needed, File templateFolder) { @@ -404,11 +359,11 @@ private void deleteAllDocumentTypes() { } public boolean isActive() { - return db.isActiveOnCurrentThread(); + return true; } public void addDocument(DocumentModel document) { - ODocument doc = new ODocument(Schema.DOCUMENTS); + MutableDocument doc = db.newDocument(Schema.DOCUMENTS); doc.fromMap(document); doc.save(); } diff --git a/jbake-core/src/main/java/org/jbake/app/Crawler.java b/jbake-core/src/main/java/org/jbake/app/Crawler.java index 0c3d8b9d6..0e7c6e964 100644 --- a/jbake-core/src/main/java/org/jbake/app/Crawler.java +++ b/jbake-core/src/main/java/org/jbake/app/Crawler.java @@ -1,6 +1,5 @@ package org.jbake.app; -import com.orientechnologies.orient.core.record.impl.ODocument; import org.apache.commons.configuration2.CompositeConfiguration; import org.apache.commons.io.FilenameUtils; import org.jbake.app.configuration.JBakeConfiguration; @@ -13,13 +12,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.Date; -import java.util.Map; +import java.io.*; +import java.net.*; +import java.nio.charset.*; +import java.util.*; /** * Crawls a file system looking for content. diff --git a/jbake-core/src/main/java/org/jbake/app/DBUtil.java b/jbake-core/src/main/java/org/jbake/app/DBUtil.java index 73fce3fdb..a2e3c18f2 100644 --- a/jbake-core/src/main/java/org/jbake/app/DBUtil.java +++ b/jbake-core/src/main/java/org/jbake/app/DBUtil.java @@ -1,11 +1,11 @@ package org.jbake.app; -import com.orientechnologies.orient.core.db.record.OTrackedList; -import com.orientechnologies.orient.core.record.OElement; -import com.orientechnologies.orient.core.sql.executor.OResult; +import com.arcadedb.query.sql.executor.Result; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.model.DocumentModel; +import java.util.*; + public class DBUtil { private static ContentStore contentStore; @@ -34,7 +34,7 @@ public static void closeDataStore() { contentStore = null; } - public static DocumentModel documentToModel(OResult doc) { + public static DocumentModel documentToModel(Result doc) { DocumentModel result = new DocumentModel(); for (String key : doc.getPropertyNames()) { @@ -53,8 +53,8 @@ public static DocumentModel documentToModel(OResult doc) { public static String[] toStringArray(Object entry) { if (entry instanceof String[]) { return (String[]) entry; - } else if (entry instanceof OTrackedList) { - OTrackedList list = (OTrackedList) entry; + } else if (entry instanceof List) { + List list = (List) entry; return list.toArray(new String[list.size()]); } return new String[0]; diff --git a/jbake-core/src/main/java/org/jbake/app/DocumentList.java b/jbake-core/src/main/java/org/jbake/app/DocumentList.java index d297ff387..5e3b0d6cc 100644 --- a/jbake-core/src/main/java/org/jbake/app/DocumentList.java +++ b/jbake-core/src/main/java/org/jbake/app/DocumentList.java @@ -1,10 +1,10 @@ package org.jbake.app; -import com.orientechnologies.orient.core.sql.executor.OResult; -import com.orientechnologies.orient.core.sql.executor.OResultSet; +import com.arcadedb.query.sql.executor.Result; +import com.arcadedb.query.sql.executor.ResultSet; import org.jbake.model.DocumentModel; -import java.util.LinkedList; +import java.util.*; /** * Wraps an OrientDB document iterator into a model usable by @@ -14,10 +14,10 @@ */ public class DocumentList extends LinkedList { - public static DocumentList wrap(OResultSet docs) { + public static DocumentList wrap(ResultSet docs) { DocumentList list = new DocumentList<>(); while (docs.hasNext()) { - OResult next = docs.next(); + Result next = docs.next(); list.add(DBUtil.documentToModel(next)); } docs.close(); diff --git a/jbake-core/src/main/java/org/jbake/template/JadeTemplateEngine.java b/jbake-core/src/main/java/org/jbake/template/JadeTemplateEngine.java index bbbff0bfb..e5c352145 100644 --- a/jbake-core/src/main/java/org/jbake/template/JadeTemplateEngine.java +++ b/jbake-core/src/main/java/org/jbake/template/JadeTemplateEngine.java @@ -1,6 +1,5 @@ package org.jbake.template; - import de.neuland.jade4j.Jade4J; import de.neuland.jade4j.JadeConfiguration; import de.neuland.jade4j.filter.CDATAFilter; @@ -11,18 +10,14 @@ import de.neuland.jade4j.template.JadeTemplate; import de.neuland.jade4j.template.TemplateLoader; import org.apache.commons.configuration2.CompositeConfiguration; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.text.StringEscapeUtils; import org.jbake.app.ContentStore; import org.jbake.app.configuration.JBakeConfiguration; import org.jbake.template.model.TemplateModel; -import java.io.File; -import java.io.IOException; -import java.io.Writer; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; +import java.io.*; +import java.text.*; +import java.util.*; /** * Renders pages using the Jade template language. @@ -105,7 +100,7 @@ public String format(Date date, String pattern) { } public String escape(String s) { - return StringEscapeUtils.escapeHtml(s); + return StringEscapeUtils.escapeHtml4(s); } } } diff --git a/jbake-core/src/main/java/org/jbake/template/ThymeleafTemplateEngine.java b/jbake-core/src/main/java/org/jbake/template/ThymeleafTemplateEngine.java index c6cc99dc2..88a672e4b 100644 --- a/jbake-core/src/main/java/org/jbake/template/ThymeleafTemplateEngine.java +++ b/jbake-core/src/main/java/org/jbake/template/ThymeleafTemplateEngine.java @@ -1,7 +1,7 @@ package org.jbake.template; import org.apache.commons.configuration2.CompositeConfiguration; -import org.apache.commons.lang.LocaleUtils; +import org.apache.commons.lang3.LocaleUtils; import org.jbake.app.ContentStore; import org.jbake.app.configuration.DefaultJBakeConfiguration; import org.jbake.app.configuration.JBakeConfiguration; @@ -13,12 +13,9 @@ import org.thymeleaf.context.LazyContextVariable; import org.thymeleaf.templateresolver.FileTemplateResolver; -import java.io.File; -import java.io.Writer; -import java.util.Date; -import java.util.Locale; -import java.util.Set; -import java.util.concurrent.locks.ReentrantLock; +import java.io.*; +import java.util.*; +import java.util.concurrent.locks.*; /** *

A template engine which renders pages using Thymeleaf.

diff --git a/jbake-core/src/test/java/org/jbake/FakeDocumentBuilder.java b/jbake-core/src/test/java/org/jbake/FakeDocumentBuilder.java index c72898db4..4526f43e7 100644 --- a/jbake-core/src/test/java/org/jbake/FakeDocumentBuilder.java +++ b/jbake-core/src/test/java/org/jbake/FakeDocumentBuilder.java @@ -1,6 +1,6 @@ package org.jbake; -import com.orientechnologies.orient.core.record.impl.ODocument; +import org.jbake.app.ContentStore; import org.jbake.model.DocumentModel; import java.math.BigInteger; @@ -55,7 +55,7 @@ public FakeDocumentBuilder withCached(boolean cached) { return this; } - public void build() { + public void build(ContentStore db) { try { if (!hasSourceUri()) { @@ -67,8 +67,7 @@ public void build() { if (!hasDate()) { this.withCurrentDate(); } - ODocument document = new ODocument("Documents").fromMap(fileModel); - document.save(); + db.addDocument(fileModel); } catch (NoSuchAlgorithmException e) { e.printStackTrace(); } diff --git a/jbake-core/src/test/java/org/jbake/app/ContentStoreTest.java b/jbake-core/src/test/java/org/jbake/app/ContentStoreTest.java index 567ced0c0..bdc48ec12 100644 --- a/jbake-core/src/test/java/org/jbake/app/ContentStoreTest.java +++ b/jbake-core/src/test/java/org/jbake/app/ContentStoreTest.java @@ -24,13 +24,13 @@ public void shouldGetCountForPublishedDocuments() throws Exception { FakeDocumentBuilder builder = new FakeDocumentBuilder(DOC_TYPE_POST); builder.withStatus("published") .withRandomSha1() - .build(); + .build(db); } FakeDocumentBuilder builder = new FakeDocumentBuilder(DOC_TYPE_POST); builder.withStatus("draft") .withRandomSha1() - .build(); + .build(db); assertEquals(6, db.getDocumentCount(DOC_TYPE_POST)); assertEquals(5, db.getPublishedCount(DOC_TYPE_POST)); diff --git a/jbake-core/src/test/java/org/jbake/app/PaginationTest.java b/jbake-core/src/test/java/org/jbake/app/PaginationTest.java index 70fefa86a..02a94c81f 100644 --- a/jbake-core/src/test/java/org/jbake/app/PaginationTest.java +++ b/jbake-core/src/test/java/org/jbake/app/PaginationTest.java @@ -29,7 +29,6 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; -import org.junit.BeforeClass; import java.util.Calendar; import java.util.Locale; @@ -66,7 +65,7 @@ public void testPagination() { builder.withCached(true) .withStatus("published") .withDate(cal.getTime()) - .build(); + .build(db); } int pageCount = 1; From b8dad2d4fd0e7411cdff419207e85d34eb7ed7c3 Mon Sep 17 00:00:00 2001 From: lvca Date: Mon, 10 Jan 2022 19:27:20 -0500 Subject: [PATCH 2/3] Using ArcadeDB v22.1.2 --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 6ef933e0f..2ea106484 100644 --- a/gradle.properties +++ b/gradle.properties @@ -24,7 +24,7 @@ jade4jVersion = 1.3.2 jsoupVersion = 1.14.3 jgitVersion = 6.0.0.202111291000-r logbackVersion = 1.2.10 -arcadeDbVersion = 21.12.1 +arcadeDbVersion = 22.1.2 pebbleVersion = 3.1.5 slf4jVersion = 1.7.32 snakeYamlVersion = 1.30 From fda9e26848a446ba15e63ebb41e1e2ce73e3ec7a Mon Sep 17 00:00:00 2001 From: lvca Date: Tue, 11 Jan 2022 14:21:36 -0500 Subject: [PATCH 3/3] Set page size 4X in the constructor of ContentStore --- .../src/main/java/org/jbake/app/ContentStore.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/jbake-core/src/main/java/org/jbake/app/ContentStore.java b/jbake-core/src/main/java/org/jbake/app/ContentStore.java index 9a8803ea2..3be5d4fcf 100644 --- a/jbake-core/src/main/java/org/jbake/app/ContentStore.java +++ b/jbake-core/src/main/java/org/jbake/app/ContentStore.java @@ -23,9 +23,11 @@ */ package org.jbake.app; +import com.arcadedb.GlobalConfiguration; import com.arcadedb.database.Database; import com.arcadedb.database.DatabaseFactory; import com.arcadedb.database.MutableDocument; +import com.arcadedb.engine.Bucket; import com.arcadedb.query.sql.executor.ResultSet; import com.arcadedb.schema.DocumentType; import com.arcadedb.schema.Type; @@ -73,8 +75,10 @@ public class ContentStore { public ContentStore(final String type, String name) { this.type = type; this.name = name; - } + // USE A 4X BIGGER PAGE THAN THE DEFAULT + GlobalConfiguration.BUCKET_DEFAULT_PAGE_SIZE.setValue(Bucket.DEF_PAGE_SIZE * 4); + } public void startup() { factory = new DatabaseFactory(name); @@ -110,7 +114,6 @@ public void resetPagination() { } public final void updateSchema() { - com.arcadedb.schema.Schema schema = db.getSchema(); if (!schema.existsType(Schema.DOCUMENTS)) { @@ -362,8 +365,8 @@ public boolean isActive() { return true; } - public void addDocument(DocumentModel document) { - MutableDocument doc = db.newDocument(Schema.DOCUMENTS); + public void addDocument(final DocumentModel document) { + final MutableDocument doc = db.newDocument(Schema.DOCUMENTS); doc.fromMap(document); doc.save(); }