diff --git a/.gitignore b/.gitignore new file mode 100755 index 0000000..61eb58b --- /dev/null +++ b/.gitignore @@ -0,0 +1,31 @@ +### gradle +**/.gradle +!scraper-nodes-dev/gradle/wrapper/gradle-wrapper.jar + +/**/build/ +/**/builds/ +/**/classes/ + +### intellij +.idea +*.iml +/**/out/ + +### vim +**/*.swp + +## ignore antlr jar whereever it is +**/antlr.jar + +### junit error logs and normal logs +**/errors.log +**/*.log + +### generated editor html file +**/resources/**/scraper-editor.html +### generated class model file +**/*class-model.json +**/class-model.js + + +/**/*nodes*link* diff --git a/build.gradle b/build.gradle new file mode 100644 index 0000000..1c12c1a --- /dev/null +++ b/build.gradle @@ -0,0 +1,45 @@ +plugins { + id 'jacoco' +} + +allprojects { + repositories { + jcenter() + } +} + +subprojects { + configurations { + providedCompile + } + + afterEvaluate { + dependencies { + providedCompile fileTree(dir: 'libs', include: '*.jar') + testImplementation fileTree(dir: 'libs', include: '*.jar') + + testImplementation 'org.junit.jupiter:junit-jupiter-api:5.6.1' + testImplementation 'org.junit.jupiter:junit-jupiter-params:5.6.1' + testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine' + } + + compileJava { + sourceSets.main.compileClasspath += configurations.providedCompile + } + } + + group = 'org.scraper' + + plugins.withType(JavaPlugin).configureEach { + java { + modularity.inferModulePath = true + } + } + + tasks.withType(Test).configureEach { + useJUnitPlatform() + } +} + +// merge coverage +apply from: 'gradle/jacoco-merge.gradle' diff --git a/gradle/jacoco-merge.gradle b/gradle/jacoco-merge.gradle new file mode 100755 index 0000000..c62fd26 --- /dev/null +++ b/gradle/jacoco-merge.gradle @@ -0,0 +1,31 @@ +configure( subprojects ) { + apply plugin: 'jacoco' +} + +tasks.register("codeCoverageReport", JacocoReport) { + // If a subproject applies the 'jacoco' plugin, add the result it to the report + subprojects { subproject -> + subproject.plugins.withType(JacocoPlugin).configureEach { + subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).configureEach { testTask -> + sourceSets subproject.sourceSets.main + executionData(testTask) + } + + // To automatically run `test` every time `./gradlew codeCoverageReport` is called, + // you may want to set up a task dependency between them as shown below. + // Note that this requires the `test` tasks to be resolved eagerly (see `forEach`) which + // may have a negative effect on the configuration time of your build. + subproject.tasks.matching({ t -> t.extensions.findByType(JacocoTaskExtension) }).forEach { + rootProject.tasks.codeCoverageReport.dependsOn(it) + } + } + } + + // enable the different report types (html, xml, csv) + reports { + xml.enabled true + html.enabled true + csv.enabled true + } +} + diff --git a/scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar old mode 100755 new mode 100644 similarity index 56% rename from scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.jar rename to gradle/wrapper/gradle-wrapper.jar index 5c2d1cf..62d4c05 Binary files a/scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties old mode 100755 new mode 100644 similarity index 79% rename from scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.properties rename to gradle/wrapper/gradle-wrapper.properties index 7c4388a..9a882f5 --- a/scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ +#Sat Jul 04 10:44:41 CEST 2020 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5.1-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-bin.zip -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME diff --git a/scraper-plugins-debugger/gradlew b/gradlew similarity index 85% rename from scraper-plugins-debugger/gradlew rename to gradlew index 83f2acf..fbd7c51 100755 --- a/scraper-plugins-debugger/gradlew +++ b/gradlew @@ -82,6 +82,7 @@ esac CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + # Determine the Java command to use to start the JVM. if [ -n "$JAVA_HOME" ] ; then if [ -x "$JAVA_HOME/jre/sh/java" ] ; then @@ -129,6 +130,7 @@ fi if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath @@ -154,19 +156,19 @@ if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then else eval `echo args$i`="\"$arg\"" fi - i=$((i+1)) + i=`expr $i + 1` done case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + 0) set -- ;; + 1) set -- "$args0" ;; + 2) set -- "$args0" "$args1" ;; + 3) set -- "$args0" "$args1" "$args2" ;; + 4) set -- "$args0" "$args1" "$args2" "$args3" ;; + 5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + 6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + 7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + 8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + 9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; esac fi @@ -175,14 +177,9 @@ save () { for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done echo " " } -APP_ARGS=$(save "$@") +APP_ARGS=`save "$@"` # Collect all arguments for the java command, following the shell quoting and substitution rules eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - exec "$JAVACMD" "$@" diff --git a/scraper-plugins-debugger/gradlew.bat b/gradlew.bat similarity index 92% rename from scraper-plugins-debugger/gradlew.bat rename to gradlew.bat index 24467a1..a9f778a 100644 --- a/scraper-plugins-debugger/gradlew.bat +++ b/gradlew.bat @@ -29,6 +29,9 @@ if "%DIRNAME%" == "" set DIRNAME=. set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" @@ -81,6 +84,7 @@ set CMD_LINE_ARGS=%* set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + @rem Execute Gradle "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% diff --git a/libs/antlr4-runtime-4.8-module.jar b/libs/antlr4-runtime-4.8-module.jar new file mode 100644 index 0000000..84939d4 Binary files /dev/null and b/libs/antlr4-runtime-4.8-module.jar differ diff --git a/libs/api-0.12.0.jar b/libs/api-0.12.0.jar new file mode 100644 index 0000000..f59f7ef Binary files /dev/null and b/libs/api-0.12.0.jar differ diff --git a/libs/application-0.12.0.jar b/libs/application-0.12.0.jar new file mode 100644 index 0000000..6ea7f71 Binary files /dev/null and b/libs/application-0.12.0.jar differ diff --git a/libs/classgraph-4.8.47.jar b/libs/classgraph-4.8.47.jar new file mode 100644 index 0000000..7a1a1f9 Binary files /dev/null and b/libs/classgraph-4.8.47.jar differ diff --git a/libs/core-0.12.0.jar b/libs/core-0.12.0.jar new file mode 100644 index 0000000..e52cae3 Binary files /dev/null and b/libs/core-0.12.0.jar differ diff --git a/libs/core-parsers-0.12.0.jar b/libs/core-parsers-0.12.0.jar new file mode 100644 index 0000000..1e4a22a Binary files /dev/null and b/libs/core-parsers-0.12.0.jar differ diff --git a/libs/jackson-annotations-2.10.4.jar b/libs/jackson-annotations-2.10.4.jar new file mode 100644 index 0000000..941440b Binary files /dev/null and b/libs/jackson-annotations-2.10.4.jar differ diff --git a/libs/jackson-core-2.10.4.jar b/libs/jackson-core-2.10.4.jar new file mode 100644 index 0000000..ef99435 Binary files /dev/null and b/libs/jackson-core-2.10.4.jar differ diff --git a/libs/jackson-databind-2.10.4.jar b/libs/jackson-databind-2.10.4.jar new file mode 100644 index 0000000..87f8f00 Binary files /dev/null and b/libs/jackson-databind-2.10.4.jar differ diff --git a/libs/jackson-dataformat-yaml-2.10.4.jar b/libs/jackson-dataformat-yaml-2.10.4.jar new file mode 100644 index 0000000..a675f9c Binary files /dev/null and b/libs/jackson-dataformat-yaml-2.10.4.jar differ diff --git a/libs/snakeyaml-1.26-module.jar b/libs/snakeyaml-1.26-module.jar new file mode 100644 index 0000000..4ce28cf Binary files /dev/null and b/libs/snakeyaml-1.26-module.jar differ diff --git a/libs/test-framework-0.12.0.jar b/libs/test-framework-0.12.0.jar new file mode 100644 index 0000000..1275986 Binary files /dev/null and b/libs/test-framework-0.12.0.jar differ diff --git a/libs/utilities-0.12.0.jar b/libs/utilities-0.12.0.jar new file mode 100644 index 0000000..dbf6c49 Binary files /dev/null and b/libs/utilities-0.12.0.jar differ diff --git a/scraper-plugins-debugger/.gitignore b/scraper-plugins-debugger/.gitignore deleted file mode 100755 index eb3cc71..0000000 --- a/scraper-plugins-debugger/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -### gradle -**/.gradle -!gradle/wrapper/gradle-wrapper.jar - -/**/build/ -/**/builds/ -/**/classes/ - -### intellij -.idea -*.iml -/**/out/ - -### vim -**/*.swp - diff --git a/scraper-plugins-debugger/build.gradle b/scraper-plugins-debugger/build.gradle index 7611fd3..9298d5e 100644 --- a/scraper-plugins-debugger/build.gradle +++ b/scraper-plugins-debugger/build.gradle @@ -1,30 +1,23 @@ -// treat this as a stand-alone project if not used in big scraper project -if (findProject(':scraper-plugins') == null) { - apply from: 'gradle/scraper-plugin.gradle' - apply from: 'gradle/gradle-module-fix.gradle' +plugins { + id 'java-library' + id 'java-library-distribution' + id("de.jjohannes.extra-java-module-info") version "0.1" } version = '0.1.0' -dependencies { - implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.8.0-beta4' - - // put your dependencies here - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.+' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.+' - - // websocket - provided 'org.java-websocket:Java-WebSocket:1.4.0' -} - -task bundleFatJar(type: Jar) { t -> - from { - configurations.provided.findAll { it.name.endsWith('jar') }.collect { zipTree(it) } - } - from { - sourceSets.main.output +extraJavaModuleInfo { + module("Java-WebSocket-1.4.0.jar", "Java.Websocket", "1.4.0") { + exports "org.java_websocket" + exports "org.java_websocket.drafts" + exports "org.java_websocket.exceptions" + exports "org.java_websocket.handshake" + exports "org.java_websocket.server" } + module("slf4j-api-1.7.25.jar", "org.slf4j", "1.7.25") {} +} - archiveFileName = "scraper-plugins-debugger-${version}.jar" - destinationDirectory = file("$rootDir/build/libs") +dependencies { + // websocket + implementation 'org.java-websocket:Java-WebSocket:1.4.0' } diff --git a/scraper-plugins-debugger/gradle/gradle-module-fix.gradle b/scraper-plugins-debugger/gradle/gradle-module-fix.gradle deleted file mode 100755 index 2f4808b..0000000 --- a/scraper-plugins-debugger/gradle/gradle-module-fix.gradle +++ /dev/null @@ -1,41 +0,0 @@ -// this is necessary until gradle provides first-class support for java modules - -compileJava { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - options.compilerArgs = [ - '--module-path', classpath.asPath, - ] - classpath = files() - } -} - -compileTestJava { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - options.compilerArgs = [ - '--module-path', classpath.asPath, - '--add-modules', 'junit', - '--add-reads', "$moduleName=junit", - '--patch-module', "$moduleName=" + files(sourceSets.test.java.srcDirs).asPath, - ] - classpath = files() - } -} - - -test { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - jvmArgs = [ - '--module-path', classpath.asPath, - '--add-modules', 'ALL-MODULE-PATH', - '--add-reads', "$moduleName=junit", - '--patch-module', "$moduleName=" + files(sourceSets.test.java.outputDir).asPath, - ] - classpath = files() - } -} diff --git a/scraper-plugins-debugger/gradle/scraper-plugin.gradle b/scraper-plugins-debugger/gradle/scraper-plugin.gradle deleted file mode 100755 index be51dd9..0000000 --- a/scraper-plugins-debugger/gradle/scraper-plugin.gradle +++ /dev/null @@ -1,13 +0,0 @@ -apply plugin: 'java' -repositories { - mavenCentral() -} - -dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) -} - -configurations { - provided - compile.extendsFrom provided -} diff --git a/scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.jar b/scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.jar deleted file mode 100755 index 5c2d1cf..0000000 Binary files a/scraper-plugins-debugger/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/scraper-plugins-debugger/libs/scraper-0.11-annotations.jar b/scraper-plugins-debugger/libs/scraper-0.11-annotations.jar deleted file mode 100644 index 425b67b..0000000 Binary files a/scraper-plugins-debugger/libs/scraper-0.11-annotations.jar and /dev/null differ diff --git a/scraper-plugins-debugger/libs/scraper-0.11-api.jar b/scraper-plugins-debugger/libs/scraper-0.11-api.jar deleted file mode 100644 index d4b813f..0000000 Binary files a/scraper-plugins-debugger/libs/scraper-0.11-api.jar and /dev/null differ diff --git a/scraper-plugins-debugger/libs/scraper-0.11.0-utils.jar b/scraper-plugins-debugger/libs/scraper-0.11.0-utils.jar deleted file mode 100644 index bf032ec..0000000 Binary files a/scraper-plugins-debugger/libs/scraper-0.11.0-utils.jar and /dev/null differ diff --git a/scraper-plugins-debugger/src/main/java/module-info.java b/scraper-plugins-debugger/src/main/java/module-info.java index 5415670..46cb9bf 100644 --- a/scraper-plugins-debugger/src/main/java/module-info.java +++ b/scraper-plugins-debugger/src/main/java/module-info.java @@ -6,7 +6,6 @@ open module scraper.plugins.debugger { // only depend on api and annotations requires scraper.api; - requires scraper.annotations; requires scraper.utils; requires java.net.http; @@ -15,11 +14,10 @@ exports scraper.plugins.debugger; // websocket - requires Java.WebSocket; + requires Java.Websocket; requires com.fasterxml.jackson.core; requires com.fasterxml.jackson.databind; - requires org.slf4j; // NodeHook, Addon, Hook, PreHook have to be provided provides NodeHook with DebuggerNodeHookAddon; diff --git a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerNodeHookAddon.java b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerNodeHookAddon.java index 68db5fd..6996b83 100644 --- a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerNodeHookAddon.java +++ b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerNodeHookAddon.java @@ -2,8 +2,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import scraper.annotations.ArgsCommand; import scraper.annotations.NotNull; import scraper.api.di.DIContainer; @@ -23,6 +21,8 @@ import java.util.Set; import java.util.regex.Pattern; +import static java.lang.System.Logger.Level.INFO; + @ArgsCommand( value = "debug", @@ -41,7 +41,7 @@ ) public class DebuggerNodeHookAddon implements NodeHook, Hook, Addon { /** Logger with the actual class name */ - private Logger l = LoggerFactory.getLogger("Debugger"); + private final System.Logger l = System.getLogger("Debugger"); private DebuggerWebsocketServer debugger; private final ObjectMapper m = new ObjectMapper(); private Set impls = new HashSet<>(); @@ -79,7 +79,7 @@ public void afterProcess(@NotNull NodeContainer n, @NotNull Flow @Override public void load(@NotNull DIContainer loadedDependencies, @NotNull String[] args) { if (StringUtil.getArgument(args, "debug") != null) { - l.warn("Debugging activated"); + l.log(System.Logger.Level.WARNING,"Debugging activated"); String debugPort = StringUtil.getArgument(args, "debug-port"); String debugIp = StringUtil.getArgument(args, "debug-ip"); String bindingIp = "0.0.0.0"; @@ -111,7 +111,7 @@ private String wrap(String type, Object data) { @SuppressWarnings("unused") // reflection public void requestSpecifications(Map data) { - l.info("Requesting specifications"); + l.log(INFO,"Requesting specifications"); for (InstanceDTO impls : impls) { debugger.get().ifPresent(client -> client.send(wrap("instance", impls))); } @@ -119,7 +119,7 @@ public void requestSpecifications(Map data) { @SuppressWarnings("unused") // reflection public void setReady(Map data) { - l.info("Debugger is ready, waking up all flows"); + l.log(INFO,"Debugger is ready, waking up all flows"); state.setReady(true); } diff --git a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerState.java b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerState.java index aff3d65..11e21e3 100644 --- a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerState.java +++ b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerState.java @@ -1,7 +1,5 @@ package scraper.plugins.debugger; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import scraper.api.node.Address; import scraper.api.node.container.NodeContainer; import scraper.api.node.type.Node; @@ -10,8 +8,11 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; +import static java.lang.System.Logger.Level.ERROR; +import static java.lang.System.Logger.Level.INFO; + public class DebuggerState { - protected Logger l = LoggerFactory.getLogger("DebuggerState"); + protected System.Logger l = System.getLogger("DebuggerState"); // flows will wait on this object private final AtomicBoolean ready = new AtomicBoolean(false); @@ -23,7 +24,7 @@ public void waitUntilReady() { try { synchronized (ready) { if(!ready.get()) { - l.info("Waiting for debugger to connect"); + l.log(INFO,"Waiting for debugger to connect"); ready.wait(); } } @@ -43,14 +44,14 @@ public void waitIfBreakpoint(NodeContainer n, Runnable onWait, R for (String breakpoint : breakpoints) { Address addr = n.addressOf(breakpoint); if(n.getAddress().equals(addr)) { - l.info("BREAKPOINT TRIGGERED: {} <-> {}", breakpoint, n.getAddress().getRepresentation()); + l.log(INFO,"BREAKPOINT TRIGGERED: {} <-> {}", breakpoint, n.getAddress().getRepresentation()); synchronized (breaking) { try { onWait.run(); breaking.wait(); break; } catch (InterruptedException e) { - l.error("Continuing because interrupt"); + l.log(ERROR,"Continuing because interrupt"); e.printStackTrace(); } finally { onContinue.run(); diff --git a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerWebsocketServer.java b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerWebsocketServer.java index 77b4ca8..e9f8eca 100644 --- a/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerWebsocketServer.java +++ b/scraper-plugins-debugger/src/main/java/scraper/plugins/debugger/DebuggerWebsocketServer.java @@ -7,8 +7,6 @@ import org.java_websocket.handshake.ClientHandshake; import org.java_websocket.handshake.ServerHandshakeBuilder; import org.java_websocket.server.WebSocketServer; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.io.IOException; import java.lang.reflect.Method; @@ -20,7 +18,7 @@ @SuppressWarnings("unchecked") // API conventions public class DebuggerWebsocketServer extends WebSocketServer { - protected Logger l = LoggerFactory.getLogger("DebuggerWebSocket"); + protected System.Logger l = System.getLogger("DebuggerWebSocket"); WebSocket debugger = null; final ReentrantLock lock = new ReentrantLock(); @@ -36,9 +34,8 @@ public DebuggerWebsocketServer(DebuggerNodeHookAddon actions, int port, String b Runtime.getRuntime().addShutdownHook(new Thread(() -> { try { - l.warn("Shutting down system"); + l.log(System.Logger.Level.WARNING,"Shutting down system"); this.stop(); - l.warn("Graceful shutdown"); } catch (IOException | InterruptedException e) { e.printStackTrace(); } @@ -68,7 +65,7 @@ public void onOpen(WebSocket conn, ClientHandshake handshake) { lock.lock(); debugger = conn; } finally { lock.unlock(); } - l.info("Debugger connected"); + l.log(System.Logger.Level.INFO,"Debugger connected"); } @Override @@ -79,12 +76,12 @@ public void onClose(WebSocket conn, int code, String reason, boolean remote) { } finally { lock.unlock(); } - l.warn("Debugger disconnected"); + l.log(System.Logger.Level.INFO,"Debugger disconnected"); } @Override public void onError(WebSocket conn, Exception ex) { - l.error("Web Socket error", ex); + l.log(System.Logger.Level.ERROR,"Web Socket error", ex); // TODO implement error handling // idea: stop execution until debugger reconnects } @@ -101,7 +98,7 @@ public void onMessage(WebSocket conn, String message) { cmdMethod.invoke(actions, data); } catch (Exception e) { - l.error("Not a valid command: {}", message.substring(0,Math.min(message.length(),100))); + l.log(System.Logger.Level.ERROR,"Not a valid command: {}", message.substring(0,Math.min(message.length(),100))); } } diff --git a/scraper-plugins-persistent-proxy-scores/.gitignore b/scraper-plugins-persistent-proxy-scores/.gitignore deleted file mode 100755 index eb3cc71..0000000 --- a/scraper-plugins-persistent-proxy-scores/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -### gradle -**/.gradle -!gradle/wrapper/gradle-wrapper.jar - -/**/build/ -/**/builds/ -/**/classes/ - -### intellij -.idea -*.iml -/**/out/ - -### vim -**/*.swp - diff --git a/scraper-plugins-persistent-proxy-scores/build.gradle b/scraper-plugins-persistent-proxy-scores/build.gradle index f0970a3..f8a5f88 100644 --- a/scraper-plugins-persistent-proxy-scores/build.gradle +++ b/scraper-plugins-persistent-proxy-scores/build.gradle @@ -1,25 +1,7 @@ -// treat this as a stand-alone project if not used in big scraper project -if (findProject(':scraper-plugins') == null) { - apply from: 'gradle/scraper-plugin.gradle' - apply from: 'gradle/gradle-module-fix.gradle' +plugins { + id 'java-library' + id 'java-library-distribution' } version = '0.1.0' -dependencies { - implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.8.0-beta4' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-core', version: '2.10.+' - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.10.+' -} - -task bundleFatJar(type: Jar) { t -> - from { - configurations.provided.findAll { it.name.endsWith('jar') }.collect { zipTree(it) } - } - from { - sourceSets.main.output - } - - archiveFileName = "scraper-plugins-persistent-proxy-scores-${version}.jar" - destinationDirectory = file("$rootDir/build/libs") -} diff --git a/scraper-plugins-persistent-proxy-scores/gradle/gradle-module-fix.gradle b/scraper-plugins-persistent-proxy-scores/gradle/gradle-module-fix.gradle deleted file mode 100755 index 2f4808b..0000000 --- a/scraper-plugins-persistent-proxy-scores/gradle/gradle-module-fix.gradle +++ /dev/null @@ -1,41 +0,0 @@ -// this is necessary until gradle provides first-class support for java modules - -compileJava { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - options.compilerArgs = [ - '--module-path', classpath.asPath, - ] - classpath = files() - } -} - -compileTestJava { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - options.compilerArgs = [ - '--module-path', classpath.asPath, - '--add-modules', 'junit', - '--add-reads', "$moduleName=junit", - '--patch-module', "$moduleName=" + files(sourceSets.test.java.srcDirs).asPath, - ] - classpath = files() - } -} - - -test { - final String moduleName = project.name.toString().replace("-", ".") - inputs.property("moduleName", moduleName) - doFirst { - jvmArgs = [ - '--module-path', classpath.asPath, - '--add-modules', 'ALL-MODULE-PATH', - '--add-reads', "$moduleName=junit", - '--patch-module', "$moduleName=" + files(sourceSets.test.java.outputDir).asPath, - ] - classpath = files() - } -} diff --git a/scraper-plugins-persistent-proxy-scores/gradle/scraper-plugin.gradle b/scraper-plugins-persistent-proxy-scores/gradle/scraper-plugin.gradle deleted file mode 100755 index be51dd9..0000000 --- a/scraper-plugins-persistent-proxy-scores/gradle/scraper-plugin.gradle +++ /dev/null @@ -1,13 +0,0 @@ -apply plugin: 'java' -repositories { - mavenCentral() -} - -dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) -} - -configurations { - provided - compile.extendsFrom provided -} diff --git a/scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.properties b/scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.properties deleted file mode 100755 index fd0b1e1..0000000 --- a/scraper-plugins-persistent-proxy-scores/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,6 +0,0 @@ -#Mon Mar 23 10:41:04 CET 2020 -distributionUrl=https\://services.gradle.org/distributions/gradle-5.6.2-all.zip -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -zipStorePath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME diff --git a/scraper-plugins-persistent-proxy-scores/gradlew b/scraper-plugins-persistent-proxy-scores/gradlew deleted file mode 100755 index 83f2acf..0000000 --- a/scraper-plugins-persistent-proxy-scores/gradlew +++ /dev/null @@ -1,188 +0,0 @@ -#!/usr/bin/env sh - -# -# Copyright 2015 the original author or authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -############################################################################## -## -## Gradle start up script for UN*X -## -############################################################################## - -# Attempt to set APP_HOME -# Resolve links: $0 may be a link -PRG="$0" -# Need this for relative symlinks. -while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG=`dirname "$PRG"`"/$link" - fi -done -SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >/dev/null -APP_HOME="`pwd -P`" -cd "$SAVED" >/dev/null - -APP_NAME="Gradle" -APP_BASE_NAME=`basename "$0"` - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD="maximum" - -warn () { - echo "$*" -} - -die () { - echo - echo "$*" - echo - exit 1 -} - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "`uname`" in - CYGWIN* ) - cygwin=true - ;; - Darwin* ) - darwin=true - ;; - MINGW* ) - msys=true - ;; - NONSTOP* ) - nonstop=true - ;; -esac - -CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD="java" - which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." -fi - -# Increase the maximum file descriptors if we can. -if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then - MAX_FD_LIMIT=`ulimit -H -n` - if [ $? -eq 0 ] ; then - if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then - MAX_FD="$MAX_FD_LIMIT" - fi - ulimit -n $MAX_FD - if [ $? -ne 0 ] ; then - warn "Could not set maximum file descriptor limit: $MAX_FD" - fi - else - warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" - fi -fi - -# For Darwin, add options to specify how the application appears in the dock -if $darwin; then - GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" -fi - -# For Cygwin or MSYS, switch paths to Windows format before running java -if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then - APP_HOME=`cygpath --path --mixed "$APP_HOME"` - CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` - JAVACMD=`cygpath --unix "$JAVACMD"` - - # We build the pattern for arguments to be converted via cygpath - ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` - SEP="" - for dir in $ROOTDIRSRAW ; do - ROOTDIRS="$ROOTDIRS$SEP$dir" - SEP="|" - done - OURCYGPATTERN="(^($ROOTDIRS))" - # Add a user-defined pattern to the cygpath arguments - if [ "$GRADLE_CYGPATTERN" != "" ] ; then - OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" - fi - # Now convert the arguments - kludge to limit ourselves to /bin/sh - i=0 - for arg in "$@" ; do - CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` - CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option - - if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition - eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` - else - eval `echo args$i`="\"$arg\"" - fi - i=$((i+1)) - done - case $i in - (0) set -- ;; - (1) set -- "$args0" ;; - (2) set -- "$args0" "$args1" ;; - (3) set -- "$args0" "$args1" "$args2" ;; - (4) set -- "$args0" "$args1" "$args2" "$args3" ;; - (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; - (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; - (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; - (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; - (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; - esac -fi - -# Escape application args -save () { - for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done - echo " " -} -APP_ARGS=$(save "$@") - -# Collect all arguments for the java command, following the shell quoting and substitution rules -eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" - -# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong -if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then - cd "$(dirname "$0")" -fi - -exec "$JAVACMD" "$@" diff --git a/scraper-plugins-persistent-proxy-scores/gradlew.bat b/scraper-plugins-persistent-proxy-scores/gradlew.bat deleted file mode 100644 index 24467a1..0000000 --- a/scraper-plugins-persistent-proxy-scores/gradlew.bat +++ /dev/null @@ -1,100 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem - -@if "%DEBUG%" == "" @echo off -@rem ########################################################################## -@rem -@rem Gradle startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal - -set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto init - -echo. -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto init - -echo. -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% -echo. -echo Please set the JAVA_HOME variable in your environment to match the -echo location of your Java installation. - -goto fail - -:init -@rem Get command-line arguments, handling Windows variants - -if not "%OS%" == "Windows_NT" goto win9xME_args - -:win9xME_args -@rem Slurp the command line arguments. -set CMD_LINE_ARGS= -set _SKIP=2 - -:win9xME_args_slurp -if "x%~1" == "x" goto execute - -set CMD_LINE_ARGS=%* - -:execute -@rem Setup the command line - -set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar - -@rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% - -:end -@rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega diff --git a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-annotations.jar b/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-annotations.jar deleted file mode 100644 index 425b67b..0000000 Binary files a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-annotations.jar and /dev/null differ diff --git a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-api.jar b/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-api.jar deleted file mode 100644 index d4b813f..0000000 Binary files a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11-api.jar and /dev/null differ diff --git a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11.0-utils.jar b/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11.0-utils.jar deleted file mode 100644 index bf032ec..0000000 Binary files a/scraper-plugins-persistent-proxy-scores/libs/scraper-0.11.0-utils.jar and /dev/null differ diff --git a/scraper-plugins-persistent-proxy-scores/src/main/java/module-info.java b/scraper-plugins-persistent-proxy-scores/src/main/java/module-info.java index d5d4d84..eef18df 100644 --- a/scraper-plugins-persistent-proxy-scores/src/main/java/module-info.java +++ b/scraper-plugins-persistent-proxy-scores/src/main/java/module-info.java @@ -4,13 +4,10 @@ open module scraper.plugins.persitentproxies { // only depend on api and annotations requires scraper.api; - requires scraper.annotations; requires scraper.utils; // export packages exports scraper.plugins.persistentproxies; - requires org.slf4j; - provides Addon with scraper.plugins.persistentproxies.PersistentProxiesAddon; } diff --git a/scraper-plugins-persistent-proxy-scores/src/main/java/scraper/plugins/persistentproxies/PersistentProxiesAddon.java b/scraper-plugins-persistent-proxy-scores/src/main/java/scraper/plugins/persistentproxies/PersistentProxiesAddon.java index 3040243..1438035 100644 --- a/scraper-plugins-persistent-proxy-scores/src/main/java/scraper/plugins/persistentproxies/PersistentProxiesAddon.java +++ b/scraper-plugins-persistent-proxy-scores/src/main/java/scraper/plugins/persistentproxies/PersistentProxiesAddon.java @@ -1,7 +1,5 @@ package scraper.plugins.persistentproxies; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import scraper.annotations.ArgsCommand; import scraper.annotations.NotNull; import scraper.api.di.DIContainer; @@ -13,6 +11,9 @@ import java.nio.file.Files; import java.nio.file.Paths; +import static java.lang.System.Logger.Level.ERROR; +import static java.lang.System.Logger.Level.INFO; + @ArgsCommand( value = "persistent-proxies:", @@ -21,7 +22,7 @@ ) public class PersistentProxiesAddon implements Addon { /** Logger with the actual class name */ - private Logger l = LoggerFactory.getLogger("PersistentProxies"); + private System.Logger l = System.getLogger("PersistentProxies"); @Override public void load(@NotNull DIContainer loadedDependencies, @NotNull String[] args) { @@ -38,7 +39,7 @@ public void load(@NotNull DIContainer loadedDependencies, @NotNull String[] args try { tryReadScores(proxy, persist); } catch (IOException e) { - l.error("Could not read input proxy scores", e); + l.log(ERROR,"Could not read input proxy scores", e); } Runtime.getRuntime().addShutdownHook(new Thread(() -> { @@ -49,7 +50,7 @@ public void load(@NotNull DIContainer loadedDependencies, @NotNull String[] args pw.println(group+">"+line); }))); } catch (FileNotFoundException e) { - l.error("Could not write scores to file"); + l.log(ERROR,"Could not write scores to file"); e.printStackTrace(); throw new RuntimeException(e); } @@ -57,9 +58,9 @@ public void load(@NotNull DIContainer loadedDependencies, @NotNull String[] args File f_old = new File(Paths.get(persist, "proxy-scores.csv").toString()); boolean renamed = f.renameTo(f_old); if(!renamed) { - l.error("Could not finish rename operation {} -> {}", f, f_old); + l.log(ERROR,"Could not finish rename operation {} -> {}", f, f_old); } else { - l.info("Persisted proxy scores successfully"); + l.log(INFO,"Persisted proxy scores successfully"); } })); } @@ -70,12 +71,12 @@ private void tryReadScores(ProxyReservation proxy, String persist) throws IOExce File f = new File(Paths.get(persist, "proxy-scores.csv").toString()); if(f.exists()) { - l.info("Reading persistent scores at {}", f); + l.log(INFO,"Reading persistent scores at {}", f); Files.lines(Paths.get(persist, "proxy-scores.csv")) .forEach(proxy::addProxyLine); } else { - l.info("No persistent scores found at {}", f); + l.log(INFO,"No persistent scores found at {}", f); } } diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 0000000..1f44f08 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,3 @@ +rootProject.name = 'scraper-nodes' + +include 'scraper-plugins-debugger','scraper-plugins-persistent-proxy-scores'