From fcc916f3c3077e09de57cbc6d06b3ed0cb98438d Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Wed, 21 Nov 2018 14:10:12 +0100 Subject: [PATCH 01/31] Adding --csv option to search and retrieve (Issue #4 ) --- build.sbt | 1 + .../de/upb/cs/swt/delphi/cli/Config.scala | 2 + .../de/upb/cs/swt/delphi/cli/CsvOutput.scala | 69 +++++++++++++++++++ .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 7 +- .../delphi/cli/artifacts/SearchResult.scala | 20 +++--- .../cs/swt/delphi/cli/commands/Command.scala | 2 + .../delphi/cli/commands/RetrieveCommand.scala | 8 ++- .../delphi/cli/commands/SearchCommand.scala | 7 +- 8 files changed, 102 insertions(+), 14 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala diff --git a/build.sbt b/build.sbt index 3838904..286164b 100644 --- a/build.sbt +++ b/build.sbt @@ -27,6 +27,7 @@ libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" libraryDependencies += "de.vandermeer" % "asciitable" % "0.3.2" libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.5" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value +libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" debianPackageDependencies := Seq("java8-runtime-headless") diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala index b487a03..47f75c5 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala @@ -26,6 +26,7 @@ package de.upb.cs.swt.delphi.cli case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://delphi.cs.uni-paderborn.de/api/"), verbose: Boolean = false, raw: Boolean = false, + csv: String = "", silent: Boolean = false, list : Boolean = false, mode: String = "", @@ -36,5 +37,6 @@ case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://d opts: List[String] = List()) { lazy val consoleOutput = new ConsoleOutput(this) + lazy val csvOutput = new CsvOutput(this) } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala new file mode 100644 index 0000000..a11c709 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala @@ -0,0 +1,69 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.cli + +import java.io.{BufferedWriter, FileWriter} + +import de.upb.cs.swt.delphi.cli.artifacts.Result +import au.com.bytecode.opencsv.CSVWriter +import de.upb.cs.swt.delphi.cli.commands.SearchCommand.information +import de.vandermeer.asciitable.{AsciiTable, CWC_LongestLine} +import de.vandermeer.skb.interfaces.transformers.textformat.TextAlignment + +import scala.collection.mutable.ListBuffer +import scala.collection.JavaConverters._ + +/** + * Export search and retrieve results to .csv file. + * + * @author Lisa Nguyen Quang Do + * @author Ben Hermann + * + */ + +class CsvOutput(config: Config) { + + def exportResult(value: Any): Unit = { + var table = value match { + case results : Seq[Result] if results.head.isInstanceOf[Result] => resultsToCsv(results) + case _ => { + println("Error: results are in unknown format.") + return + } + } + + val outputFile = new BufferedWriter(new FileWriter(config.csv, /* append = */false)) + val csvWriter = new CSVWriter(outputFile) + csvWriter.writeAll(seqAsJavaList(table)) + outputFile.close() + println("Results written to file '" + config.csv + "'") + } + + def resultsToCsv(results : Seq[Result]) : Seq[Array[String]] = { + if (results.size != 0) { + val fieldNames = results.head.fieldNames() + val tableHeader : Array[String] = (fieldNames.+:("discovered at").+:("version").+:("groupId").+:("artifactId").+:("source").+:("Id")).toArray + val tableBody: Seq[Array[String]] = results.map { + e => { + Array(e.id, e.metadata.source, e.metadata.artifactId, e.metadata.groupId, e.metadata.version, e.metadata.discovered).++(fieldNames.map(f => e.metricResults(f).toString)) + } + } + return tableBody.+:(tableHeader) + } + return Seq.empty[Array[String]] + } +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index c7c45c6..c9868c4 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -35,7 +35,7 @@ object DelphiCLI extends App { val cliParser = { new scopt.OptionParser[Config]("delphi-cli") { - head("Delphi Command Line Tool", s"(${BuildInfo.version})") + head("Delphi Command Line Tool", s"(Lala)") // ${BuildInfo.version} version("version").text("Prints the version of the command line tool.") @@ -55,7 +55,7 @@ object DelphiCLI extends App { .children( arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"), opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " + - "with the filepath given in place of the ID") + "with the filepath given in place of the ID"), opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") ) cmd("search").action((s, c) => c.copy(mode = "search")) @@ -63,7 +63,8 @@ object DelphiCLI extends App { .children( arg[String]("query").action((x,c) => c.copy(query = x)).text("The query to be used."), opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."), - opt[Unit](name="list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)") + opt[Unit](name="list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") ) } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala index ee656bd..19d927c 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResult.scala @@ -18,21 +18,23 @@ package de.upb.cs.swt.delphi.cli.artifacts import spray.json.DefaultJsonProtocol -case class RetrieveResult(val id: String, - val metadata: ArtifactMetadata, - val metricResults: Map[String, Int]) { +trait Result{ + val id: String + val metadata: ArtifactMetadata + val metricResults: Map[String, Int] + def toMavenIdentifier() : String = s"${metadata.groupId}:${metadata.artifactId}:${metadata.version}" def fieldNames() : List[String] = metricResults.keys.toList.sorted } -case class SearchResult(val id: String, - val metadata: ArtifactMetadata, - val metricResults: Map[String, Int]) { - def toMavenIdentifier() : String = s"${metadata.groupId}:${metadata.artifactId}:${metadata.version}" +case class SearchResult(id: String, + metadata: ArtifactMetadata, + metricResults: Map[String, Int]) extends Result - def fieldNames() : List[String] = metricResults.keys.toList.sorted -} +case class RetrieveResult(id: String, + metadata: ArtifactMetadata, + metricResults: Map[String, Int]) extends Result case class ArtifactMetadata(val artifactId: String, val source: String, diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala index 02cce23..30b9e22 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala @@ -80,4 +80,6 @@ trait Command { protected def error(implicit config: Config): String => Unit = config.consoleOutput.outputError _ protected def success(implicit config: Config): String => Unit = config.consoleOutput.outputSuccess _ + protected def exportResult(implicit config: Config): Any => Unit = config.csvOutput.exportResult _ + } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 7f4865c..8bc9b6d 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -23,6 +23,7 @@ import akka.stream.ActorMaterializer import de.upb.cs.swt.delphi.cli.Config import de.upb.cs.swt.delphi.cli.artifacts.RetrieveResult import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ +import de.upb.cs.swt.delphi.cli.commands.SearchCommand.information import spray.json.DefaultJsonProtocol import scala.concurrent.Await @@ -62,7 +63,9 @@ object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonPro result.map(s => { if (config.raw) { reportResult(config)(s) - } else { + } + + if (!config.raw || !config.csv.equals("")) { val unmarshalledFuture = Unmarshal(s).to[List[RetrieveResult]] unmarshalledFuture.transform { @@ -71,6 +74,9 @@ object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonPro success(config)(s"Found ${unmarshalled.size} item(s).") reportResult(config)(unmarshalled) + if(!config.csv.equals("")) + exportResult(config)(unmarshalled) + Success(unmarshalled) } case Failure(e) => { diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index f804b8c..f0756d8 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -77,7 +77,9 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto if (config.raw || result.equals("")) { reportResult(config)(result) - } else { + } + + if(!(config.raw || result.equals("")) || !config.csv.equals("")) { val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] val processFuture = unmarshalledFuture.transform { @@ -108,6 +110,9 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto reportResult(config)(results) information(config)(f"Query took $queryRuntime%.2fs.") + + if(!config.csv.equals("")) + exportResult(config)(results) } case class Query(query: String, From 21560a2838d8d8e6dcea16500dcf7db9b78f9902 Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Quang Do Date: Wed, 21 Nov 2018 14:19:22 +0100 Subject: [PATCH 02/31] Fixed the Buildinfo placeholder --- src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index c9868c4..3c1f9ca 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -35,7 +35,7 @@ object DelphiCLI extends App { val cliParser = { new scopt.OptionParser[Config]("delphi-cli") { - head("Delphi Command Line Tool", s"(Lala)") // ${BuildInfo.version} + head("Delphi Command Line Tool", s"(${BuildInfo.version})") version("version").text("Prints the version of the command line tool.") From 20427e6d4f714b498bd46c74f16f9633cedbc141 Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Thu, 22 Nov 2018 01:08:19 +0100 Subject: [PATCH 03/31] Fixing code style issues. --- .../de/upb/cs/swt/delphi/cli/CsvOutput.scala | 37 +++++++++---------- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 4 +- .../delphi/cli/commands/RetrieveCommand.scala | 4 +- .../delphi/cli/commands/SearchCommand.scala | 5 ++- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala index a11c709..65b071a 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala @@ -20,11 +20,7 @@ import java.io.{BufferedWriter, FileWriter} import de.upb.cs.swt.delphi.cli.artifacts.Result import au.com.bytecode.opencsv.CSVWriter -import de.upb.cs.swt.delphi.cli.commands.SearchCommand.information -import de.vandermeer.asciitable.{AsciiTable, CWC_LongestLine} -import de.vandermeer.skb.interfaces.transformers.textformat.TextAlignment -import scala.collection.mutable.ListBuffer import scala.collection.JavaConverters._ /** @@ -38,32 +34,35 @@ import scala.collection.JavaConverters._ class CsvOutput(config: Config) { def exportResult(value: Any): Unit = { - var table = value match { - case results : Seq[Result] if results.head.isInstanceOf[Result] => resultsToCsv(results) - case _ => { - println("Error: results are in unknown format.") - return + printToCsv( + value match { + case results : + Seq[Result] if results.headOption.getOrElse(Seq.empty[Array[String]]).isInstanceOf[Result] => resultsToCsv(results) + case _ => Seq.empty[Array[String]] } - } + ) + } + def printToCsv(table : Seq[Array[String]]): Unit = { val outputFile = new BufferedWriter(new FileWriter(config.csv, /* append = */false)) val csvWriter = new CSVWriter(outputFile) csvWriter.writeAll(seqAsJavaList(table)) outputFile.close() - println("Results written to file '" + config.csv + "'") } def resultsToCsv(results : Seq[Result]) : Seq[Array[String]] = { - if (results.size != 0) { - val fieldNames = results.head.fieldNames() - val tableHeader : Array[String] = (fieldNames.+:("discovered at").+:("version").+:("groupId").+:("artifactId").+:("source").+:("Id")).toArray - val tableBody: Seq[Array[String]] = results.map { + if (results.isEmpty) { + Seq.empty[Array[String]] + } else { + val fieldNames = results.headOption.get.fieldNames() + val tableHeader : Array[String] = + fieldNames.+:("discovered at").+:("version").+:("groupId").+:("artifactId").+:("source").+:("Id").toArray + results.map { e => { - Array(e.id, e.metadata.source, e.metadata.artifactId, e.metadata.groupId, e.metadata.version, e.metadata.discovered).++(fieldNames.map(f => e.metricResults(f).toString)) + Array(e.id, e.metadata.source, e.metadata.artifactId, e.metadata.groupId, e.metadata.version, + e.metadata.discovered).++(fieldNames.map(f => e.metricResults(f).toString)) } - } - return tableBody.+:(tableHeader) + }.+:(tableHeader) } - return Seq.empty[Array[String]] } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 3c1f9ca..64e4d14 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -32,7 +32,6 @@ object DelphiCLI extends App { implicit val system = ActorSystem() - val cliParser = { new scopt.OptionParser[Config]("delphi-cli") { head("Delphi Command Line Tool", s"(${BuildInfo.version})") @@ -55,7 +54,8 @@ object DelphiCLI extends App { .children( arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"), opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " + - "with the filepath given in place of the ID"), opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") + "with the filepath given in place of the ID"), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") ) cmd("search").action((s, c) => c.copy(mode = "search")) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 8bc9b6d..817aa00 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -74,8 +74,10 @@ object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonPro success(config)(s"Found ${unmarshalled.size} item(s).") reportResult(config)(unmarshalled) - if(!config.csv.equals("")) + if(!config.csv.equals("")) { exportResult(config)(unmarshalled) + information(config)("Results written to file '" + config.csv + "'") + } Success(unmarshalled) } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index f0756d8..55aed07 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -29,6 +29,7 @@ import akka.util.ByteString import de.upb.cs.swt.delphi.cli.Config import de.upb.cs.swt.delphi.cli.artifacts.SearchResult import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ +import de.upb.cs.swt.delphi.cli.commands.RetrieveCommand.information import spray.json.DefaultJsonProtocol import scala.concurrent.duration._ @@ -111,8 +112,10 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto information(config)(f"Query took $queryRuntime%.2fs.") - if(!config.csv.equals("")) + if(!config.csv.equals("")) { exportResult(config)(results) + information(config)("Results written to file '" + config.csv + "'") + } } case class Query(query: String, From 72add90723f510a452fd2e1a0406c99311ca87e9 Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Thu, 22 Nov 2018 04:05:29 +0100 Subject: [PATCH 04/31] Fixing last code style issue --- src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala index 65b071a..6b3e30b 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/CsvOutput.scala @@ -51,10 +51,11 @@ class CsvOutput(config: Config) { } def resultsToCsv(results : Seq[Result]) : Seq[Array[String]] = { - if (results.isEmpty) { - Seq.empty[Array[String]] + val headOption = results.headOption.getOrElse() + if (!headOption.isInstanceOf[Result]) { + Seq.empty[Array[String]] } else { - val fieldNames = results.headOption.get.fieldNames() + val fieldNames = headOption.asInstanceOf[Result].fieldNames() val tableHeader : Array[String] = fieldNames.+:("discovered at").+:("version").+:("groupId").+:("artifactId").+:("source").+:("Id").toArray results.map { From 40e8743a13e45dc1cbce2cf7df6c6fb640c22fff Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Sat, 24 Nov 2018 13:36:28 +0100 Subject: [PATCH 05/31] Catch the TimeoutException in the search command. Undeterministically generates a harmless error message from Akka in the console (https://github.com/akka/akka-http/issues/497). Future work: disable this message. --- .../de/upb/cs/swt/delphi/cli/Config.scala | 1 + .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 7 ++- .../delphi/cli/commands/SearchCommand.scala | 62 +++++++++++-------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala index 47f75c5..86d3d7e 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala @@ -33,6 +33,7 @@ case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://d query : String = "", limit : Option[Int] = None, id : String = "", + timeout : Int = 10, args: List[String] = List(), opts: List[String] = List()) { diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 64e4d14..7890162 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -53,18 +53,19 @@ object DelphiCLI extends App { .text("Retrieve a project's description, specified by ID.") .children( arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " + - "with the filepath given in place of the ID"), - opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") + "with the filepath given in place of the ID") ) cmd("search").action((s, c) => c.copy(mode = "search")) .text("Search artifact using a query.") .children( arg[String]("query").action((x,c) => c.copy(query = x)).text("The query to be used."), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."), opt[Unit](name="list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"), - opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)") + opt[Int]("timeout").action((x, c) => c.copy(timeout = x)).text("Timeout in seconds.") ) } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 55aed07..4ba6a29 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -16,6 +16,7 @@ package de.upb.cs.swt.delphi.cli.commands +import java.util.concurrent.TimeoutException import java.util.concurrent.TimeUnit import akka.actor.ActorSystem @@ -29,7 +30,6 @@ import akka.util.ByteString import de.upb.cs.swt.delphi.cli.Config import de.upb.cs.swt.delphi.cli.artifacts.SearchResult import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ -import de.upb.cs.swt.delphi.cli.commands.RetrieveCommand.information import spray.json.DefaultJsonProtocol import scala.concurrent.duration._ @@ -59,40 +59,48 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto Http().singleRequest(HttpRequest(uri = searchUri, method = HttpMethods.POST, entity = entity)) } - val response = Await.result(responseFuture, 10 seconds) - val resultFuture: Future[String] = response match { - case HttpResponse(StatusCodes.OK, headers, entity, _) => - entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => - body.utf8String + try { + val response = Await.result(responseFuture, Duration(config.timeout + " seconds")) + val resultFuture: Future[String] = response match { + case HttpResponse(StatusCodes.OK, headers, entity, _) => + entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => + body.utf8String + } + case resp@HttpResponse(code, _, _, _) => { + error(config)("Request failed, response code: " + code) + resp.discardEntityBytes() + Future("") } - case resp@HttpResponse(code, _, _, _) => { - error(config)("Request failed, response code: " + code) - resp.discardEntityBytes() - Future("") } - } - - val result = Await.result(resultFuture, Duration.Inf) - val took = (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + val result = Await.result(resultFuture, Duration.Inf) - if (config.raw || result.equals("")) { - reportResult(config)(result) - } + val took = (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) - if(!(config.raw || result.equals("")) || !config.csv.equals("")) { - val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] + if (config.raw || result.equals("")) { + reportResult(config)(result) + } - val processFuture = unmarshalledFuture.transform { - case Success(unmarshalled) => { - processResults(config, unmarshalled, took) - Success(unmarshalled) - } - case Failure(e) => { - error(config)(result) - Failure(e) + if(!(config.raw || result.equals("")) || !config.csv.equals("")) { + val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] + + val processFuture = unmarshalledFuture.transform { + case Success(unmarshalled) => { + processResults(config, unmarshalled, took) + Success(unmarshalled) + } + case Failure(e) => { + error(config)(result) + Failure(e) + } } } + } catch { + case e : TimeoutException => { + error(config)("The query timed out after " + (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + + " seconds. To set a longer timeout, use the --timeout option.") + Failure(e) + } } } From d398c5b7a6d287030bac638be01ce8757dc33c17 Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Sat, 24 Nov 2018 19:53:49 +0100 Subject: [PATCH 06/31] Fixing code style. And refactoring to bring the search function to under 50 lines. --- .../de/upb/cs/swt/delphi/cli/Config.scala | 2 +- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 3 +- .../delphi/cli/commands/SearchCommand.scala | 83 ++++++++++--------- 3 files changed, 47 insertions(+), 41 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala index 86d3d7e..cc5b016 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala @@ -33,7 +33,7 @@ case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://d query : String = "", limit : Option[Int] = None, id : String = "", - timeout : Int = 10, + timeout : Option[Int] = None, args: List[String] = List(), opts: List[String] = List()) { diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 7890162..1c5be43 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -18,7 +18,6 @@ package de.upb.cs.swt.delphi.cli import akka.actor.ActorSystem import akka.http.scaladsl.Http -import akka.stream.ActorMaterializer import de.upb.cs.swt.delphi.cli.commands.{RetrieveCommand, SearchCommand, TestCommand} import scala.concurrent.duration.Duration @@ -65,7 +64,7 @@ object DelphiCLI extends App { opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."), opt[Unit](name="list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"), - opt[Int]("timeout").action((x, c) => c.copy(timeout = x)).text("Timeout in seconds.") + opt[Int]("timeout").action((x, c) => c.copy(timeout = Some(x))).text("Timeout in seconds.") ) } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 4ba6a29..2b7ca7b 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -16,8 +16,7 @@ package de.upb.cs.swt.delphi.cli.commands -import java.util.concurrent.TimeoutException -import java.util.concurrent.TimeUnit +import java.util.concurrent.{TimeUnit, TimeoutException} import akka.actor.ActorSystem import akka.http.scaladsl.Http @@ -33,10 +32,13 @@ import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ import spray.json.DefaultJsonProtocol import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} -import scala.util.{Failure, Success} +import scala.concurrent.{Await, ExecutionContextExecutor, Future} +import scala.util.{Failure, Success, Try} object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProtocol { + + final var SEARCH_TIMEOUT: Int = 10 + /** * Executes the command implementation * @@ -59,47 +61,52 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto Http().singleRequest(HttpRequest(uri = searchUri, method = HttpMethods.POST, entity = entity)) } - try { - val response = Await.result(responseFuture, Duration(config.timeout + " seconds")) - val resultFuture: Future[String] = response match { - case HttpResponse(StatusCodes.OK, headers, entity, _) => - entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => - body.utf8String - } - case resp@HttpResponse(code, _, _, _) => { - error(config)("Request failed, response code: " + code) - resp.discardEntityBytes() - Future("") + Try(Await.result(responseFuture, Duration(config.timeout.getOrElse(SEARCH_TIMEOUT) + " seconds"))). + map(response => parseResponse(response, config, start)(ec, materializer)). + recover { + case e : TimeoutException => { + error(config)("The query timed out after " + (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + + " seconds. To set a longer timeout, use the --timeout option.") + Failure(e) } } + } - val result = Await.result(resultFuture, Duration.Inf) - - val took = (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + private def parseResponse(response: HttpResponse, config: Config, start: Long) + (implicit ec: ExecutionContextExecutor, materializer: ActorMaterializer): Unit = { - if (config.raw || result.equals("")) { - reportResult(config)(result) + val resultFuture: Future[String] = response match { + case HttpResponse(StatusCodes.OK, headers, entity, _) => + entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => + body.utf8String + } + case resp@HttpResponse(code, _, _, _) => { + error(config)("Request failed, response code: " + code) + resp.discardEntityBytes() + Future("") } + } + + val result = Await.result(resultFuture, Duration.Inf) - if(!(config.raw || result.equals("")) || !config.csv.equals("")) { - val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] - - val processFuture = unmarshalledFuture.transform { - case Success(unmarshalled) => { - processResults(config, unmarshalled, took) - Success(unmarshalled) - } - case Failure(e) => { - error(config)(result) - Failure(e) - } + val took = (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + + if (config.raw || result.equals("")) { + reportResult(config)(result) + } + + if(!(config.raw || result.equals("")) || !config.csv.equals("")) { + val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] + + val processFuture = unmarshalledFuture.transform { + case Success(unmarshalled) => { + processResults(config, unmarshalled, took) + Success(unmarshalled) + } + case Failure(e) => { + error(config)(result) + Failure(e) } - } - } catch { - case e : TimeoutException => { - error(config)("The query timed out after " + (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + - " seconds. To set a longer timeout, use the --timeout option.") - Failure(e) } } } From fb14f37aba635cb31badbae6cc1de81cddde48cd Mon Sep 17 00:00:00 2001 From: Lisa Nguyen Date: Sat, 24 Nov 2018 21:29:07 +0100 Subject: [PATCH 07/31] Fixed last style issues --- .../de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 2b7ca7b..24b6683 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -37,7 +37,7 @@ import scala.util.{Failure, Success, Try} object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProtocol { - final var SEARCH_TIMEOUT: Int = 10 + val searchTimeout: Int = 10 /** * Executes the command implementation @@ -61,7 +61,7 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto Http().singleRequest(HttpRequest(uri = searchUri, method = HttpMethods.POST, entity = entity)) } - Try(Await.result(responseFuture, Duration(config.timeout.getOrElse(SEARCH_TIMEOUT) + " seconds"))). + Try(Await.result(responseFuture, Duration(config.timeout.getOrElse(searchTimeout) + " seconds"))). map(response => parseResponse(response, config, start)(ec, materializer)). recover { case e : TimeoutException => { From 96f184256f182c751b0568cc0b1999462bcb1abc Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Tue, 15 Jan 2019 11:51:57 +0100 Subject: [PATCH 08/31] Refactoring delphi cli sttp dependency for native image generation --- build.sbt | 21 +++- project/build.properties | 2 +- project/plugins.sbt | 2 +- .../de/upb/cs/swt/delphi/cli/Config.scala | 2 +- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 110 ++++++++---------- .../cs/swt/delphi/cli/commands/Command.scala | 41 ++++++- .../delphi/cli/commands/RetrieveCommand.scala | 2 +- .../delphi/cli/commands/SearchCommand.scala | 2 +- .../swt/delphi/cli/commands/TestCommand.scala | 8 +- .../de/upb/cs/swt/delphi/cli/package.scala | 30 +++++ 10 files changed, 143 insertions(+), 77 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/package.scala diff --git a/build.sbt b/build.sbt index 286164b..31739f4 100644 --- a/build.sbt +++ b/build.sbt @@ -29,6 +29,11 @@ libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.5" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" + +libraryDependencies += "com.softwaremill.sttp" %% "core" % "1.5.4" + + + debianPackageDependencies := Seq("java8-runtime-headless") lazy val cli = (project in file(".")). @@ -38,11 +43,23 @@ lazy val cli = (project in file(".")). enablePlugins(BuildInfoPlugin). enablePlugins(DebianPlugin). enablePlugins(WindowsPlugin). - + enablePlugins(GraalVMNativeImagePlugin). + settings( + graalVMNativeImageOptions++=Seq( + "--enable-https", + "--enable-http", + "--enable-all-security-services", + "--allow-incomplete-classpath", + "--enable-url-protocols=http,https" + ) + ). + enablePlugins(JDKPackagerPlugin). settings( buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion), - buildInfoPackage := "de.upb.cs.swt.delphi.cli" + buildInfoPackage := "de.upb.cs.swt.delphi.cli", ) scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml" trapExit := false +fork := true +connectInput := true diff --git a/project/build.properties b/project/build.properties index 210243d..7609b47 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.1.1 +sbt.version = 1.2.8 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt index 705f861..86636a4 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,6 +1,6 @@ // build management and packaging addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0") -addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.2") +addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.15") // coverage addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala index cc5b016..fcdb9dc 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/Config.scala @@ -23,7 +23,7 @@ package de.upb.cs.swt.delphi.cli * @param verbose Marker if logging should be verbose * @param mode The command to be run */ -case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://delphi.cs.uni-paderborn.de/api/"), +case class Config(server: String = sys.env.getOrElse("DELPHI_SERVER", "https://delphi.cs.uni-paderborn.de/api"), verbose: Boolean = false, raw: Boolean = false, csv: String = "", diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 1c5be43..9555813 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -16,81 +16,71 @@ package de.upb.cs.swt.delphi.cli -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import de.upb.cs.swt.delphi.cli.commands.{RetrieveCommand, SearchCommand, TestCommand} - -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext} - +import de.upb.cs.swt.delphi.cli.commands.TestCommand /** * The application class for the Delphi command line interface */ -object DelphiCLI extends App { - - implicit val system = ActorSystem() +object DelphiCLI { - val cliParser = { - new scopt.OptionParser[Config]("delphi-cli") { - head("Delphi Command Line Tool", s"(${BuildInfo.version})") + private def cliParser = { + val parser = { + new scopt.OptionParser[Config]("delphi-cli") { + head("Delphi Command Line Tool", s"(${BuildInfo.version})") - version("version").text("Prints the version of the command line tool.") + version("version").text("Prints the version of the command line tool.") - help("help").text("Prints this help text.") - override def showUsageOnError = true + help("help").text("Prints this help text.") - opt[String]("server").action( (x,c) => c.copy(server = x)).text("The url to the Delphi server") - opt[Unit] (name = "raw").action((_,c) => c.copy(raw = true)).text("Output the raw results") - opt[Unit] (name = "silent").action((_,c) => c.copy(silent = true)).text("Suppress non-result output") + override def showUsageOnError = true - checkConfig(c => if (c.server.isEmpty()) failure("Option server is required.") else success) + opt[String]("server").action((x, c) => c.copy(server = x)).text("The url to the Delphi server") + opt[Unit](name = "raw").action((_, c) => c.copy(raw = true)).text("Output the raw results") + opt[Unit](name = "silent").action((_, c) => c.copy(silent = true)).text("Suppress non-result output") - cmd("test").action((_,c) => c.copy(mode = "test")) + checkConfig(c => if (c.server.isEmpty()) failure("Option server is required.") else success) - cmd("retrieve").action((s,c) => c.copy(mode = "retrieve")) - .text("Retrieve a project's description, specified by ID.") - .children( - arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"), - opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), - opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " + - "with the filepath given in place of the ID") - ) - - cmd("search").action((s, c) => c.copy(mode = "search")) - .text("Search artifact using a query.") - .children( - arg[String]("query").action((x,c) => c.copy(query = x)).text("The query to be used."), - opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), - opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."), - opt[Unit](name="list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"), - opt[Int]("timeout").action((x, c) => c.copy(timeout = Some(x))).text("Timeout in seconds.") - ) - } - } + cmd("test").action((_, c) => c.copy(mode = "test")) + cmd("retrieve").action((s, c) => c.copy(mode = "retrieve")) + .text("Retrieve a project's description, specified by ID.") + .children( + arg[String]("id").action((x, c) => c.copy(id = x)).text("The ID of the project to retrieve"), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), + opt[Unit]('f', "file").action((_, c) => c.copy(opts = List("file"))).text("Use to load the ID from file, " + + "with the filepath given in place of the ID") + ) - cliParser.parse(args, Config()) match { - case Some(config) => - if (!config.silent) cliParser.showHeader() - config.mode match { - case "test" => TestCommand.execute(config) - case "retrieve" => RetrieveCommand.execute(config) - case "search" => SearchCommand.execute(config) - case x => config.consoleOutput.outputError(s"Unknown command: $x") + cmd("search").action((s, c) => c.copy(mode = "search")) + .text("Search artifact using a query.") + .children( + arg[String]("query").action((x, c) => c.copy(query = x)).text("The query to be used."), + opt[String]("csv").action((x, c) => c.copy(csv = x)).text("Path to the output .csv file (overwrites existing file)"), + opt[Int]("limit").action((x, c) => c.copy(limit = Some(x))).text("The maximal number of results returned."), + opt[Unit](name = "list").action((_, c) => c.copy(list = true)).text("Output results as list (raw option overrides this)"), + opt[Int]("timeout").action((x, c) => c.copy(timeout = Some(x))).text("Timeout in seconds.") + ) } - - case None => + } + parser } + def main(args: Array[String]): Unit = { + + System.setProperty("java.library.path", javaLibPath.get) + System.setProperty("javax.net.ssl.trustStore", trustStorePath.get) + System.setProperty("picocli.ansi", "true") + cliParser.parse(args, config) match { + case Some(config) => + if (!config.silent) cliParser.showHeader() + config.mode match { + case "test" => TestCommand.execute + // case "retrieve" => RetrieveCommand.execute(config) + // case "search" => SearchCommand.execute(config) + case x => config.consoleOutput.outputError(s"Unknown command: $x") + } + case None => + } - val poolShutdown = Http().shutdownAllConnectionPools() - Await.result(poolShutdown, Duration.Inf) - - implicit val ec: ExecutionContext = system.dispatcher - val terminationFuture = system.terminate() - - terminationFuture.onComplete { - sys.exit(0) } -} +} \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala index 30b9e22..998ad47 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala @@ -22,11 +22,12 @@ import akka.http.scaladsl.model.Uri.Query import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes, Uri} import akka.stream.ActorMaterializer import akka.util.ByteString +import com.softwaremill.sttp._ import de.upb.cs.swt.delphi.cli.Config -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ -import scala.util.{Failure, Success} +import scala.concurrent.{Await, Future} +import scala.util.Failure /** * Represents the implementation of a command of the CLI @@ -35,17 +36,19 @@ trait Command { /** * Executes the command implementation + * * @param config The current configuration for the command */ - def execute(config: Config)(implicit system : ActorSystem): Unit + def execute(implicit config: Config): Unit = {} /** * Implements a common request type using currying to avoid code duplication + * * @param target The endpoint to perform a Get request on * @param config The current configuration for the command */ - protected def executeGet(target: String, parameters: Map[String, String] = Map())(config: Config, system : ActorSystem) : Option[String] = { - implicit val sys : ActorSystem = system + protected def executeGet(target: String, parameters: Map[String, String] = Map())(config: Config, system: ActorSystem): Option[String] = { + implicit val sys: ActorSystem = system implicit val materializer = ActorMaterializer() implicit val executionContext = sys.dispatcher @@ -65,7 +68,7 @@ trait Command { entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => Some(body.utf8String) } - case resp @ HttpResponse(code, _, _, _) => { + case resp@HttpResponse(code, _, _, _) => { error(config)("Artifact not found.") resp.discardEntityBytes() Future(None) @@ -75,9 +78,35 @@ trait Command { Await.result(resultString, Duration.Inf) } + /** + * Generic http GET request + * + * @param target Sub url in delphi server + * @param parameters Query params + * @return GET response + */ + protected def executeGet1(target: String, parameters: Map[String, String] = Map()) + (implicit config: Config, backend: SttpBackend[Id, Nothing]): Option[String] = { + + val request = sttp.get(uri"${config.server}/$target?$parameters") + config.consoleOutput.outputInformation(s"Sending request ${request.uri}") + val response = request.send() + response.body match { + case Left(value) => + error.apply(s"Request failed:\n $value") + None + case Right(value) => + Some(value) + } + } + + protected def information(implicit config: Config): String => Unit = config.consoleOutput.outputInformation _ + protected def reportResult(implicit config: Config): Any => Unit = config.consoleOutput.outputResult _ + protected def error(implicit config: Config): String => Unit = config.consoleOutput.outputError _ + protected def success(implicit config: Config): String => Unit = config.consoleOutput.outputSuccess _ protected def exportResult(implicit config: Config): Any => Unit = config.csvOutput.exportResult _ diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 817aa00..075182d 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -38,7 +38,7 @@ import scala.util.{Failure, Success} object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonProtocol { - override def execute(config: Config)(implicit system: ActorSystem): Unit = { + def execute(config: Config)(implicit system: ActorSystem): Unit = { implicit val ec = system.dispatcher implicit val materializer = ActorMaterializer() diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 24b6683..671db5e 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -44,7 +44,7 @@ object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProto * * @param config The current configuration for the command */ - override def execute(config: Config)(implicit system: ActorSystem): Unit = { + def execute(config: Config)(implicit system: ActorSystem): Unit = { implicit val ec = system.dispatcher implicit val materializer = ActorMaterializer() diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala index 2589bea..ddd8b12 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala @@ -16,17 +16,17 @@ package de.upb.cs.swt.delphi.cli.commands -import akka.actor.ActorSystem import de.upb.cs.swt.delphi.cli.Config +import de.upb.cs.swt.delphi.cli._ /** * The implementation of the test command. * Tries to connect to the Delphi server and reports on the results of the version call. */ object TestCommand extends Command { - override def execute(config: Config)(implicit system : ActorSystem): Unit = executeGet( - "/version" - )(config, system).map(s => { + override def execute(implicit config: Config): Unit = executeGet1( + "version" + ).map(s => { success(config)("Successfully contacted Delphi server. ") information(config)("Server version: " + s) }) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala new file mode 100644 index 0000000..95e9bde --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala @@ -0,0 +1,30 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package de.upb.cs.swt.delphi + +import com.softwaremill.sttp.HttpURLConnectionBackend + +package object cli { + implicit val config: Config = Config() + implicit val backend = HttpURLConnectionBackend() + + val javaLibPath = Option(System.getenv("JAVA_LIB_PATH")) + .orElse(Some("/usr/lib/jvm/default-java/lib/")) + + val trustStorePath = Option(System.getenv("JAVA_TRUSTSTORE")) + .orElse(Some("/usr/lib/jvm/default-java/lib/security/cacerts")) + +} From 3972bd46ddb968ee08b646ff4b1439ca90add9c0 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Tue, 15 Jan 2019 15:29:27 +0100 Subject: [PATCH 09/31] Refactored retrieve command to sttp --- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 55 ++++++++----- .../cs/swt/delphi/cli/commands/Command.scala | 54 +------------ .../delphi/cli/commands/RetrieveCommand.scala | 79 ++++++++++--------- .../swt/delphi/cli/commands/TestCommand.scala | 15 ++-- .../de/upb/cs/swt/delphi/cli/package.scala | 30 ------- 5 files changed, 90 insertions(+), 143 deletions(-) delete mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/package.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 9555813..55c30fe 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -16,13 +16,47 @@ package de.upb.cs.swt.delphi.cli -import de.upb.cs.swt.delphi.cli.commands.TestCommand +import com.softwaremill.sttp.{HttpURLConnectionBackend, Id, SttpBackend} +import de.upb.cs.swt.delphi.cli.commands._ /** * The application class for the Delphi command line interface */ object DelphiCLI { + + def main(args: Array[String]): Unit = { + + def getEnvOrElse(envVar: String, defaultPath: String) = sys.env.getOrElse(envVar, defaultPath) + + val javaLibPath = getEnvOrElse("JAVA_LIB_PATH", "/usr/lib/jvm/default-java/lib/") + + val trustStorePath = getEnvOrElse("JAVA_TRUSTSTORE", "/usr/lib/jvm/default-java/lib/security/cacerts") + + System.setProperty("java.library.path", javaLibPath) + System.setProperty("javax.net.ssl.trustStore", trustStorePath) + + cliParser.parse(args, Config()) match { + case Some(c) => + + implicit val config: Config = c + implicit val backend: SttpBackend[Id, Nothing] = HttpURLConnectionBackend() + + if (!config.silent) cliParser.showHeader() + + config.mode match { + case "test" => TestCommand.execute + case "retrieve" => RetrieveCommand.execute + // case "search" => SearchCommand.execute(config) + case x => config.consoleOutput.outputError(s"Unknown command: $x") + } + + + case None => + } + + } + private def cliParser = { val parser = { new scopt.OptionParser[Config]("delphi-cli") { @@ -64,23 +98,4 @@ object DelphiCLI { } parser } - - def main(args: Array[String]): Unit = { - - System.setProperty("java.library.path", javaLibPath.get) - System.setProperty("javax.net.ssl.trustStore", trustStorePath.get) - System.setProperty("picocli.ansi", "true") - cliParser.parse(args, config) match { - case Some(config) => - if (!config.silent) cliParser.showHeader() - config.mode match { - case "test" => TestCommand.execute - // case "retrieve" => RetrieveCommand.execute(config) - // case "search" => SearchCommand.execute(config) - case x => config.consoleOutput.outputError(s"Unknown command: $x") - } - case None => - } - - } } \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala index 998ad47..307c76a 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala @@ -16,19 +16,9 @@ package de.upb.cs.swt.delphi.cli.commands -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.model.Uri.Query -import akka.http.scaladsl.model.{HttpRequest, HttpResponse, StatusCodes, Uri} -import akka.stream.ActorMaterializer -import akka.util.ByteString import com.softwaremill.sttp._ import de.upb.cs.swt.delphi.cli.Config -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} -import scala.util.Failure - /** * Represents the implementation of a command of the CLI */ @@ -39,54 +29,18 @@ trait Command { * * @param config The current configuration for the command */ - def execute(implicit config: Config): Unit = {} - - /** - * Implements a common request type using currying to avoid code duplication - * - * @param target The endpoint to perform a Get request on - * @param config The current configuration for the command - */ - protected def executeGet(target: String, parameters: Map[String, String] = Map())(config: Config, system: ActorSystem): Option[String] = { - implicit val sys: ActorSystem = system - implicit val materializer = ActorMaterializer() - implicit val executionContext = sys.dispatcher - - val uri = Uri(config.server) - config.consoleOutput.outputInformation(s"Contacting server ${uri}...") + def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = {} - val responseFuture = Http().singleRequest(HttpRequest(uri = uri.withPath(uri.path + target).withQuery(Query(parameters)))) - - responseFuture.onComplete { - case Failure(_) => error(config)(s"Could not reach server ${config.server}.") - case _ => - } - - val result = Await.result(responseFuture, 30 seconds) - val resultString = result match { - case HttpResponse(StatusCodes.OK, headers, entity, _) => - entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => - Some(body.utf8String) - } - case resp@HttpResponse(code, _, _, _) => { - error(config)("Artifact not found.") - resp.discardEntityBytes() - Future(None) - } - } - - Await.result(resultString, Duration.Inf) - } /** * Generic http GET request * - * @param target Sub url in delphi server + * @param target Sub url in delphi server * @param parameters Query params * @return GET response */ - protected def executeGet1(target: String, parameters: Map[String, String] = Map()) - (implicit config: Config, backend: SttpBackend[Id, Nothing]): Option[String] = { + protected def executeGet(target: String, parameters: Map[String, String] = Map()) + (implicit config: Config, backend: SttpBackend[Id, Nothing]): Option[String] = { val request = sttp.get(uri"${config.server}/$target?$parameters") config.consoleOutput.outputInformation(s"Sending request ${request.uri}") diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 075182d..797c8c7 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -16,31 +16,21 @@ package de.upb.cs.swt.delphi.cli.commands -import akka.actor.ActorSystem -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport -import akka.http.scaladsl.unmarshalling.Unmarshal -import akka.stream.ActorMaterializer -import de.upb.cs.swt.delphi.cli.Config +import com.softwaremill.sttp.{Id, SttpBackend} +import de.upb.cs.swt.delphi.cli._ import de.upb.cs.swt.delphi.cli.artifacts.RetrieveResult -import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ -import de.upb.cs.swt.delphi.cli.commands.SearchCommand.information -import spray.json.DefaultJsonProtocol +import spray.json._ -import scala.concurrent.Await -import scala.concurrent.duration.Duration import scala.io.Source -import scala.util.{Failure, Success} /** * The implementation of the retrieve command. * Retrieves the contents of the file at the endpoint specified by the config file, and prints them to stdout */ -object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonProtocol { +object RetrieveCommand extends Command with DefaultJsonProtocol { - def execute(config: Config)(implicit system: ActorSystem): Unit = { - implicit val ec = system.dispatcher - implicit val materializer = ActorMaterializer() + override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { //Checks whether the ID should be loaded from a file or not, and either returns the first line // of the given file if it is, or the specified ID otherwise @@ -56,37 +46,54 @@ object RetrieveCommand extends Command with SprayJsonSupport with DefaultJsonPro } val result = executeGet( - s"/retrieve/$checkTarget", + s"retrieve/$checkTarget", Map("pretty" -> "") - )(config, system) + ) - result.map(s => { + result.foreach(s => { if (config.raw) { - reportResult(config)(s) + reportResult.apply(s) } - if (!config.raw || !config.csv.equals("")) { - val unmarshalledFuture = Unmarshal(s).to[List[RetrieveResult]] + import artifacts.SearchResultJson._ - unmarshalledFuture.transform { - case Success(unmarshalled) => { - val unmarshalled = Await.result(unmarshalledFuture, Duration.Inf) - success(config)(s"Found ${unmarshalled.size} item(s).") - reportResult(config)(unmarshalled) + //TODO: convertTo[List[RetrieveResult]] not working ??? - if(!config.csv.equals("")) { - exportResult(config)(unmarshalled) - information(config)("Results written to file '" + config.csv + "'") - } + val jsonArr = s.parseJson.asInstanceOf[JsArray].elements + val retrieveResults = jsonArr.map(r => r.convertTo[RetrieveResult]) - Success(unmarshalled) - } - case Failure(e) => { - error(config)(s) - Failure(e) - } + success.apply(s"Found ${retrieveResults.size} item(s).") + reportResult.apply(retrieveResults) + if (!config.csv.equals("")) { + exportResult.apply(retrieveResults) + information.apply("Results written to file '" + config.csv + "'") } } + + }) } } + +/* if (!config.raw || !config.csv.equals("")) { + val unmarshalledFuture = Unmarshal(s).to[List[RetrieveResult]] + + unmarshalledFuture.transform { + case Success(unmarshalled) => { + val unmarshalled = Await.result(unmarshalledFuture, Duration.Inf) + success.apply(s"Found ${unmarshalled.size} item(s).") + reportResult.apply(unmarshalled) + + if (!config.csv.equals("")) { + exportResult.apply(unmarshalled) + information.apply("Results written to file '" + config.csv + "'") + } + + Success(unmarshalled) + } + case Failure(e) => { + error.apply(s) + Failure(e) + } + } + }*/ \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala index ddd8b12..4101281 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala @@ -16,18 +16,19 @@ package de.upb.cs.swt.delphi.cli.commands +import com.softwaremill.sttp.{Id, SttpBackend} import de.upb.cs.swt.delphi.cli.Config -import de.upb.cs.swt.delphi.cli._ /** * The implementation of the test command. * Tries to connect to the Delphi server and reports on the results of the version call. */ object TestCommand extends Command { - override def execute(implicit config: Config): Unit = executeGet1( - "version" - ).map(s => { - success(config)("Successfully contacted Delphi server. ") - information(config)("Server version: " + s) - }) + override def execute(implicit config: Config,backend: SttpBackend[Id, Nothing]): Unit = { + executeGet("version") + .foreach(s => { + success.apply("Successfully contacted Delphi server. ") + information.apply("Server version: " + s) + }) + } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala deleted file mode 100644 index 95e9bde..0000000 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/package.scala +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (C) 2018 The Delphi Team. -// See the LICENCE file distributed with this work for additional -// information regarding copyright ownership. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package de.upb.cs.swt.delphi - -import com.softwaremill.sttp.HttpURLConnectionBackend - -package object cli { - implicit val config: Config = Config() - implicit val backend = HttpURLConnectionBackend() - - val javaLibPath = Option(System.getenv("JAVA_LIB_PATH")) - .orElse(Some("/usr/lib/jvm/default-java/lib/")) - - val trustStorePath = Option(System.getenv("JAVA_TRUSTSTORE")) - .orElse(Some("/usr/lib/jvm/default-java/lib/security/cacerts")) - -} From 1de2c18e7d1e774b1334549c23e2567aee91ed29 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Wed, 23 Jan 2019 22:35:50 +0100 Subject: [PATCH 10/31] Fixing encoded url issue for retrieve command --- build.sbt | 12 ++++--- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 4 ++- .../cs/swt/delphi/cli/commands/Command.scala | 10 +++--- .../delphi/cli/commands/RetrieveCommand.scala | 31 +++---------------- .../swt/delphi/cli/commands/TestCommand.scala | 4 +-- 5 files changed, 22 insertions(+), 39 deletions(-) diff --git a/build.sbt b/build.sbt index 31739f4..210d9a6 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ scalaVersion := "2.12.4" -name := "delphi" +name := "delphi-cli" version := "1.0.0-SNAPSHOT" maintainer := "Ben Hermann " @@ -28,10 +28,12 @@ libraryDependencies += "de.vandermeer" % "asciitable" % "0.3.2" libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.5" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" - -libraryDependencies += "com.softwaremill.sttp" %% "core" % "1.5.4" - +libraryDependencies ++= Seq( + "com.softwaremill.sttp" %% "core" % "1.5.4", + "com.softwaremill.sttp" %% "spray-json" % "1.5.4" +) debianPackageDependencies := Seq("java8-runtime-headless") @@ -45,7 +47,7 @@ lazy val cli = (project in file(".")). enablePlugins(WindowsPlugin). enablePlugins(GraalVMNativeImagePlugin). settings( - graalVMNativeImageOptions++=Seq( + graalVMNativeImageOptions ++= Seq( "--enable-https", "--enable-http", "--enable-all-security-services", diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 55c30fe..cd6185d 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -16,7 +16,6 @@ package de.upb.cs.swt.delphi.cli -import com.softwaremill.sttp.{HttpURLConnectionBackend, Id, SttpBackend} import de.upb.cs.swt.delphi.cli.commands._ /** @@ -39,6 +38,9 @@ object DelphiCLI { cliParser.parse(args, Config()) match { case Some(c) => + import com.softwaremill.sttp._ + import scala.concurrent.duration._ + implicit val config: Config = c implicit val backend: SttpBackend[Id, Nothing] = HttpURLConnectionBackend() diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala index 307c76a..d3bc8bf 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala @@ -33,16 +33,18 @@ trait Command { /** - * Generic http GET request + * Http GET request template * * @param target Sub url in delphi server * @param parameters Query params * @return GET response */ - protected def executeGet(target: String, parameters: Map[String, String] = Map()) + protected def executeGet(paths: Seq[String], parameters: Map[String, String] = Map()) (implicit config: Config, backend: SttpBackend[Id, Nothing]): Option[String] = { - - val request = sttp.get(uri"${config.server}/$target?$parameters") + val serverUrl = uri"${config.server}" + val oldPath = serverUrl.path + val reqUrl = serverUrl.path(oldPath ++ paths).params(parameters) + val request = sttp.get(reqUrl) config.consoleOutput.outputInformation(s"Sending request ${request.uri}") val response = request.send() response.body match { diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 797c8c7..6bf7f23 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -45,8 +45,9 @@ object RetrieveCommand extends Command with DefaultJsonProtocol { } } + println(config.id) val result = executeGet( - s"retrieve/$checkTarget", + Seq("retrieve", checkTarget), Map("pretty" -> "") ) @@ -57,7 +58,8 @@ object RetrieveCommand extends Command with DefaultJsonProtocol { if (!config.raw || !config.csv.equals("")) { import artifacts.SearchResultJson._ - //TODO: convertTo[List[RetrieveResult]] not working ??? + //TODO: Direct convertTo[List[RetrieveResult]] not working ??? + val jsonArr = s.parseJson.asInstanceOf[JsArray].elements val retrieveResults = jsonArr.map(r => r.convertTo[RetrieveResult]) @@ -69,31 +71,6 @@ object RetrieveCommand extends Command with DefaultJsonProtocol { information.apply("Results written to file '" + config.csv + "'") } } - - }) } } - -/* if (!config.raw || !config.csv.equals("")) { - val unmarshalledFuture = Unmarshal(s).to[List[RetrieveResult]] - - unmarshalledFuture.transform { - case Success(unmarshalled) => { - val unmarshalled = Await.result(unmarshalledFuture, Duration.Inf) - success.apply(s"Found ${unmarshalled.size} item(s).") - reportResult.apply(unmarshalled) - - if (!config.csv.equals("")) { - exportResult.apply(unmarshalled) - information.apply("Results written to file '" + config.csv + "'") - } - - Success(unmarshalled) - } - case Failure(e) => { - error.apply(s) - Failure(e) - } - } - }*/ \ No newline at end of file diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala index 4101281..2f11b3e 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/TestCommand.scala @@ -24,8 +24,8 @@ import de.upb.cs.swt.delphi.cli.Config * Tries to connect to the Delphi server and reports on the results of the version call. */ object TestCommand extends Command { - override def execute(implicit config: Config,backend: SttpBackend[Id, Nothing]): Unit = { - executeGet("version") + override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { + executeGet(Seq("version")) .foreach(s => { success.apply("Successfully contacted Delphi server. ") information.apply("Server version: " + s) From b252d4e608051ecc363a5e66c6f76f970c0990fd Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 24 Jan 2019 11:47:53 +0100 Subject: [PATCH 11/31] Refactoring search command to sttp --- .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 2 +- .../delphi/cli/commands/RetrieveCommand.scala | 3 +- .../delphi/cli/commands/SearchCommand.scala | 152 ++++++++---------- .../cs/swt/delphi/cli/commands/package.scala | 28 ++++ 4 files changed, 95 insertions(+), 90 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/commands/package.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index cd6185d..d0d9b88 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -49,7 +49,7 @@ object DelphiCLI { config.mode match { case "test" => TestCommand.execute case "retrieve" => RetrieveCommand.execute - // case "search" => SearchCommand.execute(config) + case "search" => SearchCommand.execute case x => config.consoleOutput.outputError(s"Unknown command: $x") } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 6bf7f23..d290cd6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -27,7 +27,7 @@ import scala.io.Source * The implementation of the retrieve command. * Retrieves the contents of the file at the endpoint specified by the config file, and prints them to stdout */ -object RetrieveCommand extends Command with DefaultJsonProtocol { +object RetrieveCommand extends Command { override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { @@ -45,7 +45,6 @@ object RetrieveCommand extends Command with DefaultJsonProtocol { } } - println(config.id) val result = executeGet( Seq("retrieve", checkTarget), Map("pretty" -> "") diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 671db5e..634f308 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -16,124 +16,102 @@ package de.upb.cs.swt.delphi.cli.commands -import java.util.concurrent.{TimeUnit, TimeoutException} - -import akka.actor.ActorSystem -import akka.http.scaladsl.Http -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport -import akka.http.scaladsl.marshalling.Marshal -import akka.http.scaladsl.model._ -import akka.http.scaladsl.unmarshalling.Unmarshal -import akka.stream.ActorMaterializer -import akka.util.ByteString -import de.upb.cs.swt.delphi.cli.Config +import java.util.concurrent.TimeUnit + +import com.softwaremill.sttp._ +import com.softwaremill.sttp.sprayJson._ import de.upb.cs.swt.delphi.cli.artifacts.SearchResult -import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ -import spray.json.DefaultJsonProtocol +import de.upb.cs.swt.delphi.cli.{Config, artifacts} +import spray.json._ import scala.concurrent.duration._ -import scala.concurrent.{Await, ExecutionContextExecutor, Future} -import scala.util.{Failure, Success, Try} -object SearchCommand extends Command with SprayJsonSupport with DefaultJsonProtocol { +object SearchCommand extends Command with DefaultJsonProtocol{ - val searchTimeout: Int = 10 + val searchTimeout = 10.seconds + val TIMEOUT_CODE = 408 /** * Executes the command implementation * * @param config The current configuration for the command */ - def execute(config: Config)(implicit system: ActorSystem): Unit = { - implicit val ec = system.dispatcher - implicit val materializer = ActorMaterializer() + override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { def query = config.query - information(config)(s"Searching for artifacts matching ${'"'}$query${'"'}.") - val start = System.nanoTime() + information.apply(s"Searching for artifacts matching ${'"'}$query${'"'}.") - implicit val queryFormat = jsonFormat2(Query) - val baseUri = Uri(config.server) - val prettyParam = Map("pretty" -> "") - val searchUri = baseUri.withPath(baseUri.path + "/search").withQuery(akka.http.scaladsl.model.Uri.Query(prettyParam)) - val responseFuture = Marshal(Query(query, config.limit)).to[RequestEntity] flatMap { entity => - Http().singleRequest(HttpRequest(uri = searchUri, method = HttpMethods.POST, entity = entity)) - } - Try(Await.result(responseFuture, Duration(config.timeout.getOrElse(searchTimeout) + " seconds"))). - map(response => parseResponse(response, config, start)(ec, materializer)). - recover { - case e : TimeoutException => { - error(config)("The query timed out after " + (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + - " seconds. To set a longer timeout, use the --timeout option.") - Failure(e) - } - } + val queryParams = Map("pretty" -> "") + val queryPayload: Query = Query(query,config.limit) + val searchUri = uri"${config.server}/search?$queryParams" + + val request = sttp.body(queryPayload.toJson).post(searchUri) + + val (res, time) = processRequest(request) + res.foreach(processResults(_, time)) } - private def parseResponse(response: HttpResponse, config: Config, start: Long) - (implicit ec: ExecutionContextExecutor, materializer: ActorMaterializer): Unit = { + private def processRequest(req: Request[String, Nothing]) + (implicit config: Config, + backend: SttpBackend[Id, Nothing]): (Option[String], FiniteDuration) = { + val start = System.nanoTime() + val res: Id[Response[String]] = req.readTimeout(searchTimeout).send() + val end = System.nanoTime() + val took = (end - start).nanos - val resultFuture: Future[String] = response match { - case HttpResponse(StatusCodes.OK, headers, entity, _) => - entity.dataBytes.runFold(ByteString(""))(_ ++ _).map { body => - body.utf8String - } - case resp@HttpResponse(code, _, _, _) => { - error(config)("Request failed, response code: " + code) - resp.discardEntityBytes() - Future("") - } - } + if (res.code == TIMEOUT_CODE) { - val result = Await.result(resultFuture, Duration.Inf) + error.apply(s"The query timed out after ${took.toSeconds} seconds. " + + "To set a longer timeout, use the --timeout option.") + } + val resStr = res.body match { + case Left(v) => + error.apply(s"Search request failed \n $v") + println(v) + None + case Right(v) => + Some(v) + } + (resStr, took) + } - val took = (System.nanoTime() - start).nanos.toUnit(TimeUnit.SECONDS) + private def processResults(res: String, queryRuntime: FiniteDuration)(implicit config: Config) = { - if (config.raw || result.equals("")) { - reportResult(config)(result) + if (config.raw || res.equals("")) { + reportResult.apply(res) + } + if (!(config.raw || res.equals("")) || !config.csv.equals("")) { + import artifacts.SearchResultJson._ + val jsonArr = res.parseJson.asInstanceOf[JsArray].elements + val retrieveResults = jsonArr.map(r => r.convertTo[SearchResult]).toList + onProperSearchResults(retrieveResults) } - if(!(config.raw || result.equals("")) || !config.csv.equals("")) { - val unmarshalledFuture = Unmarshal(result).to[List[SearchResult]] + def onProperSearchResults(sr: List[SearchResult]) = { - val processFuture = unmarshalledFuture.transform { - case Success(unmarshalled) => { - processResults(config, unmarshalled, took) - Success(unmarshalled) - } - case Failure(e) => { - error(config)(result) - Failure(e) + val capMessage = { + config.limit match { + case Some(limit) if (limit <= sr.size) + => s"Results may be capped by result limit set to $limit." + case None if (sr.size >= 50) + => "Results may be capped by default limit of 50 returned results. Use --limit to extend the result set." + case _ + => "" } } - } - } - private def processResults(config: Config, results: List[SearchResult], queryRuntime: Double) = { - val capMessage = { - config.limit match { - case Some(limit) if (limit <= results.size) - => s"Results may be capped by result limit set to $limit." - case None if (results.size >= 50) - => "Results may be capped by default limit of 50 returned results. Use --limit to extend the result set." - case _ - => "" - } - } - success(config)(s"Found ${results.size} item(s). $capMessage") - reportResult(config)(results) + success.apply(s"Found ${sr.size} item(s). $capMessage") + reportResult.apply(sr) - information(config)(f"Query took $queryRuntime%.2fs.") + information.apply(f"Query took ${queryRuntime.toUnit(TimeUnit.SECONDS)}%.2fs.") - if(!config.csv.equals("")) { - exportResult(config)(results) - information(config)("Results written to file '" + config.csv + "'") + if (!config.csv.equals("")) { + exportResult.apply(sr) + information.apply("Results written to file '" + config.csv + "'") + } } } - case class Query(query: String, - limit: Option[Int] = None) - } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/package.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/package.scala new file mode 100644 index 0000000..8975631 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/package.scala @@ -0,0 +1,28 @@ +// Copyright (C) 2018 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package de.upb.cs.swt.delphi.cli + +import spray.json._ + +package object commands extends DefaultJsonProtocol { + + + case class Query(query: String, + limit: Option[Int] = None) + + implicit val queryFormat = jsonFormat2(Query) + +} From 07242692c337d6ce5757f205db8ae52b1b488b8e Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Fri, 25 Jan 2019 16:58:31 +0100 Subject: [PATCH 12/31] Implementing codacy suggestions --- .../scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 3 +-- .../cs/swt/delphi/cli/commands/RetrieveCommand.scala | 2 +- .../cs/swt/delphi/cli/commands/SearchCommand.scala | 11 +++++------ 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index d0d9b88..6148ef6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -16,6 +16,7 @@ package de.upb.cs.swt.delphi.cli +import com.softwaremill.sttp._ import de.upb.cs.swt.delphi.cli.commands._ /** @@ -38,8 +39,6 @@ object DelphiCLI { cliParser.parse(args, Config()) match { case Some(c) => - import com.softwaremill.sttp._ - import scala.concurrent.duration._ implicit val config: Config = c implicit val backend: SttpBackend[Id, Nothing] = HttpURLConnectionBackend() diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index d290cd6..93a64df 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -19,6 +19,7 @@ package de.upb.cs.swt.delphi.cli.commands import com.softwaremill.sttp.{Id, SttpBackend} import de.upb.cs.swt.delphi.cli._ import de.upb.cs.swt.delphi.cli.artifacts.RetrieveResult +import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ import spray.json._ import scala.io.Source @@ -55,7 +56,6 @@ object RetrieveCommand extends Command { reportResult.apply(s) } if (!config.raw || !config.csv.equals("")) { - import artifacts.SearchResultJson._ //TODO: Direct convertTo[List[RetrieveResult]] not working ??? diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index 634f308..e7bad31 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -1,4 +1,4 @@ -// Copyright (C) 2018 The Delphi Team. + // Copyright (C) 2018 The Delphi Team. // See the LICENCE file distributed with this work for additional // information regarding copyright ownership. // @@ -20,8 +20,9 @@ import java.util.concurrent.TimeUnit import com.softwaremill.sttp._ import com.softwaremill.sttp.sprayJson._ +import de.upb.cs.swt.delphi.cli.Config import de.upb.cs.swt.delphi.cli.artifacts.SearchResult -import de.upb.cs.swt.delphi.cli.{Config, artifacts} +import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ import spray.json._ import scala.concurrent.duration._ @@ -29,7 +30,7 @@ import scala.concurrent.duration._ object SearchCommand extends Command with DefaultJsonProtocol{ val searchTimeout = 10.seconds - val TIMEOUT_CODE = 408 + val timeoutCode = 408 /** * Executes the command implementation @@ -61,7 +62,7 @@ object SearchCommand extends Command with DefaultJsonProtocol{ val end = System.nanoTime() val took = (end - start).nanos - if (res.code == TIMEOUT_CODE) { + if (res.code == timeoutCode) { error.apply(s"The query timed out after ${took.toSeconds} seconds. " + "To set a longer timeout, use the --timeout option.") @@ -69,7 +70,6 @@ object SearchCommand extends Command with DefaultJsonProtocol{ val resStr = res.body match { case Left(v) => error.apply(s"Search request failed \n $v") - println(v) None case Right(v) => Some(v) @@ -83,7 +83,6 @@ object SearchCommand extends Command with DefaultJsonProtocol{ reportResult.apply(res) } if (!(config.raw || res.equals("")) || !config.csv.equals("")) { - import artifacts.SearchResultJson._ val jsonArr = res.parseJson.asInstanceOf[JsArray].elements val retrieveResults = jsonArr.map(r => r.convertTo[SearchResult]).toList onProperSearchResults(retrieveResults) From cc3306087a31d83610f7e0a9a8237a49f847a9fa Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 7 Mar 2019 19:00:15 +0100 Subject: [PATCH 13/31] Fixing codacy issue --- .../scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index e7bad31..edb5c66 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -1,4 +1,4 @@ - // Copyright (C) 2018 The Delphi Team. +// Copyright (C) 2018 The Delphi Team. // See the LICENCE file distributed with this work for additional // information regarding copyright ownership. // From 7d8068a5b1c3bf7869e8c647bbe18f2597d4c1c1 Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 7 Mar 2019 20:38:21 +0100 Subject: [PATCH 14/31] Renaming the project to delphi --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 210d9a6..e7b264f 100644 --- a/build.sbt +++ b/build.sbt @@ -1,6 +1,6 @@ scalaVersion := "2.12.4" -name := "delphi-cli" +name := "delphi" version := "1.0.0-SNAPSHOT" maintainer := "Ben Hermann " From 2ee1c51318536992e3d90517576d2ce0dafab57a Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Wed, 20 Nov 2019 03:37:15 +0100 Subject: [PATCH 15/31] Upgrading akka to 2.5.16 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index e7b264f..b4c4ea3 100644 --- a/build.sbt +++ b/build.sbt @@ -13,7 +13,7 @@ wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424" scalastyleConfig := baseDirectory.value / "project" / "scalastyle_config.xml" -val akkaVersion = "2.5.14" +val akkaVersion = "2.5.16" val akkaHttpVersion = "10.1.5" libraryDependencies ++= Seq( From f1fcde7051b74e3a617c648963495a0b6e884f4f Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Thu, 21 Nov 2019 00:26:28 +0100 Subject: [PATCH 16/31] Upgrading sbt-codacy to 1.3.14 --- project/plugins.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/plugins.sbt b/project/plugins.sbt index 86636a4..ef657bf 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -4,7 +4,7 @@ addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.15") // coverage addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") -addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.12") +addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.14") // preparation for dependency checking addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.1") From b0d3e8d88924c20ea9bf05c6b50bba6bbd0a0797 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Thu, 21 Nov 2019 16:53:09 +0100 Subject: [PATCH 17/31] Updated sbt file --- build.sbt | 48 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 286164b..44397fb 100644 --- a/build.sbt +++ b/build.sbt @@ -1,11 +1,37 @@ -scalaVersion := "2.12.4" +ThisBuild / organization := "de.upb.cs.swt.delphi" +ThisBuild / organizationName := "Delphi Project" +ThisBuild / organizationHomepage := Some(url("https://delphi.cs.uni-paderborn.de/")) + +ThisBuild / scmInfo := Some( + ScmInfo( + url("https://github.com/delphi-hub/delphi-cli"), + "scm:git@github.com:delphi-hub/delphi-cli.git" + ) +) + +ThisBuild / developers := List( + Developer( + id = "bhermann", + name = "Ben Hermann", + email = "ben.hermann@upb.de", + url = url("https://www.thewhitespace.de") + ) +) + +ThisBuild / description := "The command line client for Delphi" +ThisBuild / licenses := List("Apache 2" -> new URL("http://www.apache.org/licenses/LICENSE-2.0.txt")) +ThisBuild / homepage := Some(url("https://delphi.cs.uni-paderborn.de/")) + +lazy val scala212 = "2.12.10" +lazy val scala213 = "2.13.1" +lazy val supportedScalaVersions = List(scala212, scala213) + +ThisBuild / scalaVersion := scala213 name := "delphi" version := "1.0.0-SNAPSHOT" maintainer := "Ben Hermann " -licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html")) - packageSummary := "Windows Package for the Delphi CLI" packageDescription := """Windows Package for the Delphi CLI""" wixProductId := "ce07be71-510d-414a-92d4-dff47631848a" @@ -13,7 +39,9 @@ wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424" scalastyleConfig := baseDirectory.value / "project" / "scalastyle_config.xml" +/* val akkaVersion = "2.5.14" + val akkaHttpVersion = "10.1.5" libraryDependencies ++= Seq( @@ -21,6 +49,17 @@ libraryDependencies ++= Seq( "com.typesafe.akka" %% "akka-http-spray-json" % akkaHttpVersion, "com.typesafe.akka" %% "akka-stream" % akkaVersion ) + */ +val http4sVersion = "0.18.21" + +// Only necessary for SNAPSHOT releases +resolvers += Resolver.sonatypeRepo("snapshots") + +libraryDependencies ++= Seq( + "org.http4s" %% "http4s-dsl" % http4sVersion, + "org.http4s" %% "http4s-blaze-client" % http4sVersion, + "org.http4s" %% "http4s-circe" % http4sVersion +) libraryDependencies += "com.github.scopt" %% "scopt" % "3.7.0" libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" @@ -41,7 +80,8 @@ lazy val cli = (project in file(".")). settings( buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, sbtVersion), - buildInfoPackage := "de.upb.cs.swt.delphi.cli" + buildInfoPackage := "de.upb.cs.swt.delphi.cli", + crossScalaVersions := supportedScalaVersions ) scalastyleConfig := baseDirectory.value / "project" / "scalastyle-config.xml" From 2835ad15d275da37e2c2efa458f5599b251ae38c Mon Sep 17 00:00:00 2001 From: Hariharan Ramanathan Date: Fri, 22 Nov 2019 02:19:19 +0100 Subject: [PATCH 18/31] - Switching back scala version to Scala 2.12 - Upgrading spray json to 1.3.5 - Upgrarding sttp to 1.7.2 --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 5dee8ed..fd9091f 100644 --- a/build.sbt +++ b/build.sbt @@ -26,7 +26,7 @@ lazy val scala212 = "2.12.10" lazy val scala213 = "2.13.1" lazy val supportedScalaVersions = List(scala212, scala213) -ThisBuild / scalaVersion := scala213 +ThisBuild / scalaVersion := scala212 name := "delphi" version := "1.0.0-SNAPSHOT" @@ -51,7 +51,7 @@ libraryDependencies ++= Seq( ) libraryDependencies += "com.github.scopt" %% "scopt" % "3.7.0" -libraryDependencies += "io.spray" %% "spray-json" % "1.3.3" +libraryDependencies += "io.spray" %% "spray-json" % "1.3.5" libraryDependencies += "de.vandermeer" % "asciitable" % "0.3.2" libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.5" libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value @@ -59,8 +59,8 @@ libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" libraryDependencies ++= Seq( - "com.softwaremill.sttp" %% "core" % "1.5.4", - "com.softwaremill.sttp" %% "spray-json" % "1.5.4" + "com.softwaremill.sttp" %% "core" % "1.7.2", + "com.softwaremill.sttp" %% "spray-json" % "1.7.2" ) From 8e7a377bca595d3538adc9119e1cf3ce3cf78758 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 24 Dec 2019 17:42:16 +0100 Subject: [PATCH 19/31] Made scala 2.12 default --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 5dee8ed..ebd41bf 100644 --- a/build.sbt +++ b/build.sbt @@ -24,9 +24,9 @@ ThisBuild / homepage := Some(url("https://delphi.cs.uni-paderborn.de/")) lazy val scala212 = "2.12.10" lazy val scala213 = "2.13.1" -lazy val supportedScalaVersions = List(scala212, scala213) +lazy val supportedScalaVersions = List(scala212) -ThisBuild / scalaVersion := scala213 +ThisBuild / scalaVersion := scala212 name := "delphi" version := "1.0.0-SNAPSHOT" From f4c3e586b0b73d3e372d8ab2e0e5c7b8975e3779 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 14:14:06 +0100 Subject: [PATCH 20/31] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bb0503b..40743e0 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,14 @@ -# Delphi Command-Line Interface (CLI) +# Delphi Command-Line Interface (CLI) The command-line interface for the Delphi platform. We are currently in pre-alpha state! There is no release and the code in this repository is purely experimental! -|branch | status | codacy | snyk | -| :---: | :---: | :---: | :---: | -| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-cli.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-cli) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/47046de0e8d64ae4b76191b7dae80075)](https://www.codacy.com/app/delphi-hub/delphi-cli?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Grade)| [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-cli/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-cli?targetFile=build.sbt) | -| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-cli.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-cli) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/47046de0e8d64ae4b76191b7dae80075?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-cli?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Grade)| [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-cli/develop/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-cli/develop?targetFile=build.sbt) +|branch | status | codacy | coverage | snyk | +| :---: | :---: | :---: | :---: | :---: | +| master | [![Build Status](https://travis-ci.org/delphi-hub/delphi-cli.svg?branch=master)](https://travis-ci.org/delphi-hub/delphi-cli) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/47046de0e8d64ae4b76191b7dae80075)](https://www.codacy.com/app/delphi-hub/delphi-cli?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Grade) | [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/47046de0e8d64ae4b76191b7dae80075)](https://www.codacy.com/manual/delphi-hub/delphi-cli?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Coverage) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-cli/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-cli?targetFile=build.sbt) | +| develop | [![Build Status](https://travis-ci.org/delphi-hub/delphi-cli.svg?branch=develop)](https://travis-ci.org/delphi-hub/delphi-cli) | [![Codacy Badge](https://api.codacy.com/project/badge/Grade/47046de0e8d64ae4b76191b7dae80075?branch=develop)](https://www.codacy.com/app/delphi-hub/delphi-cli?branch=develop&utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Grade)| [![Codacy Badge](https://api.codacy.com/project/badge/Coverage/47046de0e8d64ae4b76191b7dae80075)](https://www.codacy.com/manual/delphi-hub/delphi-cli?utm_source=github.com&utm_medium=referral&utm_content=delphi-hub/delphi-cli&utm_campaign=Badge_Coverage) | [![Known Vulnerabilities](https://snyk.io/test/github/delphi-hub/delphi-cli/develop/badge.svg?targetFile=build.sbt)](https://snyk.io/test/github/delphi-hub/delphi-cli/develop?targetFile=build.sbt) ## What is the Delphi Command-Line Interface? From f78e63ebe21d6e063265297f18a90eca92b330db Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:39:12 +0100 Subject: [PATCH 21/31] Removed coverage for now --- .travis.yml | 7 ++++--- project/plugins.sbt | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 0a2dc6c..7391247 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ language: scala scala: - - 2.12.4 + - 2.13.1 script: - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then sbt ++$TRAVIS_SCALA_VERSION test; fi' - - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION coverage test coverageReport coverageAggregate codacyCoverage; fi' + - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION test; fi' +# - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then sbt ++$TRAVIS_SCALA_VERSION coverage test coverageReport coverageAggregate codacyCoverage; fi' after_success: - - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bash <(curl -s https://codecov.io/bash); fi' +# - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then bash <(curl -s https://codecov.io/bash); fi' diff --git a/project/plugins.sbt b/project/plugins.sbt index ef657bf..58462f1 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -3,8 +3,8 @@ addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.7.0") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.15") // coverage -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") -addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.14") +// addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1") +// addSbtPlugin("com.codacy" % "sbt-codacy-coverage" % "1.3.14") // preparation for dependency checking addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.9.1") From 066f6662b2b62a1ff3d0e559549031793e9ef577 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:39:56 +0100 Subject: [PATCH 22/31] SBT version bumped to 1.3.5 --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 7609b47..8b8de72 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.2.8 \ No newline at end of file +sbt.version = 1.3.5 \ No newline at end of file From d9af2e3ee61da9c7dfd7cec867ea04eb052b2036 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:40:56 +0100 Subject: [PATCH 23/31] Bumped dependencies, moved to Scala 2.13 --- build.sbt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build.sbt b/build.sbt index 2fb614f..90db270 100644 --- a/build.sbt +++ b/build.sbt @@ -24,12 +24,12 @@ ThisBuild / homepage := Some(url("https://delphi.cs.uni-paderborn.de/")) lazy val scala212 = "2.12.10" lazy val scala213 = "2.13.1" -lazy val supportedScalaVersions = List(scala212) +lazy val supportedScalaVersions = List(scala213) -ThisBuild / scalaVersion := scala212 +ThisBuild / scalaVersion := scala213 name := "delphi" -version := "1.0.0-SNAPSHOT" +version := "0.9.5-SNAPSHOT" maintainer := "Ben Hermann " packageSummary := "Windows Package for the Delphi CLI" @@ -39,7 +39,7 @@ wixProductUpgradeId := "4552fb0e-e257-4dbd-9ecb-dba9dbacf424" scalastyleConfig := baseDirectory.value / "project" / "scalastyle_config.xml" -val http4sVersion = "0.18.21" +val http4sVersion = "0.21.0-M6" // Only necessary for SNAPSHOT releases resolvers += Resolver.sonatypeRepo("snapshots") @@ -50,13 +50,13 @@ libraryDependencies ++= Seq( "org.http4s" %% "http4s-circe" % http4sVersion ) -libraryDependencies += "com.github.scopt" %% "scopt" % "3.7.0" +libraryDependencies += "com.github.scopt" %% "scopt" % "3.7.1" libraryDependencies += "io.spray" %% "spray-json" % "1.3.5" libraryDependencies += "de.vandermeer" % "asciitable" % "0.3.2" -libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.5" -libraryDependencies += "org.scala-lang" % "scala-reflect" % scalaVersion.value +libraryDependencies += "com.lihaoyi" %% "fansi" % "0.2.7" libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" -libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" +libraryDependencies += "org.scalatest" %% "scalatest" % "3.1.0" % "test" +libraryDependencies += "joda-time" % "joda-time" % "2.10.5" libraryDependencies ++= Seq( "com.softwaremill.sttp" %% "core" % "1.7.2", From 32a45f097d544364277cfe7f3c84fead63a9dc5e Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:41:53 +0100 Subject: [PATCH 24/31] Deactivated GraalVM specific lines. Bugfix for help text --- src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 6148ef6..394c3bf 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -33,8 +33,9 @@ object DelphiCLI { val trustStorePath = getEnvOrElse("JAVA_TRUSTSTORE", "/usr/lib/jvm/default-java/lib/security/cacerts") - System.setProperty("java.library.path", javaLibPath) - System.setProperty("javax.net.ssl.trustStore", trustStorePath) + // This only is allowed to be set for GraalVM compiles... + //System.setProperty("java.library.path", javaLibPath) + //System.setProperty("javax.net.ssl.trustStore", trustStorePath) cliParser.parse(args, Config()) match { case Some(c) => @@ -43,12 +44,13 @@ object DelphiCLI { implicit val config: Config = c implicit val backend: SttpBackend[Id, Nothing] = HttpURLConnectionBackend() - if (!config.silent) cliParser.showHeader() + if (!config.silent && config.mode != "") cliParser.showHeader() config.mode match { case "test" => TestCommand.execute case "retrieve" => RetrieveCommand.execute case "search" => SearchCommand.execute + case "" => cliParser.showUsage() case x => config.consoleOutput.outputError(s"Unknown command: $x") } From 0351ee8b64d7321839fef44694beb023e355462f Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:48:06 +0100 Subject: [PATCH 25/31] Changes to reflect 0.9.5 API --- .../cs/swt/delphi/cli/commands/RetrieveCommand.scala | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala index 93a64df..7ceace6 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/RetrieveCommand.scala @@ -33,9 +33,7 @@ object RetrieveCommand extends Command { override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { - //Checks whether the ID should be loaded from a file or not, and either returns the first line - // of the given file if it is, or the specified ID otherwise - def checkTarget: String = { + val queriedId: String = { if (config.opts.contains("file")) { val source = Source.fromFile(config.args.head) val target = source.getLines.next() @@ -47,8 +45,7 @@ object RetrieveCommand extends Command { } val result = executeGet( - Seq("retrieve", checkTarget), - Map("pretty" -> "") + Seq("retrieve", queriedId) ) result.foreach(s => { @@ -56,14 +53,9 @@ object RetrieveCommand extends Command { reportResult.apply(s) } if (!config.raw || !config.csv.equals("")) { - - //TODO: Direct convertTo[List[RetrieveResult]] not working ??? - - val jsonArr = s.parseJson.asInstanceOf[JsArray].elements val retrieveResults = jsonArr.map(r => r.convertTo[RetrieveResult]) - success.apply(s"Found ${retrieveResults.size} item(s).") reportResult.apply(retrieveResults) if (!config.csv.equals("")) { exportResult.apply(retrieveResults) From f3b1a0779259fb712956f3526d806aec8af19d3f Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Tue, 7 Jan 2020 17:48:42 +0100 Subject: [PATCH 26/31] Changes to reflect 0.9.5 API --- .../delphi/cli/artifacts/SearchResults.scala | 26 ++++++++++++ .../delphi/cli/commands/SearchCommand.scala | 41 +++++++++---------- 2 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResults.scala diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResults.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResults.scala new file mode 100644 index 0000000..2617d09 --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/artifacts/SearchResults.scala @@ -0,0 +1,26 @@ +package de.upb.cs.swt.delphi.cli.artifacts + +import org.joda.time.DateTime +import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ +import org.joda.time.format.{DateTimeFormatter, ISODateTimeFormat} +import spray.json.{DefaultJsonProtocol, DeserializationException, JsString, JsValue, RootJsonFormat} + +case class SearchResults(totalHits : Long, hits : Array[SearchResult], queried : DateTime = DateTime.now()) + + + +object SearchResultsJson extends DefaultJsonProtocol { + implicit object DateJsonFormat extends RootJsonFormat[DateTime] { + + private val parserISO: DateTimeFormatter = ISODateTimeFormat.dateTime() + + override def write(obj: DateTime) = JsString(parserISO.print(obj)) + + override def read(json: JsValue): DateTime = json match { + case JsString(s) => parserISO.parseDateTime(s) + case _ => throw new DeserializationException("Error info you want here ...") + } + } + + implicit val SearchResultsFormat = jsonFormat3(SearchResults) +} diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala index edb5c66..e423d3f 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/SearchCommand.scala @@ -21,8 +21,8 @@ import java.util.concurrent.TimeUnit import com.softwaremill.sttp._ import com.softwaremill.sttp.sprayJson._ import de.upb.cs.swt.delphi.cli.Config -import de.upb.cs.swt.delphi.cli.artifacts.SearchResult -import de.upb.cs.swt.delphi.cli.artifacts.SearchResultJson._ +import de.upb.cs.swt.delphi.cli.artifacts.{SearchResult, SearchResults} +import de.upb.cs.swt.delphi.cli.artifacts.SearchResultsJson._ import spray.json._ import scala.concurrent.duration._ @@ -43,10 +43,8 @@ object SearchCommand extends Command with DefaultJsonProtocol{ information.apply(s"Searching for artifacts matching ${'"'}$query${'"'}.") - - val queryParams = Map("pretty" -> "") val queryPayload: Query = Query(query,config.limit) - val searchUri = uri"${config.server}/search?$queryParams" + val searchUri = uri"${config.server}/search" val request = sttp.body(queryPayload.toJson).post(searchUri) @@ -64,7 +62,7 @@ object SearchCommand extends Command with DefaultJsonProtocol{ if (res.code == timeoutCode) { - error.apply(s"The query timed out after ${took.toSeconds} seconds. " + + error.apply(s"The query timed out after ${took.toSeconds}%.0f seconds. " + "To set a longer timeout, use the --timeout option.") } val resStr = res.body match { @@ -83,28 +81,27 @@ object SearchCommand extends Command with DefaultJsonProtocol{ reportResult.apply(res) } if (!(config.raw || res.equals("")) || !config.csv.equals("")) { - val jsonArr = res.parseJson.asInstanceOf[JsArray].elements - val retrieveResults = jsonArr.map(r => r.convertTo[SearchResult]).toList - onProperSearchResults(retrieveResults) - } - - def onProperSearchResults(sr: List[SearchResult]) = { - + val retrieveResults = res.parseJson.convertTo[SearchResults] + val sr = retrieveResults.hits.toList val capMessage = { - config.limit match { - case Some(limit) if (limit <= sr.size) - => s"Results may be capped by result limit set to $limit." - case None if (sr.size >= 50) - => "Results may be capped by default limit of 50 returned results. Use --limit to extend the result set." - case _ - => "" + if (sr.size < retrieveResults.totalHits ) { + config.limit match { + case Some(limit) if (limit <= sr.size) + => s"Results are capped by results limit set to $limit." + case None if (sr.size >= 50) + => "Results are capped by default limit of 50 returned results. Use --limit to extend the result set." + case _ + => "" + } + } else { + "" } } - success.apply(s"Found ${sr.size} item(s). $capMessage") + success.apply(s"Found ${retrieveResults.totalHits} item(s). $capMessage") reportResult.apply(sr) - information.apply(f"Query took ${queryRuntime.toUnit(TimeUnit.SECONDS)}%.2fs.") + information.apply(f"Query roundtrip took ${queryRuntime.toUnit(TimeUnit.MILLISECONDS)}%.0fms.") if (!config.csv.equals("")) { exportResult.apply(sr) From d48bf522498c38421f03b5517d98274c15d18656 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Wed, 8 Jan 2020 00:42:31 +0100 Subject: [PATCH 27/31] Added features command. Closes #25 --- build.sbt | 3 ++ .../upb/cs/swt/delphi/cli/ConsoleOutput.scala | 2 + .../de/upb/cs/swt/delphi/cli/DelphiCLI.scala | 4 ++ .../cs/swt/delphi/cli/ResultBeautifier.scala | 35 ++++++++++++++- .../cs/swt/delphi/cli/commands/Command.scala | 2 +- .../delphi/cli/commands/FeaturesCommand.scala | 43 +++++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 src/main/scala/de/upb/cs/swt/delphi/cli/commands/FeaturesCommand.scala diff --git a/build.sbt b/build.sbt index 90db270..2f35691 100644 --- a/build.sbt +++ b/build.sbt @@ -58,6 +58,9 @@ libraryDependencies += "au.com.bytecode" % "opencsv" % "2.4" libraryDependencies += "org.scalatest" %% "scalatest" % "3.1.0" % "test" libraryDependencies += "joda-time" % "joda-time" % "2.10.5" +libraryDependencies += "de.upb.cs.swt.delphi" %% "delphi-core" % "0.9.2" +libraryDependencies += "de.upb.cs.swt.delphi" %% "delphi-client" % "0.9.2" + libraryDependencies ++= Seq( "com.softwaremill.sttp" %% "core" % "1.7.2", "com.softwaremill.sttp" %% "spray-json" % "1.7.2" diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/ConsoleOutput.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/ConsoleOutput.scala index 66f97ea..9268bd7 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/ConsoleOutput.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/ConsoleOutput.scala @@ -17,6 +17,7 @@ package de.upb.cs.swt.delphi.cli import de.upb.cs.swt.delphi.cli.artifacts.{RetrieveResult, SearchResult} +import de.upb.cs.swt.delphi.client.FieldDefinition class ConsoleOutput(config: Config) { @@ -45,6 +46,7 @@ class ConsoleOutput(config: Config) { } } case retrieveResults : Seq[RetrieveResult] if retrieveResults.head.isInstanceOf[RetrieveResult] => ResultBeautifier.beautifyRetrieveResults(retrieveResults) + case featureResults : Seq[FieldDefinition] if featureResults.head.isInstanceOf[FieldDefinition] => ResultBeautifier.beautifyFeatures(featureResults) case _ => value.toString } } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala index 394c3bf..8761820 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/DelphiCLI.scala @@ -50,6 +50,7 @@ object DelphiCLI { case "test" => TestCommand.execute case "retrieve" => RetrieveCommand.execute case "search" => SearchCommand.execute + case "features" => FeaturesCommand.execute case "" => cliParser.showUsage() case x => config.consoleOutput.outputError(s"Unknown command: $x") } @@ -79,6 +80,9 @@ object DelphiCLI { cmd("test").action((_, c) => c.copy(mode = "test")) + cmd("features").action((_, c) => c.copy(mode = "features")) + .text("Retrieve the current list of features.") + cmd("retrieve").action((s, c) => c.copy(mode = "retrieve")) .text("Retrieve a project's description, specified by ID.") .children( diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/ResultBeautifier.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/ResultBeautifier.scala index 7f6fc0c..20bbe75 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/ResultBeautifier.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/ResultBeautifier.scala @@ -17,6 +17,7 @@ package de.upb.cs.swt.delphi.cli import de.upb.cs.swt.delphi.cli.artifacts.{RetrieveResult, SearchResult} +import de.upb.cs.swt.delphi.client.FieldDefinition import de.vandermeer.asciitable.{AsciiTable, CWC_LongestLine} import de.vandermeer.asciithemes.{TA_Grid, TA_GridThemes} import de.vandermeer.skb.interfaces.transformers.textformat.TextAlignment @@ -53,13 +54,13 @@ object ResultBeautifier { at.getRenderer.setCWC(new CWC_LongestLine) at.setPaddingLeft(1) at.setPaddingRight(1) + at.getContext.setFrameTopMargin(1) at.getContext.setFrameBottomMargin(1) at.getContext().setGridTheme(TA_GridThemes.INSIDE) at.render() - //CustomAsciiTable.make(table) } } @@ -112,4 +113,36 @@ object ResultBeautifier { }.fold("")(_ + _) } + def beautifyFeatures(results : Seq[FieldDefinition]) : String = { + + if (results.size == 0) { + "" + } else { + val tableHeader = Seq ("Name", "Description") + val tableBody = results.sortBy(f => f.name).map(f => Seq(f.name, f.description)) + val table = tableBody.+:(tableHeader) + + val at = new AsciiTable() + + at.setTextAlignment(TextAlignment.JUSTIFIED_LEFT) + at.getContext().setWidth(80) + + // add header + at.addRule() + at.addRow(table.head.asJavaCollection) + at.addRule() + + // add body + table.tail.foreach { row: Iterable[String] => at.addRow(row.asJavaCollection) } + + at.setPaddingLeft(1) + at.setPaddingRight(1) + at.getContext.setFrameTopMargin(1) + at.getContext.setFrameBottomMargin(1) + //at.getContext().setGridTheme(TA_GridThemes.) + at.addRule() + at.render() + } + } + } diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala index d3bc8bf..335846b 100644 --- a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/Command.scala @@ -45,7 +45,7 @@ trait Command { val oldPath = serverUrl.path val reqUrl = serverUrl.path(oldPath ++ paths).params(parameters) val request = sttp.get(reqUrl) - config.consoleOutput.outputInformation(s"Sending request ${request.uri}") + //config.consoleOutput.outputInformation(s"Sending request ${request.uri}") val response = request.send() response.body match { case Left(value) => diff --git a/src/main/scala/de/upb/cs/swt/delphi/cli/commands/FeaturesCommand.scala b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/FeaturesCommand.scala new file mode 100644 index 0000000..93e5d6c --- /dev/null +++ b/src/main/scala/de/upb/cs/swt/delphi/cli/commands/FeaturesCommand.scala @@ -0,0 +1,43 @@ +// Copyright (C) 2020 The Delphi Team. +// See the LICENCE file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package de.upb.cs.swt.delphi.cli.commands +import com.softwaremill.sttp.{Id, SttpBackend} +import de.upb.cs.swt.delphi.cli.Config +import de.upb.cs.swt.delphi.client.FieldDefinition +import de.upb.cs.swt.delphi.client.FieldDefinitionJson._ +import spray.json._ + +object FeaturesCommand extends Command { + override def execute(implicit config: Config, backend: SttpBackend[Id, Nothing]): Unit = { + val result = executeGet(Seq("features")) + + result.map(features => { + if (config.raw) { + reportResult.apply(features) + } + if (!config.raw || !config.csv.equals("")) { + val featureList = features.parseJson.convertTo[JsArray].elements.map(e => e.convertTo[FieldDefinition]) + reportResult.apply(featureList) + + if (!config.csv.equals("")) { + exportResult.apply(featureList) + information.apply("Results written to file '" + config.csv + "'") + } + } + }) + } +} From 70cad004b54267bfa158110b279f39c5c7998e1e Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Wed, 8 Jan 2020 00:45:54 +0100 Subject: [PATCH 28/31] Updated README to new console parameters --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40743e0..6d161c2 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,8 @@ Our software is available as a binary release on [GitHub](https://github.com/del ``` $ delphi --help -Delphi Command Line Tool (1.0.0-SNAPSHOT) -Usage: delphi [test|retrieve|search] [options] ... +Delphi Command Line Tool (0.9.5-SNAPSHOT) +Usage: delphi-cli [test|features|retrieve|search] [options] ... --version Prints the version of the command line tool. --help Prints this help text. From 568818e1a9fc0df249e2fa6936fb3db665961164 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Wed, 8 Jan 2020 16:30:14 +0100 Subject: [PATCH 29/31] Made a helpful runnable docker container possible --- build.sbt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build.sbt b/build.sbt index 2f35691..63c0589 100644 --- a/build.sbt +++ b/build.sbt @@ -1,3 +1,5 @@ +import com.typesafe.sbt.packager.docker._ + ThisBuild / organization := "de.upb.cs.swt.delphi" ThisBuild / organizationName := "Delphi Project" ThisBuild / organizationHomepage := Some(url("https://delphi.cs.uni-paderborn.de/")) @@ -68,10 +70,23 @@ libraryDependencies ++= Seq( debianPackageDependencies := Seq("java8-runtime-headless") +mainClass in Compile := Some("de.upb.cs.swt.delphi.cli.DelphiCLI") +discoveredMainClasses in Compile := Seq() lazy val cli = (project in file(".")). enablePlugins(JavaAppPackaging). enablePlugins(DockerPlugin). + settings( + dockerBaseImage := "openjdk:jre-alpine", + dockerAlias := com.typesafe.sbt.packager.docker.DockerAlias(None, Some("delphihub"),"delphi-cli", Some(version.value)), + dockerEntrypoint := Seq("/bin/bash"), + dockerCommands ++= Seq( + Cmd("USER", "root"), + Cmd("RUN", "apk", "--no-cache", "add", "bash"), + Cmd("RUN", "ln", "-s", "/opt/docker/bin/delphi", "/usr/bin/delphi" ), + Cmd("USER", "daemon") + ) + ). enablePlugins(ScalastylePlugin). enablePlugins(BuildInfoPlugin). enablePlugins(DebianPlugin). From b7d59502a273064ea37c3fed481908a6d893fac6 Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Wed, 8 Jan 2020 16:41:17 +0100 Subject: [PATCH 30/31] Bumped sbt and project version --- build.sbt | 2 +- project/build.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 63c0589..40c03fc 100644 --- a/build.sbt +++ b/build.sbt @@ -31,7 +31,7 @@ lazy val supportedScalaVersions = List(scala213) ThisBuild / scalaVersion := scala213 name := "delphi" -version := "0.9.5-SNAPSHOT" +version := "0.9.5" maintainer := "Ben Hermann " packageSummary := "Windows Package for the Delphi CLI" diff --git a/project/build.properties b/project/build.properties index 8b8de72..3913aa8 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.3.5 \ No newline at end of file +sbt.version = 1.3.6 \ No newline at end of file From 518f76a6644efe7dd66f71c6d2e4e301448c84fc Mon Sep 17 00:00:00 2001 From: Ben Hermann Date: Wed, 8 Jan 2020 16:48:00 +0100 Subject: [PATCH 31/31] Reverted to sbt 1.2.8 because of travis *sigh* --- project/build.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/build.properties b/project/build.properties index 3913aa8..7609b47 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version = 1.3.6 \ No newline at end of file +sbt.version = 1.2.8 \ No newline at end of file