Skip to content

Commit

Permalink
Add execution id to api to allow for multiple executions of one simul…
Browse files Browse the repository at this point in the history 
…ation
  • Loading branch information
@franksn90
franksn90 committed May 30, 2024
1 parent 43eadb5 commit aa98c02
Show file tree
Hide file tree
Showing 3 changed files with 49 additions and 40 deletions.
15 changes: 8 additions & 7 deletions restAPI/src/main/java/restAPI/simulation_running/SimulationRunningController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,26 @@ public ResponseEntity<String> handleMultipleFilesUpload(
@RequestParam(name = "experiments", required = false) MultipartFile[] experiments,
@RequestParam(name = "loads", required = false) MultipartFile[] loads,
@RequestParam(name = "mtls", required = false) MultipartFile[] mtls,
@RequestParam("simulation_id") String id) throws IOException {
return runSimulation(architectures, scenarios, experiments, loads, mtls, id);
@RequestParam("simulation_id") String id,
@RequestParam("execution_id") String executionID) throws IOException {
return runSimulation(architectures, scenarios, experiments, loads, mtls, id, executionID);
}

// TODO: Handle this call in a non-blocking manner, taking into account that this implementation is not
// client friendly as it can time-out the request due to the long processing time.
private ResponseEntity<String> runSimulation(MultipartFile[] architectures, MultipartFile[] scenarios,
MultipartFile[] experiments, MultipartFile[] loads,
MultipartFile[] mtls,
String id) throws IOException {
String id, String executionID) throws IOException {
Path tmpFolder = null;
try {
if (TempFileUtils.existsSimulationId(id)) {
if (TempFileUtils.existsSimulationId(id, executionID)) {
return new ResponseEntity<>(String.format("Simulation ID <%s> is already in use. " +
"Please provide a unique new id.", id),
HttpStatus.BAD_REQUEST);
}
tmpFolder = TempFileUtils.createDefaultTempDir("misim-");
Path outputFolder = TempFileUtils.createOutputDir(TempFileUtils.RAW_OUTPUT_DIR, id);
Path outputFolder = TempFileUtils.createOutputDir(TempFileUtils.RAW_OUTPUT_DIR, id, executionID);

Multimap<String, String> savedFiles = ArrayListMultimap.create();
savedFiles = TempFileUtils.saveFile(savedFiles, "architecture", architectures, tmpFolder);
Expand All @@ -69,15 +70,15 @@ private ResponseEntity<String> runSimulation(MultipartFile[] architectures, Mult

//Block1
simulationRunningService.runExperiment(savedFiles, outputFolder);
if (!TempFileUtils.existsSimulationId(id)) {
if (!TempFileUtils.existsSimulationId(id, executionID)) {
return new ResponseEntity<>(
String.format("An Error happened when running the simulation with the ID: " +
"%s.", id),
HttpStatus.INTERNAL_SERVER_ERROR);
}
String rawResultsDirPath = outputFolder.toString() + TempFileUtils.SEPARATOR + "raw";
//Block2
ReportDataPointsManipulator.adjustSimulationResults(rawResultsDirPath, id);
ReportDataPointsManipulator.adjustSimulationResults(rawResultsDirPath, id, executionID);
return new ResponseEntity<>("Files have been successfully uploaded, and the simulation is running.",
HttpStatus.OK);
} catch (Exception e) {
Expand Down
57 changes: 30 additions & 27 deletions restAPI/src/main/java/restAPI/util/ReportDataPointsManipulator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,25 +19,26 @@
public class ReportDataPointsManipulator {

private static final String[] DISCRETE_VALUED_METRICS = {"Requests_InSystem", "Requests_NotComputed",
"Requests_WaitingForDependencies", "SendOff_Internal_Requests", "FailedRequests", "SuccessfulRequests",
"Load."};
"Requests_WaitingForDependencies", "SendOff_Internal_Requests", "FailedRequests", "SuccessfulRequests",
"Load."};
private static final String[] CONTINUOUS_VALUED_METRICS = {"RelativeUtilization", "Utilization",
"UtilizationBinned", "NL_latency", "ResponseTimes"};
"UtilizationBinned", "NL_latency", "ResponseTimes"};

public static void adjustSimulationResults(String rawResultsFilePath, String simulationId) throws Exception {
public static void adjustSimulationResults(String rawResultsFilePath, String simulationId, String executionId)
throws Exception {
String simulationResultsDirPath = "." + TempFileUtils.SEPARATOR + rawResultsFilePath;
Set<String> files = TempFileUtils.getFilesFromResultsDir(Path.of(simulationResultsDirPath));
String outputDir = TempFileUtils.createOutputDir("." + TempFileUtils.SEPARATOR
+ TempFileUtils.OUTPUT_DIR, simulationId).toString();
+ TempFileUtils.OUTPUT_DIR, simulationId, executionId).toString();
int i = 1;
String fileName;
for(String filePath : files) {
for (String filePath : files) {
String completePath = simulationResultsDirPath + TempFileUtils.SEPARATOR + filePath;
Table t = transformSimulationResults(completePath);
if(!t.isEmpty()) {
if (!t.isEmpty()) {
fileName = t.column(1).name();
if(fileName.isEmpty()) {
fileName = String.format("%s%sstable_%d.csv",outputDir,TempFileUtils.SEPARATOR,i);
if (fileName.isEmpty()) {
fileName = String.format("%s%sstable_%d.csv", outputDir, TempFileUtils.SEPARATOR, i);
i += 1;
}
t.write().csv(outputDir + TempFileUtils.SEPARATOR + fileName + ".csv");
Expand All @@ -50,7 +51,7 @@ public static void adjustSimulationResults(String rawResultsFilePath, String sim
}

private static Table transformSimulationResults(String filePath) {
if(stringContainsItemFromList(filePath, CONTINUOUS_VALUED_METRICS)) {
if (stringContainsItemFromList(filePath, CONTINUOUS_VALUED_METRICS)) {
return transformContinuousValuedMetric(filePath);
} else if (stringContainsItemFromList(filePath, DISCRETE_VALUED_METRICS)) {
return transformDiscreteValuedMetric(filePath);
Expand All @@ -67,10 +68,10 @@ private static Table transformSimulationResults(String filePath) {
private static String createColumnNameFromFileName(String fileName) {
String regex = ".+\\[.+\\].+\\.csv";
if (fileName.matches(regex)) {
String serviceName = StringUtils.substringBetween(fileName, "[", "]")
.replaceAll("[#-]","_");
String serviceName = StringUtils.substringBetween(fileName, "[", "]")
.replaceAll("[#-]", "_");
String metricName = StringUtils.substringBetween(fileName, "]", ".csv");
return String.format("%s%s",serviceName, metricName);
return String.format("%s%s", serviceName, metricName);
} else {
return FilenameUtils.getName(fileName);
}
Expand All @@ -82,16 +83,16 @@ private static boolean stringContainsItemFromList(String inputStr, String[] item

private static Table createTableFromFile(String filePath, ColumnType[] types) {
CsvReadOptions.Builder csvReadOptions =
CsvReadOptions.builder(filePath)
.separator(';')
.columnTypes(types);
CsvReadOptions.builder(filePath)
.separator(';')
.columnTypes(types);

return Table.read().usingOptions(csvReadOptions);
}

private static Table adjustTime(String filePath) {
ColumnType[] types = {DOUBLE, DOUBLE};
Table table = createTableFromFile(filePath,types);
Table table = createTableFromFile(filePath, types);

DoubleColumn oldColumn = (DoubleColumn) table.column("SimulationTime");
// Cast double column to int column to be categorical for the reduce/summarization step later
Expand All @@ -104,7 +105,7 @@ private static Table transformContinuousValuedMetric(String filePath) {
Table adjustedadTable = adjustTime(filePath);
// Do the aggregation (calculate the average)
Table averagedTable = adjustedadTable.summarize("Value", mean)
.by("SimulationTimeAdjusted");
.by("SimulationTimeAdjusted");
averagedTable.column(0).setName("SimulationTime");
averagedTable.column(1).setName(createColumnNameFromFileName(filePath));
return averagedTable;
Expand All @@ -115,7 +116,7 @@ private static Table transformDiscreteValuedMetric(String filePath) {
Table adjustedadTable = adjustTime(filePath);
// Do the aggregation (calculate the average)
Table averagedTable = adjustedadTable.summarize("Value", mean)
.by("SimulationTimeAdjusted");
.by("SimulationTimeAdjusted");
averagedTable.column(0).setName("SimulationTime");
DoubleColumn oldColumn = (DoubleColumn) averagedTable.column(1);
IntColumn newColumn = oldColumn.map(Math::ceil).setName(createColumnNameFromFileName(filePath)).asIntColumn();
Expand All @@ -129,28 +130,30 @@ private static Table replicateServiceInstanceCountValue(String filePath) {
ColumnType[] types = {INTEGER, INTEGER};
Table table = createTableFromFile(filePath, types);

Map<Integer,Integer> instancesCount = new LinkedHashMap<>();
table.forEach(row -> {int time = row.getInt(0);
int value = row.getInt(1);
instancesCount.put(time, value);
Map<Integer, Integer> instancesCount = new LinkedHashMap<>();
table.forEach(row -> {
int time = row.getInt(0);
int value = row.getInt(1);
instancesCount.put(time, value);
});

IntColumn simulationTimeColumn = IntColumn.create("SimulationTime");
IntColumn valueColumn = IntColumn.create(createColumnNameFromFileName(filePath));
int fromTime;
int toTime;

for(int i = 0; i <= instancesCount.size() - 2; i++) {
for (int i = 0; i <= instancesCount.size() - 2; i++) {
fromTime = (int) instancesCount.keySet().toArray()[i];
toTime = (int) instancesCount.keySet().toArray()[i+1];;
toTime = (int) instancesCount.keySet().toArray()[i + 1];

int currentValue = instancesCount.get(fromTime);

while(fromTime <= toTime) {
while (fromTime <= toTime) {
simulationTimeColumn.append(fromTime);
valueColumn.append(currentValue);
fromTime++;
}
}
return Table.create(simulationTimeColumn,valueColumn);
return Table.create(simulationTimeColumn, valueColumn);
}
}
17 changes: 11 additions & 6 deletions restAPI/src/main/java/restAPI/util/TempFileUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,13 +72,18 @@ public static Path createDefaultTempDir(String prefix) throws IOException {
return Files.createTempDirectory(prefix);
}

public static Path createOutputDir(String outputDirName, String simulationId) throws IOException {
public static Path createOutputDir(String outputDirName, String simulationId, String executionId)
throws IOException {
Path outPutDirPath = Path.of(outputDirName);
if (!Files.exists(outPutDirPath)) {
Files.createDirectory(outPutDirPath);
}
String simulationOutputDirPath = outputDirName + SEPARATOR + simulationId;
return Files.createDirectory(Path.of(simulationOutputDirPath));
Path simulationOutputDirPath = Path.of(outputDirName + SEPARATOR + simulationId);
if (!Files.exists(simulationOutputDirPath)) {
Files.createDirectory(simulationOutputDirPath);
}
String executionOutputDirPath = simulationOutputDirPath + SEPARATOR + executionId;
return Files.createDirectory(Path.of(executionOutputDirPath));
}


Expand All @@ -93,12 +98,12 @@ public static Set<String> getFilesFromResultsDir(Path dirPath) throws IOExceptio
}
}

public static boolean existsSimulationId(String simulationId) {
String simulationOutputDirPath = RAW_OUTPUT_DIR + SEPARATOR + simulationId;
public static boolean existsSimulationId(String simulationId, String executionId) {
String simulationOutputDirPath = RAW_OUTPUT_DIR + SEPARATOR + simulationId + SEPARATOR + executionId;
return Files.exists(Path.of(simulationOutputDirPath));
}

public static void cleanOutputDir(String simulationId){
public static void cleanOutputDir(String simulationId) {
String simulationOutputDirPath = RAW_OUTPUT_DIR + SEPARATOR + simulationId;
try {
FileUtils.deleteDirectory(new File(simulationOutputDirPath));
Expand Down

0 comments on commit aa98c02

Please sign in to comment.