Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Performance profile table in the DB #828

Merged
merged 9 commits into from
Aug 11, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,12 @@ public void validate(List<ExperimentResultData> experimentResultDataList, Map<St
*/
try {
LOGGER.debug("Kruize Object: {}", kruizeObject);
// fetch the Performance Profile from the DB
try {
new ExperimentDBService().loadPerformanceProfileFromDBByName(performanceProfilesMap, kruizeObject.getPerformanceProfile());
} catch (Exception e) {
LOGGER.error("Loading saved Performance Profile {} failed: {} ", expName, e.getMessage());
}
PerformanceProfile performanceProfile = performanceProfilesMap.get(kruizeObject.getPerformanceProfile());
// validate the 'resultdata' with the performance profile
errorMsg = PerformanceProfileUtil.validateResults(performanceProfile, resultData);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public class ExperimentValidation {
private boolean success;
private String errorMessage;
private Map<String, KruizeObject> mainKruizeExperimentMAP;
private Map<String, PerformanceProfile> performanceProfilesMap = new HashMap<>();
//Mandatory fields
private List<String> mandatoryFields = new ArrayList<>(Arrays.asList(
AnalyzerConstants.NAME,
Expand Down Expand Up @@ -103,7 +104,13 @@ public void validate(List<KruizeObject> kruizeExptList) {
errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.SLO_REDUNDANCY_ERROR;
validationOutputData.setErrorCode(HttpServletResponse.SC_BAD_REQUEST);
} else {
if (null == PerformanceProfilesDeployment.performanceProfilesMap.get(kruizeObject.getPerformanceProfile())) {
// fetch the Performance Profile from the DB
try {
new ExperimentDBService().loadPerformanceProfileFromDBByName(performanceProfilesMap, kruizeObject.getPerformanceProfile());
} catch (Exception e) {
LOGGER.error("Loading saved Performance Profile {} failed: {} ", expName, e.getMessage());
}
if (null == performanceProfilesMap.get(kruizeObject.getPerformanceProfile())) {
errorMsg = AnalyzerErrorConstants.AutotuneObjectErrors.MISSING_PERF_PROFILE + kruizeObject.getPerformanceProfile();
validationOutputData.setErrorCode(HttpServletResponse.SC_BAD_REQUEST);
} else
Expand Down Expand Up @@ -136,24 +143,28 @@ public void validate(List<KruizeObject> kruizeExptList) {
markFailed(validationOutputData.getMessage());
break;
}
// set performance profile metrics in the kruize Object
PerformanceProfile performanceProfile = PerformanceProfilesDeployment.performanceProfilesMap.get(kruizeObject.getPerformanceProfile());
HashMap<AnalyzerConstants.MetricName, Metric> metricsMap = new HashMap<>();
for (Metric metric : performanceProfile.getSloInfo().getFunctionVariables()) {
if (metric.getKubernetesObject().equals(KruizeConstants.JSONKeys.CONTAINER))
metricsMap.put(AnalyzerConstants.MetricName.valueOf(metric.getName()), metric);
}
List<K8sObject> k8sObjectList = new ArrayList<>();
HashMap<String, ContainerData> containerDataMap = new HashMap<>();
for (K8sObject k8sObject : kruizeObject.getKubernetes_objects()) {
for (ContainerData containerData : k8sObject.getContainerDataMap().values()) {
containerDataMap.put(containerData.getContainer_name(), new ContainerData(
containerData.getContainer_name(), containerData.getContainer_image_name(), new ContainerRecommendations(), metricsMap));
// set Performance Profile metrics in the Kruize Object
PerformanceProfile performanceProfile = performanceProfilesMap.get(kruizeObject.getPerformanceProfile());
try {
HashMap<AnalyzerConstants.MetricName, Metric> metricsMap = new HashMap<>();
for (Metric metric : performanceProfile.getSloInfo().getFunctionVariables()) {
if (metric.getKubernetesObject().equals(KruizeConstants.JSONKeys.CONTAINER))
metricsMap.put(AnalyzerConstants.MetricName.valueOf(metric.getName()), metric);
}
k8sObject.setContainerDataMap(containerDataMap);
k8sObjectList.add(k8sObject);
List<K8sObject> k8sObjectList = new ArrayList<>();
HashMap<String, ContainerData> containerDataMap = new HashMap<>();
for (K8sObject k8sObject : kruizeObject.getKubernetes_objects()) {
for (ContainerData containerData : k8sObject.getContainerDataMap().values()) {
containerDataMap.put(containerData.getContainer_name(), new ContainerData(
containerData.getContainer_name(), containerData.getContainer_image_name(), new ContainerRecommendations(), metricsMap));
}
k8sObject.setContainerDataMap(containerDataMap);
k8sObjectList.add(k8sObject);
}
kruizeObject.setKubernetes_objects(k8sObjectList);
} catch (Exception e) {
LOGGER.error("Failed to set Performance Profile Metrics to the Kruize Object: {}", e.getMessage());
}
kruizeObject.setKubernetes_objects(k8sObjectList);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,35 @@

public class PerformanceProfile {

private final String name;
private String name;

private final double profile_version;
private double profile_version;

@SerializedName("k8s_type")
private final String k8s_type;
private String k8s_type;

@SerializedName("slo")
private final SloInfo sloInfo;
private SloInfo sloInfo;

public void setName(String name) {
this.name = name;
}

public void setProfile_version(double profile_version) {
this.profile_version = profile_version;
}

public String getK8s_type() {
return k8s_type;
}

public void setK8s_type(String k8s_type) {
this.k8s_type = k8s_type;
}

public void setSloInfo(SloInfo sloInfo) {
this.sloInfo = sloInfo;
}

public PerformanceProfile(String name, double profile_version, String k8s_type, SloInfo sloInfo) {
this.name = name;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.autotune.common.data.ValidationOutputData;
import com.autotune.common.data.metrics.Metric;
import com.autotune.analyzer.kruizeObject.SloInfo;
import com.autotune.database.service.ExperimentDBService;
import com.autotune.experimentManager.utils.EMConstants;
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.analyzer.utils.AnalyzerErrorConstants;
Expand Down Expand Up @@ -90,6 +91,11 @@ private ValidationOutputData validatePerformanceProfileData(PerformanceProfile p

// If the mandatory values are present,proceed for further validation else return the validation object directly
if (validationOutputData.isSuccess()) {
try {
new ExperimentDBService().loadAllPerformanceProfiles(performanceProfilesMap);
} catch (Exception e) {
LOGGER.error("Loading saved performance profiles failed: {} ", e.getMessage());
}
StringBuilder errorString = new StringBuilder();
// check if the performance profile already exists
if (performanceProfilesMap.get(performanceProfile.getName()) != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public static String validateResults(PerformanceProfile performanceProfile, Expe

public static void addPerformanceProfile(Map<String, PerformanceProfile> performanceProfileMap, PerformanceProfile performanceProfile) {
performanceProfileMap.put(performanceProfile.getName(), performanceProfile);
LOGGER.info("Added PerformanceProfile: {} ",performanceProfile.getName());
LOGGER.debug("Added PerformanceProfile: {} ",performanceProfile.getName());
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import com.autotune.analyzer.performanceProfiles.PerformanceProfile;
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.analyzer.utils.AnalyzerErrorConstants;
import com.autotune.database.service.ExperimentDBService;
import com.google.gson.ExclusionStrategy;
import com.google.gson.FieldAttributes;
import com.google.gson.Gson;
Expand All @@ -46,6 +47,7 @@
import java.util.Date;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;

import static com.autotune.analyzer.utils.AnalyzerConstants.ServiceConstants.CHARACTER_ENCODING;
Expand All @@ -59,14 +61,11 @@ public class PerformanceProfileService extends HttpServlet {
@Serial
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = LoggerFactory.getLogger(PerformanceProfileService.class);
Map<String, PerformanceProfile> performanceProfilesMap;


@Override
public void init(ServletConfig config) throws ServletException {
super.init(config);
this.performanceProfilesMap = (HashMap<String, PerformanceProfile>) getServletContext()
.getAttribute(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_MAP);
}

/**
Expand All @@ -80,13 +79,21 @@ public void init(ServletConfig config) throws ServletException {
@Override
protected void doPost(HttpServletRequest request, HttpServletResponse response) throws IOException {
try {
Map<String, PerformanceProfile> performanceProfilesMap = new ConcurrentHashMap<>();
String inputData = request.getReader().lines().collect(Collectors.joining());
PerformanceProfile performanceProfile = Converters.KruizeObjectConverters.convertInputJSONToCreatePerfProfile(inputData);
ValidationOutputData validationOutputData = PerformanceProfileUtil.validateAndAddProfile(performanceProfilesMap, performanceProfile);
if (validationOutputData.isSuccess()) {
LOGGER.debug("Added Performance Profile : {} into the map with version: {}",
performanceProfile.getName(), performanceProfile.getProfile_version());
sendSuccessResponse(response, "Performance Profile : "+performanceProfile.getName()+" created successfully.");
ValidationOutputData addedToDB;
addedToDB = new ExperimentDBService().addPerformanceProfileToDB(performanceProfile);
if (addedToDB.isSuccess()) {
LOGGER.debug("Added Performance Profile : {} into the DB with version: {}",
performanceProfile.getName(), performanceProfile.getProfile_version());
sendSuccessResponse(response, "Performance Profile : " + performanceProfile.getName() + " created successfully.");
}
else {
sendErrorResponse(response, null, HttpServletResponse.SC_BAD_REQUEST, addedToDB.getMessage());
}
}
else
sendErrorResponse(response, null, validationOutputData.getErrorCode(), validationOutputData.getMessage());
Expand All @@ -110,7 +117,14 @@ protected void doGet(HttpServletRequest req, HttpServletResponse response) throw
response.setCharacterEncoding(CHARACTER_ENCODING);
response.setStatus(HttpServletResponse.SC_OK);
String gsonStr = "[]";
if (this.performanceProfilesMap.size() > 0) {
Map<String, PerformanceProfile> performanceProfilesMap = new ConcurrentHashMap<>();
// Fetch all profiles from the DB
try {
new ExperimentDBService().loadAllPerformanceProfiles(performanceProfilesMap);
} catch (Exception e) {
LOGGER.error("Failed to load saved experiment data: {} ", e.getMessage());
}
if (performanceProfilesMap.size() > 0) {
Collection<PerformanceProfile> values = performanceProfilesMap.values();
Gson gsonObj = new GsonBuilder()
.disableHtmlEscaping()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,14 +55,12 @@
public class UpdateResults extends HttpServlet {
private static final long serialVersionUID = 1L;
private static final Logger LOGGER = LoggerFactory.getLogger(UpdateResults.class);
Map<String, PerformanceProfile> performanceProfilesMap;
private Map<String, PerformanceProfile> performanceProfilesMap = new HashMap<>();

@Override
public void init(ServletConfig config) throws ServletException {
super.init(config);

this.performanceProfilesMap = (HashMap<String, PerformanceProfile>) getServletContext()
.getAttribute(AnalyzerConstants.PerformanceProfileConstants.PERF_PROFILE_MAP);
int totalResultsCount = 0;
getServletContext().setAttribute(AnalyzerConstants.RESULTS_COUNT, totalResultsCount);
}
Expand Down
10 changes: 10 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAO.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import com.autotune.analyzer.utils.AnalyzerConstants;
import com.autotune.common.data.ValidationOutputData;
import com.autotune.database.table.KruizeExperimentEntry;
import com.autotune.database.table.KruizePerformanceProfileEntry;
import com.autotune.database.table.KruizeRecommendationEntry;
import com.autotune.database.table.KruizeResultsEntry;

Expand All @@ -20,6 +21,9 @@ public interface ExperimentDAO {
// Add recommendation to DB
public ValidationOutputData addRecommendationToDB(KruizeRecommendationEntry recommendationEntry);

// Add Performance Profile to DB
public ValidationOutputData addPerformanceProfileToDB(KruizePerformanceProfileEntry kruizePerformanceProfileEntry);

// Update experiment status
public boolean updateExperimentStatus(KruizeObject kruizeObject, AnalyzerConstants.ExperimentStatus status);

Expand All @@ -35,6 +39,9 @@ public interface ExperimentDAO {
// If Kruize restarts load all recommendations
List<KruizeRecommendationEntry> loadAllRecommendations() throws Exception;

// If Kruize restarts load all performance profiles
List<KruizePerformanceProfileEntry> loadAllPerformanceProfiles() throws Exception;

// Load a single experiment based on experimentName
List<KruizeExperimentEntry> loadExperimentByName(String experimentName) throws Exception;

Expand All @@ -44,4 +51,7 @@ public interface ExperimentDAO {
// Load all recommendations of a particular experiment
List<KruizeRecommendationEntry> loadRecommendationsByExperimentName(String experimentName) throws Exception;

// Load a single Performance Profile based on name
List<KruizePerformanceProfileEntry> loadPerformanceProfileByName(String performanceProfileName) throws Exception;

}
51 changes: 51 additions & 0 deletions src/main/java/com/autotune/database/dao/ExperimentDAOImpl.java
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.autotune.database.helper.DBConstants;
import com.autotune.database.init.KruizeHibernateUtil;
import com.autotune.database.table.KruizeExperimentEntry;
import com.autotune.database.table.KruizePerformanceProfileEntry;
import com.autotune.database.table.KruizeRecommendationEntry;
import com.autotune.database.table.KruizeResultsEntry;
import com.autotune.utils.MetricsConfig;
Expand Down Expand Up @@ -107,6 +108,31 @@ public ValidationOutputData addRecommendationToDB(KruizeRecommendationEntry reco
return validationOutputData;
}

@Override
public ValidationOutputData addPerformanceProfileToDB(KruizePerformanceProfileEntry kruizePerformanceProfileEntry) {
ValidationOutputData validationOutputData = new ValidationOutputData(false, null, null);
Transaction tx = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
try {
tx = session.beginTransaction();
session.persist(kruizePerformanceProfileEntry);
tx.commit();
validationOutputData.setSuccess(true);
} catch (HibernateException e) {
LOGGER.error("Not able to save performance profile due to {}", e.getMessage());
if (tx != null) tx.rollback();
e.printStackTrace();
validationOutputData.setSuccess(false);
validationOutputData.setMessage(e.getMessage());
//todo save error to API_ERROR_LOG
}
} catch (Exception e) {
LOGGER.error("Not able to save performance profile due to {}", e.getMessage());
validationOutputData.setMessage(e.getMessage());
}
return validationOutputData;
}


@Override
public boolean updateExperimentStatus(KruizeObject kruizeObject, AnalyzerConstants.ExperimentStatus status) {
Expand Down Expand Up @@ -213,6 +239,18 @@ public List<KruizeRecommendationEntry> loadAllRecommendations() throws Exception
return recommendationEntries;
}

@Override
public List<KruizePerformanceProfileEntry> loadAllPerformanceProfiles() throws Exception {
List<KruizePerformanceProfileEntry> entries = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_PERFORMANCE_PROFILE, KruizePerformanceProfileEntry.class).list();
} catch (Exception e) {
LOGGER.error("Not able to load Performance Profile due to {}", e.getMessage());
throw new Exception("Error while loading existing Performance Profile from database due to : " + e.getMessage());
}
return entries;
}

@Override
public List<KruizeExperimentEntry> loadExperimentByName(String experimentName) throws Exception {
//todo load only experimentStatus=inprogress , playback may not require completed experiments
Expand Down Expand Up @@ -262,4 +300,17 @@ public List<KruizeRecommendationEntry> loadRecommendationsByExperimentName(Strin
}
return recommendationEntries;
}

@Override
public List<KruizePerformanceProfileEntry> loadPerformanceProfileByName(String performanceProfileName) throws Exception {
List<KruizePerformanceProfileEntry> entries = null;
try (Session session = KruizeHibernateUtil.getSessionFactory().openSession()) {
entries = session.createQuery(DBConstants.SQLQUERY.SELECT_FROM_PERFORMANCE_PROFILE_BY_NAME, KruizePerformanceProfileEntry.class)
.setParameter("name", performanceProfileName).list();
} catch (Exception e) {
LOGGER.error("Not able to load Performance Profile {} due to {}", performanceProfileName, e.getMessage());
throw new Exception("Error while loading existing profile from database due to : " + e.getMessage());
}
return entries;
}
}
3 changes: 3 additions & 0 deletions src/main/java/com/autotune/database/helper/DBConstants.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@ public static final class SQLQUERY {
public static final String SELECT_FROM_RESULTS_BY_EXP_NAME = "from KruizeResultsEntry k WHERE k.experiment_name = :experimentName";
public static final String SELECT_FROM_RECOMMENDATIONS_BY_EXP_NAME = "from KruizeRecommendationEntry k WHERE k.experiment_name = :experimentName";
public static final String SELECT_FROM_RECOMMENDATIONS = "from KruizeRecommendationEntry";
public static final String SELECT_FROM_PERFORMANCE_PROFILE = "from KruizePerformanceProfileEntry";
public static final String SELECT_FROM_PERFORMANCE_PROFILE_BY_NAME = "from KruizePerformanceProfileEntry k WHERE k.name = :name";


public static final String DELETE_FROM_EXPERIMENTS_BY_EXP_NAME = "DELETE FROM KruizeExperimentEntry k WHERE k.experiment_name = :experimentName";
public static final String DELETE_FROM_RESULTS_BY_EXP_NAME = "DELETE FROM KruizeResultsEntry k WHERE k.experiment_name = :experimentName";
Expand Down
Loading