Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Redesign execution module #74

Open
wants to merge 37 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
8a970b2
Fixing test pod deletion
cmendesce Aug 11, 2024
47eb19c
forcing action to run
cmendesce Aug 11, 2024
c01a929
Fixing test pod deletion
cmendesce Aug 11, 2024
13b5c74
Fixing test pod deletion
cmendesce Aug 11, 2024
ab2ab66
Removing unused method
cmendesce Aug 11, 2024
a1a5f82
adding guard when env is null
cmendesce Aug 11, 2024
0c3c826
adding guard when params are null
cmendesce Aug 11, 2024
217ddfc
adding guard when params are null
cmendesce Aug 11, 2024
78147e9
adding istio timeout
cmendesce Aug 11, 2024
4583f97
adding code formatting
cmendesce Aug 31, 2024
1952d17
removing unused files
cmendesce Aug 31, 2024
47a4e57
adding flow to restore deployment env
cmendesce Sep 1, 2024
2dc4215
fixing container name
cmendesce Sep 1, 2024
b5faa74
saving before change
cmendesce Sep 1, 2024
9403d43
Refactor environment step to restart pods and update deployment
cmendesce Sep 7, 2024
e67e20e
Refactor environment step to log container environment variables
cmendesce Sep 7, 2024
17fce38
increase wait time for pod restart
cmendesce Sep 8, 2024
8ab0739
make environment step to wait until deployment is ready (not pods)
cmendesce Sep 8, 2024
8ae51c6
Update README.md
cmendesce Sep 8, 2024
9212dfe
environment step to wait until deployment is ready (not pods)
cmendesce Sep 9, 2024
cb44744
Merge branch 'main' of github.com:cmendesce/resilience-bench-operator
cmendesce Sep 9, 2024
0bfbeb7
use asText instead of toString
cmendesce Sep 20, 2024
8288c69
add waitUntilCondition for pod readiness check
cmendesce Sep 20, 2024
c887deb
improving logging
cmendesce Sep 24, 2024
2264c61
remove hardcoded strings
cmendesce Sep 26, 2024
85659e0
add samples
cmendesce Sep 26, 2024
b64edac
reducing excessive checks
cmendesce Sep 27, 2024
03df8ef
Update README.md
cmendesce Sep 27, 2024
26098a6
adding ApplicationReadinessStep
cmendesce Sep 27, 2024
2663518
adding ApplicationReadinessStep
cmendesce Sep 27, 2024
654e280
adding ApplicationReadinessStep
cmendesce Sep 27, 2024
4961c6c
Redesign execution module
cmendesce Sep 28, 2024
740371c
Remove retry
cmendesce Sep 28, 2024
a974394
Redesign execution module
cmendesce Sep 28, 2024
8a87488
Redesign execution module
cmendesce Sep 28, 2024
05d53d7
Redesign execution module
cmendesce Sep 28, 2024
d560941
Redesign execution module
cmendesce Sep 28, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
adding flow to restore deployment env
  • Loading branch information
cmendesce committed Sep 1, 2024
commit 47a4e576d87993e191c3af1710d1cd1cf0b1e6e6
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package io.resiliencebench.execution.steps;

import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.resiliencebench.resources.scenario.Scenario;
import io.resiliencebench.resources.service.ResilientService;
import io.resiliencebench.support.CustomResourceRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;
import java.util.concurrent.TimeUnit;

abstract class AbstractEnvironmentStep extends ExecutorStep<Deployment> {

private final static Logger logger = LoggerFactory.getLogger(AbstractEnvironmentStep.class);
protected final CustomResourceRepository<ResilientService> resilientServiceRepository;

public AbstractEnvironmentStep(KubernetesClient kubernetesClient,
CustomResourceRepository<ResilientService> resilientServiceRepository) {
super(kubernetesClient);
this.resilientServiceRepository = resilientServiceRepository;
}

protected Deployment getDeployment(Scenario scenario, ResilientService resilientService) {
return kubernetesClient()
.apps()
.deployments()
.inNamespace(scenario.getMetadata().getNamespace())
.withLabelSelector(resilientService.getSpec().getSelector())
.list()
.getItems()
.stream()
.findFirst()
.orElse(null);
}

protected void updateDeployment(Deployment targetDeployment) {
kubernetesClient().apps().deployments()
.inNamespace(targetDeployment.getMetadata().getNamespace())
.resource(targetDeployment)
.update();
}

protected void restartPods(Deployment targetDeployment) {
logger.info("Waiting for the pods to restart");
kubernetesClient().pods()
.inNamespace(targetDeployment.getMetadata().getNamespace())
.withLabel("app", targetDeployment.getMetadata().getName())
.waitUntilReady(60, TimeUnit.SECONDS);
logger.info("Pods restarted successfully");
}

protected List<EnvVar> getActualEnv(Deployment targetDeployment, String containerName) {
var container = targetDeployment.getSpec().getTemplate().getSpec().getContainers().stream()
.filter(c -> c.getName().equals(containerName))
.findFirst()
.orElseThrow(() -> new RuntimeException("Container not found: " + containerName));
return container.getEnv();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package io.resiliencebench.execution.steps;

import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.resiliencebench.resources.queue.ExecutionQueue;
import io.resiliencebench.resources.scenario.Scenario;
import io.resiliencebench.resources.service.ResilientService;
import io.resiliencebench.support.CustomResourceRepository;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import static io.resiliencebench.support.Annotations.CONTAINER;

@Service
public class EnvironmentPostStep extends AbstractEnvironmentStep {

private final static Logger logger = LoggerFactory.getLogger(EnvironmentPostStep.class);

public EnvironmentPostStep(KubernetesClient kubernetesClient,
CustomResourceRepository<ResilientService> resilientServiceRepository) {
super(kubernetesClient, resilientServiceRepository);
}

@Override
protected boolean isApplicable(Scenario scenario) {
return scenario
.getSpec()
.getConnectors()
.stream()
.anyMatch(connector -> connector.getDestination().getEnvs() != null ||
connector.getSource().getEnvs() != null);
}

@Override
protected Deployment internalExecute(Scenario scenario, ExecutionQueue queue) {
for (var connector : scenario.getSpec().getConnectors()) {
restoreEnvironment(scenario, connector.getSource());
restoreEnvironment(scenario, connector.getDestination());
}
return null;
}

public void restoreEnvironment(Scenario scenario, io.resiliencebench.resources.scenario.Service service) {
var resilientService = resilientServiceRepository.get(scenario.getMetadata().getNamespace(), service.getName());
var containerName = resilientService.getMetadata().getAnnotations().get(CONTAINER);

var env = resilientService.getSpec().getEnvs();
if (env == null) {
return;
}

var deployment = kubernetesClient()
.apps()
.deployments()
.inNamespace(scenario.getMetadata().getNamespace())
.withLabelSelector(resilientService.getSpec().getSelector())
.list()
.getItems()
.stream()
.findFirst();

if (deployment.isPresent()) {
var container = deployment.get().getSpec().getTemplate().getSpec().getContainers().stream()
.filter(c -> c.getName().equals(containerName))
.findFirst()
.orElseThrow(() -> new RuntimeException("Container not found: " + containerName));

container.setEnv(env);
updateDeployment(deployment.get());
restartPods(deployment.get());
} else {
logger.warn("Deployment not found for ResilientService {}", service.getName());
}
}
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
package io.resiliencebench.execution.steps;

import com.fasterxml.jackson.databind.JsonNode;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.resiliencebench.resources.queue.ExecutionQueue;
Expand All @@ -11,32 +11,28 @@
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;

import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.List;

import static io.resiliencebench.support.Annotations.CONTAINER;

@Service
public class EnvironmentStep extends ExecutorStep<Deployment> {
public class EnvironmentStep extends AbstractEnvironmentStep {

private final static Logger logger = LoggerFactory.getLogger(EnvironmentStep.class);

private final CustomResourceRepository<ResilientService> resilientServiceRepository;

public EnvironmentStep(KubernetesClient kubernetesClient,
CustomResourceRepository<ResilientService> resilientServiceRepository) {
super(kubernetesClient);
this.resilientServiceRepository = resilientServiceRepository;
CustomResourceRepository<ResilientService> resilientServiceRepository) {
super(kubernetesClient, resilientServiceRepository);
}

@Override
protected boolean isApplicable(Scenario scenario) {
return scenario
.getSpec()
.getConnectors()
.stream()
.anyMatch(connector -> connector.getDestination().getEnvs() != null ||
connector.getSource().getEnvs() != null);
.getSpec()
.getConnectors()
.stream()
.anyMatch(connector -> connector.getDestination().getEnvs() != null ||
connector.getSource().getEnvs() != null);
}

public void applyServiceEnvironment(Scenario scenario, io.resiliencebench.resources.scenario.Service service) {
Expand All @@ -48,17 +44,28 @@ public void applyServiceEnvironment(Scenario scenario, io.resiliencebench.resour
var containerName = resilientService.getMetadata().getAnnotations().get(CONTAINER);

var deployment = kubernetesClient()
.apps()
.deployments()
.inNamespace(scenario.getMetadata().getNamespace())
.withLabelSelector(resilientService.getSpec().getSelector())
.list()
.getItems()
.stream()
.findFirst();
.apps()
.deployments()
.inNamespace(scenario.getMetadata().getNamespace())
.withLabelSelector(resilientService.getSpec().getSelector())
.list()
.getItems()
.stream()
.findFirst();

if (deployment.isPresent()) {
applyNewVariables(deployment.get(), containerName, env);
var deploymentVars = getActualEnv(deployment.get(), CONTAINER);

for (var variable : deploymentVars) {
var newValue = env.get(variable.getName());
if (newValue != null) {
logger.info("Container {}. EnvVar {}={}", containerName, variable.getName(), newValue);
variable.setValue(newValue.toString());
}
}
updateDeployment(deployment.get());
restartPods(deployment.get());
saveActualEnv(deploymentVars, resilientService);
} else {
logger.warn("Deployment not found for ResilientService {}", service.getName());
}
Expand All @@ -73,41 +80,10 @@ protected Deployment internalExecute(Scenario scenario, ExecutionQueue queue) {
return null;
}

public void applyNewVariables(Deployment targetDeployment, String containerName, Map<String, JsonNode> env) {
var container = targetDeployment.getSpec().getTemplate().getSpec().getContainers().stream()
.filter(c -> c.getName().equals(containerName))
.findFirst()
.orElseThrow(() -> new RuntimeException("Container not found: " + containerName));
var deploymentVars = container
.getEnv()
.stream()
.toList();

for (var variable : deploymentVars) {
var newValue = env.get(variable.getName());
if (newValue != null) {
logger.info("Container {}. EnvVar {}={}", containerName, variable.getName(), newValue);
variable.setValue(newValue.toString());
}
}

updateDeployment(targetDeployment);
restartPods(targetDeployment);
}

private void updateDeployment(Deployment targetDeployment) {
kubernetesClient().apps().deployments()
.inNamespace(targetDeployment.getMetadata().getNamespace())
.resource(targetDeployment)
.update();
private void saveActualEnv(List<EnvVar> deploymentVars, ResilientService resilientService) {
resilientService.getSpec().getEnvs().clear();
resilientService.getSpec().getEnvs().addAll(deploymentVars);
resilientServiceRepository.update(resilientService);
}

private void restartPods(Deployment targetDeployment) {
logger.info("Waiting for the pods to restart");
kubernetesClient().pods()
.inNamespace(targetDeployment.getMetadata().getNamespace())
.withLabel("app", targetDeployment.getMetadata().getName())
.waitUntilReady(60, TimeUnit.SECONDS);
logger.info("Pods restarted successfully");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,25 @@ public class StepRegister {
private final List<ExecutorStep<?>> postExecutionSteps;

public StepRegister(UpdateStatusQueueStep updateStatusQueueStep,
ResultLocalFileStep resultLocalFileStep,
IstioCircuitBreakerStep istioCircuitBreakerStep,
IstioRetryStep istioRetryStep,
IstioTimeoutStep istioTimeoutStep,
IstioFaultStep istioFaultStep,
EnvironmentStep environmentStep) {
ResultLocalFileStep resultLocalFileStep,
IstioCircuitBreakerStep istioCircuitBreakerStep,
IstioRetryStep istioRetryStep,
IstioTimeoutStep istioTimeoutStep,
IstioFaultStep istioFaultStep,
EnvironmentStep environmentStep,
EnvironmentPostStep environmentPostStep) {

preparationSteps = List.of(
updateStatusQueueStep,
istioRetryStep,
istioCircuitBreakerStep,
istioTimeoutStep,
istioFaultStep,
environmentStep
);
updateStatusQueueStep,
istioRetryStep,
istioCircuitBreakerStep,
istioTimeoutStep,
istioFaultStep,
environmentStep);
postExecutionSteps = List.of(
updateStatusQueueStep,
resultLocalFileStep
);
updateStatusQueueStep,
resultLocalFileStep,
environmentPostStep);
}

public List<ExecutorStep<?>> getPostExecutionSteps() {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
package io.resiliencebench.resources.service;

import java.util.List;
import io.fabric8.kubernetes.api.model.EnvVar;
import io.fabric8.kubernetes.api.model.LabelSelector;

public class ResilientServiceSpec {

private LabelSelector selector;
private List<EnvVar> envs;

public LabelSelector getSelector() {
return selector;
Expand All @@ -13,4 +16,12 @@ public LabelSelector getSelector() {
public void setSelector(LabelSelector selector) {
this.selector = selector;
}

public List<EnvVar> getEnvs() {
return envs;
}

public void setEnvs(List<EnvVar> envs) {
this.envs = envs;
}
}