Skip to content
This repository has been archived by the owner on Jan 4, 2022. It is now read-only.

Commit

Permalink
Merge pull request #8 from kanedafromparis/testing-registry
Browse files Browse the repository at this point in the history
Merge kanendafromparis's new pipeline to deploy the operator to k8s for integration tets
  • Loading branch information
adam-sandor authored Sep 2, 2021
2 parents a699b58 + f15ae6a commit 4e7df10
Show file tree
Hide file tree
Showing 9 changed files with 278 additions and 26 deletions.
141 changes: 131 additions & 10 deletions .github/workflows/TomcatIntegrationTest.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,27 +12,148 @@ jobs:

- name: Create kind cluster
uses: helm/[email protected]
with:
cluster_name: tomcat-integration

- name: Apply CRDs
run: kubectl apply -f tomcat/k8s/crd.yaml

- name: Set up Java and Maven
uses: actions/setup-java@v1
uses: actions/setup-java@v2
with:
java-version: 15
distribution: adopt-hotspot

- name: Cache Maven packages
- name: cache
uses: actions/cache@v2
if: ${{ !env.ACT }}
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-m2
- name: Set up Maven
uses: stCarolas/setup-maven@v4
if: ${{ env.ACT }}
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-m2
maven-version: 3.8.1

- name: Run unit tests
if: ${{ env.ACT }}
run: mvn --version

- name: Run unit tests
run: mvn -B test --file tomcat/pom.xml
run: mvn -B test -q --file tomcat/pom.xml

tomcat_local_apply_setup_test:
runs-on: ubuntu-latest
env:
KIND_CL_NAME: tomcat-local-apply
steps:
- name: Checkout
uses: actions/checkout@v2

- name: Create Kubernetes KinD Cluster
uses: container-tools/[email protected]
with:
cluster_name: tomcat-local-apply
registry: false

# for DIMG in "tomcat-local-apply-control-plane kind-registry tomcat_local_apply_setup_test "; do docker stop $DIMG ; docker rm $DIMG ; done ; sleep 1

- name: Set up Java and Maven
uses: actions/setup-java@v2
with:
# java-version: ${{ matrix.java }}
java-version: 15
distribution: adopt-hotspot

- name: cache
uses: actions/cache@v2
if: ${{ !env.ACT }}
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
restore-keys: |
${{ runner.os }}-maven-
- name: Dump state
if: ${{ failure() }}
- name: Set up Maven for local ACT
uses: stCarolas/setup-maven@v4
if: ${{ env.ACT }}
with:
maven-version: 3.8.1

- name: build jib
run: |
mvn -B package jib:dockerBuild jib:buildTar -Djib-maven-image=tomcat-operator --file tomcat/pom.xml -DskipTests
kind load image-archive tomcat/target/jib-image.tar --name=${{ env.KIND_CL_NAME }}
- name: Apply CRDs
run: kubectl apply -f tomcat/k8s/crd.yaml

- name: install tomcat operator
run: |
kubectl apply -f tomcat/k8s/operator.yaml
- name: create ns tomcatoperator-sample
run: kubectl create ns tomcatoperator-sample

- name: debug local kind
if: ${{ env.ACT }}
run: |
kubectl get pods -n tomcat-operator -l app=tomcat-operator -o yaml | tee -a debug.log
- name: wait for operators ready
run: |
LOOP=0 &&\
while [[ $(kubectl get pods -n tomcat-operator -l app=tomcat-operator -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do \
echo "waiting for pod" &&\
kubectl logs -n tomcat-operator -l app=tomcat-operator &&\
(( LOOP++ )) &&\
if [[ $LOOP -gt 10 ]]; then exit 1; fi &&\
echo "loop number $LOOP" &&\
sleep 5; \
done
- name: install sample operators
run: |
for sample in $(ls tomcat/k8s/*sample*); do
kubectl -n tomcatoperator-sample apply -f $sample;
done
- name: check pod correctly started
run: |
LOOP=0 &&\
while [[ $(kubectl get pods -n tomcatoperator-sample -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True True True True True True" ]]; do \
echo "waiting for pod" &&\
kubectl logs -n tomcat-operator -l app=tomcat-operator &&\
kubectl get pods -n tomcatoperator-sample &&\
(( LOOP++ )) &&\
if [[ $LOOP -gt 10 ]]; then exit 1; fi &&\
echo "loop number $LOOP" &&\
sleep 10; \
done
#Waiting 5 seconds for Tomcat to unpack the downloaded war
sleep 5;
- name: Get webapps
run: |
kubectl get tomcats,webapps -A -o yaml | tee -a debug
kubectl -n tomcatoperator-sample -c tomcat logs -l app=test-tomcat1 | grep startup
- name: check code
run: |
kubectl get all -n tomcat-test -o yaml
kubectl logs curl
kubectl -n tomcatoperator-sample run sample1 --labels=app=curl --image=curlimages/curl:7.78.0 --restart=Never --timeout=30s --command -- curl -s -v http://test-tomcat1/mysample/;
kubectl -n tomcatoperator-sample run sample2 --labels=app=curl --image=curlimages/curl:7.78.0 --restart=Never --timeout=30s --command -- curl -s -v http://test-tomcat2/othercontext/;
LOOP=0 &&\
while [[ $(kubectl get pods -n tomcatoperator-sample -l app=curl -o 'jsonpath={..status.phase}') != "Succeeded Succeeded" ]]; do \
echo "waiting for pod" &&\
kubectl logs -n tomcatoperator-sample -l app=curl &&\
(( LOOP++ )) &&\
if [[ $LOOP -gt 5 ]]; then exit 1; fi &&\
echo "loop number $LOOP" &&\
sleep 5; \
done
if [[ $(kubectl logs -n tomcatoperator-sample sample1 --tail=500 | grep tomcat.gif | wc -l) -ne 1 ]]; then exit 1; fi
if [[ $(kubectl logs -n tomcatoperator-sample sample2 --tail=500 | grep dog.jpeg | wc -l) -ne 1 ]]; then exit 1; fi
4 changes: 4 additions & 0 deletions tomcat/k8s/crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,10 @@ spec:
properties:
deployedArtifact:
type: string
deploymentStatus:
type: array
items:
type: string
required: [spec]
# either Namespaced or Cluster
scope: Namespaced
Expand Down
19 changes: 19 additions & 0 deletions tomcat/k8s/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,27 @@ metadata:
rules:
- apiGroups:
- ""
- "extensions"
- "apps"
resources:
- deployments
- services
- pods
- pods/exec
verbs:
- '*'
- apiGroups:
- "apiextensions.k8s.io"
resources:
- customresourcedefinitions
verbs:
- '*'
- apiGroups:
- "tomcatoperator.io"
resources:
- tomcats
- tomcats/status
- webapps
- webapps/status
verbs:
- '*'
2 changes: 1 addition & 1 deletion tomcat/k8s/webapp-sample2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ metadata:
spec:
tomcat: test-tomcat2
url: charlottemach.com/assets/jax.war
contextPath: mysample
contextPath: othercontext
2 changes: 1 addition & 1 deletion tomcat/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<jib-maven-plugin.version>2.7.1</jib-maven-plugin.version>
<jib-maven-plugin.version>3.1.4</jib-maven-plugin.version>
</properties>

<dependencies>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,23 @@
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.apps.Deployment;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.dsl.ExecListener;
import io.fabric8.kubernetes.client.dsl.ExecWatch;
import io.javaoperatorsdk.operator.api.*;
import io.javaoperatorsdk.operator.processing.event.EventSourceManager;
import okhttp3.Response;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.nio.charset.Charset;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

@Controller
public class WebappController implements ResourceController<Webapp> {
Expand Down Expand Up @@ -47,17 +56,21 @@ public UpdateControl<Webapp> createOrUpdateResource(Webapp webapp, Context<Webap
if (tomcat.getStatus() != null && Objects.equals(tomcat.getSpec().getReplicas(), tomcat.getStatus().getReadyReplicas())) {
log.info("Tomcat is ready and webapps not yet deployed. Commencing deployment of {} in Tomcat {}", webapp.getMetadata().getName(), tomcat.getMetadata().getName());
String[] command = new String[]{"wget", "-O", "/data/" + webapp.getSpec().getContextPath() + ".war", webapp.getSpec().getUrl()};
if(log.isInfoEnabled()){
command = new String[]{"time", "wget", "-O", "/data/" + webapp.getSpec().getContextPath() + ".war", webapp.getSpec().getUrl()};
}

executeCommandInAllPods(kubernetesClient, webapp, command);
String[] commandStatusInAllPods = executeCommandInAllPods(kubernetesClient, webapp, command);

if (webapp.getStatus() == null) {
webapp.setStatus(new WebappStatus());
}
webapp.getStatus().setDeployedArtifact(webapp.getSpec().getUrl());
webapp.getStatus().setDeploymentStatus(commandStatusInAllPods);
return UpdateControl.updateStatusSubResource(webapp);
} else {
log.info("WebappController invoked but Tomcat not ready yet ({}/{})", tomcat.getSpec().getReplicas(),
tomcat.getStatus() != null ? tomcat.getStatus().getReadyReplicas() : 0);
log.info("WebappController invoked but Tomcat not ready yet ({}/{})",
tomcat.getStatus() != null ? tomcat.getStatus().getReadyReplicas() : 0, tomcat.getSpec().getReplicas());
return UpdateControl.noUpdate();
}
}
Expand All @@ -66,15 +79,18 @@ public UpdateControl<Webapp> createOrUpdateResource(Webapp webapp, Context<Webap
public DeleteControl deleteResource(Webapp webapp, Context<Webapp> context) {

String[] command = new String[] {"rm", "/data/" + webapp.getSpec().getContextPath() + ".war"};
executeCommandInAllPods(kubernetesClient, webapp, command);
String[] commandStatusInAllPods = executeCommandInAllPods(kubernetesClient, webapp, command);
if (webapp.getStatus() != null) {
webapp.getStatus().setDeployedArtifact(null);
webapp.getStatus().setDeploymentStatus(commandStatusInAllPods);
}
return DeleteControl.DEFAULT_DELETE;
}

private void executeCommandInAllPods(
private String[] executeCommandInAllPods(
KubernetesClient kubernetesClient, Webapp webapp, String[] command) {
String[] status = new String[0];

Deployment deployment =
kubernetesClient
.apps()
Expand All @@ -91,20 +107,67 @@ private void executeCommandInAllPods(
.withLabels(deployment.getSpec().getSelector().getMatchLabels())
.list()
.getItems();
for (Pod pod : pods) {
status = new String[pods.size()];
for (int i=0; i<pods.size();i++) {
Pod pod = pods.get(i);
log.info(
"Executing command {} in Pod {}",
String.join(" ", command),
pod.getMetadata().getName());
kubernetesClient
.pods()
.inNamespace(deployment.getMetadata().getNamespace())

CompletableFuture<String> data = new CompletableFuture<>();
try (ExecWatch execWatch = execCmd(pod, data, command)) {
status[i] = ""+pod.getMetadata().getName()+":"+data.get(30, TimeUnit.SECONDS);;
} catch (ExecutionException e) {
status[i] = ""+pod.getMetadata().getName()+": ExecutionException - "+e.getMessage();
} catch (InterruptedException e) {
status[i] = ""+pod.getMetadata().getName()+": InterruptedException - "+e.getMessage();
} catch (TimeoutException e) {
status[i] = ""+pod.getMetadata().getName()+": TimeoutException - "+e.getMessage();
}
}
}
return status;
}

private ExecWatch execCmd(Pod pod, CompletableFuture<String> data, String... command) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
return kubernetesClient.pods()
.inNamespace(pod.getMetadata().getNamespace())
.withName(pod.getMetadata().getName())
.inContainer("war-downloader")
.writingOutput(new ByteArrayOutputStream())
.writingError(new ByteArrayOutputStream())
.writingOutput(baos)
.writingError(baos)
.usingListener(new SimpleListener(data, baos))
.exec(command);
}
}

static class SimpleListener implements ExecListener {

private CompletableFuture<String> data;
private ByteArrayOutputStream baos;
private final Logger log = LoggerFactory.getLogger(getClass());
public SimpleListener(CompletableFuture<String> data, ByteArrayOutputStream baos) {
this.data = data;
this.baos = baos;
}

@Override
public void onOpen(Response response) {
log.debug("Reading data... " + response.message());
}

@Override
public void onFailure(Throwable t, Response response) {
log.debug(t.getMessage() + " " + response.message());
data.completeExceptionally(t);
}

@Override
public void onClose(int code, String reason) {
log.debug("Exit with: " + code + " and with reason: " + reason);
data.complete(baos.toString());
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,14 @@ public String getDeployedArtifact() {
public void setDeployedArtifact(String deployedArtifact) {
this.deployedArtifact = deployedArtifact;
}

private String[] deploymentStatus;

public String[] getDeploymentStatus() {
return deploymentStatus;
}

public void setDeploymentStatus(String[] deploymentStatus) {
this.deploymentStatus = deploymentStatus;
}
}
Loading

0 comments on commit 4e7df10

Please sign in to comment.