Improvements to the testsuite

Author: andreaTP

.github/workflows/e2e-new-test-mysql.yml

@@ -0,0 +1,44 @@
+# End to end integration test which deploys the MySQL Schema Operator operator to a Kubernetes
+# (Minikube) cluster and creates custom resources to verify the operator's functionality
+name: MySQL Schema Operator New End to End test
+on:
+  pull_request:
+    branches: [ main ]
+  push:
+    branches:
+      - main
+jobs:
+  mysql_e2e_test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Minikube-Kubernetes
+        uses: manusa/[email protected]
+        with:
+          minikube version: v1.24.0
+          kubernetes version: v1.23.0
+          github token: ${{ secrets.GITHUB_TOKEN }}
+          driver: docker
+
+      - name: Set up Java and Maven
+        uses: actions/setup-java@v2
+        with:
+          java-version: 11
+          distribution: adopt-hotspot
+          cache: 'maven'
+
+      - name: Build SDK
+        run: mvn install -DskipTests
+
+      - name: Run E2E Tests in Local Mode
+        working-directory: sample-operators/mysql-schema
+        run: |
+          mvn test -Dtest=MySQLSchemaOperatorNewE2E
+
+      - name: Run E2E Tests as a deployment
+        working-directory: sample-operators/mysql-schema
+        run: |
+          eval $(minikube -p minikube docker-env)
+          mvn jib:dockerBuild test -Dtest=MySQLSchemaOperatorNewE2E -Djosdk.it.localOperatorDeployment=false

operator-framework-core/pom.xml

@@ -56,6 +56,7 @@
     <dependency>
       <groupId>io.fabric8</groupId>
       <artifactId>openshift-client</artifactId>
+      <version>${fabric8-client.version}</version>
     </dependency>
 
     <dependency>
@@ -98,6 +99,7 @@
     <dependency>
       <groupId>io.fabric8</groupId>
       <artifactId>kubernetes-server-mock</artifactId>
+      <version>${fabric8-client.version}</version>
       <scope>test</scope>
     </dependency>
     <dependency>

operator-framework-junit5/src/main/java/io/javaoperatorsdk/operator/junit/OperatorExtension.java

@@ -1,9 +1,11 @@
 package io.javaoperatorsdk.operator.junit;
 
 import java.io.InputStream;
+import java.time.Duration;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -19,6 +21,7 @@
 import io.fabric8.kubernetes.api.model.HasMetadata;
 import io.fabric8.kubernetes.api.model.KubernetesResourceList;
 import io.fabric8.kubernetes.api.model.NamespaceBuilder;
+import io.fabric8.kubernetes.api.model.rbac.ClusterRoleBinding;
 import io.fabric8.kubernetes.client.DefaultKubernetesClient;
 import io.fabric8.kubernetes.client.KubernetesClient;
 import io.fabric8.kubernetes.client.dsl.NonNamespaceOperation;
@@ -47,24 +50,42 @@ public class OperatorExtension
   private final KubernetesClient kubernetesClient;
   private final ConfigurationService configurationService;
   private final Operator operator;
+  private final List<HasMetadata> operatorDeployment;
+  private final Duration deploymentTimeout;
   private final List<ReconcilerSpec> reconcilers;
+  private final List<HasMetadata> infrastructure;
+  private final Duration infrastructureTimeout;
   private final boolean preserveNamespaceOnError;
   private final boolean waitForNamespaceDeletion;
+  private final boolean oneNamespacePerClass;
+  private final boolean localOperatorDeployment;
 
   private String namespace;
 
   private OperatorExtension(
       ConfigurationService configurationService,
       List<ReconcilerSpec> reconcilers,
+      List<HasMetadata> operatorDeployment,
+      Duration deploymentTimeout,
+      List<HasMetadata> infrastructure,
+      Duration infrastructureTimeout,
       boolean preserveNamespaceOnError,
-      boolean waitForNamespaceDeletion) {
+      boolean waitForNamespaceDeletion,
+      boolean oneNamespacePerClass,
+      boolean localOperatorDeployment) {
 
     this.kubernetesClient = new DefaultKubernetesClient();
     this.configurationService = configurationService;
     this.reconcilers = reconcilers;
+    this.operatorDeployment = operatorDeployment;
+    this.deploymentTimeout = deploymentTimeout;
+    this.infrastructure = infrastructure;
+    this.infrastructureTimeout = infrastructureTimeout;
     this.operator = new Operator(this.kubernetesClient, this.configurationService);
     this.preserveNamespaceOnError = preserveNamespaceOnError;
     this.waitForNamespaceDeletion = waitForNamespaceDeletion;
+    this.oneNamespacePerClass = oneNamespacePerClass;
+    this.localOperatorDeployment = localOperatorDeployment;
   }
 
   /**
@@ -78,22 +99,22 @@ public static Builder builder() {
 
   @Override
   public void beforeAll(ExtensionContext context) throws Exception {
-    before(context);
+    beforeAllImpl(context);
   }
 
   @Override
   public void beforeEach(ExtensionContext context) throws Exception {
-    before(context);
+    beforeEachImpl(context);
   }
 
   @Override
   public void afterAll(ExtensionContext context) throws Exception {
-    after(context);
+    afterAllImpl(context);
   }
 
   @Override
   public void afterEach(ExtensionContext context) throws Exception {
-    after(context);
+    afterEachImpl(context);
   }
 
   @Override
@@ -118,6 +139,10 @@ public Reconciler getFirstReconciler() {
         .findFirst().orElseThrow();
   }
 
+  public boolean isLocal() {
+    return this.localOperatorDeployment;
+  }
+
   @SuppressWarnings({"rawtypes"})
   public <T extends Reconciler> T getControllerOfType(Class<T> type) {
     return operator.getControllers().stream()
@@ -151,19 +176,47 @@ public <T extends HasMetadata> boolean delete(Class<T> type, T resource) {
   }
 
   @SuppressWarnings("unchecked")
-  protected void before(ExtensionContext context) {
-    namespace = context.getRequiredTestClass().getSimpleName();
-    namespace += "-";
-    namespace += context.getRequiredTestMethod().getName();
-    namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US);
-    namespace = namespace.substring(0, Math.min(namespace.length(), 63));
+  protected void beforeAllImpl(ExtensionContext context) {
+    if (oneNamespacePerClass) {
+      namespace = context.getRequiredTestClass().getSimpleName();
+      namespace += "-";
+      namespace += UUID.randomUUID();
+      namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US);
+      namespace = namespace.substring(0, Math.min(namespace.length(), 63));
+
+      before(context);
+    }
+  }
 
+  @SuppressWarnings("unchecked")
+  protected void beforeEachImpl(ExtensionContext context) {
+    if (!oneNamespacePerClass) {
+      namespace = context.getRequiredTestClass().getSimpleName();
+      namespace += "-";
+      namespace += context.getRequiredTestMethod().getName();
+      namespace += "-";
+      namespace += UUID.randomUUID();
+      namespace = KubernetesResourceUtil.sanitizeName(namespace).toLowerCase(Locale.US);
+      namespace = namespace.substring(0, Math.min(namespace.length(), 63));
+
+      before(context);
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  protected void before(ExtensionContext context) {
     LOGGER.info("Initializing integration test in namespace {}", namespace);
 
     kubernetesClient
         .namespaces()
         .create(new NamespaceBuilder().withNewMetadata().withName(namespace).endMetadata().build());
 
+    kubernetesClient
+        .resourceList(infrastructure)
+        .createOrReplace();
+    kubernetesClient
+        .resourceList(infrastructure)
+        .waitUntilReady(infrastructureTimeout.toMillis(), TimeUnit.MILLISECONDS);
 
     for (var ref : reconcilers) {
       final var config = configurationService.getConfigurationFor(ref.reconciler);
@@ -191,14 +244,50 @@ protected void before(ExtensionContext context) {
       this.operator.register(ref.reconciler, oconfig.build());
     }
 
-    this.operator.start();
+    if (isLocal()) {
+      LOGGER.debug("Starting the operator locally");
+      this.operator.start();
+    } else {
+      LOGGER.debug("Deploying the operator into Kubernetes");
+      operatorDeployment.stream().forEach(hm -> {
+        hm.getMetadata().setNamespace(namespace);
+        if (hm.getKind().toLowerCase(Locale.ROOT).equals("clusterrolebinding")) {
+          var crb = (ClusterRoleBinding) hm;
+          for (var subject : crb.getSubjects()) {
+            subject.setNamespace(namespace);
+          }
+        }
+      });
+
+      kubernetesClient
+          .resourceList(operatorDeployment)
+          .inNamespace(namespace)
+          .createOrReplace();
+      kubernetesClient
+          .resourceList(operatorDeployment)
+          .waitUntilReady(deploymentTimeout.toMillis(), TimeUnit.MILLISECONDS);
+    }
+  }
+
+  protected void afterAllImpl(ExtensionContext context) {
+    if (oneNamespacePerClass) {
+      after(context);
+    }
+  }
+
+  protected void afterEachImpl(ExtensionContext context) {
+    if (!oneNamespacePerClass) {
+      after(context);
+    }
   }
 
   protected void after(ExtensionContext context) {
     if (namespace != null) {
       if (preserveNamespaceOnError && context.getExecutionException().isPresent()) {
         LOGGER.info("Preserving namespace {}", namespace);
       } else {
+        kubernetesClient.resourceList(infrastructure).delete();
+        kubernetesClient.resourceList(operatorDeployment).inNamespace(namespace).delete();
         LOGGER.info("Deleting namespace {} and stopping operator", namespace);
         kubernetesClient.namespaces().withName(namespace).delete();
         if (waitForNamespaceDeletion) {
@@ -212,7 +301,9 @@ protected void after(ExtensionContext context) {
     }
 
     try {
-      this.operator.stop();
+      if (isLocal()) {
+        this.operator.stop();
+      }
     } catch (Exception e) {
       // ignored
     }
@@ -221,13 +312,23 @@ protected void after(ExtensionContext context) {
   @SuppressWarnings("rawtypes")
   public static class Builder {
     private final List<ReconcilerSpec> reconcilers;
+    private final List<HasMetadata> operatorDeployment;
+    private Duration deploymentTimeout;
+    private boolean localOperatorDeployment;
+    private final List<HasMetadata> infrastructure;
+    private Duration infrastructureTimeout;
     private ConfigurationService configurationService;
     private boolean preserveNamespaceOnError;
     private boolean waitForNamespaceDeletion;
+    private boolean oneNamespacePerClass;
 
     protected Builder() {
       this.configurationService = new BaseConfigurationService(Version.UNKNOWN);
       this.reconcilers = new ArrayList<>();
+      this.operatorDeployment = new ArrayList<>();
+      this.infrastructure = new ArrayList<>();
+      this.deploymentTimeout = Duration.ofMinutes(1);
+      this.infrastructureTimeout = Duration.ofMinutes(1);
 
       this.preserveNamespaceOnError = Utils.getSystemPropertyOrEnvVar(
           "josdk.it.preserveNamespaceOnError",
@@ -236,6 +337,14 @@ protected Builder() {
       this.waitForNamespaceDeletion = Utils.getSystemPropertyOrEnvVar(
           "josdk.it.waitForNamespaceDeletion",
           true);
+
+      this.oneNamespacePerClass = Utils.getSystemPropertyOrEnvVar(
+          "josdk.it.oneNamespacePerClass",
+          false);
+
+      this.localOperatorDeployment = Utils.getSystemPropertyOrEnvVar(
+          "josdk.it.localOperatorDeployment",
+          true);
     }
 
     public Builder preserveNamespaceOnError(boolean value) {
@@ -248,11 +357,31 @@ public Builder waitForNamespaceDeletion(boolean value) {
       return this;
     }
 
+    public Builder oneNamespacePerClass(boolean value) {
+      this.oneNamespacePerClass = value;
+      return this;
+    }
+
+    public Builder localOperatorDeployment(boolean value) {
+      this.localOperatorDeployment = value;
+      return this;
+    }
+
     public Builder withConfigurationService(ConfigurationService value) {
       configurationService = value;
       return this;
     }
 
+    public Builder withDeploymentTimeout(Duration value) {
+      deploymentTimeout = value;
+      return this;
+    }
+
+    public Builder withInfrastructureTimeout(Duration value) {
+      infrastructureTimeout = value;
+      return this;
+    }
+
     @SuppressWarnings("rawtypes")
     public Builder withReconciler(Reconciler value) {
       reconcilers.add(new ReconcilerSpec(value, null));
@@ -275,12 +404,38 @@ public Builder withReconciler(Class<? extends Reconciler> value) {
       return this;
     }
 
+    public Builder withInfrastructure(List<HasMetadata> hm) {
+      infrastructure.addAll(hm);
+      return this;
+    }
+
+    public Builder withInfrastructure(HasMetadata hm) {
+      infrastructure.add(hm);
+      return this;
+    }
+
+    public Builder withDeployment(List<HasMetadata> hm) {
+      operatorDeployment.addAll(hm);
+      return this;
+    }
+
+    public Builder withDeployment(HasMetadata hm) {
+      operatorDeployment.add(hm);
+      return this;
+    }
+
     public OperatorExtension build() {
       return new OperatorExtension(
           configurationService,
           reconcilers,
+          operatorDeployment,
+          deploymentTimeout,
+          infrastructure,
+          infrastructureTimeout,
           preserveNamespaceOnError,
-          waitForNamespaceDeletion);
+          waitForNamespaceDeletion,
+          oneNamespacePerClass,
+          localOperatorDeployment);
     }
   }