[ci] Fix workflows stable for ARM64 tests (#2553)

Signed-off-by: Viet Nguyen Duc <[email protected]>

Author: VietND96

.circleci/config.yml

@@ -134,8 +134,8 @@ workflows:
           test-strategy: test_parallel
           use-random-user: false
           platforms: linux/arm64
-          machine-type: ubuntu2204arm64
-          firefox-install-lang-package: true
+          machine-type: ubuntu2204arm64large
+          firefox-install-lang-package: false
           enable-managed-downloads: true
       - docker-test:
           name: "Docker test - Node relay commands"
@@ -202,7 +202,7 @@ jobs:
           command: |
             N=3
             while [ $N -gt 0 ]; do
-              output=$(eval "PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build")
+              PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
               status=$?
               if [ $status -eq 0 ]; then
                 echo "Build images passed"
@@ -213,26 +213,31 @@ jobs:
                 sleep 10
               fi
             done
-            exit 1
+            exit $status
       - run:
           name: "Test Docker images"
+          no_output_timeout: 60m
           command: |
-            N=3
-            while [ $N -gt 0 ]; do
-              output=$(eval "USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \
+            max_iteration=3
+            iteration=1
+            until USE_RANDOM_USER_ID=${USE_RANDOM_USER} PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} \
               TEST_FIREFOX_INSTALL_LANG_PACKAGE=${TEST_FIREFOX_INSTALL_LANG_PACKAGE} SELENIUM_ENABLE_MANAGED_DOWNLOADS=${SELENIUM_ENABLE_MANAGED_DOWNLOADS} \
-              make ${TEST_STRATEGY}")
-              status=$?
-              if [ $status -eq 0 ]; then
-                echo "Tests passed"
-                exit 0
-              else
-                echo "Tests failed. Retrying..."
-                N=$((N-1))
-                sleep 10
+              make ${TEST_STRATEGY} >& /dev/null; [[ $? -eq 0 ]];
+            do
+              echo "Result unsuccessful"
+              if [[ $iteration -eq $max_iteration ]]
+              then
+                break
               fi
+              sleep 1
+              ((iteration++))  
             done
-            exit 1
+            if [[ $iteration -eq $max_iteration ]]
+            then
+              echo "All of the $max_iteration trials failed!!!"
+            else
+              echo "Result successful"
+            fi
 
   kubernetes-test:
     parameters:
@@ -263,6 +268,8 @@ jobs:
       HELM_VERSION: << parameters.helm-version >>
       DOCKER_VERSION: << parameters.docker-version >>
       TEST_UPGRADE_CHART: << parameters.test-upgrade >>
+      TEST_MULTIPLE_VERSIONS: false
+      TEST_MULTIPLE_PLATFORMS: false
     steps:
       - run:
           name: "Prepare workflow environment variables"
@@ -288,7 +295,7 @@ jobs:
           command: |
             N=3
             while [ $N -gt 0 ]; do
-              output=$(eval "PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build")
+              PLATFORMS=${PLATFORMS} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} make build
               status=$?
               if [ $status -eq 0 ]; then
                 echo "Build images passed"
@@ -299,7 +306,7 @@ jobs:
                 sleep 10
               fi
             done
-            exit 1
+            exit $status
       - run:
           name: "Build Helm charts"
           command: |
@@ -309,28 +316,28 @@ jobs:
             source $BASH_ENV
       - run:
           name: "Test Selenium Grid on Kubernetes"
-          no_output_timeout: 30m
+          no_output_timeout: 60m
           command: |
-            if [ "${TEST_STRATEGY}" == "playwright_connect_grid" ]; then
-              PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} && make test_video_integrity
-              exit $?
-            fi
-            N=3
-            while [ $N -gt 0 ]; do
-              output=$(eval "PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false make chart_test_autoscaling_${TEST_STRATEGY} && make test_video_integrity")
-              status=$?
-              if [ $status -eq 0 ]; then
-                echo "${output}"
-                echo "Tests passed"
-                exit 0
-              else
-                echo "${output}"
-                echo "Tests failed. Retrying..."
-                N=$((N-1))
-                sleep 10
+            max_iteration=3
+            iteration=1
+            until PLATFORMS=${PLATFORMS} NAME=${IMAGE_REGISTRY} VERSION=${BRANCH} BUILD_DATE=${BUILD_DATE} TEST_UPGRADE_CHART=false \
+              TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS} TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS} \
+              make chart_test_autoscaling_${TEST_STRATEGY} && make test_video_integrity >& /dev/null; [[ $? -eq 0 ]];
+            do
+              echo "Result unsuccessful"
+              if [[ $iteration -eq $max_iteration ]]
+              then
+                break
               fi
+              sleep 1
+              ((iteration++))  
             done
-            exit 1
+            if [[ $iteration -eq $max_iteration ]]
+            then
+              echo "All of the $max_iteration trials failed!!!"
+            else
+              echo "Result successful"
+            fi
       - run:
           name: "Clean-up Kubernetes environment"
           command: |

.github/workflows/helm-chart-test.yml

@@ -213,7 +213,9 @@ jobs:
         uses: actions/upload-artifact@main
         with:
           name: ${{ env.ARTIFACT_NAME }}-videos
-          path: ./tests/videos/
+          path: |
+            ./tests/videos/
+            !./tests/videos/**/*.hprof
       - name: Test chart upgrade
         if: (matrix.test-upgrade == true)
         run: |

tests/SeleniumTests/__init__.py

@@ -59,6 +59,8 @@ class SeleniumGenericTests(unittest.TestCase):
 
     def test_title(self):
         self.driver.get('https://the-internet.herokuapp.com')
+        wait = WebDriverWait(self.driver, WEB_DRIVER_WAIT_TIMEOUT)
+        wait.until(EC.title_is('The Internet'))
         self.assertTrue(self.driver.title == 'The Internet')
 
     # https://github.com/tourdedave/elemental-selenium-tips/blob/master/03-work-with-frames/python/frames.py
@@ -167,7 +169,7 @@ def setUp(self):
                 browser_version = random.choice(LIST_CHROMIUM_VERSIONS)
                 if browser_version:
                     options.set_capability('browserVersion', browser_version)
-                    options.set_capability('platformName', 'Linux')
+                options.set_capability('platformName', 'Linux')
             if TEST_NODE_RELAY == 'Android':
                 options.set_capability('platformName', TEST_NODE_RELAY)
                 options.set_capability('appium:platformVersion', TEST_ANDROID_PLATFORM_API)

tests/charts/make/chart_test.sh

@@ -67,6 +67,20 @@ TEST_MULTIPLE_VERSIONS=${TEST_MULTIPLE_VERSIONS:-"false"}
 TEST_MULTIPLE_VERSIONS_EXPLICIT=${TEST_MULTIPLE_VERSIONS_EXPLICIT:-"true"}
 TEST_MULTIPLE_PLATFORMS=${TEST_MULTIPLE_PLATFORMS:-"false"}
 
+wait_for_terminated() {
+  # Wait until no pods are in "Terminating" state
+  while true; do
+    terminating_pods=$(kubectl get pods -n ${SELENIUM_NAMESPACE} --no-headers | grep Terminating | wc -l)
+    if [ $terminating_pods -eq 0 ]; then
+      echo "No pods in 'Terminating' state."
+      break
+    else
+      echo "Waiting for $terminating_pods pod(s) to terminate..."
+      sleep 2
+    fi
+  done
+}
+
 cleanup() {
   # Get the list of pods
   pods=$(kubectl get pods -n ${SELENIUM_NAMESPACE} -o jsonpath='{.items[*].metadata.name}')
@@ -77,7 +91,9 @@ cleanup() {
   done
   if [ "${SKIP_CLEANUP}" = "false" ] || [ "${CI:-false}" != "false" ]; then
     echo "Clean up chart release and namespace"
-    helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --wait
+    helm delete ${RELEASE_NAME} --namespace ${SELENIUM_NAMESPACE} --no-hooks || true
+    wait_for_terminated
+    sudo chmod -R 777 ${HOST_PATH}/logs
   fi
 }
 
@@ -94,13 +110,13 @@ on_failure() {
     kubectl describe all -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt
     kubectl describe pod -n ${SELENIUM_NAMESPACE} >> tests/tests/describe_all_resources_${MATRIX_BROWSER}.txt
     echo "There is step failed with exit status $exit_status"
-    cleanup
     sudo chmod -R 777 ${HOST_PATH}/logs
     exit $exit_status
 }
 
 # Trap ERR signal and call on_failure function
 trap 'on_failure' ERR EXIT
+trap 'cleanup' ERR
 
 if [ "${RENDER_HELM_TEMPLATE_ONLY}" != "true" ]; then
   rm -rf tests/tests/*
@@ -496,16 +512,6 @@ else
   ./tests/bootstrap.sh ${MATRIX_BROWSER}
 fi
 
-# Wait until no pods are in "Terminating" state
-while true; do
-  terminating_pods=$(kubectl get pods -n ${SELENIUM_NAMESPACE} --no-headers | grep Terminating | wc -l)
-  if [ $terminating_pods -eq 0 ]; then
-    echo "No pods in 'Terminating' state."
-    break
-  else
-    echo "Waiting for $terminating_pods pod(s) to terminate..."
-    sleep 2
-  fi
-done
+wait_for_terminated
 
 cleanup

tests/docker-compose-v3-test-parallel.yml

@@ -12,6 +12,7 @@ services:
       replicas: 5
     image: selenium/node-${NODE_CHROME}:${TAG}
     user: ${UID}
+    shm_size: 2gb
     depends_on:
       - selenium-hub
     volumes:
@@ -28,7 +29,9 @@ services:
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=info
       - SE_NODE_GRACEFUL_SHUTDOWN=true
-      - SE_DRAIN_AFTER_SESSION_COUNT=1
+      - SE_NODE_OVERRIDE_MAX_SESSIONS=true
+      - SE_NODE_MAX_SESSIONS=2
+      - SE_DRAIN_AFTER_SESSION_COUNT=4
       - SE_ENABLE_TLS=true
       - SE_JAVA_OPTS=-Dwebdriver.httpclient.readTimeout=${REQUEST_TIMEOUT}
       - SE_RECORD_VIDEO=true
@@ -61,7 +64,9 @@ services:
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=info
       - SE_NODE_GRACEFUL_SHUTDOWN=true
-      - SE_DRAIN_AFTER_SESSION_COUNT=3
+      - SE_NODE_OVERRIDE_MAX_SESSIONS=true
+      - SE_NODE_MAX_SESSIONS=2
+      - SE_DRAIN_AFTER_SESSION_COUNT=2
       - SE_ENABLE_TLS=true
       - SE_JAVA_OPTS=-Dwebdriver.httpclient.readTimeout=${REQUEST_TIMEOUT}
       - SE_RECORD_VIDEO=true
@@ -92,7 +97,9 @@ services:
       - SE_LOG_LEVEL=${LOG_LEVEL}
       - SE_SUPERVISORD_LOG_LEVEL=info
       - SE_NODE_GRACEFUL_SHUTDOWN=true
-      - SE_DRAIN_AFTER_SESSION_COUNT=2
+      - SE_NODE_OVERRIDE_MAX_SESSIONS=true
+      - SE_NODE_MAX_SESSIONS=3
+      - SE_DRAIN_AFTER_SESSION_COUNT=3
       - SE_ENABLE_TLS=true
       - SE_JAVA_OPTS=-Dwebdriver.httpclient.readTimeout=${REQUEST_TIMEOUT}
       - SE_RECORD_VIDEO=true