diff --git a/.ci/functions/cleanup.sh b/.ci/functions/cleanup.sh
index 4c25166f..f9efc0b1 100644
--- a/.ci/functions/cleanup.sh
+++ b/.ci/functions/cleanup.sh
@@ -13,6 +13,18 @@ function cleanup_volume {
     (docker volume rm "$1") || true
   fi
 }
+function cleanup_container {
+  if [[ "$(docker container ls -q -f name=$1)" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
+    (docker container rm --force --volumes "$1") || true
+  fi
+}
+function container_exists {
+  if [[ -n "$(docker container ls -a -q -f name=$1)" ]]; then
+    return 0;
+    else return 1;
+  fi
+}
 function container_running {
   if [[ "$(docker ps -q -f name=$1)" ]]; then
     return 0;
@@ -20,11 +32,8 @@ function container_running {
   fi
 }
 function cleanup_node {
-  if container_running "$1"; then
-    echo -e "\033[34;1mINFO:\033[0m Removing container $1\033[0m"
-    (docker container rm --force --volumes "$1") || true
-  fi
   if [[ -n "$1" ]]; then
+    cleanup_container "$1"
     echo -e "\033[34;1mINFO:\033[0m Removing volume $1-${suffix}-data\033[0m"
     cleanup_volume "$1-${suffix}-data"
   fi
@@ -58,6 +67,7 @@ function cleanup_all_in_network {
     echo -e "\033[34;1mINFO:\033[0m $1 is already deleted\033[0m"
     return 0
   fi
+  # Note that this will only clean up containers attached to the network (i.e. running)
   containers=$(docker network inspect -f '{{ range $key, $value := .Containers }}{{ printf "%s\n" .Name}}{{ end }}' $1)
   while read -r container; do
     cleanup_node "$container"
diff --git a/.ci/functions/wait-for-container.sh b/.ci/functions/wait-for-container.sh
index 1a721b58..0ea0bae8 100644
--- a/.ci/functions/wait-for-container.sh
+++ b/.ci/functions/wait-for-container.sh
@@ -17,8 +17,8 @@ function wait_for_container {
     sleep 2;
   done;
 
-  # Always show logs if the container is running, this is very useful both on CI as well as while developing
-  if container_running $1; then
+  # Always show logs, this is very useful both on CI as well as while developing
+  if container_exists $1; then
     docker logs $1
   fi
 
diff --git a/.ci/run-elasticsearch.sh b/.ci/run-elasticsearch.sh
index b781de75..29920d74 100755
--- a/.ci/run-elasticsearch.sh
+++ b/.ci/run-elasticsearch.sh
@@ -78,12 +78,13 @@ fi
 
 # Pull the container, retry on failures up to 5 times with
 # short delays between each attempt. Fixes most transient network errors.
-docker_pull_attempts=0
-until [ "$docker_pull_attempts" -ge 5 ]
+DOCKER_PULL_ATTEMPTS=${DOCKER_PULL_ATTEMPTS-5}
+docker_pull_attempt=0
+until [ "$docker_pull_attempt" -ge "$DOCKER_PULL_ATTEMPTS" ]
 do
    docker pull docker.elastic.co/elasticsearch/"$elasticsearch_container" && break
-   docker_pull_attempts=$((docker_pull_attempts+1))
-   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempts/5)..."
+   docker_pull_attempt=$((docker_pull_attempt+1))
+   echo "Failed to pull image, retrying in 10 seconds (retry $docker_pull_attempt/5)..."
    sleep 10
 done
 
@@ -107,6 +108,12 @@ END
   # make sure we detach for all but the last node if DETACH=false (default) so all nodes are started
   local_detach="true"
   if [[ "$i" == "$((NUMBER_OF_NODES-1))" ]]; then local_detach=$DETACH; fi
+  # make sure we delete any existing container from previous runs
+  if [[ -n "$(docker container ls -a -q -f name="^$node_name\$")" ]]; then
+    echo -e "\033[34;1mINFO:\033[0m Deleting previously existing node $node_name \033[0m"
+    (docker container rm --force --volumes "$node_name") || true
+  fi
+  # start the new container
   echo -e "\033[34;1mINFO:\033[0m Starting container $node_name \033[0m"
   set -x
   docker run \
@@ -123,7 +130,6 @@ END
     --health-interval=2s \
     --health-retries=20 \
     --health-timeout=2s \
-    --rm \
     docker.elastic.co/elasticsearch/"$elasticsearch_container";
 
   set +x