Merge remote-tracking branch 'es/6.x' into ccr-6.x

* es/6.x: (155 commits)
  Make persistent tasks work. Made persistent tasks executors pluggable.
  Removed ClientHelper dependency from PersistentTasksService.
  Added AllocatedPersistentTask#waitForPersistentTaskStatus(...) that delegates to PersistentTasksService#waitForPersistentTaskStatus(...)
  Add adding ability to associate an ID with tasks.
  Remove InternalClient and InternalSecurityClient (#3054)
  Make the persistent task status available to PersistentTasksExecutor.nodeOperation(...) method
  Refactor/to x content fragments2 (#2329)
  Make AllocatedPersistentTask members volatile (#2297)
  Moves more classes over to ToXContentObject/Fragment (#2283)
  Adapt to upstream changes made to AbstractStreamableXContentTestCase (#2117)
  Move tribe to a module (#2088)
  Persistent Tasks: remove unused isCurrentStatus method (#2076)
  Call initialising constructor of BaseTasksRequest (#1771)
  Always Accumulate Transport Exceptions (#1619)
  Pass down the provided timeout.
  Fix static / version based BWC tests (#1456)
  Don't call ClusterService.state() in a ClusterStateUpdateTask
  Separate publishing from applying cluster states
  Persistent tasks: require allocation id on task completion (#1107)
  Fixes compile errors in Eclipse due to generics
  ...

Author: martijnvg

Vagrantfile

@@ -189,6 +189,7 @@ subprojects {
     "org.elasticsearch:elasticsearch:${version}": ':server',
     "org.elasticsearch:elasticsearch-cli:${version}": ':server:cli',
     "org.elasticsearch:elasticsearch-core:${version}": ':libs:elasticsearch-core',
+    "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',    
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',

build.gradle

@@ -62,6 +62,9 @@ class RandomizedTestingPlugin implements Plugin<Project> {
         RandomizedTestingTask newTestTask = tasks.create(properties)
         newTestTask.classpath = oldTestTask.classpath
         newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
+        // since gradle 4.5, tasks immutable dependencies are "hidden" (do not show up in dependsOn)
+        // so we must explicitly add a dependency on generating the test classpath
+        newTestTask.dependsOn('testClasses')
 
         // hack so check task depends on custom test
         Task checkTask = tasks.findByPath('check')

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -567,6 +567,9 @@ class BuildPlugin implements Plugin<Project> {
             File heapdumpDir = new File(project.buildDir, 'heapdump')
             heapdumpDir.mkdirs()
             jvmArg '-XX:HeapDumpPath=' + heapdumpDir
+            if (project.runtimeJavaVersion >= JavaVersion.VERSION_1_9) {
+                jvmArg '--illegal-access=warn'
+            }
             argLine System.getProperty('tests.jvm.argline')
 
             // we use './temp' since this is per JVM and tests are forbidden from writing to CWD

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -123,6 +123,14 @@ class ClusterConfiguration {
         return tmpFile.exists()
     }
 
+    /**
+     * The maximum number of seconds to wait for nodes to complete startup, which includes writing
+     * the ports files for the transports and the pid file. This wait time occurs before the wait
+     * condition is executed.
+     */
+    @Input
+    int nodeStartupWaitSeconds = 30
+
     public ClusterConfiguration(Project project) {
         this.project = project
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -24,7 +24,6 @@ import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.MetaPluginBuildPlugin
-import org.elasticsearch.gradle.plugin.MetaPluginPropertiesExtension
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.elasticsearch.gradle.plugin.PluginPropertiesExtension
 import org.gradle.api.AntBuilder
@@ -120,7 +119,7 @@ class ClusterFormationTasks {
             startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, nodes.get(0)))
         }
 
-        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks)
+        Task wait = configureWaitTask("${prefix}#wait", project, nodes, startTasks, config.nodeStartupWaitSeconds)
         runner.dependsOn(wait)
 
         return nodes
@@ -581,10 +580,10 @@ class ClusterFormationTasks {
         return start
     }
 
-    static Task configureWaitTask(String name, Project project, List<NodeInfo> nodes, List<Task> startTasks) {
+    static Task configureWaitTask(String name, Project project, List<NodeInfo> nodes, List<Task> startTasks, int waitSeconds) {
         Task wait = project.tasks.create(name: name, dependsOn: startTasks)
         wait.doLast {
-            ant.waitfor(maxwait: '30', maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
+            ant.waitfor(maxwait: "${waitSeconds}", maxwaitunit: 'second', checkevery: '500', checkeveryunit: 'millisecond', timeoutproperty: "failed${name}") {
                 or {
                     for (NodeInfo node : nodes) {
                         resourceexists {
@@ -614,6 +613,17 @@ class ClusterFormationTasks {
                 waitFailed(project, nodes, logger, 'Failed to start elasticsearch')
             }
 
+            // make sure all files exist otherwise we haven't fully started up
+            boolean missingFile = false
+            for (NodeInfo node : nodes) {
+                missingFile |= node.pidFile.exists() == false
+                missingFile |= node.httpPortsFile.exists() == false
+                missingFile |= node.transportPortsFile.exists() == false
+            }
+            if (missingFile) {
+                waitFailed(project, nodes, logger, 'Elasticsearch did not complete startup in time allotted')
+            }
+
             // go through each node checking the wait condition
             for (NodeInfo node : nodes) {
                 // first bind node info to the closure, then pass to the ant runner so we can get good logging

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -162,7 +162,7 @@ public class RestIntegTestTask extends DefaultTask {
                 if (line.startsWith("[")) {
                     inExcerpt = false // clear with the next log message
                 }
-                if (line =~ /(\[WARN\])|(\[ERROR\])/) {
+                if (line =~ /(\[WARN *\])|(\[ERROR *\])/) {
                     inExcerpt = true // show warnings and errors
                 }
                 if (inStartup || inExcerpt) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy

@@ -82,3 +82,56 @@ org.joda.time.DateTime#<init>(int, int, int, int, int, int)
 org.joda.time.DateTime#<init>(int, int, int, int, int, int, int)
 org.joda.time.DateTime#now()
 org.joda.time.DateTimeZone#getDefault()
+
+@defaultMessage Local times may be ambiguous or nonexistent in a specific time zones. Use ZoneRules#getValidOffsets() instead.
+java.time.LocalDateTime#atZone(java.time.ZoneId)
+java.time.ZonedDateTime#of(int, int, int, int, int, int, int, java.time.ZoneId)
+java.time.ZonedDateTime#of(java.time.LocalDate, java.time.LocalTime, java.time.ZoneId)
+java.time.ZonedDateTime#of(java.time.LocalDateTime, java.time.ZoneId)
+java.time.ZonedDateTime#truncatedTo(java.time.temporal.TemporalUnit)
+java.time.ZonedDateTime#of(int, int, int, int, int, int, int, java.time.ZoneId)
+java.time.ZonedDateTime#of(java.time.LocalDate, java.time.LocalTime, java.time.ZoneId)
+java.time.ZonedDateTime#of(java.time.LocalDateTime, java.time.ZoneId)
+java.time.ZonedDateTime#ofLocal(java.time.LocalDateTime, java.time.ZoneId, java.time.ZoneOffset)
+java.time.OffsetDateTime#atZoneSimilarLocal(java.time.ZoneId)
+java.time.zone.ZoneRules#getOffset(java.time.LocalDateTime)
+
+@defaultMessage Manipulation of an OffsetDateTime may yield a time that is not valid in the desired time zone. Use ZonedDateTime instead.
+java.time.OffsetDateTime#minus(long, java.time.temporal.TemporalUnit)
+java.time.OffsetDateTime#minus(long, java.time.temporal.TemporalUnit)
+java.time.OffsetDateTime#minus(java.time.temporal.TemporalAmount)
+java.time.OffsetDateTime#minusDays(long)
+java.time.OffsetDateTime#minusHours(long)
+java.time.OffsetDateTime#minusMinutes(long)
+java.time.OffsetDateTime#minusMonths(long)
+java.time.OffsetDateTime#minusNanos(long)
+java.time.OffsetDateTime#minusSeconds(long)
+java.time.OffsetDateTime#minusWeeks(long)
+java.time.OffsetDateTime#minusYears(long)
+java.time.OffsetDateTime#plus(long, java.time.temporal.TemporalUnit)
+java.time.OffsetDateTime#plus(java.time.temporal.TemporalAmount)
+java.time.OffsetDateTime#plusDays(long)
+java.time.OffsetDateTime#plusHours(long)
+java.time.OffsetDateTime#plusMinutes(long)
+java.time.OffsetDateTime#plusMonths(long)
+java.time.OffsetDateTime#plusNanos(long)
+java.time.OffsetDateTime#plusSeconds(long)
+java.time.OffsetDateTime#plusWeeks(long)
+java.time.OffsetDateTime#plusYears(long)
+java.time.OffsetDateTime#with(java.time.temporal.TemporalAdjuster)
+java.time.OffsetDateTime#with(java.time.temporal.TemporalField, long)
+java.time.OffsetDateTime#withDayOfMonth(int)
+java.time.OffsetDateTime#withDayOfYear(int)
+java.time.OffsetDateTime#withHour(int)
+java.time.OffsetDateTime#withMinute(int)
+java.time.OffsetDateTime#withMonth(int)
+java.time.OffsetDateTime#withNano(int)
+java.time.OffsetDateTime#withOffsetSameInstant(java.time.ZoneOffset)
+java.time.OffsetDateTime#withOffsetSameLocal(java.time.ZoneOffset)
+java.time.OffsetDateTime#withSecond(int)
+java.time.OffsetDateTime#withYear(int)
+
+@defaultMessage Daylight saving is not the only reason for a change in timezone offset.
+java.time.zone.ZoneRules#getStandardOffset(java.time.Instant)
+java.time.zone.ZoneRules#getDaylightSavings(java.time.Instant)
+java.time.zone.ZoneRules#isDaylightSavings(java.time.Instant)