Support ES version 7.8.0 (#246)

* Support ES version 7.8.0

* Bump ODFE version to 1.9.0
* Bump Elasticsearch version to 7.8.0
* Bump Gradle version to 6.5
* Changes to make compatible with Elasticsearch 7.8.0
* Fix jacocoagent path to work with gradle 6.5 for integration tests.

* Add release notes for ODFE 1.9.0

* Updating Release notes, rebasing on snapshot fixes

* Release notes for set index priority action

Author: setiah

build-tools/esplugin-coverage.gradle

@@ -71,7 +71,7 @@ allprojects{
         jacocoTestReport.dependsOn integTest.runner
 
         testClusters.integTest {
-            jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}"
+            jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}".replace('javaagent:','javaagent:/')
             systemProperty 'com.sun.management.jmxremote', "true"
             systemProperty 'com.sun.management.jmxremote.authenticate', "false"
             systemProperty 'com.sun.management.jmxremote.port', "7777"

build.gradle

@@ -15,7 +15,7 @@
 
 buildscript {
     ext {
-        es_version = System.getProperty("es.version", "7.7.0")
+        es_version = System.getProperty("es.version", "7.8.0")
         kotlin_version = System.getProperty("kotlin.version", "1.3.72")
     }
 
@@ -79,12 +79,12 @@ configurations.testCompile {
 
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${es_version}"
-    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.8.0.0"
+    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.9.0.0"
     compile "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
     compile 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.2.1'
     compile "org.jetbrains:annotations:13.0"
-    compile "com.amazon.opendistroforelasticsearch:notification:1.8.0.0"
+    compile "com.amazon.opendistroforelasticsearch:notification:1.9.0.0"
 
     testCompile "org.elasticsearch.test:framework:${es_version}"
     testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
@@ -125,6 +125,7 @@ javadoc.enabled = false // turn off javadoc as it barfs on Kotlin code
 licenseHeaders.enabled = true
 dependencyLicenses.enabled = false
 thirdPartyAudit.enabled = false
+validateNebulaPom.enabled = false
 
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
 es_tmp_dir.mkdirs()

gradle.properties

@@ -13,4 +13,4 @@
 #   permissions and limitations under the License.
 #
 
-version = 1.8.0
+version = 1.9.0

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 #Tue May 19 22:58:14 PDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

release-notes/opendistro-elasticsearch-index-management-1.9.0.md

@@ -0,0 +1,14 @@
+
+## Version 1.9.0.0 (2020-6-25)
+
+Compatible with Elasticsearch 7.8.0, Adds support for ODFE 1.9.0
+
+### New Features
+* Adds support for Elasticsearch 7.8.0 [PR #246](https://github.com/opendistro-for-elasticsearch/index-management/pull/246)
+
+### Enhancement
+* Implement set index priority action [PR #241](https://github.com/opendistro-for-elasticsearch/index-management/pull/241)
+
+### Bug Fixes
+* Fixes snapshot bugs [PR #244](https://github.com/opendistro-for-elasticsearch/index-management/pull/244)
+

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/IndexStateManagementHistory.kt

@@ -121,7 +121,7 @@ class IndexStateManagementHistory(
             .mapping(_DOC, IndexStateManagementIndices.indexStateManagementHistoryMappings, XContentType.JSON)
         request.addMaxIndexDocsCondition(historyMaxDocs)
         request.addMaxIndexAgeCondition(historyMaxAge)
-        val response = client.admin().indices().rolloversIndex(request).actionGet()
+        val response = client.admin().indices().rolloverIndex(request).actionGet()
         if (!response.isRolledOver) {
             logger.info("${IndexStateManagementIndices.HISTORY_WRITE_INDEX_ALIAS} not rolled over. Conditions were: ${response.conditionStatus}")
         }
@@ -135,13 +135,13 @@ class IndexStateManagementHistory(
         val clusterStateRequest = ClusterStateRequest()
             .clear()
             .indices(IndexStateManagementIndices.HISTORY_ALL)
-            .metaData(true)
+            .metadata(true)
             .local(true)
             .indicesOptions(IndicesOptions.strictExpand())
 
         val clusterStateResponse = client.admin().cluster().state(clusterStateRequest).actionGet()
 
-        for (entry in clusterStateResponse.state.metaData.indices()) {
+        for (entry in clusterStateResponse.state.metadata.indices()) {
             val indexMetaData = entry.value
             val creationTime = indexMetaData.creationDate
 

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/IndexStateManagementIndices.kt

@@ -89,7 +89,7 @@ class IndexStateManagementIndices(
     /**
      * ============== History =============
      */
-    fun indexStateManagementIndexHistoryExists(): Boolean = clusterService.state().metaData.hasAlias(HISTORY_WRITE_INDEX_ALIAS)
+    fun indexStateManagementIndexHistoryExists(): Boolean = clusterService.state().metadata.hasAlias(HISTORY_WRITE_INDEX_ALIAS)
 
     suspend fun initHistoryIndex() {
         if (!indexStateManagementIndexHistoryExists())

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/IndexStateManagementPlugin.kt

@@ -51,6 +51,7 @@ import org.elasticsearch.env.Environment
 import org.elasticsearch.env.NodeEnvironment
 import org.elasticsearch.plugins.ActionPlugin
 import org.elasticsearch.plugins.Plugin
+import org.elasticsearch.repositories.RepositoriesService
 import org.elasticsearch.rest.RestController
 import org.elasticsearch.rest.RestHandler
 import org.elasticsearch.script.ScriptService
@@ -139,7 +140,8 @@ internal class IndexStateManagementPlugin : JobSchedulerExtension, ActionPlugin,
         environment: Environment,
         nodeEnvironment: NodeEnvironment,
         namedWriteableRegistry: NamedWriteableRegistry,
-        indexNameExpressionResolver: IndexNameExpressionResolver
+        indexNameExpressionResolver: IndexNameExpressionResolver,
+        repositoriesServiceSupplier: Supplier<RepositoriesService>
     ): Collection<Any> {
         val settings = environment.settings()
         this.clusterService = clusterService

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/ManagedIndexCoordinator.kt

@@ -161,9 +161,9 @@ class ManagedIndexCoordinator(
         if (!event.localNodeMaster()) return
 
         // TODO: Look into event.isNewCluster, can we return early if true?
-        if (event.isNewCluster) { }
+        // if (event.isNewCluster) { }
 
-        if (!event.metaDataChanged()) return
+        if (!event.metadataChanged()) return
 
         launch { sweepClusterChangedEvent(event) }
     }
@@ -200,7 +200,7 @@ class ManagedIndexCoordinator(
     private suspend fun reenableJobs() {
         val clusterStateRequest = ClusterStateRequest()
             .clear()
-            .metaData(true)
+            .metadata(true)
             .local(false)
             .indices("*")
             .indicesOptions(IndicesOptions.strictExpand())
@@ -214,7 +214,7 @@ class ManagedIndexCoordinator(
          *   2. Does not have a completed Policy
          *   3. Does not have a failed Policy
          */
-        val updateManagedIndicesRequests: List<DocWriteRequest<*>> = response.state.metaData.indices.mapNotNull {
+        val updateManagedIndicesRequests: List<DocWriteRequest<*>> = response.state.metadata.indices.mapNotNull {
             val managedIndexMetaData = it.value.getManagedIndexMetaData()
             if (!(managedIndexMetaData == null || managedIndexMetaData.isPolicyCompleted || managedIndexMetaData.isFailed)) {
                 updateEnableManagedIndexRequest(it.value.indexUUID)
@@ -231,7 +231,7 @@ class ManagedIndexCoordinator(
     @OpenForTesting
     suspend fun sweepClusterChangedEvent(event: ClusterChangedEvent) {
         val indicesDeletedRequests = event.indicesDeleted()
-                    .filter { event.previousState().metaData().index(it)?.getPolicyID() != null }
+                    .filter { event.previousState().metadata().index(it)?.getPolicyID() != null }
                     .map { deleteManagedIndexRequest(it.uuid) }
 
         /*
@@ -245,8 +245,8 @@ class ManagedIndexCoordinator(
         var hasCreateRequests = false
         val updateManagedIndicesRequests = mutableListOf<DocWriteRequest<*>>()
         val indicesToRemoveManagedIndexMetaDataFrom = mutableListOf<Index>()
-        event.state().metaData().indices().forEach {
-            val previousIndexMetaData = event.previousState().metaData().index(it.value.index)
+        event.state().metadata().indices().forEach {
+            val previousIndexMetaData = event.previousState().metadata().index(it.value.index)
             val policyID = it.value.getPolicyID()
             val request: DocWriteRequest<*>? = when {
                 it.value.shouldCreateManagedIndexConfig(previousIndexMetaData) && policyID != null -> {
@@ -373,7 +373,7 @@ class ManagedIndexCoordinator(
      */
     @OpenForTesting
     fun sweepClusterState(clusterState: ClusterState): Map<String, ClusterStateManagedIndexConfig> {
-        return clusterState.metaData().indices().values()
+        return clusterState.metadata().indices().values()
                 .mapNotNull {
                     val clusterConfig = it.value.getClusterStateManagedIndexConfig()
                     clusterConfig?.run {
@@ -412,7 +412,7 @@ class ManagedIndexCoordinator(
     /** Returns a list of [Index]es that need to have their [ManagedIndexMetaData] removed. */
     @OpenForTesting
     fun getIndicesToDeleteManagedIndexMetaDataFrom(clusterState: ClusterState): List<Index> {
-        return clusterState.metaData().indices().values().mapNotNull {
+        return clusterState.metadata().indices().values().mapNotNull {
             if (it.value.shouldDeleteManagedIndexMetaData()) it.value.index else null
         }
     }

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/ManagedIndexRunner.kt

@@ -82,7 +82,7 @@ import org.elasticsearch.client.Client
 import org.elasticsearch.cluster.block.ClusterBlockException
 import org.elasticsearch.cluster.health.ClusterHealthStatus
 import org.elasticsearch.cluster.health.ClusterStateHealth
-import org.elasticsearch.cluster.metadata.IndexMetaData
+import org.elasticsearch.cluster.metadata.IndexMetadata
 import org.elasticsearch.cluster.service.ClusterService
 import org.elasticsearch.common.settings.Settings
 import org.elasticsearch.common.unit.TimeValue
@@ -678,19 +678,19 @@ object ManagedIndexRunner : ScheduledJobRunner,
 
     private fun clusterIsRed(): Boolean = ClusterStateHealth(clusterService.state()).status == ClusterHealthStatus.RED
 
-    private suspend fun getIndexMetaData(index: String): IndexMetaData? {
-        var indexMetaData: IndexMetaData? = null
+    private suspend fun getIndexMetaData(index: String): IndexMetadata? {
+        var indexMetaData: IndexMetadata? = null
         try {
             val clusterStateRequest = ClusterStateRequest()
                 .clear()
                 .indices(index)
-                .metaData(true)
+                .metadata(true)
                 .local(false)
                 .indicesOptions(IndicesOptions.strictExpand())
 
             val response: ClusterStateResponse = client.admin().cluster().suspendUntil { state(clusterStateRequest, it) }
 
-            indexMetaData = response.state.metaData.indices.firstOrNull()?.value
+            indexMetaData = response.state.metadata.indices.firstOrNull()?.value
             if (indexMetaData == null) {
                 logger.error("Could not find IndexMetaData in master cluster state for $index")
             }