Skip to content
This repository was archived by the owner on Aug 2, 2022. It is now read-only.

Commit 4a4d707

Browse files
dabr-grapeupDaniel BrylaJavierAbrego
authored
Snapshot implementation (#135)
* Snapshot implementation Action has two steps: attempt_snapshot and wait_for_snapshot * Add timestamp to snapshot * added missing license headers replaced LinkedHashMap in SnapshotAction handle waitForSnapshotStep is not completed added usermetadata added hash at the end of the name set explicit waitForCompletion using CONDITION_NOT_MET * added snapshot name hash * implemented isIdempotent method * Do not fail snapshot if another one is in progress + Restore snapshot name with timestamp * replace wildcard imports and set isIdempotent to false * removed includeGlobalState from Snapshotaction Co-authored-by: Daniel Bryla <[email protected]> Co-authored-by: Javier Abrego <[email protected]>
1 parent a2d607d commit 4a4d707

File tree

11 files changed

+500
-2
lines changed

11 files changed

+500
-2
lines changed

‎src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/action/SnapshotAction.kt‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
/*
2+
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistroforelasticsearch.indexstatemanagement.action
17+
18+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.ManagedIndexMetaData
19+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.action.ActionConfig.ActionType
20+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.action.SnapshotActionConfig
21+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.step.Step
22+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.step.snapshot.AttemptSnapshotStep
23+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.step.snapshot.WaitForSnapshotStep
24+
import org.elasticsearch.client.Client
25+
import org.elasticsearch.cluster.service.ClusterService
26+
27+
class SnapshotAction(
28+
clusterService: ClusterService,
29+
client: Client,
30+
managedIndexMetaData: ManagedIndexMetaData,
31+
config: SnapshotActionConfig
32+
) : Action(ActionType.SNAPSHOT, config, managedIndexMetaData) {
33+
private val attemptSnapshotStep = AttemptSnapshotStep(clusterService, client, config, managedIndexMetaData)
34+
private val waitForSnapshotStep = WaitForSnapshotStep(clusterService, client, config, managedIndexMetaData)
35+
36+
override fun getSteps(): List<Step> = listOf(attemptSnapshotStep, waitForSnapshotStep)
37+
38+
@Suppress("ReturnCount")
39+
override fun getStepToExecute(): Step {
40+
// If stepMetaData is null, return the first step
41+
val stepMetaData = managedIndexMetaData.stepMetaData ?: return attemptSnapshotStep
42+
43+
// If the current step has completed, return the next step
44+
if (stepMetaData.stepStatus == Step.StepStatus.COMPLETED) {
45+
return when (stepMetaData.name) {
46+
AttemptSnapshotStep.name -> waitForSnapshotStep
47+
else -> attemptSnapshotStep
48+
}
49+
}
50+
51+
return when (stepMetaData.name) {
52+
AttemptSnapshotStep.name -> attemptSnapshotStep
53+
else -> waitForSnapshotStep
54+
}
55+
}
56+
}

‎src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/model/action/ActionConfig.kt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ abstract class ActionConfig(
6161
READ_WRITE("read_write"),
6262
REPLICA_COUNT("replica_count"),
6363
FORCE_MERGE("force_merge"),
64-
NOTIFICATION("notification");
64+
NOTIFICATION("notification"),
65+
SNAPSHOT("snapshot");
6566

6667
override fun toString(): String {
6768
return type
@@ -94,6 +95,7 @@ abstract class ActionConfig(
9495
ActionType.REPLICA_COUNT.type -> actionConfig = ReplicaCountActionConfig.parse(xcp, index)
9596
ActionType.FORCE_MERGE.type -> actionConfig = ForceMergeActionConfig.parse(xcp, index)
9697
ActionType.NOTIFICATION.type -> actionConfig = NotificationActionConfig.parse(xcp, index)
98+
ActionType.SNAPSHOT.type -> actionConfig = SnapshotActionConfig.parse(xcp, index)
9799
else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in Action.")
98100
}
99101
}

‎src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/model/action/SnapshotActionConfig.kt‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
/*
2+
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistroforelasticsearch.indexstatemanagement.model.action
17+
18+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.action.Action
19+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.action.SnapshotAction
20+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.ManagedIndexMetaData
21+
import org.elasticsearch.client.Client
22+
import org.elasticsearch.cluster.service.ClusterService
23+
import org.elasticsearch.common.xcontent.ToXContent
24+
import org.elasticsearch.common.xcontent.ToXContentObject
25+
import org.elasticsearch.common.xcontent.XContentBuilder
26+
import org.elasticsearch.common.xcontent.XContentParser
27+
import org.elasticsearch.common.xcontent.XContentParserUtils
28+
import org.elasticsearch.script.ScriptService
29+
import java.io.IOException
30+
31+
data class SnapshotActionConfig(
32+
val repository: String?,
33+
val snapshot: String?,
34+
val index: Int
35+
) : ToXContentObject, ActionConfig(ActionType.SNAPSHOT, index) {
36+
37+
init {
38+
require(repository != null) { "SnapshotActionConfig repository must be specified" }
39+
require(snapshot != null) { "SnapshotActionConfig snapshot must be specified" }
40+
}
41+
42+
override fun toXContent(builder: XContentBuilder, params: ToXContent.Params): XContentBuilder {
43+
builder.startObject()
44+
super.toXContent(builder, params)
45+
.startObject(ActionType.SNAPSHOT.type)
46+
if (repository != null) builder.field(REPOSITORY_FIELD, repository)
47+
if (snapshot != null) builder.field(SNAPSHOT_FIELD, snapshot)
48+
return builder.endObject().endObject()
49+
}
50+
51+
override fun isFragment(): Boolean = super<ToXContentObject>.isFragment()
52+
53+
override fun toAction(
54+
clusterService: ClusterService,
55+
scriptService: ScriptService,
56+
client: Client,
57+
managedIndexMetaData: ManagedIndexMetaData
58+
): Action = SnapshotAction(clusterService, client, managedIndexMetaData, this)
59+
60+
companion object {
61+
const val REPOSITORY_FIELD = "repository"
62+
const val SNAPSHOT_FIELD = "snapshot"
63+
const val INCLUDE_GLOBAL_STATE = "include_global_state"
64+
65+
@JvmStatic
66+
@Throws(IOException::class)
67+
fun parse(xcp: XContentParser, index: Int): SnapshotActionConfig {
68+
var repository: String? = null
69+
var snapshot: String? = null
70+
71+
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp::getTokenLocation)
72+
while (xcp.nextToken() != XContentParser.Token.END_OBJECT) {
73+
val fieldName = xcp.currentName()
74+
xcp.nextToken()
75+
76+
when (fieldName) {
77+
REPOSITORY_FIELD -> repository = xcp.text()
78+
SNAPSHOT_FIELD -> snapshot = xcp.text()
79+
else -> throw IllegalArgumentException("Invalid field: [$fieldName] found in SnapshotActionConfig.")
80+
}
81+
}
82+
83+
return SnapshotActionConfig(
84+
repository = repository,
85+
snapshot = snapshot,
86+
index = index
87+
)
88+
}
89+
}
90+
}

‎src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/step/snapshot/AttemptSnapshotStep.kt‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/*
2+
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistroforelasticsearch.indexstatemanagement.step.snapshot
17+
18+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.elasticapi.suspendUntil
19+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.ManagedIndexMetaData
20+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.action.SnapshotActionConfig
21+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.managedindexmetadata.StepMetaData
22+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.step.Step
23+
import org.apache.logging.log4j.LogManager
24+
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequest
25+
import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse
26+
import org.elasticsearch.client.Client
27+
import org.elasticsearch.cluster.service.ClusterService
28+
import org.elasticsearch.rest.RestStatus
29+
import org.elasticsearch.snapshots.ConcurrentSnapshotExecutionException
30+
import java.time.LocalDateTime
31+
import java.time.ZoneId
32+
import java.time.format.DateTimeFormatter
33+
import java.util.Locale
34+
35+
class AttemptSnapshotStep(
36+
val clusterService: ClusterService,
37+
val client: Client,
38+
val config: SnapshotActionConfig,
39+
managedIndexMetaData: ManagedIndexMetaData
40+
) : Step(name, managedIndexMetaData) {
41+
42+
private val logger = LogManager.getLogger(javaClass)
43+
private var stepStatus = StepStatus.STARTING
44+
private var info: Map<String, Any>? = null
45+
46+
override fun isIdempotent() = false
47+
48+
@Suppress("TooGenericExceptionCaught")
49+
override suspend fun execute() {
50+
try {
51+
logger.info("Executing snapshot on ${managedIndexMetaData.index}")
52+
val snapshotName = config
53+
.snapshot
54+
.plus("-")
55+
.plus(LocalDateTime
56+
.now(ZoneId.of("UTC"))
57+
.format(DateTimeFormatter.ofPattern("uuuu.MM.dd-HH:mm:ss.SSS", Locale.ROOT)))
58+
val mutableInfo = mutableMapOf("snapshotName" to snapshotName)
59+
60+
val createSnapshotRequest = CreateSnapshotRequest()
61+
.userMetadata(mapOf("snapshot_created" to "Open Distro for Elasticsearch Index Management"))
62+
.indices(managedIndexMetaData.index)
63+
.snapshot(snapshotName)
64+
.repository(config.repository)
65+
.waitForCompletion(false)
66+
67+
val response: CreateSnapshotResponse = client.admin().cluster().suspendUntil { createSnapshot(createSnapshotRequest, it) }
68+
when (response.status()) {
69+
RestStatus.ACCEPTED -> {
70+
stepStatus = StepStatus.COMPLETED
71+
mutableInfo["message"] = "Snapshot creation started and is still in progress for index: ${managedIndexMetaData.index}"
72+
}
73+
RestStatus.OK -> {
74+
stepStatus = StepStatus.COMPLETED
75+
mutableInfo["message"] = "Snapshot created for index: ${managedIndexMetaData.index}"
76+
}
77+
else -> {
78+
stepStatus = StepStatus.FAILED
79+
mutableInfo["message"] = "There was an error during snapshot creation for index: ${managedIndexMetaData.index}"
80+
mutableInfo["cause"] = response.toString()
81+
}
82+
}
83+
info = mutableInfo.toMap()
84+
} catch (e: ConcurrentSnapshotExecutionException) {
85+
val message = "Snapshot creation already in progress."
86+
logger.debug(message, e)
87+
stepStatus = StepStatus.CONDITION_NOT_MET
88+
info = mapOf("message" to message)
89+
} catch (e: Exception) {
90+
val message = "Failed to create snapshot for index: ${managedIndexMetaData.index}"
91+
logger.error(message, e)
92+
stepStatus = StepStatus.FAILED
93+
val mutableInfo = mutableMapOf("message" to message)
94+
val errorMessage = e.message
95+
if (errorMessage != null) mutableInfo["cause"] = errorMessage
96+
info = mutableInfo.toMap()
97+
}
98+
}
99+
100+
override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData {
101+
return currentMetaData.copy(
102+
stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus),
103+
transitionTo = null,
104+
info = info
105+
)
106+
}
107+
108+
companion object {
109+
const val name = "attempt_snapshot"
110+
}
111+
}

‎src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/step/snapshot/WaitForSnapshotStep.kt‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
/*
2+
* Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package com.amazon.opendistroforelasticsearch.indexstatemanagement.step.snapshot
17+
18+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.elasticapi.suspendUntil
19+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.ManagedIndexMetaData
20+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.action.SnapshotActionConfig
21+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.model.managedindexmetadata.StepMetaData
22+
import com.amazon.opendistroforelasticsearch.indexstatemanagement.step.Step
23+
import org.apache.logging.log4j.LogManager
24+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotStatus
25+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest
26+
import org.elasticsearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse
27+
import org.elasticsearch.client.Client
28+
import org.elasticsearch.cluster.service.ClusterService
29+
30+
class WaitForSnapshotStep(
31+
val clusterService: ClusterService,
32+
val client: Client,
33+
val config: SnapshotActionConfig,
34+
managedIndexMetaData: ManagedIndexMetaData
35+
) : Step(name, managedIndexMetaData) {
36+
private val logger = LogManager.getLogger(javaClass)
37+
private var stepStatus = StepStatus.STARTING
38+
private var info: Map<String, Any>? = null
39+
40+
override fun isIdempotent() = true
41+
42+
override suspend fun execute() {
43+
logger.info("Waiting for snapshot to complete...")
44+
val request = SnapshotsStatusRequest()
45+
.snapshots(arrayOf(managedIndexMetaData.info?.get("snapshotName").toString()))
46+
.repository(config.repository)
47+
val response: SnapshotsStatusResponse = client.admin().cluster().suspendUntil { snapshotsStatus(request, it) }
48+
val status: SnapshotStatus? = response
49+
.snapshots
50+
.find { snapshotStatus ->
51+
snapshotStatus.snapshot.snapshotId.name == managedIndexMetaData.info?.get("snapshotName").toString() &&
52+
snapshotStatus.snapshot.repository == config.repository
53+
}
54+
if (status != null) {
55+
if (status.state.completed()) {
56+
stepStatus = StepStatus.COMPLETED
57+
info = mapOf("message" to "Snapshot created for index: ${managedIndexMetaData.index}")
58+
} else {
59+
stepStatus = StepStatus.CONDITION_NOT_MET
60+
info = mapOf("message" to "Creating snapshot in progress for index: ${managedIndexMetaData.index}")
61+
}
62+
} else {
63+
stepStatus = StepStatus.FAILED
64+
info = mapOf("message" to "Snapshot doesn't exist for index: ${managedIndexMetaData.index}")
65+
}
66+
}
67+
68+
override fun getUpdatedManagedIndexMetaData(currentMetaData: ManagedIndexMetaData): ManagedIndexMetaData {
69+
return currentMetaData.copy(
70+
stepMetaData = StepMetaData(name, getStepStartTime().toEpochMilli(), stepStatus),
71+
transitionTo = null,
72+
info = info
73+
)
74+
}
75+
76+
companion object {
77+
const val name = "wait_for_snapshot"
78+
}
79+
}

0 commit comments

Comments
 (0)