Skip to content

Commit f03cffc

Browse files
shwstpprDaanHoogland
shwstppr
and
DaanHoogland
authored
feature: instance boot group and readiness rules (#13907)
* feature: instance boot group and readiness rules Feature to add the concept of instance boot group and readiness rules. Design spec: https://cwiki.apache.org/confluence/spaces/CLOUDSTACK/pages/449282465/Instance+Boot+Group+and+Readiness+Rules Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * address review comments Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix ui build Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fixes Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix vm action warning Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * minor label fixes Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * add tests Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fixes, address comments Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * remove unused logs Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fixes Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fixes and more tests Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * force stop option and smoke test Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * minor config description Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * address comments Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * address comments; consistent API description Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix failing test Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * address missed Copilot review Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * engine-schema: upgrade path for 4.24.0 Adds a 4.23.0 to 4.24.0 upgrade path. Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * move schema changes to 4.24, update order column name Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * remove unused rule type from presentation Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * test: fix testRestoreVirtualMachineWhenHostRemoved failure Build was failing with, 18:53:06 [ERROR] Errors: 18:53:06 [ERROR] UserVmManagerImplTest.testRestoreVirtualMachineWhenHostRemoved:1886 ? NullPointer Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix imports Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * add since version for api commands Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fixes around members and ui Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix ui issue on updating readiness rule Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * fix wrong reporting of the attempt count Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * address copilot review Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> * prevent instancegroup vms delete/remove only for one with rules Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> --------- Signed-off-by: Abhishek Kumar <abhishek.mrt22@gmail.com> Co-authored-by: dahn <daan@onecht.net>
1 parent 17294ca commit f03cffc

121 files changed

Lines changed: 16568 additions & 15 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎api/src/main/java/com/cloud/event/EventTypes.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,8 @@
4949
import org.apache.cloudstack.storage.sharedfs.SharedFS;
5050
import org.apache.cloudstack.usage.Usage;
5151
import org.apache.cloudstack.schedule.ResourceSchedule;
52+
import org.apache.cloudstack.vm.bootgroup.InstanceBootGroup;
53+
import org.apache.cloudstack.vm.bootgroup.readiness.InstanceBootGroupReadinessRule;
5254

5355
import com.cloud.dc.DataCenter;
5456
import com.cloud.dc.DataCenterGuestIpv6Prefix;
@@ -899,6 +901,21 @@ public class EventTypes {
899901
public static final String EVENT_DNS_RECORD_DELETE = "DNS.RECORD.DELETE";
900902
public static final String EVENT_DNS_NAME_COLLISION = "DNS.NAME.COLLISION";
901903

904+
// Instance Boot Group
905+
public static final String EVENT_INSTANCE_BOOT_GROUP_CREATE = "INSTANCE.BOOT.GROUP.CREATE";
906+
public static final String EVENT_INSTANCE_BOOT_GROUP_DELETE = "INSTANCE.BOOT.GROUP.DELETE";
907+
public static final String EVENT_INSTANCE_BOOT_GROUP_UPDATE = "INSTANCE.BOOT.GROUP.UPDATE";
908+
public static final String EVENT_INSTANCE_BOOT_GROUP_START = "INSTANCE.BOOT.GROUP.START";
909+
public static final String EVENT_INSTANCE_BOOT_GROUP_STOP = "INSTANCE.BOOT.GROUP.STOP";
910+
public static final String EVENT_INSTANCE_BOOT_GROUP_REBOOT = "INSTANCE.BOOT.GROUP.REBOOT";
911+
public static final String EVENT_INSTANCE_BOOT_GROUP_MEMBER_ADD = "INSTANCE.BOOT.GROUP.MEMBER.ADD";
912+
public static final String EVENT_INSTANCE_BOOT_GROUP_MEMBER_REMOVE = "INSTANCE.BOOT.GROUP.MEMBER.REMOVE";
913+
public static final String EVENT_INSTANCE_BOOT_GROUP_MEMBER_REORDER = "INSTANCE.BOOT.GROUP.MEMBER.REORDER";
914+
public static final String EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_CREATE = "INSTANCE.BOOT.GROUP.READINESS.RULE.CREATE";
915+
public static final String EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_UPDATE = "INSTANCE.BOOT.GROUP.READINESS.RULE.UPDATE";
916+
public static final String EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_DELETE = "INSTANCE.BOOT.GROUP.READINESS.RULE.DELETE";
917+
918+
902919
static {
903920

904921
// TODO: need a way to force author adding event types to declare the entity details as well, with out braking
@@ -1468,6 +1485,19 @@ public class EventTypes {
14681485
entityEventDetails.put(EVENT_DNS_RECORD_CREATE, DnsRecord.class);
14691486
entityEventDetails.put(EVENT_DNS_RECORD_DELETE, DnsRecord.class);
14701487

1488+
// Instance Boot Group
1489+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_CREATE, InstanceBootGroup.class);
1490+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_DELETE, InstanceBootGroup.class);
1491+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_UPDATE, InstanceBootGroup.class);
1492+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_START, InstanceBootGroup.class);
1493+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_STOP, InstanceBootGroup.class);
1494+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_REBOOT, InstanceBootGroup.class);
1495+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_MEMBER_ADD, InstanceBootGroup.class);
1496+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_MEMBER_REMOVE, InstanceBootGroup.class);
1497+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_MEMBER_REORDER, InstanceBootGroup.class);
1498+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_CREATE, InstanceBootGroupReadinessRule.class);
1499+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_UPDATE, InstanceBootGroupReadinessRule.class);
1500+
entityEventDetails.put(EVENT_INSTANCE_BOOT_GROUP_READINESS_RULE_DELETE, InstanceBootGroupReadinessRule.class);
14711501
}
14721502

14731503
public static boolean isNetworkEvent(String eventType) {

‎api/src/main/java/org/apache/cloudstack/api/ApiCommandResourceType.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.Map;
2323

2424
import org.apache.cloudstack.region.PortableIp;
25+
import org.apache.cloudstack.vm.bootgroup.readiness.InstanceBootGroupReadinessRule;
2526
import org.apache.commons.collections.CollectionUtils;
2627
import org.apache.commons.lang3.EnumUtils;
2728
import org.apache.commons.lang3.StringUtils;
@@ -91,7 +92,9 @@ public enum ApiCommandResourceType {
9192
Extension(org.apache.cloudstack.extension.Extension.class),
9293
ExtensionCustomAction(org.apache.cloudstack.extension.ExtensionCustomAction.class),
9394
KmsKey(org.apache.cloudstack.kms.KMSKey.class),
94-
HsmProfile(org.apache.cloudstack.kms.HSMProfile.class);
95+
HsmProfile(org.apache.cloudstack.kms.HSMProfile.class),
96+
InstanceBootGroup(org.apache.cloudstack.vm.bootgroup.InstanceBootGroup.class),
97+
InstanceBootGroupReadinessRule(InstanceBootGroupReadinessRule.class);
9598

9699
private final Class<?> clazz;
97100

‎api/src/main/java/org/apache/cloudstack/api/ApiConstants.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,23 @@ public class ApiConstants {
9393
public static final String CAPACITY = "capacity";
9494
public static final String CATEGORY = "category";
9595
public static final String CAN_REVERT = "canrevert";
96+
public static final String BOOT_GROUP_ID = "bootgroupid";
97+
public static final String BOOT_ORDER = "order";
98+
public static final String MEMBER_TYPE = "membertype";
99+
public static final String MEMBER_ID = "memberid";
100+
public static final String MEMBER_NAME = "membername";
101+
public static final String MEMBER_STATE = "memberstate";
102+
public static final String CHILDREN = "children";
103+
public static final String RULE_TYPE = "ruletype";
104+
public static final String INHERITED = "inherited";
105+
public static final String READINESS_MODE = "readinessmode";
106+
public static final String READINESS_STATUS = "readinessstatus";
107+
public static final String READINESS_MESSAGE = "readinessmessage";
108+
public static final String READINESS_ATTEMPT_TIMEOUT_SECONDS = "readinessattempttimeoutseconds";
109+
public static final String READINESS_MAX_RETRY_ATTEMPTS = "readinessmaxretryattempts";
110+
public static final String READINESS_REBOOT_ON_RETRY = "readinessrebootonretry";
111+
public static final String READINESS_INITIAL_DELAY_SECONDS = "readinessinitialdelayseconds";
112+
public static final String IGNORE_INSTANCE_STATE = "ignoreinstancestate";
96113
public static final String CA_CERTIFICATES = "cacertificates";
97114
public static final String CERTIFICATE = "certificate";
98115
public static final String CERTIFICATE_CHAIN = "certchain";
@@ -322,6 +339,7 @@ public class ApiConstants {
322339
public static final String INTERNAL_DNS2 = "internaldns2";
323340
public static final String INTERNET_PROTOCOL = "internetprotocol";
324341
public static final String INTERVAL_TYPE = "intervaltype";
342+
public static final String INSTANCE_GROUP_ID = "instancegroupid";
325343
public static final String INSTANCE_LEASE_DURATION = "leaseduration";
326344
public static final String INSTANCE_LEASE_ENABLED = "instanceleaseenabled";
327345
public static final String INSTANCE_LEASE_EXPIRY_ACTION = "leaseexpiryaction";

‎api/src/main/java/org/apache/cloudstack/api/command/user/bootgroup/AddMemberToInstanceBootGroupCmd.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.cloudstack.api.command.user.bootgroup;
19+
20+
import javax.inject.Inject;
21+
22+
import org.apache.cloudstack.acl.RoleType;
23+
import org.apache.cloudstack.api.APICommand;
24+
import org.apache.cloudstack.api.ApiCommandResourceType;
25+
import org.apache.cloudstack.api.ApiConstants;
26+
import org.apache.cloudstack.api.ApiErrorCode;
27+
import org.apache.cloudstack.api.BaseCmd;
28+
import org.apache.cloudstack.api.Parameter;
29+
import org.apache.cloudstack.api.ServerApiException;
30+
import org.apache.cloudstack.api.command.user.UserCmd;
31+
import org.apache.cloudstack.api.response.InstanceBootGroupMemberResponse;
32+
import org.apache.cloudstack.api.response.InstanceBootGroupResponse;
33+
import org.apache.cloudstack.api.response.InstanceGroupResponse;
34+
import org.apache.cloudstack.api.response.UserVmResponse;
35+
import org.apache.cloudstack.context.CallContext;
36+
import org.apache.cloudstack.vm.bootgroup.InstanceBootGroupMember;
37+
import org.apache.cloudstack.vm.bootgroup.InstanceBootGroupService;
38+
39+
@APICommand(name = "addMemberToInstanceBootGroup",
40+
description = "Adds an Instance or instance group to an instance boot group. Exactly one of virtualmachineid or instancegroupid must be specified.",
41+
since = "24.0",
42+
responseObject = InstanceBootGroupMemberResponse.class,
43+
entityType = {InstanceBootGroupMember.class},
44+
requestHasSensitiveInfo = false,
45+
responseHasSensitiveInfo = false,
46+
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
47+
public class AddMemberToInstanceBootGroupCmd extends BaseCmd implements UserCmd {
48+
49+
@Inject
50+
InstanceBootGroupService instanceBootGroupService;
51+
52+
@Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = InstanceBootGroupResponse.class, required = true, description = "The ID of the instance boot group")
53+
private Long id;
54+
55+
@Parameter(name = ApiConstants.VIRTUAL_MACHINE_ID, type = CommandType.UUID, entityType = UserVmResponse.class, description = "The ID of the Instance to add (exclusive with instancegroupid)")
56+
private Long virtualMachineId;
57+
58+
@Parameter(name = ApiConstants.INSTANCE_GROUP_ID, type = CommandType.UUID, entityType = InstanceGroupResponse.class, description = "The ID of the Instance Group to add (exclusive with virtualmachineid)")
59+
private Long instanceGroupId;
60+
61+
@Parameter(name = ApiConstants.BOOT_ORDER, type = CommandType.INTEGER, required = true,
62+
description = "The boot order value for this member (0 or greater; non-contiguous values are allowed). "
63+
+ "Any existing member already at or past this value is shifted one slot later to make room")
64+
private int order;
65+
66+
public Long getId() {
67+
return id;
68+
}
69+
70+
public Long getVirtualMachineId() {
71+
return virtualMachineId;
72+
}
73+
74+
public Long getInstanceGroupId() {
75+
return instanceGroupId;
76+
}
77+
78+
public int getOrder() {
79+
return order;
80+
}
81+
82+
@Override
83+
public long getEntityOwnerId() {
84+
return CallContext.current().getCallingAccount().getId();
85+
}
86+
87+
@Override
88+
public Long getApiResourceId() {
89+
return id;
90+
}
91+
92+
@Override
93+
public ApiCommandResourceType getApiResourceType() {
94+
return ApiCommandResourceType.InstanceBootGroup;
95+
}
96+
97+
@Override
98+
public void execute() {
99+
InstanceBootGroupMember result = instanceBootGroupService.addMemberToInstanceBootGroup(this);
100+
if (result == null) {
101+
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to add member to instance boot group");
102+
}
103+
InstanceBootGroupMemberResponse response = instanceBootGroupService.createInstanceBootGroupMemberResponse(result);
104+
response.setResponseName(getCommandName());
105+
setResponseObject(response);
106+
}
107+
}

‎api/src/main/java/org/apache/cloudstack/api/command/user/bootgroup/CreateInstanceBootGroupCmd.java‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.cloudstack.api.command.user.bootgroup;
19+
20+
import javax.inject.Inject;
21+
22+
import org.apache.cloudstack.acl.RoleType;
23+
import org.apache.cloudstack.api.APICommand;
24+
import org.apache.cloudstack.api.ApiCommandResourceType;
25+
import org.apache.cloudstack.api.ApiConstants;
26+
import org.apache.cloudstack.api.ApiErrorCode;
27+
import org.apache.cloudstack.api.BaseCmd;
28+
import org.apache.cloudstack.api.Parameter;
29+
import org.apache.cloudstack.api.ServerApiException;
30+
import org.apache.cloudstack.api.command.user.UserCmd;
31+
import org.apache.cloudstack.api.response.DomainResponse;
32+
import org.apache.cloudstack.api.response.InstanceBootGroupResponse;
33+
import org.apache.cloudstack.api.response.ProjectResponse;
34+
import org.apache.cloudstack.context.CallContext;
35+
import org.apache.cloudstack.vm.bootgroup.InstanceBootGroup;
36+
import org.apache.cloudstack.vm.bootgroup.InstanceBootGroupService;
37+
38+
@APICommand(name = "createInstanceBootGroup",
39+
description = "Creates an Instance Boot Group",
40+
since = "24.0",
41+
responseObject = InstanceBootGroupResponse.class,
42+
entityType = {InstanceBootGroup.class},
43+
requestHasSensitiveInfo = false,
44+
responseHasSensitiveInfo = false,
45+
authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User})
46+
public class CreateInstanceBootGroupCmd extends BaseCmd implements UserCmd {
47+
48+
@Inject
49+
InstanceBootGroupService instanceBootGroupService;
50+
51+
@Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = true, description = "The name of the Instance Boot Group")
52+
private String name;
53+
54+
@Parameter(name = ApiConstants.DESCRIPTION, type = CommandType.STRING, description = "The description of the Instance Boot Group")
55+
private String description;
56+
57+
@Parameter(name = ApiConstants.ACCOUNT, type = CommandType.STRING, description = "The account of the Instance Boot Group. Must be used with domainid")
58+
private String accountName;
59+
60+
@Parameter(name = ApiConstants.DOMAIN_ID, type = CommandType.UUID, entityType = DomainResponse.class, description = "The domain ID of the account owning the Instance Boot Group")
61+
private Long domainId;
62+
63+
@Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "The project of the Instance Boot Group")
64+
private Long projectId;
65+
66+
@Parameter(name = ApiConstants.READINESS_ATTEMPT_TIMEOUT_SECONDS, type = CommandType.LONG,
67+
description = "Per-boot-group override of the global timeout (seconds) for each readiness retry attempt")
68+
private Long readinessAttemptTimeoutSeconds;
69+
70+
@Parameter(name = ApiConstants.READINESS_MAX_RETRY_ATTEMPTS, type = CommandType.LONG,
71+
description = "Per-boot-group override of the global maximum number of readiness retry attempts")
72+
private Long readinessMaxRetryAttempts;
73+
74+
@Parameter(name = ApiConstants.READINESS_REBOOT_ON_RETRY, type = CommandType.BOOLEAN,
75+
description = "Per-boot-group override of whether an instance is rebooted between readiness retry attempts")
76+
private Boolean readinessRebootOnRetry;
77+
78+
@Parameter(name = ApiConstants.READINESS_INITIAL_DELAY_SECONDS, type = CommandType.LONG,
79+
description = "Per-boot-group override of the global delay (seconds) after starting or rebooting an instance before its first readiness check of that attempt")
80+
private Long readinessInitialDelaySeconds;
81+
82+
public String getName() {
83+
return name;
84+
}
85+
86+
public String getDescription() {
87+
return description;
88+
}
89+
90+
public String getAccountName() {
91+
return accountName;
92+
}
93+
94+
public Long getDomainId() {
95+
return domainId;
96+
}
97+
98+
public Long getProjectId() {
99+
return projectId;
100+
}
101+
102+
public Long getReadinessAttemptTimeoutSeconds() {
103+
return readinessAttemptTimeoutSeconds;
104+
}
105+
106+
public Long getReadinessMaxRetryAttempts() {
107+
return readinessMaxRetryAttempts;
108+
}
109+
110+
public Boolean getReadinessRebootOnRetry() {
111+
return readinessRebootOnRetry;
112+
}
113+
114+
public Long getReadinessInitialDelaySeconds() {
115+
return readinessInitialDelaySeconds;
116+
}
117+
118+
@Override
119+
public long getEntityOwnerId() {
120+
Long accountId = _accountService.finalizeAccountId(accountName, domainId, projectId, true);
121+
if (accountId == null) {
122+
return CallContext.current().getCallingAccount().getId();
123+
}
124+
return accountId;
125+
}
126+
127+
@Override
128+
public ApiCommandResourceType getApiResourceType() {
129+
return ApiCommandResourceType.InstanceBootGroup;
130+
}
131+
132+
@Override
133+
public void execute() {
134+
InstanceBootGroup result = instanceBootGroupService.createInstanceBootGroup(this);
135+
if (result == null) {
136+
throw new ServerApiException(ApiErrorCode.INTERNAL_ERROR, "Failed to create instance boot group");
137+
}
138+
InstanceBootGroupResponse response = instanceBootGroupService.createInstanceBootGroupResponse(result.getId());
139+
response.setResponseName(getCommandName());
140+
setResponseObject(response);
141+
}
142+
}

0 commit comments

Comments
 (0)