Skip to content

Commit 389a9ef

Browse files
rootnomadrootnomad
committed
metrics: add host usage history
Store host stats samples in a new host_stats table and expose them through a new listHostsUsageHistory API, the host counterpart of listVirtualMachinesUsageHistory, listSystemVmsUsageHistory and listVolumesUsageHistory. Samples are written on every host stats collection and pruned by the new host.stats.max.retention.time setting (minutes, default 720). Setting it to 0 or less disables storing and pruning.
1 parent 602d9ec commit 389a9ef

14 files changed

Lines changed: 924 additions & 0 deletions

File tree

‎core/src/main/java/com/cloud/agent/api/HostStatsEntryBase.java‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
//
2+
// Licensed to the Apache Software Foundation (ASF) under one
3+
// or more contributor license agreements. See the NOTICE file
4+
// distributed with this work for additional information
5+
// regarding copyright ownership. The ASF licenses this file
6+
// to you under the Apache License, Version 2.0 (the
7+
// "License"); you may not use this file except in compliance
8+
// with the License. You may obtain a copy of the License at
9+
//
10+
// http://www.apache.org/licenses/LICENSE-2.0
11+
//
12+
// Unless required by applicable law or agreed to in writing,
13+
// software distributed under the License is distributed on an
14+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
// KIND, either express or implied. See the License for the
16+
// specific language governing permissions and limitations
17+
// under the License.
18+
//
19+
20+
package com.cloud.agent.api;
21+
22+
import com.cloud.host.HostStats;
23+
24+
/**
25+
* Serializable host-stats payload persisted as JSON in {@code host_stats.host_stats_data}. Unlike
26+
* {@link HostStatsEntry} it carries no {@code HostVO}, so serialization does not pull in a host entity.
27+
*/
28+
public class HostStatsEntryBase implements HostStats {
29+
30+
private long hostId;
31+
private String entityType;
32+
private double cpuUtilization;
33+
private double averageLoad;
34+
private double networkReadKBs;
35+
private double networkWriteKBs;
36+
private double totalMemoryKBs;
37+
private double freeMemoryKBs;
38+
39+
public HostStatsEntryBase() {
40+
}
41+
42+
public HostStatsEntryBase(long hostId, String entityType, double cpuUtilization, double averageLoad,
43+
double networkReadKBs, double networkWriteKBs, double totalMemoryKBs, double freeMemoryKBs) {
44+
this.hostId = hostId;
45+
this.entityType = entityType;
46+
this.cpuUtilization = cpuUtilization;
47+
this.averageLoad = averageLoad;
48+
this.networkReadKBs = networkReadKBs;
49+
this.networkWriteKBs = networkWriteKBs;
50+
this.totalMemoryKBs = totalMemoryKBs;
51+
this.freeMemoryKBs = freeMemoryKBs;
52+
}
53+
54+
public long getHostId() {
55+
return hostId;
56+
}
57+
58+
public void setHostId(long hostId) {
59+
this.hostId = hostId;
60+
}
61+
62+
@Override
63+
public String getEntityType() {
64+
return entityType;
65+
}
66+
67+
public void setEntityType(String entityType) {
68+
this.entityType = entityType;
69+
}
70+
71+
@Override
72+
public double getCpuUtilization() {
73+
return cpuUtilization;
74+
}
75+
76+
public void setCpuUtilization(double cpuUtilization) {
77+
this.cpuUtilization = cpuUtilization;
78+
}
79+
80+
@Override
81+
public double getLoadAverage() {
82+
return averageLoad;
83+
}
84+
85+
public void setAverageLoad(double averageLoad) {
86+
this.averageLoad = averageLoad;
87+
}
88+
89+
@Override
90+
public double getNetworkReadKBs() {
91+
return networkReadKBs;
92+
}
93+
94+
public void setNetworkReadKBs(double networkReadKBs) {
95+
this.networkReadKBs = networkReadKBs;
96+
}
97+
98+
@Override
99+
public double getNetworkWriteKBs() {
100+
return networkWriteKBs;
101+
}
102+
103+
public void setNetworkWriteKBs(double networkWriteKBs) {
104+
this.networkWriteKBs = networkWriteKBs;
105+
}
106+
107+
@Override
108+
public double getTotalMemoryKBs() {
109+
return totalMemoryKBs;
110+
}
111+
112+
public void setTotalMemoryKBs(double totalMemoryKBs) {
113+
this.totalMemoryKBs = totalMemoryKBs;
114+
}
115+
116+
@Override
117+
public double getFreeMemoryKBs() {
118+
return freeMemoryKBs;
119+
}
120+
121+
public void setFreeMemoryKBs(double freeMemoryKBs) {
122+
this.freeMemoryKBs = freeMemoryKBs;
123+
}
124+
125+
@Override
126+
public double getUsedMemory() {
127+
return (totalMemoryKBs - freeMemoryKBs) * 1024;
128+
}
129+
130+
@Override
131+
public HostStats getHostStats() {
132+
return this;
133+
}
134+
}

‎engine/schema/src/main/java/com/cloud/host/HostStatsVO.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.host;
18+
19+
import java.util.Date;
20+
21+
import javax.persistence.Column;
22+
import javax.persistence.Entity;
23+
import javax.persistence.Id;
24+
import javax.persistence.Table;
25+
import javax.persistence.Temporal;
26+
import javax.persistence.TemporalType;
27+
28+
import org.apache.cloudstack.utils.reflectiontostringbuilderutils.ReflectionToStringBuilderUtils;
29+
30+
/** One persisted historical host-stats sample. */
31+
@Entity
32+
@Table(name = "host_stats")
33+
public class HostStatsVO {
34+
35+
@Id
36+
@Column(name = "id", updatable = false, nullable = false)
37+
protected long id;
38+
39+
@Column(name = "host_id", updatable = false, nullable = false)
40+
protected Long hostId;
41+
42+
@Column(name = "mgmt_server_id", updatable = false, nullable = false)
43+
protected Long mgmtServerId;
44+
45+
@Column(name = "timestamp", updatable = false)
46+
@Temporal(value = TemporalType.TIMESTAMP)
47+
protected Date timestamp;
48+
49+
@Column(name = "host_stats_data", updatable = false, nullable = false, length = 65535)
50+
protected String hostStatsData;
51+
52+
public HostStatsVO(Long hostId, Long mgmtServerId, Date timestamp, String hostStatsData) {
53+
this.hostId = hostId;
54+
this.mgmtServerId = mgmtServerId;
55+
this.timestamp = timestamp;
56+
this.hostStatsData = hostStatsData;
57+
}
58+
59+
public HostStatsVO() {
60+
61+
}
62+
63+
public long getId() {
64+
return id;
65+
}
66+
67+
public Long getHostId() {
68+
return hostId;
69+
}
70+
71+
public Long getMgmtServerId() {
72+
return mgmtServerId;
73+
}
74+
75+
public Date getTimestamp() {
76+
return timestamp;
77+
}
78+
79+
public String getHostStatsData() {
80+
return hostStatsData;
81+
}
82+
83+
@Override
84+
public String toString() {
85+
return ReflectionToStringBuilderUtils.reflectOnlySelectedFields(this, "hostId", "mgmtServerId", "timestamp", "hostStatsData");
86+
}
87+
88+
}

‎engine/schema/src/main/java/com/cloud/host/dao/HostStatsDao.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.host.dao;
18+
19+
import java.util.Date;
20+
import java.util.List;
21+
22+
import com.cloud.host.HostStatsVO;
23+
import com.cloud.utils.db.GenericDao;
24+
25+
/** DAO for the host_stats table. */
26+
public interface HostStatsDao extends GenericDao<HostStatsVO, Long> {
27+
28+
List<HostStatsVO> findByHostId(long hostId);
29+
30+
List<HostStatsVO> findByHostIdAndTimestampGreaterThanEqual(long hostId, Date time);
31+
32+
List<HostStatsVO> findByHostIdAndTimestampLessThanEqual(long hostId, Date time);
33+
34+
List<HostStatsVO> findByHostIdAndTimestampBetween(long hostId, Date startTime, Date endTime);
35+
36+
/**
37+
* Expunges all host stats older than {@code limitDate}.
38+
* @param limitPerQuery max rows removed per query; 0 or negative means no limit.
39+
*/
40+
void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery);
41+
42+
}

‎engine/schema/src/main/java/com/cloud/host/dao/HostStatsDaoImpl.java‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
package com.cloud.host.dao;
18+
19+
import java.util.Date;
20+
import java.util.List;
21+
22+
import javax.annotation.PostConstruct;
23+
24+
import org.apache.logging.log4j.LogManager;
25+
import org.apache.logging.log4j.Logger;
26+
import org.springframework.stereotype.Component;
27+
28+
import com.cloud.host.HostStatsVO;
29+
import com.cloud.utils.db.GenericDaoBase;
30+
import com.cloud.utils.db.SearchBuilder;
31+
import com.cloud.utils.db.SearchCriteria;
32+
import com.cloud.utils.db.SearchCriteria.Op;
33+
34+
/** DAO for the host_stats table. */
35+
@Component
36+
public class HostStatsDaoImpl extends GenericDaoBase<HostStatsVO, Long> implements HostStatsDao {
37+
38+
protected Logger logger = LogManager.getLogger(getClass());
39+
40+
protected SearchBuilder<HostStatsVO> hostIdSearch;
41+
protected SearchBuilder<HostStatsVO> hostIdTimestampGreaterThanEqualSearch;
42+
protected SearchBuilder<HostStatsVO> hostIdTimestampLessThanEqualSearch;
43+
protected SearchBuilder<HostStatsVO> hostIdTimestampBetweenSearch;
44+
protected SearchBuilder<HostStatsVO> timestampSearch;
45+
46+
@PostConstruct
47+
protected void init() {
48+
hostIdSearch = createSearchBuilder();
49+
hostIdSearch.and("hostId", hostIdSearch.entity().getHostId(), Op.EQ);
50+
hostIdSearch.done();
51+
52+
hostIdTimestampGreaterThanEqualSearch = createSearchBuilder();
53+
hostIdTimestampGreaterThanEqualSearch.and("hostId", hostIdTimestampGreaterThanEqualSearch.entity().getHostId(), Op.EQ);
54+
hostIdTimestampGreaterThanEqualSearch.and("timestamp", hostIdTimestampGreaterThanEqualSearch.entity().getTimestamp(), Op.GTEQ);
55+
hostIdTimestampGreaterThanEqualSearch.done();
56+
57+
hostIdTimestampLessThanEqualSearch = createSearchBuilder();
58+
hostIdTimestampLessThanEqualSearch.and("hostId", hostIdTimestampLessThanEqualSearch.entity().getHostId(), Op.EQ);
59+
hostIdTimestampLessThanEqualSearch.and("timestamp", hostIdTimestampLessThanEqualSearch.entity().getTimestamp(), Op.LTEQ);
60+
hostIdTimestampLessThanEqualSearch.done();
61+
62+
hostIdTimestampBetweenSearch = createSearchBuilder();
63+
hostIdTimestampBetweenSearch.and("hostId", hostIdTimestampBetweenSearch.entity().getHostId(), Op.EQ);
64+
hostIdTimestampBetweenSearch.and("timestamp", hostIdTimestampBetweenSearch.entity().getTimestamp(), Op.BETWEEN);
65+
hostIdTimestampBetweenSearch.done();
66+
67+
timestampSearch = createSearchBuilder();
68+
timestampSearch.and("timestamp", timestampSearch.entity().getTimestamp(), Op.LT);
69+
timestampSearch.done();
70+
}
71+
72+
@Override
73+
public List<HostStatsVO> findByHostId(long hostId) {
74+
SearchCriteria<HostStatsVO> sc = hostIdSearch.create();
75+
sc.setParameters("hostId", hostId);
76+
return listBy(sc);
77+
}
78+
79+
@Override
80+
public List<HostStatsVO> findByHostIdAndTimestampGreaterThanEqual(long hostId, Date time) {
81+
SearchCriteria<HostStatsVO> sc = hostIdTimestampGreaterThanEqualSearch.create();
82+
sc.setParameters("hostId", hostId);
83+
sc.setParameters("timestamp", time);
84+
return listBy(sc);
85+
}
86+
87+
@Override
88+
public List<HostStatsVO> findByHostIdAndTimestampLessThanEqual(long hostId, Date time) {
89+
SearchCriteria<HostStatsVO> sc = hostIdTimestampLessThanEqualSearch.create();
90+
sc.setParameters("hostId", hostId);
91+
sc.setParameters("timestamp", time);
92+
return listBy(sc);
93+
}
94+
95+
@Override
96+
public List<HostStatsVO> findByHostIdAndTimestampBetween(long hostId, Date startTime, Date endTime) {
97+
SearchCriteria<HostStatsVO> sc = hostIdTimestampBetweenSearch.create();
98+
sc.setParameters("hostId", hostId);
99+
sc.setParameters("timestamp", startTime, endTime);
100+
return listBy(sc);
101+
}
102+
103+
@Override
104+
public void removeAllByTimestampLessThan(Date limitDate, long limitPerQuery) {
105+
SearchCriteria<HostStatsVO> sc = timestampSearch.create();
106+
sc.setParameters("timestamp", limitDate);
107+
108+
logger.debug(String.format("Starting to remove all host_stats rows older than [%s].", limitDate));
109+
110+
long totalRemoved = batchExpunge(sc, limitPerQuery);
111+
112+
logger.info(String.format("Removed a total of [%s] host_stats rows older than [%s].", totalRemoved, limitDate));
113+
}
114+
115+
}

‎engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@
227227
<bean id="vMRootDiskTagDaoImpl" class="org.apache.cloudstack.engine.cloud.entity.api.db.dao.VMRootDiskTagDaoImpl" />
228228
<bean id="vMSnapshotDetailsDaoImpl" class="com.cloud.vm.snapshot.dao.VMSnapshotDetailsDaoImpl" />
229229
<bean id="vmStatsDaoImpl" class="com.cloud.vm.dao.VmStatsDaoImpl" />
230+
<bean id="hostStatsDaoImpl" class="com.cloud.host.dao.HostStatsDaoImpl" />
230231
<bean id="vMTemplateDetailsDaoImpl" class="com.cloud.storage.dao.VMTemplateDetailsDaoImpl" />
231232
<bean id="vMTemplatePoolDaoImpl" class="com.cloud.storage.dao.VMTemplatePoolDaoImpl" />
232233
<bean id="vMTemplateZoneDaoImpl" class="com.cloud.storage.dao.VMTemplateZoneDaoImpl" />

0 commit comments

Comments
 (0)