diff --git a/docs/querying/sql-metadata-tables.md b/docs/querying/sql-metadata-tables.md index 6d59462fd1e9..271190f9a48c 100644 --- a/docs/querying/sql-metadata-tables.md +++ b/docs/querying/sql-metadata-tables.md @@ -238,6 +238,9 @@ Servers table lists all discovered servers in the cluster. |start_time|STRING|Timestamp in ISO8601 format when the server was announced in the cluster| |version|VARCHAR|Druid version running on the server| |labels|VARCHAR|Labels for the server configured using the property [`druid.labels`](../configuration/index.md)| +|available_processors|BIGINT|Total number of CPU processors available to the server| +|total_memory|BIGINT|Total memory in bytes available to the server| + To retrieve information about all servers, use the query: ```sql diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java index 429c4e315595..d33c1891b0d2 100644 --- a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java +++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/auth/AbstractAuthConfigurationTest.java @@ -1030,9 +1030,8 @@ private static String getServerUrl(EmbeddedDruidServer server) /** * curr_size on historicals changes because cluster state is not isolated across - * different - * integration tests, zero it out for consistent test results - * version and start_time are not configurable therefore we zero them as well + * different integration tests, zero it out for consistent test results. + * version, start_time, available_processors, total_memory are not configurable therefore we zero them as well */ protected static List> getServersWithoutNonConfigurableFields(List> servers) { @@ -1043,6 +1042,8 @@ protected static List> getServersWithoutNonConfigurableField newServer.put("curr_size", 0); newServer.put("start_time", "0"); newServer.put("version", "0.0.0"); + newServer.put("available_processors", 0); + newServer.put("total_memory", 0); return newServer; } ); diff --git a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json index ddbd92e3111e..1e7f9474cd3a 100644 --- a/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json +++ b/integration-tests/src/test/resources/results/auth_test_sys_schema_servers.json @@ -11,7 +11,9 @@ "is_leader": null, "start_time": "0", "version": "0.0.0", - "labels": null + "labels": null, + "available_processors": 0, + "total_memory": 0 }, { "server": "%%BROKER%%:8282", @@ -25,6 +27,8 @@ "is_leader": null, "start_time": "0", "version": "0.0.0", - "labels": null + "labels": null, + "available_processors": 0, + "total_memory": 0 } ] diff --git a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java index c5c502ca156f..57814fe23723 100644 --- a/processing/src/main/java/org/apache/druid/utils/JvmUtils.java +++ b/processing/src/main/java/org/apache/druid/utils/JvmUtils.java @@ -21,6 +21,7 @@ import com.google.common.primitives.Ints; import com.google.inject.Inject; +import com.sun.management.OperatingSystemMXBean; import java.io.File; import java.lang.management.ManagementFactory; @@ -44,6 +45,7 @@ public class JvmUtils private static RuntimeInfo RUNTIME_INFO = new RuntimeInfo(); private static final ThreadMXBean THREAD_MX_BEAN = ManagementFactory.getThreadMXBean(); + private static final OperatingSystemMXBean OPERATING_SYSTEM_MX_BEAN = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); private static int computeMajorVersion() { @@ -139,4 +141,15 @@ public static List systemClassPath() ).collect(Collectors.toList()); return jobURLs; } + + /** + * Get the total memory of the machine it is running on. This function is container aware. + * If the machine is running in a container, the function will return the total memory of the container. + * If the machine is not running in a container, the function will return the total memory of the machine. + * @return the total memory of the machine it is running on in bytes. + */ + public static long getTotalMemory() + { + return OPERATING_SYSTEM_MX_BEAN.getTotalPhysicalMemorySize(); + } } diff --git a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java index 3a62e5344b11..7d24c203b2e0 100644 --- a/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java +++ b/server/src/main/java/org/apache/druid/discovery/DiscoveryDruidNode.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import org.apache.druid.client.DruidServer; import org.apache.druid.jackson.StringObjectPairList; @@ -32,6 +33,7 @@ import org.apache.druid.java.util.common.NonnullPair; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.DruidNode; +import org.apache.druid.utils.JvmUtils; import org.joda.time.DateTime; import javax.annotation.Nullable; @@ -50,10 +52,13 @@ public class DiscoveryDruidNode { private static final Logger LOG = new Logger(DiscoveryDruidNode.class); + private static final int UNKNOWN_VALUE = -1; private final DruidNode druidNode; private final NodeRole nodeRole; private final DateTime startTime; + private final Integer availableProcessors; + private final Long totalMemory; /** * Map of service name -> DruidServices. @@ -65,20 +70,36 @@ public class DiscoveryDruidNode */ private final Map services = new HashMap<>(); + /** + * Constructor for tests. In production, the @Inject constructor is used instead. + */ + @VisibleForTesting + public DiscoveryDruidNode( + DruidNode druidNode, + NodeRole nodeRole, + Map services, + DateTime startTime + ) + { + this(druidNode, nodeRole, services, startTime, Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory()); + } + public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, Map services ) { - this(druidNode, nodeRole, services, DateTimes.nowUtc()); + this(druidNode, nodeRole, services, DateTimes.nowUtc(), Runtime.getRuntime().availableProcessors(), JvmUtils.getTotalMemory()); } public DiscoveryDruidNode( DruidNode druidNode, NodeRole nodeRole, Map services, - DateTime startTime + DateTime startTime, + Integer availableProcessors, + Long totalMemory ) { this.druidNode = druidNode; @@ -88,6 +109,10 @@ public DiscoveryDruidNode( this.services.putAll(services); } this.startTime = startTime; + + // Happens if service is running older version of Druid + this.availableProcessors = availableProcessors != null ? availableProcessors : UNKNOWN_VALUE; + this.totalMemory = totalMemory != null ? totalMemory : UNKNOWN_VALUE; } @JsonCreator @@ -96,6 +121,8 @@ private static DiscoveryDruidNode fromJson( @JsonProperty("nodeType") NodeRole nodeRole, @JsonProperty("services") Map rawServices, @JsonProperty("startTime") DateTime startTime, + @JsonProperty("availableProcessors") Integer availableProcessors, + @JsonProperty("totalMemory") Long totalMemory, @JacksonInject ObjectMapper jsonMapper ) { @@ -111,7 +138,7 @@ private static DiscoveryDruidNode fromJson( } } } - return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime); + return new DiscoveryDruidNode(druidNode, nodeRole, services, startTime, availableProcessors, totalMemory); } /** @@ -188,6 +215,18 @@ public DateTime getStartTime() return startTime; } + @JsonProperty + public Integer getAvailableProcessors() + { + return availableProcessors; + } + + @JsonProperty + public Long getTotalMemory() + { + return totalMemory; + } + @Nullable @JsonIgnore public T getService(String key, Class clazz) @@ -235,13 +274,15 @@ public boolean equals(Object o) DiscoveryDruidNode that = (DiscoveryDruidNode) o; return Objects.equals(druidNode, that.druidNode) && Objects.equals(nodeRole, that.nodeRole) && - Objects.equals(services, that.services); + Objects.equals(services, that.services) && + Objects.equals(availableProcessors, that.availableProcessors) && + Objects.equals(totalMemory, that.totalMemory); } @Override public int hashCode() { - return Objects.hash(druidNode, nodeRole, services); + return Objects.hash(druidNode, nodeRole, services, availableProcessors, totalMemory); } @Override @@ -252,6 +293,8 @@ public String toString() ", nodeRole='" + nodeRole + '\'' + ", services=" + services + '\'' + ", startTime=" + startTime + + ", availableProcessors=" + availableProcessors + + ", totalMemory=" + totalMemory + '}'; } } diff --git a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java index b9c99a4fcb7d..9f6e13f72d89 100644 --- a/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java +++ b/server/src/test/java/org/apache/druid/discovery/DiscoveryDruidNodeTest.java @@ -58,7 +58,7 @@ public DiscoveryDruidNodeTest() public void testEquals() { EqualsVerifier.forClass(DiscoveryDruidNode.class) - .withNonnullFields("druidNode", "nodeRole", "services") + .withNonnullFields("druidNode", "nodeRole", "services", "availableProcessors", "totalMemory") .withIgnoredFields("startTime") .usingGetClass() .verify(); @@ -156,7 +156,9 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 3,\n" + + " \"totalMemory\" : 1234\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -175,7 +177,10 @@ public void testDeserializeWithDataNodeServiceWithAWrongPropertyOrder() throws J ImmutableMap.of( "dataNodeService", new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0) - ) + ), + null, + 3, + 1234L ), mapper.readValue(json, DiscoveryDruidNode.class) ); @@ -206,7 +211,9 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 6,\n" + + " \"totalMemory\" : 5432\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -225,7 +232,10 @@ public void testDeserialize_duplicateProperties_shouldSucceedToDeserialize() thr ImmutableMap.of( "dataNodeService", new DataNodeService("_default_tier", 1000000000, ServerType.BROKER, 0) - ) + ), + null, + 6, + 5432L ), mapper.readValue(json, DiscoveryDruidNode.class) ); @@ -257,7 +267,9 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode + " \"serverType\" : \"broker\",\n" + " \"priority\" : 0\n" + " }\n" - + " }\n" + + " },\n" + + " \"availableProcessors\" : 4,\n" + + " \"totalMemory\" : 246810\n" + "}"; Assert.assertEquals( new DiscoveryDruidNode( @@ -273,7 +285,10 @@ public void testDeserialize_duplicateKeysWithDifferentValus_shouldIgnoreDataNode null ), NodeRole.BROKER, - ImmutableMap.of() + ImmutableMap.of(), + null, + 4, + 246810L ), mapper.readValue(json, DiscoveryDruidNode.class) ); diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java index 1343ace28fe4..47b769f29467 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/schema/SystemSchema.java @@ -183,6 +183,8 @@ public class SystemSchema extends AbstractSchema .add("start_time", ColumnType.STRING) .add("version", ColumnType.STRING) .add("labels", ColumnType.STRING) + .add("available_processors", ColumnType.LONG) + .add("total_memory", ColumnType.LONG) .build(); static final RowSignature SERVER_SEGMENTS_SIGNATURE = RowSignature @@ -648,7 +650,9 @@ private Object[] buildRowForNonDataServer(DiscoveryDruidNode discoveryDruidNode) null, toStringOrNull(discoveryDruidNode.getStartTime()), node.getVersion(), - node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()) + node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } @@ -673,7 +677,9 @@ private Object[] buildRowForNonDataServerWithLeadership( isLeader ? 1L : 0L, toStringOrNull(discoveryDruidNode.getStartTime()), node.getVersion(), - node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()) + node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } @@ -710,7 +716,9 @@ private Object[] buildRowForDiscoverableDataServer( null, toStringOrNull(discoveryDruidNode.getStartTime()), node.getVersion(), - node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()) + node.getLabels() == null ? null : JacksonUtils.writeValueAsString(jsonMapper, node.getLabels()), + (long) discoveryDruidNode.getAvailableProcessors(), + discoveryDruidNode.getTotalMemory() }; } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java index dd5af77e92f9..3fe3068a370d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/schema/SystemSchemaTest.java @@ -111,6 +111,7 @@ import org.apache.druid.timeline.SegmentStatusInCluster; import org.apache.druid.timeline.partition.NumberedShardSpec; import org.apache.druid.timeline.partition.ShardSpec; +import org.apache.druid.utils.JvmUtils; import org.easymock.EasyMock; import org.jboss.netty.handler.codec.http.HttpResponse; import org.joda.time.DateTime; @@ -381,6 +382,8 @@ public void setUp(@TempDir File tmpDir) throws Exception final List realtimeSegments = ImmutableList.of(segment2, segment4, segment5); private final DateTime startTime = DateTimes.nowUtc(); + private final long availableProcessors = Runtime.getRuntime().availableProcessors(); + private final long totalMemory = JvmUtils.getTotalMemory(); private final String version = GuavaUtils.firstNonNull( SystemSchemaTest.class.getPackage().getImplementationVersion(), @@ -558,7 +561,7 @@ public void testGetTableMap() final SystemSchema.ServersTable serversTable = (SystemSchema.ServersTable) schema.getTableMap().get("servers"); final RelDataType serverRowType = serversTable.getRowType(new JavaTypeFactoryImpl()); final List serverFields = serverRowType.getFieldList(); - Assert.assertEquals(12, serverFields.size()); + Assert.assertEquals(14, serverFields.size()); Assert.assertEquals("server", serverFields.get(0).getName()); Assert.assertEquals(SqlTypeName.VARCHAR, serverFields.get(0).getType().getSqlTypeName()); } @@ -860,7 +863,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -876,7 +881,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -892,7 +899,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -908,7 +917,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -924,7 +935,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add(createExpectedRow( @@ -939,7 +952,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory )); expectedRows.add( createExpectedRow( @@ -954,7 +969,9 @@ public void testServersTable() throws URISyntaxException 1L, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -970,7 +987,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - "{\"brokerKey\":\"brokerValue\",\"brokerKey2\":\"brokerValue2\"}" + "{\"brokerKey\":\"brokerValue\",\"brokerKey2\":\"brokerValue2\"}", + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -986,7 +1005,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1002,7 +1023,9 @@ public void testServersTable() throws URISyntaxException 1L, startTimeStr, version, - "{\"overlordKey\":\"overlordValue\"}" + "{\"overlordKey\":\"overlordValue\"}", + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1018,7 +1041,9 @@ public void testServersTable() throws URISyntaxException 0L, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1034,7 +1059,9 @@ public void testServersTable() throws URISyntaxException 0L, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1050,7 +1077,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add( @@ -1066,7 +1095,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory ) ); expectedRows.add(createExpectedRow( @@ -1081,7 +1112,9 @@ public void testServersTable() throws URISyntaxException nonLeader, startTimeStr, version, - null + null, + availableProcessors, + totalMemory )); Assert.assertEquals(expectedRows.size(), rows.size()); for (int i = 0; i < rows.size(); i++) { @@ -1116,7 +1149,9 @@ private Object[] createExpectedRow( @Nullable Long isLeader, String startTime, String version, - String labels + String labels, + long availableProcessors, + long totalMemory ) { return new Object[]{ @@ -1131,7 +1166,9 @@ private Object[] createExpectedRow( isLeader, startTime, version, - labels + labels, + availableProcessors, + totalMemory }; } diff --git a/web-console/src/utils/general.tsx b/web-console/src/utils/general.tsx index 3547738ef80d..f206ed024575 100644 --- a/web-console/src/utils/general.tsx +++ b/web-console/src/utils/general.tsx @@ -292,8 +292,8 @@ export function formatRate(n: NumberLike) { return numeral(n).format('0,0.0') + '/s'; } -export function formatBytes(n: NumberLike): string { - return numeral(n).format('0.00 b'); +export function formatBytes(n: NumberLike, useBinaryBytes = false): string { + return numeral(n).format(useBinaryBytes ? '0.00 ib' : '0.00 b'); } export function formatByteRate(n: NumberLike): string { diff --git a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap index 38e99faa1d20..44aed574fa07 100644 --- a/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap +++ b/web-console/src/views/services-view/__snapshots__/services-view.spec.tsx.snap @@ -59,6 +59,8 @@ exports[`ServicesView renders data 1`] = ` "Usage", "Start time", "Version", + "CPU processors", + "Total memory", "Labels", "Detail", ] @@ -220,6 +222,26 @@ exports[`ServicesView renders data 1`] = ` "show": true, "width": 200, }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "CPU processors", + "accessor": "available_processors", + "className": "padded", + "filterable": false, + "show": true, + "width": 120, + }, + { + "Aggregated": [Function], + "Cell": [Function], + "Header": "Total memory", + "accessor": "total_memory", + "className": "padded", + "filterable": false, + "show": true, + "width": 120, + }, { "Aggregated": [Function], "Cell": [Function], diff --git a/web-console/src/views/services-view/services-view.tsx b/web-console/src/views/services-view/services-view.tsx index f0d97f356127..867d7337c4f6 100644 --- a/web-console/src/views/services-view/services-view.tsx +++ b/web-console/src/views/services-view/services-view.tsx @@ -89,6 +89,8 @@ const TABLE_COLUMNS_BY_MODE: Record '', }, + { + Header: 'CPU processors', + show: visibleColumns.shown('CPU processors'), + accessor: 'available_processors', + className: 'padded', + filterable: false, + width: 120, + Cell: ({ value }) => (value === null ? '' : value), + Aggregated: ({ subRows }) => { + const originalRows: ServiceResultRow[] = subRows.map(r => r._original); + const totalAvailableProcessors = sum(originalRows, s => s.available_processors); + return totalAvailableProcessors; + }, + }, + { + Header: 'Total memory', + show: visibleColumns.shown('Total memory'), + accessor: 'total_memory', + className: 'padded', + width: 120, + filterable: false, + Cell: ({ value }) => { + if (value === null) return ''; + return formatBytes(value, true); + }, + Aggregated: ({ subRows }) => { + const originalRows: ServiceResultRow[] = subRows.map(r => r._original); + const totalMemory = sum(originalRows, s => s.total_memory); + return formatBytes(totalMemory, true); + }, + }, { Header: 'Labels', show: visibleColumns.shown('Labels'), diff --git a/website/.spelling b/website/.spelling index 2eaa410bace9..6c4357e824a6 100644 --- a/website/.spelling +++ b/website/.spelling @@ -716,6 +716,8 @@ is_realtime java.sql.Types last_compaction_state max_size +available_processors +total_memory num_replicas num_rows num_segments