EHT-1238: Granular metrics

lustre-collector/src/llite/mod.rs

@@ -13,10 +13,7 @@ pub(crate) const LLITE: &str = "llite";
 pub(crate) const STATS: &str = "stats";
 
 pub(crate) fn params() -> Vec<String> {
-    [STATS]
-        .into_iter()
-        .map(|x| format!("{LLITE}.*.{x}"))
-        .collect()
+    vec![format!("{LLITE}.*.{STATS}")]
 }
 
 fn target_name<I>() -> impl Parser<I, Output = Target>

lustre-collector/src/osd_parser.rs

@@ -6,7 +6,7 @@ use crate::{
     QuotaKind, QuotaStatsOsd,
     base_parsers::{digits, param, period, target, till_newline, till_period},
     brw_stats_parser::brw_stats,
-    quota::quota_parser::quota_stats_osd,
+    quota::quota_stats_osd,
     stats_parser::stats,
     types::{BrwStats, Param, Record, Stat, Target, TargetStat, TargetStats, TargetVariant},
 };

lustre-collector/src/parser.rs

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a MIT-style
 // license that can be found in the LICENSE file.
 
+use std::collections::BTreeSet;
+
 use crate::{
     ldlm, llite, mdd_parser,
     mds::{self, client_count_parser},
@@ -11,22 +13,61 @@ use crate::{
 };
 use combine::{Parser, Stream, choice, error::ParseError, many};
 
-pub fn params() -> Vec<String> {
-    top_level_parser::top_level_params()
+#[derive(Copy, Clone)]
+pub enum Component {
+    Clients,
+    Osd,
+    Mgs,
+    Oss,
+    Mds,
+    Ldlm,
+    Llite,
+    Mdd,
+    Quota,
+    Nodemap,
+}
+
+pub fn get_params(components: &[Component]) -> Vec<String> {
+    components
+        .iter()
+        .fold(
+            BTreeSet::from_iter(top_level_parser::top_level_params()),
+            |mut acc, e| {
+                acc.extend(match e {
+                    Component::Clients => client_count_parser::params(),
+                    Component::Osd => osd_parser::params(),
+                    Component::Mgs => mgs_parser::params(),
+                    Component::Oss => oss::params(),
+                    Component::Mds => mds::params(),
+                    Component::Ldlm => ldlm::params(),
+                    Component::Llite => llite::params(),
+                    Component::Mdd => mdd_parser::params(),
+                    Component::Quota => quota::params(),
+                    Component::Nodemap => nodemap::params(),
+                });
+
+                acc
+            },
+        )
         .into_iter()
-        .chain(client_count_parser::params())
-        .chain(osd_parser::params())
-        .chain(mgs_parser::params())
-        .chain(oss::params())
-        .chain(mds::params())
-        .chain(ldlm::params())
-        .chain(llite::params())
-        .chain(mdd_parser::params())
-        .chain(quota::params())
-        .chain(nodemap::params())
         .collect()
 }
 
+pub fn params() -> Vec<String> {
+    get_params(&[
+        Component::Clients,
+        Component::Osd,
+        Component::Mgs,
+        Component::Oss,
+        Component::Mds,
+        Component::Ldlm,
+        Component::Llite,
+        Component::Mdd,
+        Component::Quota,
+        Component::Nodemap,
+    ])
+}
+
 pub fn parse<I>() -> impl Parser<I, Output = Vec<Record>>
 where
     I: Stream<Token = char>,

lustre-collector/src/quota/quota_parser.rs → lustre-collector/src/quota.rs

@@ -6,32 +6,37 @@ use crate::{
     QuotaKind, QuotaStat, QuotaStatLimits, QuotaStatOsd, QuotaStatUsage, QuotaStats,
     TargetQuotaStat,
     base_parsers::{digits, param, period, target},
-    quota::QMT,
     types::{Param, Record, Target, TargetStats},
 };
 use combine::{
-    Parser, Stream, between, choice, eof,
-    error::ParseError,
-    many1, optional,
+    ParseError, Parser, Stream, between, choice, eof, many1, optional,
     parser::{
         char::{newline, spaces, string},
         repeat::take_until,
     },
     token,
 };
 
+pub(crate) const QMT: &str = "qmt";
+
 pub(crate) const USR_QUOTAS: &str = "usr";
 pub(crate) const PRJ_QUOTAS: &str = "prj";
 pub(crate) const GRP_QUOTAS: &str = "grp";
-pub(crate) const QMT_STATS: [&str; 3] = [USR_QUOTAS, PRJ_QUOTAS, GRP_QUOTAS];
+
+const PARAMS: [&str; 3] = ["qmt.*.*.glb-usr", "qmt.*.*.glb-prj", "qmt.*.*.glb-grp"];
 
 /// Takes QMT_STATS and produces a list of params for
 /// consumption in proper ltcl get_param format.
 pub(crate) fn params() -> Vec<String> {
-    QMT_STATS
-        .iter()
-        .map(|x| format!("{QMT}.*.*.glb-{x}"))
-        .collect()
+    PARAMS.into_iter().map(String::from).collect()
+}
+
+pub fn parse<I>() -> impl Parser<I, Output = Record>
+where
+    I: Stream<Token = char>,
+    I::Error: ParseError<I::Token, I::Range, I::Position>,
+{
+    (string(QMT), period()).with(qmt_parse())
 }
 
 /// Parses a target name
@@ -225,7 +230,7 @@ where
 
 #[cfg(test)]
 mod tests {
-    use crate::quota::quota_parser::{params, quota_stats, quota_stats_osd};
+    use crate::quota::{params, quota_stats, quota_stats_osd};
     use combine::Parser as _;
 
     #[test]

lustre-collector/src/snapshots/lustre_collector__parser__tests__params.snap

@@ -3,49 +3,7 @@ source: lustre-collector/src/parser.rs
 expression: params()
 ---
 [
-    "memused",
-    "memused_max",
-    "lnet_memused",
     "health_check",
-    "mdt.*.exports.*.uuid",
-    "osd-*.*.stats",
-    "osd-*.*.filesfree",
-    "osd-*.*.filestotal",
-    "osd-*.*.fstype",
-    "osd-*.*.kbytesavail",
-    "osd-*.*.kbytesfree",
-    "osd-*.*.kbytestotal",
-    "osd-*.*.brw_stats",
-    "osd-*.*.quota_slave.acct_group",
-    "osd-*.*.quota_slave.acct_user",
-    "osd-*.*.quota_slave.acct_project",
-    "mgs.*.mgs.stats",
-    "mgs.*.mgs.threads_max",
-    "mgs.*.mgs.threads_min",
-    "mgs.*.mgs.threads_started",
-    "mgs.*.num_exports",
-    "obdfilter.*OST*.stats",
-    "obdfilter.*OST*.num_exports",
-    "obdfilter.*OST*.tot_dirty",
-    "obdfilter.*OST*.tot_granted",
-    "obdfilter.*OST*.tot_pending",
-    "obdfilter.*OST*.exports.*.stats",
-    "ost.OSS.ost.stats",
-    "ost.OSS.ost_io.stats",
-    "ost.OSS.ost_create.stats",
-    "ost.OSS.ost_out.stats",
-    "ost.OSS.ost_seq.stats",
-    "mds.MDS.mdt.stats",
-    "mds.MDS.mdt_fld.stats",
-    "mds.MDS.mdt_io.stats",
-    "mds.MDS.mdt_out.stats",
-    "mds.MDS.mdt_readpage.stats",
-    "mds.MDS.mdt_seqm.stats",
-    "mds.MDS.mdt_seqs.stats",
-    "mds.MDS.mdt_setattr.stats",
-    "mdt.*.md_stats",
-    "mdt.*MDT*.num_exports",
-    "mdt.*MDT*.exports.*.stats",
     "ldlm.namespaces.{mdt-,filter-}*.contended_locks",
     "ldlm.namespaces.{mdt-,filter-}*.contention_seconds",
     "ldlm.namespaces.{mdt-,filter-}*.ctime_age_limit",
@@ -61,10 +19,52 @@ expression: params()
     "ldlm.services.ldlm_canceld.stats",
     "ldlm.services.ldlm_cbd.stats",
     "llite.*.stats",
+    "lnet_memused",
     "mdd.*.changelog_users",
-    "qmt.*.*.glb-usr",
-    "qmt.*.*.glb-prj",
-    "qmt.*.*.glb-grp",
+    "mds.MDS.mdt.stats",
+    "mds.MDS.mdt_fld.stats",
+    "mds.MDS.mdt_io.stats",
+    "mds.MDS.mdt_out.stats",
+    "mds.MDS.mdt_readpage.stats",
+    "mds.MDS.mdt_seqm.stats",
+    "mds.MDS.mdt_seqs.stats",
+    "mds.MDS.mdt_setattr.stats",
+    "mdt.*.exports.*.uuid",
+    "mdt.*.md_stats",
+    "mdt.*MDT*.exports.*.stats",
+    "mdt.*MDT*.num_exports",
+    "memused",
+    "memused_max",
+    "mgs.*.mgs.stats",
+    "mgs.*.mgs.threads_max",
+    "mgs.*.mgs.threads_min",
+    "mgs.*.mgs.threads_started",
+    "mgs.*.num_exports",
     "nodemap.*.dt_stats",
     "nodemap.*.md_stats",
+    "obdfilter.*OST*.exports.*.stats",
+    "obdfilter.*OST*.num_exports",
+    "obdfilter.*OST*.stats",
+    "obdfilter.*OST*.tot_dirty",
+    "obdfilter.*OST*.tot_granted",
+    "obdfilter.*OST*.tot_pending",
+    "osd-*.*.brw_stats",
+    "osd-*.*.filesfree",
+    "osd-*.*.filestotal",
+    "osd-*.*.fstype",
+    "osd-*.*.kbytesavail",
+    "osd-*.*.kbytesfree",
+    "osd-*.*.kbytestotal",
+    "osd-*.*.quota_slave.acct_group",
+    "osd-*.*.quota_slave.acct_project",
+    "osd-*.*.quota_slave.acct_user",
+    "osd-*.*.stats",
+    "ost.OSS.ost.stats",
+    "ost.OSS.ost_create.stats",
+    "ost.OSS.ost_io.stats",
+    "ost.OSS.ost_out.stats",
+    "ost.OSS.ost_seq.stats",
+    "qmt.*.*.glb-grp",
+    "qmt.*.*.glb-prj",
+    "qmt.*.*.glb-usr",
 ]

lustre-collector/src/snapshots/lustre_collector__tests__params.snap

@@ -2,4 +2,4 @@
 source: lustre-collector/src/lib.rs
 expression: "xs.join(\" \")"
 ---
-memused memused_max lnet_memused health_check mdt.*.exports.*.uuid osd-*.*.stats osd-*.*.filesfree osd-*.*.filestotal osd-*.*.fstype osd-*.*.kbytesavail osd-*.*.kbytesfree osd-*.*.kbytestotal osd-*.*.brw_stats osd-*.*.quota_slave.acct_group osd-*.*.quota_slave.acct_user osd-*.*.quota_slave.acct_project mgs.*.mgs.stats mgs.*.mgs.threads_max mgs.*.mgs.threads_min mgs.*.mgs.threads_started mgs.*.num_exports obdfilter.*OST*.stats obdfilter.*OST*.num_exports obdfilter.*OST*.tot_dirty obdfilter.*OST*.tot_granted obdfilter.*OST*.tot_pending obdfilter.*OST*.exports.*.stats ost.OSS.ost.stats ost.OSS.ost_io.stats ost.OSS.ost_create.stats ost.OSS.ost_out.stats ost.OSS.ost_seq.stats mds.MDS.mdt.stats mds.MDS.mdt_fld.stats mds.MDS.mdt_io.stats mds.MDS.mdt_out.stats mds.MDS.mdt_readpage.stats mds.MDS.mdt_seqm.stats mds.MDS.mdt_seqs.stats mds.MDS.mdt_setattr.stats mdt.*.md_stats mdt.*MDT*.num_exports mdt.*MDT*.exports.*.stats ldlm.namespaces.{mdt-,filter-}*.contended_locks ldlm.namespaces.{mdt-,filter-}*.contention_seconds ldlm.namespaces.{mdt-,filter-}*.ctime_age_limit ldlm.namespaces.{mdt-,filter-}*.early_lock_cancel ldlm.namespaces.{mdt-,filter-}*.lock_count ldlm.namespaces.{mdt-,filter-}*.lock_timeouts ldlm.namespaces.{mdt-,filter-}*.lock_unused_count ldlm.namespaces.{mdt-,filter-}*.lru_max_age ldlm.namespaces.{mdt-,filter-}*.lru_size ldlm.namespaces.{mdt-,filter-}*.max_nolock_bytes ldlm.namespaces.{mdt-,filter-}*.max_parallel_ast ldlm.namespaces.{mdt-,filter-}*.resource_count ldlm.services.ldlm_canceld.stats ldlm.services.ldlm_cbd.stats llite.*.stats mdd.*.changelog_users qmt.*.*.glb-usr qmt.*.*.glb-prj qmt.*.*.glb-grp nodemap.*.dt_stats nodemap.*.md_stats
+health_check ldlm.namespaces.{mdt-,filter-}*.contended_locks ldlm.namespaces.{mdt-,filter-}*.contention_seconds ldlm.namespaces.{mdt-,filter-}*.ctime_age_limit ldlm.namespaces.{mdt-,filter-}*.early_lock_cancel ldlm.namespaces.{mdt-,filter-}*.lock_count ldlm.namespaces.{mdt-,filter-}*.lock_timeouts ldlm.namespaces.{mdt-,filter-}*.lock_unused_count ldlm.namespaces.{mdt-,filter-}*.lru_max_age ldlm.namespaces.{mdt-,filter-}*.lru_size ldlm.namespaces.{mdt-,filter-}*.max_nolock_bytes ldlm.namespaces.{mdt-,filter-}*.max_parallel_ast ldlm.namespaces.{mdt-,filter-}*.resource_count ldlm.services.ldlm_canceld.stats ldlm.services.ldlm_cbd.stats llite.*.stats lnet_memused mdd.*.changelog_users mds.MDS.mdt.stats mds.MDS.mdt_fld.stats mds.MDS.mdt_io.stats mds.MDS.mdt_out.stats mds.MDS.mdt_readpage.stats mds.MDS.mdt_seqm.stats mds.MDS.mdt_seqs.stats mds.MDS.mdt_setattr.stats mdt.*.exports.*.uuid mdt.*.md_stats mdt.*MDT*.exports.*.stats mdt.*MDT*.num_exports memused memused_max mgs.*.mgs.stats mgs.*.mgs.threads_max mgs.*.mgs.threads_min mgs.*.mgs.threads_started mgs.*.num_exports nodemap.*.dt_stats nodemap.*.md_stats obdfilter.*OST*.exports.*.stats obdfilter.*OST*.num_exports obdfilter.*OST*.stats obdfilter.*OST*.tot_dirty obdfilter.*OST*.tot_granted obdfilter.*OST*.tot_pending osd-*.*.brw_stats osd-*.*.filesfree osd-*.*.filestotal osd-*.*.fstype osd-*.*.kbytesavail osd-*.*.kbytesfree osd-*.*.kbytestotal osd-*.*.quota_slave.acct_group osd-*.*.quota_slave.acct_project osd-*.*.quota_slave.acct_user osd-*.*.stats ost.OSS.ost.stats ost.OSS.ost_create.stats ost.OSS.ost_io.stats ost.OSS.ost_out.stats ost.OSS.ost_seq.stats qmt.*.*.glb-grp qmt.*.*.glb-prj qmt.*.*.glb-usr

lustrefs-exporter/Cargo.toml

@@ -31,6 +31,7 @@ combine.workspace = true
 commandeer-test.workspace = true
 const_format.workspace = true
 criterion = { workspace = true, features = ["html_reports", "async_tokio"] }
+futures-util = "0.3.31"
 iai-callgrind.workspace = true
 insta = { workspace = true, features = ["glob"] }
 pretty_assertions.workspace = true
@@ -39,6 +40,7 @@ reqwest.workspace = true
 serde_json.workspace = true
 serial_test.workspace = true
 sysinfo.workspace = true
+test-case = "3.3.1"
 
 [lib]
 bench = false