From f6b412ca5b9c40d8ad226298eac0368dac6b8c47 Mon Sep 17 00:00:00 2001 From: Patrick MARIE Date: Tue, 23 Feb 2021 23:10:26 +0100 Subject: [PATCH] Implementing "stats" command. --- src/main.rs | 102 +++++++++++++++++++++++++++++++++++++++++++++++++- src/metric.rs | 27 ++++++++++--- src/stage.rs | 32 ++++++---------- 3 files changed, 134 insertions(+), 27 deletions(-) diff --git a/src/main.rs b/src/main.rs index 662f7e8..5255a63 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,6 +4,7 @@ * Author: Patrick MARIE */ +use std::collections::HashMap; use std::convert::TryFrom; use std::error; @@ -21,6 +22,7 @@ mod timerange; use crate::cassandra::*; use crate::session::Session; use crate::stage::*; +use crate::metric::Metric; #[allow(dead_code)] @@ -259,6 +261,14 @@ fn main() -> Result<(), Box> { .arg(Arg::with_name("metric") .index(1) .required(true))) + .subcommand(SubCommand::with_name("stats") + .about("Stats") + .arg(Arg::with_name("start-key") + .long("start-key") + .takes_value(true)) + .arg(Arg::with_name("end-key") + .long("end-key") + .takes_value(true))) .get_matches(); let contact_points_metadata = "tag--cstars07--cassandra-cstars07.query.consul.preprod.crto.in"; @@ -346,7 +356,36 @@ fn main() -> Result<(), Box> { } metric_delete(&session, &metric)?; - } + }, + Some("stats") => { + let matches = matches.subcommand_matches("stats").unwrap(); + let start_key = matches.value_of("start-key"); // 0 + let end_key = matches.value_of("end-key"); // 100000000000000 + + let start_key = match start_key { + None => 0, + Some(s) => match s.parse::() { + Ok(n) => n, + Err(_) => { + eprintln!("Could not parse {}", s); + return Ok(()) + } + } + }; + + let end_key = match end_key { + None => 100000000000000, + Some(s) => match s.parse::() { + Ok(n) => n, + Err(_) => { + eprintln!("Could not parse {}", s); + return Ok(()) + } + } + }; + + metric_stats(&session, start_key, end_key)?; + }, None => { eprintln!("No command was used."); return Ok(()); @@ -356,3 +395,64 @@ fn main() -> Result<(), Box> { Ok(()) } + + +fn metric_stats(session: &Session, start_key: i64, end_key: i64) -> Result<(), Error> { + let q = + "SELECT id, name, token(name), config, created_on, updated_on, read_on \ + FROM biggraphite_metadata.metrics_metadata WHERE token(name) > ? LIMIT 1000"; + + let mut current_token = start_key; + let mut n = 0; + let mut points : u64 = 0; + + let mut stats : HashMap = HashMap::new(); + + while current_token < end_key { + let mut query = stmt!(q); + query.bind(0, current_token)?; + + let results = session.metadata_session().execute(&query).wait()?; + + for row in results.iter() { + current_token = row.get_column(2)?.get_i64()?; + + let metric : Metric = row.into(); + let stages = match metric.stages() { + Ok(stages) => stages, + Err(_) => continue, + }; + + for stage in stages { + points += stage.points() as u64; + } + + let parts = metric.name().split(".").collect::>(); + + *stats.entry(String::from(parts[0])).or_insert(0) += 1; + + // println!("{}: {}", current_token, metric.name()); + n += 1; + } + } + + let p : f64 = ((current_token - start_key) / std::i64::MAX) as f64; + + println!("Range: {} -> {} ({:.4}%)", start_key, current_token, 100. * p); + println!("{} metrics", n); + println!("{} points", points); + println!("-----"); + + let mut vec : Vec<(&String, &usize)> = stats.iter().collect(); + vec.sort_by(|a, b| b.1.cmp(a.1)); + + for (id, v) in vec.iter().enumerate() { + println!("{} {}", v.0, v.1); + + if id == 10 { + break; + } + } + + Ok(()) +} diff --git a/src/metric.rs b/src/metric.rs index 0a15286..704ccac 100644 --- a/src/metric.rs +++ b/src/metric.rs @@ -25,6 +25,10 @@ impl Metric { &self.id } + pub fn name(self: &Self) -> &String { + &self.name + } + pub fn config(self: &Self, name: String) -> Result { let res = self.config.get(&name); if let Some(v) = res { @@ -78,20 +82,31 @@ impl From for Metric { impl From for Metric { fn from(row: Row) -> Self { - let config_collection = row.get_column_by_name("config".to_string()).unwrap().get_map().unwrap(); let mut config : HashMap = HashMap::new(); - config_collection - .map(|(k, v)| config.insert(k.to_string(), v.to_string())) - .count(); + match row.get_column_by_name("config".to_string()).unwrap().get_map() { + Ok(v) => { + v.map(|(k, v)| config.insert(k.to_string(), v.to_string())) + .count(); + }, + Err(_) => {}, + }; let created_on = row.get_column_by_name("created_on".to_string()).unwrap(); - let created_on_timestamp = created_on.get_uuid().unwrap().timestamp(); + let created_on_timestamp = match created_on.get_uuid() { + Err(_) => 0, + Ok(v) => v.timestamp(), + }; let updated_on = row.get_column_by_name("updated_on".to_string()).unwrap(); let updated_on_timestamp = updated_on.get_uuid().unwrap().timestamp(); + let uuid = match row.get_column_by_name("id".to_string()).unwrap().get_uuid() { + Ok(v) => v.to_string(), + Err(_) => String::from(""), + }; + Self { - id: row.get_column_by_name("id".to_string()).unwrap().get_uuid().unwrap().to_string(), + id: uuid, name: row.get_column_by_name("name".to_string()).unwrap().to_string(), config: config, created_on: created_on_timestamp, diff --git a/src/stage.rs b/src/stage.rs index 648f7c8..1d2b79f 100644 --- a/src/stage.rs +++ b/src/stage.rs @@ -7,8 +7,6 @@ use std::fmt; use std::convert::TryFrom; use std::string::String; -use regex::Regex; - #[derive(Copy,Clone,Debug)] pub struct Stage { points: u32, @@ -20,25 +18,15 @@ impl TryFrom<&str> for Stage { type Error = &'static str; fn try_from(stage: &str) -> Result { - let re = Regex::new(r"^(\d+)\*(\d+)(.)"); - - if let Err(_) = re { - return Err("regex initialisation failed"); + let parts = stage.split("*").collect::>(); + if parts.len() != 2 { + return Err("invalid retention string"); } - let captures = match re.unwrap().captures(&stage) { - None => return Err("invalid regex capture"), - Some(c) => c, - }; + let points = parts[0].parse::().unwrap(); + let precision = &parts[1][0..parts[1].len()-1]; - let points = captures.get(1).unwrap().as_str().parse::().unwrap(); - - let factor = captures.get(3).unwrap().as_str(); - if factor.len() != 1 { - return Err("invalid factor length") - } - - let factor = factor.chars().nth(0).unwrap(); + let factor = &parts[1][parts[1].len()-1..].chars().nth(0).unwrap(); match factor { 's' | 'm' | 'h' | 'd' | 'w' | 'y' => {}, @@ -47,12 +35,12 @@ impl TryFrom<&str> for Stage { } }; - let precision = captures.get(2).unwrap().as_str().parse::().unwrap(); + let precision = precision.parse::().unwrap(); Ok(Stage { points: points, precision: precision, - factor: factor, + factor: *factor, }) } } @@ -72,6 +60,10 @@ impl Stage { factor * self.precision as i64 } + pub fn points(self: &Self) -> u32 { + self.points + } + pub fn to_string(self: &Self) -> String { format!("{}*{}{}", self.points, self.precision, self.factor) }