use std::fmt; use std::fs; use crate::config::*; const fn build_lookup_table() -> [usize; 128] { let mut result = [0; 128]; let mut i = 0; while i < NUM_KEYS { result[KEY_CHARS[i] as usize] = i; i += 1; } return result; } const CHAR_TO_INDEX: [usize; 128] = build_lookup_table(); const NUM_BIGRAMS: usize = NUM_KEYS * NUM_KEYS; pub struct Corpus { bigram_count: [u32; NUM_BIGRAMS], character_count: [u32; 128], total_count: u32, index_threshold: u32, } fn pair_to_index(x: u8, y: u8) -> usize { let xi = CHAR_TO_INDEX[x as usize]; let yi = CHAR_TO_INDEX[y as usize]; xi + NUM_KEYS * yi } impl Corpus { pub fn get_character_count(&self, c: u8) -> u32 { self.character_count[c as usize] } pub fn get_bigram_count(&self, x: u8, y: u8) -> u32 { self.bigram_count[pair_to_index(x, y)] } pub fn get_index_threshold(&self) -> u32 { self.index_threshold } pub fn get_total_count(&self) -> u32 { self.total_count } pub fn load(path: &str) -> Result { let contents = fs::read_to_string(path)?; let mut bigram_count = [0; NUM_BIGRAMS]; let mut character_count = [0; 128]; let mut total_count = 0; let mut last_char: Option = None; for c in contents.chars() { if let Some(c) = canonicalise(c) { if let Some(lc) = last_char { // We don't count these anyway if lc != c { bigram_count[pair_to_index(c, lc)] += 1; bigram_count[pair_to_index(lc, c)] += 1; } } last_char = Some(c); character_count[c as usize] += 1; total_count += 1; } else { last_char = None; } } let index_threshold = (total_count as f32 * DESIRED_INDEX_USAGE_PERCENT) as u32; return Ok(Corpus { bigram_count, character_count, total_count, index_threshold, }); } } fn dump_bigrams(corpus: &Corpus) -> Vec<(String, u32)> { let mut result: Vec<(String, u32)> = Vec::new(); for (i, &x) in KEY_CHARS.iter().enumerate() { for &y in &KEY_CHARS[i..] { let mut pair = String::from(x); pair.push(y); result.push((pair, corpus.bigram_count[pair_to_index(x as u8, y as u8)])); } } result.sort_by(|(_, c1), (_, c2)| c1.cmp(c2).reverse()); return result; } impl fmt::Display for Corpus { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str(&format!( "Counted {} characters, top 5 bigrams {:?}", self.total_count, &dump_bigrams(self)[0..5] )) } }