From f6d4be2195a9103e42b00c9aa362c62d079c3f4e Mon Sep 17 00:00:00 2001 From: tslil clingman Date: Sat, 22 Oct 2022 13:12:04 +0200 Subject: stats loading directly from json file --- srchr/src/config.rs | 3 ++ srchr/src/corpus.rs | 75 +++++++++++++++++++++++++++++++++++++++++-------- srchr/src/evaluation.rs | 8 ++---- srchr/src/main.rs | 9 ++++-- 4 files changed, 75 insertions(+), 20 deletions(-) (limited to 'srchr/src') diff --git a/srchr/src/config.rs b/srchr/src/config.rs index c9ca302..8badd52 100644 --- a/srchr/src/config.rs +++ b/srchr/src/config.rs @@ -20,6 +20,9 @@ // Input parameters pub const CORPUS_FILE_NAME: &str = "corpus.txt"; +pub const STATS_FILE_NAME: &str = "shai.json"; +pub const LOAD_STATS: bool = true; + pub const STARTING_LAYOUT_STRING: &str = " A B C D E F G H I J K L M N O P Q R S T diff --git a/srchr/src/corpus.rs b/srchr/src/corpus.rs index 306018c..102aabd 100644 --- a/srchr/src/corpus.rs +++ b/srchr/src/corpus.rs @@ -2,6 +2,7 @@ use std::fmt; use std::fs; use crate::config::*; +extern crate json; const fn build_lookup_table() -> [usize; 128] { let mut result = [0; 128]; @@ -20,8 +21,6 @@ pub struct Corpus { bigram_percs: [f32; NUM_BIGRAMS], skipgram_percs: [f32; NUM_BIGRAMS], character_percs: [f32; 128], - total_count: u32, - total_bigrams: u32, } fn pair_to_index(x: u8, y: u8) -> usize { @@ -43,11 +42,63 @@ impl Corpus { self.skipgram_percs[pair_to_index(x, y)] } - pub fn get_total_bigrams(&self) -> u32 { - self.total_bigrams + pub fn load_from_json_file(path: &str) -> Result { + let contents = fs::read_to_string(path)?; + + let json = json::parse(&contents).unwrap(); + + let mut character_percs = [0.0; 128]; + let mut bigram_percs = [0.0; NUM_BIGRAMS]; + let mut skipgram_percs = [0.0; NUM_BIGRAMS]; + + for (key, value) in json.entries() { + if key == "characters" { + for (c, p) in value.entries() { + if let Some(c) = canonicalise(c.as_bytes()[0] as char) { + character_percs[c as usize] = p.as_f32().unwrap(); + } + } + } else if key == "bigrams" { + for (b, p) in value.entries() { + let canon: Vec> = b + .as_bytes() + .iter() + .map(|&b| canonicalise(b as char)) + .collect(); + if let Some(c1) = canon[0] { + if let Some(c2) = canon[1] { + bigram_percs[pair_to_index(c1, c2)] += p.as_f32().unwrap(); + bigram_percs[pair_to_index(c2, c1)] += p.as_f32().unwrap(); + } + } + } + } else if key == "skipgrams" { + for (t, p) in value.entries() { + let canon: Vec> = t + .as_bytes() + .iter() + .map(|&b| canonicalise(b as char)) + .collect(); + if let Some(c1) = canon[0] { + if let Some(c3) = canon[1] { + skipgram_percs[pair_to_index(c1, c3)] += p.as_f32().unwrap(); + skipgram_percs[pair_to_index(c3, c1)] += p.as_f32().unwrap(); + } + } + } + } + } + + println!("Chars: {:?}\nBigs:{:?}", character_percs, bigram_percs); + + return Ok(Corpus { + skipgram_percs, + bigram_percs, + character_percs, + }); } - pub fn load(path: &str) -> Result { + pub fn load_from_text_file(path: &str) -> Result { let contents = fs::read_to_string(path)?; let mut bigram_count = [0; NUM_BIGRAMS]; @@ -100,8 +151,6 @@ impl Corpus { skipgram_percs, bigram_percs, character_percs, - total_count, - total_bigrams, }); } } @@ -109,11 +158,14 @@ impl Corpus { fn dump_bigrams(corpus: &Corpus) -> Vec<(String, f32)> { let mut result: Vec<(String, f32)> = Vec::new(); - for (i, &x) in KEY_CHARS.iter().enumerate() { - for &y in &KEY_CHARS[i..] { + for &x in KEY_CHARS.iter() { + for &y in KEY_CHARS.iter() { let mut pair = String::from(x); pair.push(y); - result.push((pair, corpus.bigram_percs[pair_to_index(x as u8, y as u8)])); + let perc = corpus.bigram_percs[pair_to_index(x as u8, y as u8)]; + if perc > 0.0 { + result.push((pair, perc)); + } } } @@ -131,8 +183,7 @@ fn dump_bigrams(corpus: &Corpus) -> Vec<(String, f32)> { impl fmt::Display for Corpus { fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result { formatter.write_str(&format!( - "Counted {} characters, top 5 bigrams {:?}", - self.total_count, + "Corpus has top 5 bigrams {:?}", &dump_bigrams(self)[0..5] )) } diff --git a/srchr/src/evaluation.rs b/srchr/src/evaluation.rs index b412d88..3d171b0 100644 --- a/srchr/src/evaluation.rs +++ b/srchr/src/evaluation.rs @@ -92,7 +92,6 @@ pub fn prelayout_fitness(corpus: &Corpus, layout: &Prelayout) -> f32 { pub struct Evaluation { keypress_perc: [f32; NUM_KEYS], - total_bigrams: u32, sfb: [f32; 8], dsfb: [f32; 8], } @@ -183,7 +182,6 @@ impl Evaluation { return Evaluation { keypress_perc, - total_bigrams: corpus.get_total_bigrams(), sfb, dsfb, }; @@ -191,8 +189,6 @@ impl Evaluation { fn output_eval(&self) -> String { let mut result = String::new(); - let big = self.total_bigrams as f32; - let tig = self.total_bigrams as f32; result += &"Percent per key:\n"; result += &format_block_output(self.keypress_perc.into_iter().map(|u| u * 100.0)); @@ -228,8 +224,8 @@ impl Evaluation { let sfb = self.sfb.iter().sum::(); let dsfb = self.dsfb.iter().sum::(); - result += &format!("\nTotal sfb: {:.3}% ({:.0})", sfb * 100.0, sfb * big); - result += &format!("\nTotal dsfb: {:.3}% ({:.0})", dsfb * 100.0, dsfb * tig); + result += &format!("\nTotal sfb: {:.3}%", sfb * 100.0); + result += &format!("\nTotal dsfb: {:.3}%", dsfb * 100.0); return result; } diff --git a/srchr/src/main.rs b/srchr/src/main.rs index 667b675..b4a26f9 100644 --- a/srchr/src/main.rs +++ b/srchr/src/main.rs @@ -67,7 +67,7 @@ impl<'a> Tournament<'a> { }); let result; - if tournament[0].1 < best { + if tournament[0].1 + 0.00001 < best { result = Some(tournament[0].0); } else { result = None; @@ -82,7 +82,12 @@ impl<'a> Tournament<'a> { } fn main() { - let corpus = Corpus::load(CORPUS_FILE_NAME).unwrap(); + let corpus; + if LOAD_STATS { + corpus = Corpus::load_from_json_file(STATS_FILE_NAME).unwrap(); + } else { + corpus = Corpus::load_from_text_file(CORPUS_FILE_NAME).unwrap(); + } println!("{}", corpus); let seed_layout = Layout::from_verbose(STARTING_LAYOUT_STRING).unwrap(); -- cgit v1.3.1