diff options
Diffstat (limited to 'src/learner.rs')
| -rw-r--r-- | src/learner.rs | 201 |
1 files changed, 114 insertions, 87 deletions
diff --git a/src/learner.rs b/src/learner.rs index 91b24fe..59db567 100644 --- a/src/learner.rs +++ b/src/learner.rs @@ -5,24 +5,15 @@ use serde::{Deserialize, Serialize}; use crate::{config::Config, trajectory::Action}; -fn key(a: &str, b: &str) -> (String, String) { - let a = a.to_string(); - let b = b.to_string(); - if a < b { (a, b) } else { (b, a) } -} - -fn compute_sum( - cand: &str, - known_keys: &HashSet<&String>, - map: &HashMap<(String, String), f32>, -) -> f32 { - known_keys - .iter() - .filter_map(|k| map.get(&key(k, cand))) - .sum() +fn compute_weight(past: &HashSet<String>, now: &HashSet<&String>) -> f32 { + let inter = now.iter().filter(|&&p| past.contains(p)).count(); + if inter == 0 { + return 0.0; + } + (inter as f32) / ((now.len() as f32).sqrt() * (past.len() as f32).sqrt()) } -fn softmax_sample(items: &[(&String, f32)], temperature: f32) -> Option<(String, f32)> { +fn softmax_sample(items: &[(String, f32)], temperature: f32) -> Option<(String, f32)> { if items.is_empty() { return None; } @@ -35,7 +26,7 @@ fn softmax_sample(items: &[(&String, f32)], temperature: f32) -> Option<(String, let exp_values: Vec<_> = items .iter() - .map(|&(k, val)| (k, ((val - max_val) / temperature).exp())) + .map(|(k, val)| (k, ((val - max_val) / temperature).exp())) .collect(); let sum: f32 = exp_values.iter().map(|(_, v)| *v).sum(); @@ -48,80 +39,79 @@ fn softmax_sample(items: &[(&String, f32)], temperature: f32) -> Option<(String, .ok() } +#[derive(Serialize, Deserialize)] +enum Episode { + Continue { + group: HashSet<String>, + avoid: Option<String>, + committed: String, + }, + Escape { + from: HashSet<String>, + to: String, + committed: String, + }, +} + +impl Episode { + fn prune_mut(&mut self, valid: &HashSet<String>) -> bool { + match self { + Episode::Continue { group, avoid, .. } => { + if let Some(a) = avoid + && valid.contains(a) + { + } else { + *avoid = None; + }; + group.retain(|g| valid.contains(g)); + !group.is_empty() + } + Episode::Escape { from, to, .. } => { + if !valid.contains(to) { + return false; + }; + from.retain(|f| valid.contains(f)); + !from.is_empty() + } + } + } +} + #[derive(Serialize, Deserialize, Default)] pub struct Learner { - similar: HashMap<(String, String), f32>, - different: HashMap<(String, String), f32>, + history: Vec<Episode>, } -#[derive(Debug)] pub enum Learning { - SkipExtend, - MoreExtend(Vec<String>, String), SkipToMore(Vec<String>, String), MoreToSkip(Vec<String>, String), } impl Learner { pub fn prune(&mut self, valid: &HashSet<String>) { - self.similar - .retain(|(a, b), _v| valid.contains(a) && valid.contains(b)); - self.different - .retain(|(a, b), _v| valid.contains(a) && valid.contains(b)); + self.history.retain_mut(|e| e.prune_mut(valid)); } +} - pub fn learn(&mut self, config: &Config, learning: &Learning) { - // global decay - for (_k, v) in self.similar.iter_mut().chain(self.different.iter_mut()) { - *v *= 1.0 - config.decay_rate; - } - // update +impl Learner { + pub fn learn(&mut self, learning: &Learning) { + let stamp = chrono::Utc::now().format("U%Y%m%d-%H%M%S").to_string(); match learning { - Learning::SkipExtend => { - // This is the least information carrying case, it does not - // follow that the trajectory contains similar or different - // items, we may simply be seeking something in particular. - } - Learning::MoreExtend(trajectory, new) => { - // We're continuing a good run so `new` is similar to everything - // in `trajectory`, but to account for the possibilty that our - // mood has changed over the course of this streak we damp - // sub-linearly that update by a proxy of temporal distance. - for (distance, e) in trajectory.iter().rev().enumerate() { - let damp = ((distance + 1) as f32).powf(-0.5); - let v = self.similar.entry(key(e, new)).or_default(); - *v = (1.0 - config.learning_rate) * *v + config.learning_rate * damp; - } - } Learning::SkipToMore(trajectory, new) => { - // We have learnt that `new` is different to everything in - // `trajectory`, the strongest signal we have. - for e in trajectory { - let v = self.different.entry(key(e, new)).or_default(); - *v = (1.0 - config.learning_rate) * *v + config.learning_rate; - } + self.history.push(Episode::Escape { + from: trajectory.iter().map(String::clone).collect(), + to: new.clone(), + committed: stamp, + }); } Learning::MoreToSkip(trajectory, new) => { - // `new` could be different to everything in `trajectory`, or we - // simply changed our minds, so we have only weak evidence of - // difference. The positive coherence of trajectory was taken - // care of during MoreExtend above. - let damp = (trajectory.len() + 1) as f32; - for e in trajectory { - let v = self.different.entry(key(e, new)).or_default(); - *v = (1.0 - config.learning_rate) * *v + config.learning_rate / damp; - } + self.history.push(Episode::Continue { + group: trajectory.iter().map(String::clone).collect(), + avoid: Some(new.clone()), + committed: stamp, + }); } }; - // global threshold drop - self.similar = self - .similar - .extract_if(|_k, v| (*v).abs() > config.min_score_thresh) - .collect(); - self.different = self - .different - .extract_if(|_k, v| (*v).abs() > config.min_score_thresh) - .collect(); } pub fn sample( @@ -129,10 +119,9 @@ impl Learner { trajectory: &[String], action: &Action, candidates: &HashSet<String>, - temperature: f32, + config: &Config, ) -> Option<(String, f32)> { let trajectory: HashSet<_> = trajectory.iter().collect(); - let normalisation: f32 = f32::max(trajectory.len() as f32, 1.0); let candidates: Vec<_> = candidates .iter() @@ -143,19 +132,57 @@ impl Learner { return None; } - let items = candidates - .iter() - .map(|&c| { - let sim = compute_sum(c, &trajectory, &self.similar); - let dif = compute_sum(c, &trajectory, &self.different); - let score = match action { - Action::Skip => dif - sim, - Action::More => sim - dif, - }; - (c, score / normalisation) - }) - .collect::<Vec<_>>(); + let mut items: HashMap<String, f32> = + candidates.into_iter().map(|c| (c.clone(), 0.0)).collect(); + + for episode in self.history.iter() { + match (episode, action) { + (Episode::Escape { from, to, .. }, Action::Skip) => { + let w = compute_weight(from, &trajectory); + if let Some(to_w) = items.get_mut(to) { + *to_w += config.max_weight * w; + } + for f in from { + if let Some(from_weight) = items.get_mut(f) { + *from_weight -= config.mid_weight * w; + } + } + } + (Episode::Escape { from, to, .. }, Action::More) => { + let w = compute_weight(from, &trajectory); + if trajectory.contains(to) { + for f in from { + if let Some(f_w) = items.get_mut(f) { + *f_w -= config.low_weight * w; + } + } + } + } + (Episode::Continue { group, avoid, .. }, Action::More) => { + let w = compute_weight(group, &trajectory); + for g in group { + if let Some(g_w) = items.get_mut(g) { + *g_w += config.max_weight * w; + } + } + if let Some(a) = avoid + && let Some(a_w) = items.get_mut(a) + { + *a_w -= config.low_weight * w; + } + } + (Episode::Continue { group, .. }, Action::Skip) => { + let w = compute_weight(group, &trajectory); + for g in group { + if let Some(v) = items.get_mut(g) { + *v -= config.high_weight * w; + } + } + } + } + } - softmax_sample(&items, temperature) + let pairs: Vec<_> = items.into_iter().collect(); + softmax_sample(&pairs, config.temperature) } } |