lossy compression for code context: fit a 1M token monorepo into 8k without losing the load bearing parts.
1 //! token counting without a model in the loop.
2 //! cl100k-ish heuristic: bytes / 3.6, corrected per language.
3
4 const CORRECTION: &[(&str, f32)] = &[
5 ("rs", 1.08),
6 ("ts", 1.02),
7 ("py", 0.97),
8 ("md", 0.88),
9 ];
10
11 pub fn count(file: &RepoFile) -> usize {
12 let base = file.bytes.len() as f32 / 3.6;
13 let k = CORRECTION
14 .iter()
15 .find(|(ext, _)| *ext == file.ext())
16 .map(|(_, k)| *k)
17 .unwrap_or(1.0);
18 (base * k).ceil() as usize
19 }