lossy compression for code context: fit a 1M token monorepo into 8k without losing the load bearing parts.
1 //! ctx-compress: fit repositories into context windows.
2
3 mod heat;
4 mod tokenize;
5
6 pub struct Budget(pub usize);
7
8 #[derive(Default)]
9 pub struct Packed {
10 pub summary: String,
11 pub kept: Vec<PathBuf>,
12 pub elided: usize,
13 pub tokens: usize,
14 }
15
16 pub fn pack(root: &Path, budget: Budget) -> Result<Packed, PackError> {
17 let tree = scan(root)?;
18 let ranked = heat::rank(&tree);
19
20 let mut out = Packed::default();
21 for file in ranked {
22 let cost = tokenize::count(&file);
23 if out.tokens + cost > budget.0 {
24 out.elided += 1;
25 continue;
26 }
27 out.kept.push(file.path.clone());
28 out.tokens += cost;
29 }
30
31 out.summary = render(&tree, &out.kept);
32 Ok(out)
33 }