sqz_engine/
table_compactor.rs

1//! Generic aligned-column table compactor.
2//!
3//! Many CLIs pad output into aligned columns (`docker ps`, `kubectl get`,
4//! `ps aux`, database shells). The padding is pure token waste: the
5//! alignment carries no information a model needs. This stage detects a
6//! shared column grid and collapses each separator run to exactly two
7//! spaces, preserving every cell byte-for-byte — cells keep their single
8//! internal spaces, so columns stay parseable.
9//!
10//! The detector is deliberately strict so indentation-significant content
11//! (source code, YAML, diffs, pretty JSON) never triggers it:
12//!
13//! * at least 5 non-empty lines, none starting with whitespace
14//! * a column position must be gap-covered by every line (or past the
15//!   line's end) to count as a separator
16//! * at least one separator band where 80% of lines have a real run of
17//!   two or more spaces
18
19/// Result of a successful compaction.
20#[derive(Debug, Clone)]
21pub struct TableCompactResult {
22    pub text: String,
23    /// Number of lines that had at least one separator collapsed.
24    pub rows_compacted: usize,
25}
26
27const MIN_ROWS: usize = 5;
28const SEPARATOR: &str = "  ";
29
30/// Detect an aligned-column table in `text` and collapse the alignment
31/// padding. Returns `None` when the content does not look like a table
32/// or when compaction would not shrink it.
33pub fn compact_aligned_table(text: &str) -> Option<TableCompactResult> {
34    let lines: Vec<&str> = text.lines().collect();
35    let non_empty: Vec<&str> = lines
36        .iter()
37        .copied()
38        .filter(|l| !l.trim().is_empty())
39        .collect();
40    if non_empty.len() < MIN_ROWS {
41        return None;
42    }
43    // Leading whitespace anywhere = indentation-significant content
44    // (code, YAML, diff context lines, pretty JSON). Never touch it.
45    if non_empty
46        .iter()
47        .any(|l| l.starts_with(' ') || l.starts_with('\t'))
48    {
49        return None;
50    }
51    // Embedded tabs make column positions unreliable.
52    if non_empty.iter().any(|l| l.contains('\t')) {
53        return None;
54    }
55
56    let rows: Vec<Vec<char>> = non_empty.iter().map(|l| l.chars().collect()).collect();
57    let max_len = rows.iter().map(|r| r.len()).max()?;
58
59    // A column position is a separator candidate when EVERY line either
60    // ended before it or has a 2+ space gap covering it.
61    let gap_maps: Vec<Vec<bool>> = rows.iter().map(|r| gap_map(r)).collect();
62    let mut is_separator = vec![false; max_len];
63    for (p, sep) in is_separator.iter_mut().enumerate() {
64        *sep = rows.iter().zip(&gap_maps).all(|(row, gaps)| {
65            p >= row.len() || gaps[p]
66        });
67    }
68
69    // Group consecutive separator positions into bands; a band must not
70    // start at column 0 (that would be leading indentation, excluded
71    // above anyway) and must be at least 2 columns wide.
72    let mut bands: Vec<(usize, usize)> = Vec::new();
73    let mut start: Option<usize> = None;
74    for p in 0..max_len {
75        match (is_separator[p], start) {
76            (true, None) => start = Some(p),
77            (false, Some(s)) => {
78                if p - s >= 2 && s > 0 {
79                    bands.push((s, p));
80                }
81                start = None;
82            }
83            _ => {}
84        }
85    }
86    // A band running to max_len is trailing padding shared by every
87    // line; collapsing it saves nothing meaningful, so drop it.
88    if bands.is_empty() {
89        return None;
90    }
91    // Require at least one band that is a REAL gap (2+ spaces present,
92    // not just end-of-line) in 80% of lines — otherwise ragged prose
93    // with one accidental alignment could sneak through.
94    let strong_band = bands.iter().any(|&(s, e)| {
95        let real = rows
96            .iter()
97            .zip(&gap_maps)
98            .filter(|(row, gaps)| {
99                (s..e).any(|p| p < row.len() && gaps[p])
100            })
101            .count();
102        real * 10 >= rows.len() * 8
103    });
104    if !strong_band {
105        return None;
106    }
107
108    // Rewrite every line: any run of 2+ spaces that overlaps a band
109    // collapses to the two-space separator. Runs elsewhere are left
110    // alone (they are inside a cell, wherever a CLI allows that).
111    let mut rows_compacted = 0usize;
112    let rewritten: Vec<String> = lines
113        .iter()
114        .map(|line| {
115            if line.trim().is_empty() {
116                return (*line).to_string();
117            }
118            let chars: Vec<char> = line.chars().collect();
119            let gaps = gap_map(&chars);
120            let mut out = String::with_capacity(line.len());
121            let mut i = 0;
122            let mut changed = false;
123            while i < chars.len() {
124                if gaps[i] {
125                    let run_start = i;
126                    while i < chars.len() && gaps[i] {
127                        i += 1;
128                    }
129                    let overlaps_band = bands
130                        .iter()
131                        .any(|&(s, e)| run_start < e && i > s);
132                    if overlaps_band {
133                        out.push_str(SEPARATOR);
134                        if i - run_start > SEPARATOR.len() {
135                            changed = true;
136                        }
137                    } else {
138                        out.extend(&chars[run_start..i]);
139                    }
140                } else {
141                    out.push(chars[i]);
142                    i += 1;
143                }
144            }
145            if changed {
146                rows_compacted += 1;
147            }
148            out
149        })
150        .collect();
151
152    if rows_compacted == 0 {
153        return None;
154    }
155    let mut text_out = rewritten.join("\n");
156    if text.ends_with('\n') {
157        text_out.push('\n');
158    }
159    if text_out.len() >= text.len() {
160        return None;
161    }
162    Some(TableCompactResult {
163        text: text_out,
164        rows_compacted,
165    })
166}
167
168/// Per-position map of "this char is part of a 2+ space run".
169fn gap_map(chars: &[char]) -> Vec<bool> {
170    let mut map = vec![false; chars.len()];
171    let mut i = 0;
172    while i < chars.len() {
173        if chars[i] == ' ' {
174            let start = i;
175            while i < chars.len() && chars[i] == ' ' {
176                i += 1;
177            }
178            if i - start >= 2 {
179                for slot in &mut map[start..i] {
180                    *slot = true;
181                }
182            }
183        } else {
184            i += 1;
185        }
186    }
187    map
188}
189
190// ── Tests ─────────────────────────────────────────────────────────────────
191
192#[cfg(test)]
193mod tests {
194    use super::*;
195
196    const DOCKER_PS: &str = "\
197CONTAINER ID   IMAGE          COMMAND                  STATUS          PORTS                    NAMES
1983f4a5b6c7d8e   nginx:latest   \"/docker-entrypoint.\"   Up 2 hours      0.0.0.0:8080->80/tcp     web-frontend
1999a8b7c6d5e4f   postgres:16    \"docker-entrypoint.s\"   Up 2 hours      5432/tcp                 db-primary
2001c2d3e4f5a6b   redis:7        \"docker-entrypoint.s\"   Up 45 minutes   6379/tcp                 cache
2017e8f9a0b1c2d   grafana/graf   \"/run.sh\"                Up 12 minutes   0.0.0.0:3000->3000/tcp   metrics-ui
2025a4b3c2d1e0f   alpine:3.20    \"sleep infinity\"         Up 3 seconds    <none>                   scratchpad
203";
204
205    #[test]
206    fn docker_ps_compacts_and_preserves_cells() {
207        let result = compact_aligned_table(DOCKER_PS).expect("docker ps must compact");
208        assert!(result.text.len() < DOCKER_PS.len());
209        assert!(result.rows_compacted >= 5);
210        // Every cell value survives byte-exact, including cells with
211        // single internal spaces.
212        for cell in [
213            "CONTAINER ID",
214            "Up 2 hours",
215            "Up 45 minutes",
216            "0.0.0.0:8080->80/tcp",
217            "web-frontend",
218            "db-primary",
219            "\"sleep infinity\"",
220        ] {
221            assert!(result.text.contains(cell), "lost cell {cell:?}:\n{}", result.text);
222        }
223        // Separators collapsed to exactly two spaces: no 3+ space runs remain.
224        assert!(!result.text.contains("   "), "residual padding:\n{}", result.text);
225    }
226
227    #[test]
228    fn kubectl_get_pods_compacts() {
229        let input = "\
230NAME                          READY   STATUS    RESTARTS   AGE
231api-6d4cf56db6-2m5xk          1/1     Running   0          4d20h
232api-6d4cf56db6-9lq2v          1/1     Running   0          4d20h
233worker-7c9f8b5d44-abcde       1/1     Running   2          12h
234worker-7c9f8b5d44-fghij       0/1     Pending   0          3m
235ingress-nginx-controller-x    1/1     Running   0          30d
236";
237        let result = compact_aligned_table(input).expect("kubectl output must compact");
238        assert!(result.text.contains("api-6d4cf56db6-2m5xk  1/1  Running  0  4d20h"));
239    }
240
241    #[test]
242    fn source_code_never_triggers() {
243        let input = "\
244fn main() {
245    let x = 1;
246    let y = 2;
247    if x < y {
248        println!(\"{}\", x);
249    }
250    let aligned    = 3;
251    let alignment  = 4;
252}
253";
254        assert!(compact_aligned_table(input).is_none(), "indented code must not compact");
255    }
256
257    #[test]
258    fn pretty_json_never_triggers() {
259        let input = "\
260{
261  \"name\": \"sqz\",
262  \"version\": \"1.6.1\",
263  \"nested\": {
264    \"a\":  1,
265    \"b\":  2,
266    \"c\":  3
267  }
268}
269";
270        assert!(compact_aligned_table(input).is_none());
271    }
272
273    #[test]
274    fn diff_context_never_triggers() {
275        let input = "\
276+added line one with content
277-removed line with content
278 context line kept as-is
279+added line two with content
280 another context line here
281 third context line present
282";
283        assert!(compact_aligned_table(input).is_none(), "diff context lines start with a space");
284    }
285
286    #[test]
287    fn too_few_rows_never_triggers() {
288        let input = "\
289NAME     AGE
290first    1d
291second   2d
292";
293        assert!(compact_aligned_table(input).is_none());
294    }
295
296    #[test]
297    fn prose_without_shared_grid_never_triggers() {
298        let input = "\
299This is an ordinary paragraph of text without any columns.
300It has several lines, more than the minimum row count requires.
301None of them share  a consistent gap position with the others.
302Some lines have a  double space here and there, but never aligned.
303The detector must not touch this content at all, ever.
304Final line to push the count over the minimum threshold.
305";
306        assert!(compact_aligned_table(input).is_none());
307    }
308
309    #[test]
310    fn ragged_last_column_still_compacts() {
311        let input = "\
312PID    COMMAND         TIME
313101    /usr/bin/a      0:01
3142      b               12:44
31530303  /opt/long/c     0:59
31644     dd              1:02
3175      e               100:00
318";
319        let result = compact_aligned_table(input).expect("ragged table must compact");
320        assert!(result.text.contains("101  /usr/bin/a  0:01"));
321        assert!(result.text.contains("30303  /opt/long/c  0:59"));
322    }
323}
sqz_engine/table_compactor.rs

sqz_engine/
table_compactor.rs
