sqz_engine/
entropy_analyzer.rs

1//! Entropy Analyzer — classifies logical blocks by Shannon entropy.
2//!
3//! Builds on top of
4//! [`file_reader::compute_entropy`](crate::file_reader::compute_entropy) and
5//! [`file_reader::analyze_block_entropies`](crate::file_reader::analyze_block_entropies)
6//! to provide higher-level classification (HighInfo / MediumInfo / LowInfo)
7//! with configurable percentile thresholds.
8
9use std::ops::Range;
10
11use serde::{Deserialize, Serialize};
12
13use crate::file_reader::{analyze_block_entropies, BlockEntropy};
14
15// ── InfoLevel ─────────────────────────────────────────────────────────────
16
17/// Classification of a block's information density.
18#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
19pub enum InfoLevel {
20    /// Above the high-percentile threshold — retain.
21    HighInfo,
22    /// Between low and high thresholds — compress.
23    MediumInfo,
24    /// Below the low-percentile threshold — discard (boilerplate).
25    LowInfo,
26}
27
28// ── AnalyzedBlock ─────────────────────────────────────────────────────────
29
30/// A logical block with its entropy score and classification.
31#[derive(Debug, Clone, Serialize, Deserialize)]
32pub struct AnalyzedBlock {
33    /// The text content of the block.
34    pub text: String,
35    /// Shannon entropy in bits per character.
36    pub entropy: f64,
37    /// Classification based on percentile thresholds.
38    pub info_level: InfoLevel,
39    /// Line range (start inclusive, end exclusive).
40    pub line_range: Range<usize>,
41}
42
43// ── EntropyAnalyzer ───────────────────────────────────────────────────────
44
45/// Configurable entropy analyzer that classifies blocks into three tiers.
46///
47/// - Blocks at or above `high_percentile` → `HighInfo`
48/// - Blocks at or above `low_percentile` but below `high_percentile` → `MediumInfo`
49/// - Blocks below `low_percentile` → `LowInfo`
50pub struct EntropyAnalyzer {
51    /// Percentile threshold for HighInfo (default 60.0).
52    high_percentile: f64,
53    /// Percentile threshold separating LowInfo from MediumInfo (default 25.0).
54    low_percentile: f64,
55}
56
57impl Default for EntropyAnalyzer {
58    fn default() -> Self {
59        Self::new()
60    }
61}
62
63impl EntropyAnalyzer {
64    /// Create an analyzer with default thresholds (high=60, low=25).
65    pub fn new() -> Self {
66        Self {
67            high_percentile: 60.0,
68            low_percentile: 25.0,
69        }
70    }
71
72    /// Create an analyzer with custom percentile thresholds.
73    ///
74    /// `high_percentile` — blocks at or above this are `HighInfo`.
75    /// `low_percentile`  — blocks below this are `LowInfo`.
76    pub fn with_thresholds(high_percentile: f64, low_percentile: f64) -> Self {
77        Self {
78            high_percentile,
79            low_percentile,
80        }
81    }
82
83    /// Analyze source text and return classified blocks.
84    pub fn analyze(&self, source: &str) -> Vec<AnalyzedBlock> {
85        let blocks = analyze_block_entropies(source);
86        if blocks.is_empty() {
87            return Vec::new();
88        }
89
90        let (high_thresh, low_thresh) = Self::compute_thresholds(&blocks, self.high_percentile, self.low_percentile);
91
92        blocks
93            .into_iter()
94            .map(|b| {
95                let info_level = if b.entropy >= high_thresh {
96                    InfoLevel::HighInfo
97                } else if b.entropy >= low_thresh {
98                    InfoLevel::MediumInfo
99                } else {
100                    InfoLevel::LowInfo
101                };
102                AnalyzedBlock {
103                    text: b.text,
104                    entropy: b.entropy,
105                    info_level,
106                    line_range: b.start_line..b.end_line,
107                }
108            })
109            .collect()
110    }
111
112    /// Return only the high-info blocks from the analysis.
113    pub fn high_info_blocks(&self, source: &str) -> Vec<AnalyzedBlock> {
114        self.analyze(source)
115            .into_iter()
116            .filter(|b| b.info_level == InfoLevel::HighInfo)
117            .collect()
118    }
119
120    /// Get the configured high percentile.
121    pub fn high_percentile(&self) -> f64 {
122        self.high_percentile
123    }
124
125    /// Get the configured low percentile.
126    pub fn low_percentile(&self) -> f64 {
127        self.low_percentile
128    }
129
130    // ── internal ──────────────────────────────────────────────────────────
131
132    /// Compute the entropy value at a given percentile from a set of blocks.
133    fn percentile_value(blocks: &[BlockEntropy], pct: f64) -> f64 {
134        let mut entropies: Vec<f64> = blocks.iter().map(|b| b.entropy).collect();
135        entropies.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
136        let idx = ((pct / 100.0) * (entropies.len() as f64 - 1.0)).round() as usize;
137        entropies[idx.min(entropies.len() - 1)]
138    }
139
140    fn compute_thresholds(blocks: &[BlockEntropy], high_pct: f64, low_pct: f64) -> (f64, f64) {
141        let high = Self::percentile_value(blocks, high_pct);
142        let low = Self::percentile_value(blocks, low_pct);
143        (high, low)
144    }
145}
146
147// ── Tests ─────────────────────────────────────────────────────────────────
148
149#[cfg(test)]
150mod tests {
151    use super::*;
152
153    fn sample_source() -> &'static str {
154        r#"use std::collections::HashMap;
155use std::path::Path;
156
157/// A configuration struct.
158pub struct Config {
159    pub name: String,
160    pub value: i32,
161}
162
163impl Config {
164    pub fn new(name: &str, value: i32) -> Self {
165        Self {
166            name: name.to_string(),
167            value,
168        }
169    }
170
171    pub fn validate(&self) -> bool {
172        !self.name.is_empty() && self.value > 0
173    }
174}
175
176pub fn process(config: &Config) -> String {
177    let mut result = String::new();
178    for i in 0..config.value {
179        result.push_str(&format!("item {}: {}\n", i, config.name));
180    }
181    result
182}
183
184// boilerplate
185// boilerplate
186// boilerplate
187"#
188    }
189
190    #[test]
191    fn test_analyze_returns_blocks() {
192        let analyzer = EntropyAnalyzer::new();
193        let blocks = analyzer.analyze(sample_source());
194        assert!(!blocks.is_empty(), "should produce at least one block");
195    }
196
197    #[test]
198    fn test_all_blocks_have_valid_info_level() {
199        let analyzer = EntropyAnalyzer::new();
200        let blocks = analyzer.analyze(sample_source());
201        for block in &blocks {
202            assert!(
203                block.info_level == InfoLevel::HighInfo
204                    || block.info_level == InfoLevel::MediumInfo
205                    || block.info_level == InfoLevel::LowInfo
206            );
207        }
208    }
209
210    #[test]
211    fn test_high_info_blocks_subset() {
212        let analyzer = EntropyAnalyzer::new();
213        let all = analyzer.analyze(sample_source());
214        let high = analyzer.high_info_blocks(sample_source());
215        assert!(high.len() <= all.len());
216        for h in &high {
217            assert_eq!(h.info_level, InfoLevel::HighInfo);
218        }
219    }
220
221    #[test]
222    fn test_empty_input() {
223        let analyzer = EntropyAnalyzer::new();
224        let blocks = analyzer.analyze("");
225        assert!(blocks.is_empty());
226    }
227
228    #[test]
229    fn test_single_block() {
230        let analyzer = EntropyAnalyzer::new();
231        let blocks = analyzer.analyze("fn main() { println!(\"hello\"); }");
232        assert_eq!(blocks.len(), 1);
233        // Single block is always at the 100th percentile of itself → HighInfo
234        assert_eq!(blocks[0].info_level, InfoLevel::HighInfo);
235    }
236
237    #[test]
238    fn test_custom_thresholds() {
239        let analyzer = EntropyAnalyzer::with_thresholds(90.0, 10.0);
240        assert!((analyzer.high_percentile() - 90.0).abs() < f64::EPSILON);
241        assert!((analyzer.low_percentile() - 10.0).abs() < f64::EPSILON);
242        let blocks = analyzer.analyze(sample_source());
243        assert!(!blocks.is_empty());
244    }
245
246    #[test]
247    fn test_line_ranges_are_valid() {
248        let analyzer = EntropyAnalyzer::new();
249        let blocks = analyzer.analyze(sample_source());
250        for block in &blocks {
251            assert!(
252                block.line_range.start < block.line_range.end,
253                "line range should be non-empty: {:?}",
254                block.line_range
255            );
256        }
257    }
258
259    #[test]
260    fn test_entropy_is_non_negative() {
261        let analyzer = EntropyAnalyzer::new();
262        let blocks = analyzer.analyze(sample_source());
263        for block in &blocks {
264            assert!(block.entropy >= 0.0, "entropy should be non-negative");
265        }
266    }
267
268    // ── Property-based tests ──────────────────────────────────────────
269
270    mod prop_tests {
271        use super::*;
272        use proptest::prelude::*;
273
274        // Feature: sqz, Property 38: Entropy analysis preserves high-information content
275        // **Validates: Requirements 33.1, 33.2**
276
277        /// Generate a high-entropy code block (function/class with varied content).
278        fn high_entropy_block() -> impl Strategy<Value = String> {
279            prop::collection::vec(
280                prop::sample::select(vec![
281                    "fn process(x: i32) -> Result<String, Error> {",
282                    "    let mut map = HashMap::new();",
283                    "    for (key, val) in items.iter().enumerate() {",
284                    "        if val > threshold { map.insert(key, val * 2); }",
285                    "    }",
286                    "    match result { Ok(v) => v, Err(e) => return Err(e) }",
287                    "}",
288                    "pub struct Config { name: String, value: i32, enabled: bool }",
289                    "impl Config { pub fn validate(&self) -> bool { !self.name.is_empty() } }",
290                    "type ResultMap = HashMap<String, Vec<(usize, f64)>>;",
291                ]),
292                3..8,
293            )
294            .prop_map(|lines| lines.join("\n"))
295        }
296
297        /// Generate a low-entropy block (repetitive boilerplate).
298        fn low_entropy_block() -> impl Strategy<Value = String> {
299            prop::collection::vec(
300                prop::sample::select(vec![
301                    "// comment",
302                    "// comment",
303                    "// comment",
304                    "// ------",
305                    "// ------",
306                ]),
307                3..6,
308            )
309            .prop_map(|lines| lines.join("\n"))
310        }
311
312        /// Generate source text with multiple blocks separated by blank lines,
313        /// mixing high-entropy and low-entropy content.
314        fn multi_block_source() -> impl Strategy<Value = String> {
315            (
316                prop::collection::vec(high_entropy_block(), 1..4),
317                prop::collection::vec(low_entropy_block(), 1..4),
318            )
319                .prop_map(|(high_blocks, low_blocks)| {
320                    let mut all = Vec::new();
321                    for b in &high_blocks {
322                        all.push(b.as_str());
323                    }
324                    for b in &low_blocks {
325                        all.push(b.as_str());
326                    }
327                    all.join("\n\n")
328                })
329        }
330
331        proptest! {
332            #![proptest_config(ProptestConfig::with_cases(100))]
333
334            /// Property: The highest-entropy block in any multi-block input
335            /// is always classified as HighInfo.
336            #[test]
337            fn highest_entropy_block_is_high_info(source in multi_block_source()) {
338                let analyzer = EntropyAnalyzer::new();
339                let blocks = analyzer.analyze(&source);
340
341                // Only check inputs that produce multiple blocks
342                if blocks.len() >= 2 {
343                    let max_entropy = blocks
344                        .iter()
345                        .map(|b| b.entropy)
346                        .fold(f64::NEG_INFINITY, f64::max);
347
348                    let highest_block = blocks
349                        .iter()
350                        .find(|b| (b.entropy - max_entropy).abs() < f64::EPSILON)
351                        .unwrap();
352
353                    prop_assert_eq!(
354                        highest_block.info_level,
355                        InfoLevel::HighInfo,
356                        "Highest-entropy block (entropy={:.4}) should be HighInfo, got {:?}",
357                        highest_block.entropy,
358                        highest_block.info_level
359                    );
360                }
361            }
362
363            /// Property: For any source with multiple blocks, high-entropy
364            /// blocks are never classified as LowInfo.
365            /// A block is considered "high-entropy" if its entropy is above
366            /// the median entropy of all blocks.
367            #[test]
368            fn high_entropy_blocks_never_low_info(source in multi_block_source()) {
369                let analyzer = EntropyAnalyzer::new();
370                let blocks = analyzer.analyze(&source);
371
372                if blocks.len() >= 2 {
373                    // Compute median entropy
374                    let mut entropies: Vec<f64> = blocks.iter().map(|b| b.entropy).collect();
375                    entropies.sort_by(|a, b| a.partial_cmp(b).unwrap());
376                    let median = entropies[entropies.len() / 2];
377
378                    for block in &blocks {
379                        if block.entropy >= median {
380                            prop_assert_ne!(
381                                block.info_level,
382                                InfoLevel::LowInfo,
383                                "Block with entropy {:.4} (>= median {:.4}) should not be LowInfo",
384                                block.entropy,
385                                median
386                            );
387                        }
388                    }
389                }
390            }
391        }
392    }
393}
sqz_engine/entropy_analyzer.rs

sqz_engine/
entropy_analyzer.rs
