sqz_engine/
lib.rs

1//! # sqz_engine
2//!
3//! The core compression engine behind sqz. Takes text — JSON, CLI output, code,
4//! logs, prose — and squeezes it down to use fewer LLM tokens while keeping the
5//! important stuff intact.
6//!
7//! ## Quick start
8//!
9//! ```rust
10//! use sqz_engine::SqzEngine;
11//! # let dir = tempfile::tempdir().unwrap();
12//! # std::env::set_var("SQZ_DB_PATH", dir.path().join("sessions.db"));
13//!
14//! let engine = SqzEngine::new().expect("failed to init engine");
15//!
16//! // Compress some text
17//! let result = engine.compress("hello world").unwrap();
18//! println!("compressed: {}", result.data);
19//! println!("tokens: {} → {}", result.tokens_original, result.tokens_compressed);
20//!
21//! // JSON gets TOON-encoded automatically
22//! let json_result = engine.compress(r#"{"name": "Alice", "age": 30}"#).unwrap();
23//! assert!(json_result.data.starts_with("TOON:"));
24//! ```
25//!
26//! ## How it works
27//!
28//! Content flows through a multi-stage pipeline:
29//!
30//! 1. **Content routing** — the confidence router classifies input (JSON, code,
31//!    logs, prose) and picks a compression mode (safe, default, aggressive).
32//! 2. **Stage pipeline** — configurable stages run in priority order: ANSI
33//!    stripping, null removal, repeated-line condensing, git diff folding,
34//!    string truncation, array collapsing, and custom transforms.
35//! 3. **Post-processing** — RLE compression, sliding-window dedup, entropy
36//!    truncation, and token pruning for prose.
37//! 4. **TOON encoding** — JSON gets encoded into Token-Optimized Object
38//!    Notation, which drops unnecessary quotes and whitespace for 30-60%
39//!    fewer tokens.
40//! 5. **Verification** — a two-pass verifier checks that error lines, JSON
41//!    keys, and diff hunks survived compression. If confidence is low, the
42//!    engine falls back to safe mode.
43//!
44//! ## Key types
45//!
46//! - [`SqzEngine`] — top-level facade, wires everything together
47//! - [`CompressionPipeline`] — the stage-based compression orchestrator
48//! - [`CacheManager`] — SHA-256 content-hash dedup cache
49//! - [`SessionStore`] — SQLite-backed session and cache persistence
50//! - [`Preset`] — TOML-configurable compression settings
51//! - [`ToonEncoder`] — JSON → TOON lossless encoding
52//! - [`CompressedContent`] — compression result with token counts and metadata
53
54pub mod adaptive_tree;
55pub mod advanced_search;
56pub mod ansi_strip;
57pub mod api_proxy;
58pub mod ast_delta;
59pub mod ast_parser;
60pub mod benchmarks;
61#[cfg(test)]
62mod cmd_formatter_bench;
63pub mod bpe_compressor;
64pub mod cascade_compressor;
65pub mod claude_md_integration;
66pub mod cmd_formatters;
67pub mod codex_integration;
68pub mod compression_quality;
69pub mod confidence_router;
70pub mod context_evictor;
71pub mod crp_engine;
72pub mod dashboard;
73pub mod delta_encoder;
74pub mod dependency_mapper;
75pub mod dict_compressor;
76pub mod entropy_analyzer;
77pub mod entropy_truncator;
78pub mod exit_marker;
79pub mod file_reader;
80pub mod image_compressor;
81pub mod json_projection;
82pub mod kv_cache_optimizer;
83pub mod litm_positioner;
84pub mod mdl_selector;
85pub mod minhash_lsh;
86pub mod ngram_abbreviator;
87pub mod opencode_plugin;
88pub mod rle_compressor;
89pub mod simhash;
90pub mod table_compactor;
91pub mod textrank;
92pub mod token_pruner;
93pub mod tool_hooks;
94pub mod tool_selector;
95pub mod engine;
96pub mod hook_manager;
97pub mod budget_tracker;
98pub mod cache_manager;
99pub mod correction_log;
100pub mod cost_calculator;
101pub mod ctx_format;
102pub mod error;
103pub mod model_router;
104pub mod parse_tree_compressor;
105pub mod pipeline;
106pub mod pin_manager;
107pub mod plugin_api;
108pub mod preset;
109pub mod progressive_throttle;
110pub mod prompt_cache;
111pub mod regret_tracker;
112pub mod sandbox_executor;
113pub mod session_continuity;
114pub mod session_store;
115pub mod shell_scan;
116pub mod kiro_integration;
117pub mod stages;
118pub mod structural_summary;
119pub mod text_boundary;
120mod text_search;
121pub mod zed_integration;
122pub mod tee_mode;
123pub mod terse_mode;
124pub mod token_counter;
125pub mod toon;
126pub mod transparency;
127pub mod types;
128pub mod url_indexer;
129pub mod verifier;
130
131pub use advanced_search::{AdvancedSearch, SearchResult};
132pub use ansi_strip::AnsiStripper;
133pub use api_proxy::{compress_request, parse_http_request, build_http_response, ApiFormat, ProxyConfig, ProxyStats};
134pub use ast_parser::{AstParser, ClassDefinition, CodeSummary, FunctionSignature, ImportDecl, TypeDeclaration};
135pub use bpe_compressor::{bpe_compress, BpeConfig, BpeResult};
136pub use compression_quality::{measure_quality, format_quality_report, CompressionQuality, QualityGrade};
137pub use confidence_router::{ConfidenceRouter, CompressionMode};
138pub use context_evictor::{evict, should_evict, ContextItem, EvictionConfig, EvictionResult};
139pub use cmd_formatters::format_command;
140pub use delta_encoder::{DeltaConfig, DeltaEncoder, DeltaResult};
141pub use dependency_mapper::DependencyMapper;
142pub use dict_compressor::{DictCompressor, DictConfig, DictCompressResult};
143pub use entropy_analyzer::{EntropyAnalyzer, InfoLevel, AnalyzedBlock};
144pub use entropy_truncator::{EntropyTruncator, EntropyTruncConfig, EntropyTruncResult, EntropyTruncArrayResult};
145pub use file_reader::{FileReadMode, FileReader, ReadResult, BlockEntropy, compute_entropy, analyze_block_entropies};
146pub use image_compressor::{ImageCompressor, ImageDescription};
147pub use json_projection::{project_json, ProjectionConfig, ProjectionResult};
148pub use litm_positioner::{ContextSection, LitmPositioner, LitmStrategy, SectionType};
149pub use ngram_abbreviator::{NgramAbbreviator, AbbreviatorConfig, AbbreviationResult};
150pub use rle_compressor::{rle_compress, sliding_window_dedup, RleResult, SlidingWindowResult};
151pub use simhash::{simhash, SimHashFingerprint};
152pub use structural_summary::{summarize as structural_summarize, summarize_multi, SummaryConfig, StructuralSummaryResult};
153pub use textrank::{textrank_compress, TextRankConfig, TextRankResult};
154pub use mdl_selector::{select_stages, profile_content, ContentProfile, MdlSelection};
155pub use tool_hooks::{process_hook, process_hook_cursor, process_hook_gemini, process_hook_kiro, process_hook_windsurf, process_hook_copilot, generate_hook_configs, has_sqz_rules, refresh_sqz_rules, remove_sqz_rules, sqz_rules_change, sqz_rules_content, sqz_rules_path, sqz_rules_state, install_tool_hooks, install_tool_hooks_scoped, install_tool_hooks_scoped_filtered, claude_user_settings_path, claude_project_settings_needs_update, remove_claude_global_hook, remove_claude_project_hook, claude_settings_stale, refresh_claude_settings, claude_post_tool_use_target, claude_hooks_compress, PostToolUseTarget, CLAUDE_ALLOW_RULES, copilot_hooks_path, install_copilot_hook, remove_copilot_hook, ci_hooks_path, ci_hook_content, install_ci_hook, remove_ci_hook, canonicalize_tool_name, parse_tool_list, integration_tier, InstallScope, IntegrationTier, ToolFilter, ToolHookConfig, HookScope, HookPlatform, SUPPORTED_TOOL_NAMES};
156pub use tool_hooks::{
157    gemini_settings_has_sqz, gemini_settings_needs_update, merge_sqz_gemini_hook,
158    remove_sqz_from_gemini_settings, resolve_sqz_binary, claude_settings_have_sqz_hooks,
159    claude_user_hooks_installed, remove_claude_project_hooks_only, sqz_hook_binaries,
160    legacy_sqz_hook_files, remove_legacy_sqz_hooks, claude_settings_have_sqz,
161};
162pub use opencode_plugin::{
163    generate_opencode_plugin, install_opencode_plugin, update_opencode_config,
164    update_opencode_config_detailed, find_opencode_config,
165    opencode_config_has_comments,
166    plan_opencode_config_change, PlannedOpencodeChange,
167    remove_sqz_from_opencode_config, strip_jsonc_comments,
168    process_opencode_hook, opencode_plugin_path,
169    opencode_plugin_stale, refresh_opencode_plugin,
170};
171pub use codex_integration::{
172    agents_md_guidance_block, agents_md_guidance_state, agents_md_path,
173    codex_config_path, codex_mcp_registered, guidance_blocks, guidance_change,
174    install_agents_md_guidance, install_codex_mcp_config,
175    refresh_agents_md_guidance, remove_agents_md_guidance, remove_all_sqz_guidance,
176    remove_codex_mcp_config, sqz_guidance, GuidanceBlock, GuidanceChange,
177};
178pub use claude_md_integration::{
179    claude_mcp_config_managed, claude_md_block_outdated, claude_md_guidance_block, claude_md_guidance_state,
180    claude_md_has_guidance, claude_md_path, claude_user_json_path, install_claude_md_guidance,
181    install_claude_mcp_config, refresh_claude_md_guidance, remove_claude_md_guidance,
182    remove_claude_mcp_config,
183};
184pub use zed_integration::{
185    install_zed_guidance, install_zed_mcp_config, remove_zed_guidance,
186    remove_zed_mcp_config, zed_guidance_block, zed_guidance_target,
187    zed_mcp_snippet, zed_settings_path, ZedMcpInstall, ZedMcpRemove,
188};
189pub use kiro_integration::{
190    install_kiro_mcp_config, install_kiro_steering, install_kiro_user_steering,
191    kiro_legacy_hook_path, kiro_mcp_path, kiro_steering_path, kiro_steering_state,
192    kiro_user_root, kiro_user_steering_path, refresh_kiro_steering, remove_kiro_legacy_hook,
193    remove_kiro_mcp_config, remove_kiro_steering, KiroMcpInstall, KiroMcpRemove,
194    SteeringState,
195};
196pub use token_pruner::{TokenPruner, PrunerConfig, PruneResult};
197pub use tool_selector::{ToolDefinition, ToolSelector};
198pub use budget_tracker::{
199    AgentBudget, BudgetTracker, BudgetWarning, UsagePrediction, UsageReport,
200};
201pub use cache_manager::{parse_ref_token, CacheManager, CacheResult, DedupHit, ExpandResult, SliceHit};
202pub use correction_log::ContextWindow;
203pub use crp_engine::{CrpEngine, CrpLevel};
204pub use cost_calculator::{
205    CostBreakdown, CostCalculator, ModelPricing, PricingConfig, SavedCostEstimate,
206    SavingsAssumptions, SessionCostSummary, TokenUsage, ToolCost, estimate_saved_cost,
207};
208pub use ctx_format::{CtxEnvelope, CtxFormat, CtxMetadata};
209pub use error::{Result, SqzError, SourceLocation};
210pub use exit_marker::{
211    posix_rewrite, posix_rewrite_list, powershell_rewrite, strip_exit_marker,
212    strip_exit_marker_bytes,
213};
214pub use model_router::{ModelRouter, RoutingDecision, TaskContext};
215pub use pipeline::{CompressionPipeline, SessionContext};
216pub use pin_manager::PinManager;
217pub use plugin_api::{PluginLoader, PluginManifest, PluginSource, SqzPlugin};
218pub use prompt_cache::{CacheBoundary, Message, PromptCacheDetector};
219pub use session_store::{CommandStats, CompressionStats, DailyGain, RecallHit, RegretStats, SessionStore, SessionSummary};
220pub use session_continuity::{
221    SessionContinuityManager, SessionGuide, Snapshot, SnapshotEvent, SnapshotEventType,
222};
223pub use toon::ToonEncoder;
224pub use tee_mode::{TeeMode, TeeManager, TeeEntry};
225pub use terse_mode::TerseMode;
226pub use token_counter::TokenCounter;
227pub use types::*;
228pub use preset::{
229    BudgetConfig, CollapseArraysConfig, CompressionConfig, CondenseConfig, CustomTransformsConfig,
230    FlattenConfig, GitDiffFoldConfig, KeepFieldsConfig, ModelConfig, ModelPricingConfig, Preset,
231    PresetHeader, PresetMeta, PresetParser, StripFieldsConfig, StripNullsConfig, TerseLevel,
232    TerseModeConfig, ToolSelectionConfig, TruncateStringsConfig,
233};
234pub use progressive_throttle::{ProgressiveThrottler, ThrottleConfig, ThrottleLevel};
235pub use dashboard::{
236    CommandBreakdown, DashboardConfig, DashboardHtml, DashboardMetrics, DashboardServer,
237    SessionHistoryEntry, ToolBreakdown,
238};
239pub use engine::{SqzEngine, ToolResultCompression, TOOL_RESULT_MIN_SAVING};
240pub use hook_manager::{
241    generate_platform_config, known_platforms, Hook, HookAction, HookContext, HookManager, HookType,
242};
243pub use sandbox_executor::{SandboxExecutor, SandboxResult, RuntimeInfo, FilteredOutput};
244pub use url_indexer::{ContentFetcher, IndexedChunk, IndexResult, UrlIndexer};
245pub use verifier::Verifier;
246
247pub use adaptive_tree::{compress_to_budget, build_tree, SemanticNode};
248pub use ast_delta::{ast_diff, encode_delta, AstDelta, AstChange, ChangeKind};
249pub use kv_cache_optimizer::{compress_with_sinks, compress_with_custom_sinks};
250pub use minhash_lsh::{MinHashLsh, MinHashSignature};
251pub use parse_tree_compressor::{compress_code, char_entropy};
252pub use cascade_compressor::{cascade_compress, CascadeLevel, CascadeThresholds, CascadeResult};
253pub use regret_tracker::{RegretTracker, RegretEvent, RegretKind, FileProfile};
254pub use transparency::{CompressionAnnotation};
sqz_engine/lib.rs

sqz_engine/
lib.rs
