Skip to main content

tufazip_preprocess/
case_space.rs

1use std::io::Write;
2
3/// Control byte suppressing the implicit leading space before the next word.
4pub const CH_NO_SPACE: u8 = 1;
5
6/// Control byte marking the next word as all uppercase.
7pub const CH_TO_UPPER: u8 = 2;
8
9/// Control byte capitalizing the first ASCII letter of the next word.
10pub const CH_FIRST_UPPER: u8 = 3;
11
12#[inline]
13fn is_word_char(c: u8) -> bool {
14    c.is_ascii_lowercase() || c.is_ascii_uppercase() || c >= 128
15}
16
17#[inline]
18fn is_upper(c: u8) -> bool {
19    c.is_ascii_uppercase()
20}
21
22#[inline]
23fn is_lower(c: u8) -> bool {
24    c.is_ascii_lowercase() || c >= 128
25}
26
27/// Applies Fabrice Bellard's reversible case-and-space encoding to raw input bytes.
28///
29/// Word runs are lowercased with space-elision and case markers
30/// ([`CH_NO_SPACE`], [`CH_TO_UPPER`], [`CH_FIRST_UPPER`]), returning a `u16`
31/// symbol buffer whose values are in `0..=255`.
32///
33/// Matches `preprocess.c` verbatim (including its quirk that a single leading ASCII
34/// space at byte 0 followed by a word is absorbed by the decoder's initial `has_space = false`).
35#[must_use]
36pub fn case_space_encode(input: &[u8]) -> Vec<u16> {
37    let mut obuf: Vec<u16> = Vec::with_capacity(input.len());
38    let mut i = 0usize;
39    while i < input.len() {
40        if is_word_char(input[i]) {
41            let mut j = i + 1;
42            let ch_type = if is_lower(input[i]) {
43                while j < input.len() && is_lower(input[j]) {
44                    j += 1;
45                }
46                0u8
47            } else if j < input.len() && is_upper(input[j]) {
48                while j < input.len() && is_upper(input[j]) {
49                    j += 1;
50                }
51                CH_TO_UPPER
52            } else {
53                while j < input.len() && is_lower(input[j]) {
54                    j += 1;
55                }
56                CH_FIRST_UPPER
57            };
58            let len = j - i;
59            let has_space = if obuf.is_empty() {
60                true
61            } else if obuf.last() == Some(&(u16::from(b' '))) {
62                obuf.pop();
63                true
64            } else {
65                false
66            };
67
68            if !has_space {
69                obuf.push(u16::from(CH_NO_SPACE));
70            }
71            if ch_type != 0 {
72                obuf.push(u16::from(ch_type));
73            }
74            obuf.push(u16::from(b' '));
75            for &b in &input[i..i + len] {
76                let lowered = if b.is_ascii_uppercase() {
77                    b - b'A' + b'a'
78                } else {
79                    b
80                };
81                obuf.push(u16::from(lowered));
82            }
83            i += len;
84        } else {
85            obuf.push(u16::from(input[i]));
86            i += 1;
87        }
88    }
89    obuf
90}
91
92/// Stateful decoder inverting [`case_space_encode`] across streamed dictionary entries.
93#[derive(Debug, Clone, Default)]
94pub struct CaseSpaceDecoder {
95    ch_type: u8,
96    ch_type1: u8,
97    has_space: bool,
98}
99
100impl CaseSpaceDecoder {
101    /// Creates a decoder initialized at the start of a stream.
102    #[must_use]
103    pub fn new() -> Self {
104        Self::default()
105    }
106
107    /// Feeds one dictionary entry's raw bytes and writes the decoded output bytes to `out`.
108    ///
109    /// # Errors
110    ///
111    /// Returns any `std::io::Error` produced by `out`.
112    pub fn push_bytes(&mut self, bytes: &[u8], out: &mut impl Write) -> std::io::Result<()> {
113        for &raw in bytes {
114            if raw == CH_TO_UPPER || raw == CH_FIRST_UPPER {
115                self.ch_type = raw;
116            } else if raw == CH_NO_SPACE {
117                self.has_space = false;
118            } else if raw == b' ' {
119                self.ch_type1 = self.ch_type;
120                self.ch_type = 0;
121                if self.has_space {
122                    out.write_all(b" ")?;
123                }
124                self.has_space = true;
125            } else {
126                let mut c = raw;
127                if self.ch_type1 == CH_TO_UPPER || self.ch_type1 == CH_FIRST_UPPER {
128                    if c.is_ascii_lowercase() {
129                        c = c - b'a' + b'A';
130                    }
131                    if self.ch_type1 == CH_FIRST_UPPER {
132                        self.ch_type1 = 0;
133                    }
134                }
135                out.write_all(&[c])?;
136            }
137        }
138        Ok(())
139    }
140}
141
142/// Decodes a byte buffer produced by [`case_space_encode`] back to original bytes.
143#[must_use]
144pub fn case_space_decode(encoded: &[u8]) -> Vec<u8> {
145    let mut out = Vec::with_capacity(encoded.len());
146    let mut decoder = CaseSpaceDecoder::new();
147    // Writing to Vec<u8> is infallible.
148    let _ = decoder.push_bytes(encoded, &mut out);
149    out
150}
151
152#[cfg(test)]
153mod tests {
154    use super::*;
155
156    #[test]
157    fn roundtrip_various_case_and_space_patterns() {
158        let samples: &[&[u8]] = &[
159            b"",
160            b"Hello, world!",
161            b"USA and NASA launched 42 rockets.",
162            b"camelCase PascalCase snake_case kebab-case",
163            b"leading   multiple  spaces\tand\nnewlines\n\n",
164            b"Mixed Case: XML <title>Wikipedia</title> [[Link]]!",
165            "UTF-8 café naïve 日本語 — em-dash".as_bytes(),
166        ];
167
168        for sample in samples {
169            let encoded_u16 = case_space_encode(sample);
170            let encoded_u8: Vec<u8> = encoded_u16.iter().map(|&v| v as u8).collect();
171            let decoded = case_space_decode(&encoded_u8);
172            assert_eq!(
173                decoded.as_slice(),
174                *sample,
175                "failed roundtrip for {:?}",
176                String::from_utf8_lossy(sample)
177            );
178        }
179    }
180}