Skip to main content

tufazip_preprocess/
dictionary.rs

1use std::fs::File;
2use std::io::{BufReader, BufWriter, Read, Write};
3use std::path::Path;
4
5use crate::error::PreprocessError;
6
7const MAX_WORD_BYTES: usize = 4096;
8
9/// NNCP token dictionary mapping 16-bit symbol IDs (`0..vocab_size`) to raw word byte slices.
10#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Dictionary {
12    words: Vec<Vec<u8>>,
13}
14
15impl Dictionary {
16    /// Creates a dictionary directly from its ordered word byte entries.
17    #[must_use]
18    pub fn from_words(words: Vec<Vec<u8>>) -> Self {
19        Self { words }
20    }
21
22    /// Loads a `.voc` dictionary file in Bellard's escaped line format.
23    ///
24    /// # Errors
25    ///
26    /// Returns [`PreprocessError`] on I/O failure, invalid escape sequences, or entries
27    /// longer than 4096 bytes.
28    pub fn load(path: impl AsRef<Path>) -> Result<Self, PreprocessError> {
29        let file = File::open(path)?;
30        Self::from_reader(BufReader::new(file))
31    }
32
33    /// Parses a `.voc` dictionary from any [`Read`] source.
34    ///
35    /// # Errors
36    ///
37    /// Returns [`PreprocessError`] on read errors, invalid escapes, or oversized entries.
38    pub fn from_reader<R: Read>(reader: R) -> Result<Self, PreprocessError> {
39        let mut bytes = BufReader::new(reader).bytes();
40        let mut words = Vec::new();
41        let mut current = Vec::new();
42
43        while let Some(res) = bytes.next() {
44            let mut b = res?;
45            if b == b'\n' {
46                if !current.is_empty() {
47                    words.push(std::mem::take(&mut current));
48                }
49            } else {
50                if b == b'\\' {
51                    let Some(esc_res) = bytes.next() else {
52                        break;
53                    };
54                    let esc = esc_res?;
55                    b = match esc {
56                        b'n' => b'\n',
57                        b'\\' => b'\\',
58                        other => return Err(PreprocessError::InvalidEscape(other)),
59                    };
60                }
61                if current.len() >= MAX_WORD_BYTES {
62                    return Err(PreprocessError::WordTooLong);
63                }
64                current.push(b);
65            }
66        }
67
68        Ok(Self { words })
69    }
70
71    /// Writes this dictionary to `path` in Bellard's escaped line format.
72    ///
73    /// # Errors
74    ///
75    /// Returns [`PreprocessError::Io`] on write failure.
76    pub fn save(&self, path: impl AsRef<Path>) -> Result<(), PreprocessError> {
77        let file = File::create(path)?;
78        let mut writer = BufWriter::new(file);
79        self.write_to(&mut writer)?;
80        writer.flush()?;
81        Ok(())
82    }
83
84    /// Serializes this dictionary to `writer` in Bellard's escaped line format.
85    ///
86    /// # Errors
87    ///
88    /// Returns [`PreprocessError::Io`] on write failure.
89    pub fn write_to(&self, writer: &mut impl Write) -> Result<(), PreprocessError> {
90        for word in &self.words {
91            for &b in word {
92                match b {
93                    b'\n' => writer.write_all(b"\\n")?,
94                    b'\\' => writer.write_all(b"\\\\")?,
95                    other => writer.write_all(&[other])?,
96                }
97            }
98            writer.write_all(b"\n")?;
99        }
100        Ok(())
101    }
102
103    /// Returns all dictionary entries in symbol-ID order.
104    #[must_use]
105    pub fn words(&self) -> &[Vec<u8>] {
106        &self.words
107    }
108
109    /// Returns the number of entries in the dictionary.
110    #[must_use]
111    pub fn len(&self) -> usize {
112        self.words.len()
113    }
114
115    /// Returns `true` if the dictionary contains zero entries.
116    #[must_use]
117    pub fn is_empty(&self) -> bool {
118        self.words.is_empty()
119    }
120
121    /// Looks up the raw byte sequence for symbol ID `symbol`.
122    #[must_use]
123    pub fn get(&self, symbol: u16) -> Option<&[u8]> {
124        self.words.get(usize::from(symbol)).map(Vec::as_slice)
125    }
126}
127
128#[cfg(test)]
129mod tests {
130    use std::path::PathBuf;
131
132    use sha2::{Digest, Sha256};
133
134    use super::*;
135
136    #[test]
137    fn enwik9_voc_roundtrip_matches_exact_bytes_and_sha256() {
138        let voc_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../assets/enwik9.voc");
139        let original = std::fs::read(&voc_path).expect("assets/enwik9.voc should exist");
140        let dict = Dictionary::load(&voc_path).unwrap();
141        assert_eq!(dict.len(), 4097);
142        assert_eq!(dict.get(0), Some(&[0x00u8][..]));
143        assert_eq!(dict.get(10), Some(&[0x0Au8][..]));
144        assert_eq!(dict.get(92), Some(&[0x5Cu8][..]));
145
146        let mut reserialized = Vec::with_capacity(original.len());
147        dict.write_to(&mut reserialized).unwrap();
148        assert_eq!(reserialized.len(), 34_421);
149        assert_eq!(reserialized, original);
150
151        let digest = format!("{:x}", Sha256::digest(&reserialized));
152        assert_eq!(
153            digest,
154            "a63e517783e84a778f3f08c1f2321f3e2035e64d34c8e9206f03591460e44629"
155        );
156    }
157
158    #[test]
159    fn dictionary_rejects_invalid_escape_and_oversized_word() {
160        let bad_escape = b"abc\\xdef\n";
161        assert!(matches!(
162            Dictionary::from_reader(&bad_escape[..]),
163            Err(PreprocessError::InvalidEscape(b'x'))
164        ));
165
166        let mut too_long = vec![b'a'; 4097];
167        too_long.push(b'\n');
168        assert!(matches!(
169            Dictionary::from_reader(&too_long[..]),
170            Err(PreprocessError::WordTooLong)
171        ));
172    }
173}