tufazip_preprocess/
dictionary.rs1use std::fs::File;
2use std::io::{BufReader, BufWriter, Read, Write};
3use std::path::Path;
4
5use crate::error::PreprocessError;
6
7const MAX_WORD_BYTES: usize = 4096;
8
9#[derive(Debug, Clone, PartialEq, Eq)]
11pub struct Dictionary {
12 words: Vec<Vec<u8>>,
13}
14
15impl Dictionary {
16 #[must_use]
18 pub fn from_words(words: Vec<Vec<u8>>) -> Self {
19 Self { words }
20 }
21
22 pub fn load(path: impl AsRef<Path>) -> Result<Self, PreprocessError> {
29 let file = File::open(path)?;
30 Self::from_reader(BufReader::new(file))
31 }
32
33 pub fn from_reader<R: Read>(reader: R) -> Result<Self, PreprocessError> {
39 let mut bytes = BufReader::new(reader).bytes();
40 let mut words = Vec::new();
41 let mut current = Vec::new();
42
43 while let Some(res) = bytes.next() {
44 let mut b = res?;
45 if b == b'\n' {
46 if !current.is_empty() {
47 words.push(std::mem::take(&mut current));
48 }
49 } else {
50 if b == b'\\' {
51 let Some(esc_res) = bytes.next() else {
52 break;
53 };
54 let esc = esc_res?;
55 b = match esc {
56 b'n' => b'\n',
57 b'\\' => b'\\',
58 other => return Err(PreprocessError::InvalidEscape(other)),
59 };
60 }
61 if current.len() >= MAX_WORD_BYTES {
62 return Err(PreprocessError::WordTooLong);
63 }
64 current.push(b);
65 }
66 }
67
68 Ok(Self { words })
69 }
70
71 pub fn save(&self, path: impl AsRef<Path>) -> Result<(), PreprocessError> {
77 let file = File::create(path)?;
78 let mut writer = BufWriter::new(file);
79 self.write_to(&mut writer)?;
80 writer.flush()?;
81 Ok(())
82 }
83
84 pub fn write_to(&self, writer: &mut impl Write) -> Result<(), PreprocessError> {
90 for word in &self.words {
91 for &b in word {
92 match b {
93 b'\n' => writer.write_all(b"\\n")?,
94 b'\\' => writer.write_all(b"\\\\")?,
95 other => writer.write_all(&[other])?,
96 }
97 }
98 writer.write_all(b"\n")?;
99 }
100 Ok(())
101 }
102
103 #[must_use]
105 pub fn words(&self) -> &[Vec<u8>] {
106 &self.words
107 }
108
109 #[must_use]
111 pub fn len(&self) -> usize {
112 self.words.len()
113 }
114
115 #[must_use]
117 pub fn is_empty(&self) -> bool {
118 self.words.is_empty()
119 }
120
121 #[must_use]
123 pub fn get(&self, symbol: u16) -> Option<&[u8]> {
124 self.words.get(usize::from(symbol)).map(Vec::as_slice)
125 }
126}
127
128#[cfg(test)]
129mod tests {
130 use std::path::PathBuf;
131
132 use sha2::{Digest, Sha256};
133
134 use super::*;
135
136 #[test]
137 fn enwik9_voc_roundtrip_matches_exact_bytes_and_sha256() {
138 let voc_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../assets/enwik9.voc");
139 let original = std::fs::read(&voc_path).expect("assets/enwik9.voc should exist");
140 let dict = Dictionary::load(&voc_path).unwrap();
141 assert_eq!(dict.len(), 4097);
142 assert_eq!(dict.get(0), Some(&[0x00u8][..]));
143 assert_eq!(dict.get(10), Some(&[0x0Au8][..]));
144 assert_eq!(dict.get(92), Some(&[0x5Cu8][..]));
145
146 let mut reserialized = Vec::with_capacity(original.len());
147 dict.write_to(&mut reserialized).unwrap();
148 assert_eq!(reserialized.len(), 34_421);
149 assert_eq!(reserialized, original);
150
151 let digest = format!("{:x}", Sha256::digest(&reserialized));
152 assert_eq!(
153 digest,
154 "a63e517783e84a778f3f08c1f2321f3e2035e64d34c8e9206f03591460e44629"
155 );
156 }
157
158 #[test]
159 fn dictionary_rejects_invalid_escape_and_oversized_word() {
160 let bad_escape = b"abc\\xdef\n";
161 assert!(matches!(
162 Dictionary::from_reader(&bad_escape[..]),
163 Err(PreprocessError::InvalidEscape(b'x'))
164 ));
165
166 let mut too_long = vec![b'a'; 4097];
167 too_long.push(b'\n');
168 assert!(matches!(
169 Dictionary::from_reader(&too_long[..]),
170 Err(PreprocessError::WordTooLong)
171 ));
172 }
173}