pub fn group_token_blocks(
streams: &[Vec<u32>],
block_size: usize,
) -> Vec<Vec<u32>>Expand description
Concatenate tokenized streams into contiguous block_size chunks, dropping any short tail
remainder (exact match to group_texts in utils/datautils.py).