How to decode large file without runing out of memory? - rust

Question

I need to convert a ~4GB IBM866 encoded xml file into UTF-8. I tried this
and this crates, but with both of them I run out of memory.
I tried to do it this way:
fn ibm866_to_utf8(ibm866: &[u8]) -> Result<String, MyError> {
use encoding_rs::IBM866;
let (utf8, _, had_error) = IBM866.decode(ibm866);
if (had_error == true) {
Err(MyError::DecodingError)
} else {
Ok(utf8.to_string())
}
}
fn main() {
let path = "ibm866_file";
let mut file = File::open(path).unwrap();
let mut vec: Vec<u8> = Vec::with_capacity(file.metadata().unwrap().len() as usize);
file.read_to_end(&mut vec);
let utf8_string = ibm866_to_utf8(&vec).unwrap();
// write to file
}
I also tried to iterate file by lines like this:
fn main() {
let path = "ibm866_file";
let mut file = File::open(path).unwrap();
let mut reader = BufReader::new(file);
let mut utf8_string = String::new();
for line in reader.lines() {
let utf8_line = ibm866_to_utf8(line.unwrap().as_bytes())
utf8_string = format!("{}{}", utf8_string, utf8_line.unwrap());
}
// write to file
}
But it panics when reader meets non UTF-8 character.
How to decode large files properly?
link to file: https://drive.google.com/file/d/1fHFS5GWPhApoNRl3CRK-pRMNthIakcZY/view?usp=sharing

Answer 1

You need to use the streaming functionality of encoding_rs.
This requires a bit of boilerplate code, though, to properly feed the chunks read from a file into the conversion function.
This code seems to work on a simple example, as well as on your multi-GB large legends.xml file and reports no conversion errors.
use std::{
fs::File,
io::{Read, Write},
};
use encoding_rs::{CoderResult, IBM866};
const BUF_SIZE: usize = 4096;
struct ConversionBuffers {
buf1: [u8; BUF_SIZE],
buf2: [u8; BUF_SIZE],
buf1_active: bool,
content: usize,
}
impl ConversionBuffers {
fn new() -> Self {
Self {
buf1: [0; BUF_SIZE],
buf2: [0; BUF_SIZE],
buf1_active: true,
content: 0,
}
}
fn move_leftovers_and_flip(&mut self, consumed: usize) {
let (src, dst) = if self.buf1_active {
(&mut self.buf1, &mut self.buf2)
} else {
(&mut self.buf2, &mut self.buf1)
};
let leftover = self.content - consumed;
dst[..leftover].clone_from_slice(&src[consumed..self.content]);
self.buf1_active = !self.buf1_active;
self.content = leftover;
}
fn append(&mut self, append_action: impl FnOnce(&mut [u8]) -> usize) {
let buf = if self.buf1_active {
&mut self.buf1[self.content..]
} else {
&mut self.buf2[self.content..]
};
let appended = append_action(buf);
self.content += appended;
}
fn get_data(&mut self) -> &[u8] {
if self.buf1_active {
&self.buf1[..self.content]
} else {
&self.buf2[..self.content]
}
}
}
fn main() {
let mut decoder = IBM866.new_decoder();
let mut file_in = File::open("test_ibm866.txt").unwrap();
let mut file_out = File::create("out_utf8-2.txt").unwrap();
let mut buffer_in = ConversionBuffers::new();
let mut buffer_out = vec![0u8; decoder.max_utf8_buffer_length(BUF_SIZE).unwrap_or(BUF_SIZE)];
let mut file_eof = false;
let mut errors = false;
loop {
if !file_eof {
buffer_in.append(|buf| {
let num_read = file_in.read(buf).unwrap();
if num_read == 0 {
file_eof = true;
}
num_read
});
}
let (result, num_consumed, num_produced, had_error) =
decoder.decode_to_utf8(buffer_in.get_data(), &mut buffer_out, file_eof);
if had_error {
errors = true;
}
let produced_data = &buffer_out[..num_produced];
file_out.write_all(produced_data).unwrap();
if file_eof && result == CoderResult::InputEmpty {
break;
}
buffer_in.move_leftovers_and_flip(num_consumed);
}
println!("Had conversion errors: {:?}", errors);
}
As #BurntSushi5 pointed out, there is the encoding_rs_io crate that allows us to skip all the boilerplate code:
use std::fs::File;
use encoding_rs::IBM866;
use encoding_rs_io::DecodeReaderBytesBuilder;
fn main() {
let file_in = File::open("test_ibm866.txt").unwrap();
let mut file_out = File::create("out_utf8.txt").unwrap();
let mut decoded_stream = DecodeReaderBytesBuilder::new()
.encoding(Some(IBM866))
.build(file_in);
std::io::copy(&mut decoded_stream, &mut file_out).unwrap();
}