steel_utils/serial/
prefixed_read.rs

1use std::io::{Cursor, Error, ErrorKind, Read, Result};
2
3use crate::{
4    codec::VarInt,
5    serial::{PrefixedRead, ReadFrom},
6};
7
8/// Reads a Minecraft UTF-8 string whose decoded length is bounded in Java
9/// UTF-16 code units.
10///
11/// Minecraft permits up to three encoded bytes per allowed UTF-16 code unit
12/// and replaces malformed UTF-8 with U+FFFD while decoding.
13///
14/// # Errors
15///
16/// Returns an error when the encoded byte length is negative, exceeds the
17/// maximum encoded length, is not fully available, or decodes to more than
18/// `max_utf16_units` UTF-16 code units.
19pub fn read_utf(data: &mut Cursor<&[u8]>, max_utf16_units: usize) -> Result<String> {
20    let encoded_len = VarInt::read(data)?.0;
21    let Ok(encoded_len) = usize::try_from(encoded_len) else {
22        return Err(Error::new(
23            ErrorKind::InvalidData,
24            "encoded string length is negative",
25        ));
26    };
27    let Some(max_encoded_len) = max_utf16_units.checked_mul(3) else {
28        return Err(Error::new(
29            ErrorKind::InvalidInput,
30            "maximum UTF-16 length is too large",
31        ));
32    };
33    if encoded_len > max_encoded_len {
34        return Err(Error::new(
35            ErrorKind::InvalidData,
36            "encoded string length exceeds maximum",
37        ));
38    }
39
40    let start = usize::try_from(data.position()).map_err(|_| {
41        Error::new(
42            ErrorKind::UnexpectedEof,
43            "encoded string starts beyond the addressable input",
44        )
45    })?;
46    let end = start.checked_add(encoded_len).ok_or_else(|| {
47        Error::new(
48            ErrorKind::UnexpectedEof,
49            "encoded string length exceeds the addressable input",
50        )
51    })?;
52    let Some(bytes) = data.get_ref().get(start..end) else {
53        return Err(Error::new(
54            ErrorKind::UnexpectedEof,
55            "encoded string is truncated",
56        ));
57    };
58
59    let decoded = decode_java_utf8_lossy(bytes);
60    let end = u64::try_from(end).map_err(|_| {
61        Error::new(
62            ErrorKind::InvalidData,
63            "encoded string end exceeds the cursor position range",
64        )
65    })?;
66    data.set_position(end);
67    if decoded.encode_utf16().count() > max_utf16_units {
68        return Err(Error::new(
69            ErrorKind::InvalidData,
70            "decoded string length exceeds maximum",
71        ));
72    }
73    Ok(decoded)
74}
75
76fn decode_java_utf8_lossy(bytes: &[u8]) -> String {
77    let mut decoded = String::with_capacity(bytes.len());
78    let mut offset = 0;
79
80    while offset < bytes.len() {
81        let first = bytes[offset];
82        if first.is_ascii() {
83            decoded.push(char::from(first));
84            offset += 1;
85            continue;
86        }
87
88        let remaining = bytes.len() - offset;
89        match first {
90            0xC2..=0xDF => {
91                if remaining >= 2 && is_utf8_continuation(bytes[offset + 1]) {
92                    let code_point =
93                        (u32::from(first & 0x1F) << 6) | u32::from(bytes[offset + 1] & 0x3F);
94                    decoded.push(char::from_u32(code_point).unwrap_or(char::REPLACEMENT_CHARACTER));
95                    offset += 2;
96                } else {
97                    decoded.push(char::REPLACEMENT_CHARACTER);
98                    offset += 1;
99                }
100            }
101            0xE0..=0xEF => {
102                if remaining < 3 {
103                    let malformed_len = if remaining == 1
104                        || malformed_three_byte_prefix(first, bytes[offset + 1])
105                    {
106                        1
107                    } else {
108                        remaining
109                    };
110                    decoded.push(char::REPLACEMENT_CHARACTER);
111                    offset += malformed_len;
112                    continue;
113                }
114
115                let second = bytes[offset + 1];
116                let third = bytes[offset + 2];
117                if malformed_three_byte_sequence(first, second, third) {
118                    decoded.push(char::REPLACEMENT_CHARACTER);
119                    offset += malformed_three_byte_length(first, second);
120                    continue;
121                }
122
123                let code_point = (u32::from(first & 0x0F) << 12)
124                    | (u32::from(second & 0x3F) << 6)
125                    | u32::from(third & 0x3F);
126                if let Some(character) = char::from_u32(code_point) {
127                    decoded.push(character);
128                } else {
129                    decoded.push(char::REPLACEMENT_CHARACTER);
130                }
131                offset += 3;
132            }
133            0xF0..=0xF7 => {
134                if remaining < 4 {
135                    let malformed_len = malformed_four_byte_prefix_length(
136                        first,
137                        bytes.get(offset + 1).copied(),
138                        bytes.get(offset + 2).copied(),
139                        remaining,
140                    );
141                    decoded.push(char::REPLACEMENT_CHARACTER);
142                    offset += malformed_len;
143                    continue;
144                }
145
146                let second = bytes[offset + 1];
147                let third = bytes[offset + 2];
148                let fourth = bytes[offset + 3];
149                if malformed_four_byte_sequence(first, second, third, fourth) {
150                    decoded.push(char::REPLACEMENT_CHARACTER);
151                    offset += malformed_four_byte_length(first, second, third);
152                    continue;
153                }
154
155                let code_point = (u32::from(first & 0x07) << 18)
156                    | (u32::from(second & 0x3F) << 12)
157                    | (u32::from(third & 0x3F) << 6)
158                    | u32::from(fourth & 0x3F);
159                decoded.push(char::from_u32(code_point).unwrap_or(char::REPLACEMENT_CHARACTER));
160                offset += 4;
161            }
162            _ => {
163                decoded.push(char::REPLACEMENT_CHARACTER);
164                offset += 1;
165            }
166        }
167    }
168
169    decoded
170}
171
172const fn is_utf8_continuation(byte: u8) -> bool {
173    byte & 0xC0 == 0x80
174}
175
176const fn malformed_three_byte_prefix(first: u8, second: u8) -> bool {
177    (first == 0xE0 && second & 0xE0 != 0xA0) || !is_utf8_continuation(second)
178}
179
180const fn malformed_three_byte_sequence(first: u8, second: u8, third: u8) -> bool {
181    malformed_three_byte_prefix(first, second) || !is_utf8_continuation(third)
182}
183
184const fn malformed_three_byte_length(first: u8, second: u8) -> usize {
185    if malformed_three_byte_prefix(first, second) {
186        1
187    } else {
188        2
189    }
190}
191
192const fn malformed_four_byte_second(first: u8, second: u8) -> bool {
193    (first == 0xF0 && (second < 0x90 || second > 0xBF))
194        || (first == 0xF4 && (second < 0x80 || second > 0x8F))
195        || first > 0xF4
196        || !is_utf8_continuation(second)
197}
198
199const fn malformed_four_byte_prefix_length(
200    first: u8,
201    second: Option<u8>,
202    third: Option<u8>,
203    remaining: usize,
204) -> usize {
205    let Some(second) = second else {
206        return 1;
207    };
208    if malformed_four_byte_second(first, second) {
209        return 1;
210    }
211    if let Some(third) = third
212        && !is_utf8_continuation(third)
213    {
214        return 2;
215    }
216    remaining
217}
218
219const fn malformed_four_byte_sequence(first: u8, second: u8, third: u8, fourth: u8) -> bool {
220    malformed_four_byte_second(first, second)
221        || !is_utf8_continuation(third)
222        || !is_utf8_continuation(fourth)
223}
224
225const fn malformed_four_byte_length(first: u8, second: u8, third: u8) -> usize {
226    if malformed_four_byte_second(first, second) {
227        1
228    } else if !is_utf8_continuation(third) {
229        2
230    } else {
231        3
232    }
233}
234
235impl PrefixedRead for String {
236    fn read_prefixed_bound<P: TryInto<usize> + ReadFrom>(
237        data: &mut Cursor<&[u8]>,
238        bound: usize,
239    ) -> Result<Self> {
240        let len: usize = P::read(data)?
241            .try_into()
242            .map_err(|_| Error::other("Invalid Prefix"))?;
243
244        if len > bound {
245            Err(Error::other("To long"))?;
246        }
247
248        let mut buf = vec![0; len];
249        data.read_exact(&mut buf)?;
250        String::from_utf8(buf).map_err(Error::other)
251    }
252}
253
254impl<T: ReadFrom> PrefixedRead for Vec<T> {
255    fn read_prefixed_bound<P: TryInto<usize> + ReadFrom>(
256        data: &mut Cursor<&[u8]>,
257        bound: usize,
258    ) -> Result<Self> {
259        let len: usize = P::read(data)?
260            .try_into()
261            .map_err(|_| Error::other("Invalid Prefix"))?;
262
263        if len > bound {
264            Err(Error::other("To long"))?;
265        }
266        let mut items = Vec::with_capacity(len);
267        for _ in 0..len {
268            items.push(T::read(data)?);
269        }
270        Ok(items)
271    }
272}
273
274impl<T: PrefixedRead> PrefixedRead for Option<T> {
275    fn read_prefixed_bound<P: TryInto<usize> + ReadFrom>(
276        data: &mut Cursor<&[u8]>,
277        bound: usize,
278    ) -> Result<Self> {
279        if bool::read(data)? {
280            Ok(Some(T::read_prefixed_bound::<P>(data, bound)?))
281        } else {
282            Ok(None)
283        }
284    }
285}
286
287#[cfg(test)]
288mod tests {
289    use std::io::{Cursor, ErrorKind};
290
291    use crate::{
292        codec::VarInt,
293        serial::{WriteTo as _, prefixed_read::read_utf},
294    };
295
296    fn encoded_string(bytes: &[u8]) -> Vec<u8> {
297        let mut encoded = Vec::with_capacity(VarInt::MAX_SIZE + bytes.len());
298        VarInt(i32::try_from(bytes.len()).expect("test string length should fit in a VarInt"))
299            .write(&mut encoded)
300            .expect("test string length should encode");
301        encoded.extend_from_slice(bytes);
302        encoded
303    }
304
305    #[test]
306    fn truncated_string_leaves_cursor_after_prefix() {
307        let bytes = [3, b'a'];
308        let mut cursor = Cursor::new(bytes.as_slice());
309
310        let error = read_utf(&mut cursor, 3).expect_err("the declared body is incomplete");
311
312        assert_eq!(error.kind(), ErrorKind::UnexpectedEof);
313        assert_eq!(cursor.position(), 1);
314    }
315
316    #[test]
317    fn length_failures_match_vanilla_cursor_consumption() {
318        let encoded_too_long = encoded_string(b"abcd");
319        let mut cursor = Cursor::new(encoded_too_long.as_slice());
320        let error = read_utf(&mut cursor, 1).expect_err("four encoded bytes exceed the bound");
321        assert_eq!(error.kind(), ErrorKind::InvalidData);
322        assert_eq!(cursor.position(), 1);
323
324        let decoded_too_long = encoded_string(b"ab");
325        let mut cursor = Cursor::new(decoded_too_long.as_slice());
326        let error = read_utf(&mut cursor, 1).expect_err("two UTF-16 units exceed the bound");
327        assert_eq!(error.kind(), ErrorKind::InvalidData);
328        assert_eq!(cursor.position(), decoded_too_long.len() as u64);
329    }
330
331    #[test]
332    fn accepts_maximum_encoded_length() {
333        const MAX_UTF16_UNITS: usize = 32_767;
334        let bytes = [0xE0, 0xA0, 0x80].repeat(MAX_UTF16_UNITS);
335        let encoded = encoded_string(&bytes);
336        let mut cursor = Cursor::new(encoded.as_slice());
337
338        let decoded =
339            read_utf(&mut cursor, MAX_UTF16_UNITS).expect("the exact encoded bound is valid");
340
341        assert_eq!(decoded.encode_utf16().count(), MAX_UTF16_UNITS);
342        assert_eq!(cursor.position(), encoded.len() as u64);
343    }
344
345    #[test]
346    fn malformed_sequences_match_java_replacement_grouping() {
347        let cases: &[(&[u8], &str)] = &[
348            (&[0xC0, 0x80], "\u{FFFD}\u{FFFD}"),
349            (&[0xE0, 0x80, 0x80], "\u{FFFD}\u{FFFD}\u{FFFD}"),
350            (&[0xED, 0xA0, 0x80], "\u{FFFD}"),
351            (&[0xE1, 0x80], "\u{FFFD}"),
352            (&[0xE1, 0x80, b'A'], "\u{FFFD}A"),
353            (&[0xF0, 0x90, 0x80], "\u{FFFD}"),
354            (
355                &[0xF4, 0x90, 0x80, 0x80],
356                "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}",
357            ),
358        ];
359
360        for &(bytes, expected) in cases {
361            let encoded = encoded_string(bytes);
362            let decoded = read_utf(&mut Cursor::new(encoded.as_slice()), 16)
363                .expect("malformed UTF-8 is decoded lossily");
364            assert_eq!(decoded, expected, "unexpected decoding for {bytes:X?}");
365        }
366    }
367
368    #[test]
369    fn rejects_impossible_maximum_without_reading_the_body() {
370        let encoded = encoded_string(b"");
371        let mut cursor = Cursor::new(encoded.as_slice());
372
373        let error =
374            read_utf(&mut cursor, usize::MAX).expect_err("the encoded bound should overflow");
375
376        assert_eq!(error.kind(), ErrorKind::InvalidInput);
377        assert_eq!(cursor.position(), 1);
378    }
379}
steel_utils/serial/prefixed_read.rs

steel_utils/serial/
prefixed_read.rs
