//! A small RFC 4180 reader for the `/csv` batch endpoint. //! //! Values may contain commas and are then double-quoted with `""` escaping, //! which is why the batch cannot be split on `,`. The reader is deliberately //! lenient: blank lines are skipped and a stray quote inside an unquoted field //! is kept as data, because a malformed byte in one row must not cost the whole //! batch. /// Split CSV text into records, honouring quoting, `""` escapes, embedded /// commas and newlines, CRLF and LF line endings, and a leading BOM. pub(crate) fn parse(input: &str) -> Vec> { let input = input.strip_prefix('\u{feff}').unwrap_or(input); let mut records: Vec> = Vec::new(); let mut record: Vec = Vec::new(); let mut field = String::new(); let mut quoted = false; let mut chars = input.chars().peekable(); while let Some(c) = chars.next() { if quoted { if c == '"' { if chars.peek() == Some(&'"') { chars.next(); field.push('"'); } else { quoted = false; } } else { field.push(c); } continue; } match c { '"' if field.is_empty() => quoted = true, ',' => record.push(std::mem::take(&mut field)), '\n' | '\r' => { if c == '\r' && chars.peek() == Some(&'\n') { chars.next(); } record.push(std::mem::take(&mut field)); records.push(std::mem::take(&mut record)); } _ => field.push(c), } } if !field.is_empty() || !record.is_empty() { record.push(field); records.push(record); } records.retain(|r| r.iter().any(|f| !f.is_empty())); records } #[cfg(test)] mod tests { use super::parse; #[test] fn splits_plain_rows() { let records = parse("a,b,c\n1,2,3\n"); assert_eq!(records, vec![vec!["a", "b", "c"], vec!["1", "2", "3"]]); } #[test] fn keeps_empty_fields() { let records = parse("99999999,,,,,,,,,,,,\n"); assert_eq!(records.len(), 1); assert_eq!(records[0].len(), 13); } #[test] fn reads_quoted_field_with_comma() { let records = parse("cep,logradouro\n01001000,\"Rua Um, Dois\"\n"); assert_eq!(records[1][1], "Rua Um, Dois"); } #[test] fn unescapes_doubled_quotes() { let records = parse("a\n\"say \"\"hi\"\"\"\n"); assert_eq!(records[1][0], "say \"hi\""); } #[test] fn accepts_embedded_newline_and_crlf() { let records = parse("a,b\r\n\"line\nbreak\",2\r\n"); assert_eq!(records[1][0], "line\nbreak"); assert_eq!(records[1][1], "2"); } #[test] fn ignores_bom_and_blank_lines() { let records = parse("\u{feff}a,b\n\n1,2"); assert_eq!(records, vec![vec!["a", "b"], vec!["1", "2"]]); } }