1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
|
//! A small RFC 4180 reader for the `/csv` batch endpoint.
//!
//! Values may contain commas and are then double-quoted with `""` escaping,
//! which is why the batch cannot be split on `,`. The reader is deliberately
//! lenient: blank lines are skipped and a stray quote inside an unquoted field
//! is kept as data, because a malformed byte in one row must not cost the whole
//! batch.
/// Split CSV text into records, honouring quoting, `""` escapes, embedded
/// commas and newlines, CRLF and LF line endings, and a leading BOM.
pub(crate) fn parse(input: &str) -> Vec<Vec<String>> {
let input = input.strip_prefix('\u{feff}').unwrap_or(input);
let mut records: Vec<Vec<String>> = Vec::new();
let mut record: Vec<String> = Vec::new();
let mut field = String::new();
let mut quoted = false;
let mut chars = input.chars().peekable();
while let Some(c) = chars.next() {
if quoted {
if c == '"' {
if chars.peek() == Some(&'"') {
chars.next();
field.push('"');
} else {
quoted = false;
}
} else {
field.push(c);
}
continue;
}
match c {
'"' if field.is_empty() => quoted = true,
',' => record.push(std::mem::take(&mut field)),
'\n' | '\r' => {
if c == '\r' && chars.peek() == Some(&'\n') {
chars.next();
}
record.push(std::mem::take(&mut field));
records.push(std::mem::take(&mut record));
}
_ => field.push(c),
}
}
if !field.is_empty() || !record.is_empty() {
record.push(field);
records.push(record);
}
records.retain(|r| r.iter().any(|f| !f.is_empty()));
records
}
#[cfg(test)]
mod tests {
use super::parse;
#[test]
fn splits_plain_rows() {
let records = parse("a,b,c\n1,2,3\n");
assert_eq!(records, vec![vec!["a", "b", "c"], vec!["1", "2", "3"]]);
}
#[test]
fn keeps_empty_fields() {
let records = parse("99999999,,,,,,,,,,,,\n");
assert_eq!(records.len(), 1);
assert_eq!(records[0].len(), 13);
}
#[test]
fn reads_quoted_field_with_comma() {
let records = parse("cep,logradouro\n01001000,\"Rua Um, Dois\"\n");
assert_eq!(records[1][1], "Rua Um, Dois");
}
#[test]
fn unescapes_doubled_quotes() {
let records = parse("a\n\"say \"\"hi\"\"\"\n");
assert_eq!(records[1][0], "say \"hi\"");
}
#[test]
fn accepts_embedded_newline_and_crlf() {
let records = parse("a,b\r\n\"line\nbreak\",2\r\n");
assert_eq!(records[1][0], "line\nbreak");
assert_eq!(records[1][1], "2");
}
#[test]
fn ignores_bom_and_blank_lines() {
let records = parse("\u{feff}a,b\n\n1,2");
assert_eq!(records, vec![vec!["a", "b"], vec!["1", "2"]]);
}
}
|