1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
|
//! A small RFC 4180 reader for the `/csv` batch endpoint.
//!
//! Values may contain commas and are then double-quoted with `""` escaping,
//! which is why the batch cannot be split on `,`. The reader is deliberately
//! lenient: blank lines are skipped and a stray quote inside an unquoted field
//! is kept as data, because a malformed byte in one row must not cost the whole
//! batch.
const std = @import("std");
const Allocator = std.mem.Allocator;
/// One parsed row: a slice of owned field values.
pub const Record = [][]const u8;
/// Split CSV text into records, honouring quoting, `""` escapes, embedded
/// commas and newlines, CRLF and LF line endings, and a leading BOM.
///
/// ## Ownership
///
/// Every field, every record and the outer slice are allocated from `gpa` and
/// owned by the caller. Release the lot with `free`; `input` is only borrowed
/// for the duration of the call.
pub fn parse(gpa: Allocator, input: []const u8) Allocator.Error![]Record {
const text = if (std.mem.startsWith(u8, input, "\xEF\xBB\xBF")) input[3..] else input;
var records: std.ArrayList(Record) = .empty;
errdefer free(gpa, records.items);
errdefer records.deinit(gpa);
var record: std.ArrayList([]const u8) = .empty;
errdefer freeRecord(gpa, record.items);
errdefer record.deinit(gpa);
var field: std.ArrayList(u8) = .empty;
defer field.deinit(gpa);
var quoted = false;
var i: usize = 0;
while (i < text.len) : (i += 1) {
const c = text[i];
if (quoted) {
if (c == '"') {
if (i + 1 < text.len and text[i + 1] == '"') {
try field.append(gpa, '"');
i += 1;
} else {
quoted = false;
}
} else {
try field.append(gpa, c);
}
continue;
}
switch (c) {
'"' => if (field.items.len == 0) {
quoted = true;
} else {
try field.append(gpa, c);
},
',' => try endField(gpa, &record, &field),
'\r', '\n' => {
if (c == '\r' and i + 1 < text.len and text[i + 1] == '\n') i += 1;
try endField(gpa, &record, &field);
try endRecord(gpa, &records, &record);
},
else => try field.append(gpa, c),
}
}
if (field.items.len != 0 or record.items.len != 0) {
try endField(gpa, &record, &field);
try endRecord(gpa, &records, &record);
}
return records.toOwnedSlice(gpa);
}
/// Release everything `parse` returned.
pub fn free(gpa: Allocator, records: []Record) void {
for (records) |record| freeRecord(gpa, record);
gpa.free(records);
}
fn freeRecord(gpa: Allocator, record: []const []const u8) void {
for (record) |value| gpa.free(value);
gpa.free(record);
}
fn endField(
gpa: Allocator,
record: *std.ArrayList([]const u8),
field: *std.ArrayList(u8),
) Allocator.Error!void {
const owned = try gpa.dupe(u8, field.items);
errdefer gpa.free(owned);
try record.append(gpa, owned);
field.clearRetainingCapacity();
}
/// Append the record unless every one of its fields is empty. A blank line
/// carries no data and would otherwise be mistaken for a row of empties, which
/// is exactly what the batch endpoint uses to mean "not found".
fn endRecord(
gpa: Allocator,
records: *std.ArrayList(Record),
record: *std.ArrayList([]const u8),
) Allocator.Error!void {
var any = false;
for (record.items) |value| {
if (value.len != 0) any = true;
}
if (!any) {
freeRecord(gpa, try record.toOwnedSlice(gpa));
return;
}
const owned = try record.toOwnedSlice(gpa);
errdefer freeRecord(gpa, owned);
try records.append(gpa, owned);
}
const testing = std.testing;
fn expectRecords(input: []const u8, expected: []const []const []const u8) !void {
const records = try parse(testing.allocator, input);
defer free(testing.allocator, records);
try testing.expectEqual(expected.len, records.len);
for (expected, records) |want_record, got_record| {
try testing.expectEqual(want_record.len, got_record.len);
for (want_record, got_record) |want, got| {
try testing.expectEqualStrings(want, got);
}
}
}
test "splits plain rows" {
try expectRecords("a,b,c\n1,2,3\n", &.{
&.{ "a", "b", "c" },
&.{ "1", "2", "3" },
});
}
test "keeps empty fields" {
const records = try parse(testing.allocator, "99999999,,,,,,,,,,,,\n");
defer free(testing.allocator, records);
try testing.expectEqual(@as(usize, 1), records.len);
try testing.expectEqual(@as(usize, 13), records[0].len);
try testing.expectEqualStrings("99999999", records[0][0]);
try testing.expectEqualStrings("", records[0][12]);
}
test "reads a quoted field containing a comma" {
try expectRecords("cep,logradouro\n01001000,\"Rua Um, Dois\"\n", &.{
&.{ "cep", "logradouro" },
&.{ "01001000", "Rua Um, Dois" },
});
}
test "unescapes doubled quotes" {
try expectRecords("a\n\"say \"\"hi\"\"\"\n", &.{
&.{"a"},
&.{"say \"hi\""},
});
}
test "accepts an embedded newline and CRLF endings" {
try expectRecords("a,b\r\n\"line\nbreak\",2\r\n", &.{
&.{ "a", "b" },
&.{ "line\nbreak", "2" },
});
}
test "ignores a BOM and blank lines" {
try expectRecords("\xEF\xBB\xBFa,b\n\n1,2", &.{
&.{ "a", "b" },
&.{ "1", "2" },
});
}
test "a final row without a trailing newline is kept" {
try expectRecords("a,b\n1,2", &.{
&.{ "a", "b" },
&.{ "1", "2" },
});
}
test "a stray quote inside an unquoted field is data" {
try expectRecords("a\n12\"34\n", &.{
&.{"a"},
&.{"12\"34"},
});
}
test "empty input yields no records" {
try expectRecords("", &.{});
try expectRecords("\n\n", &.{});
}
test "UTF-8 passes through untouched" {
try expectRecords("logradouro\nPraça da Sé\n", &.{
&.{"logradouro"},
&.{"Praça da Sé"},
});
}
|