diff options
| -rw-r--r-- | .gitignore | 10 | ||||
| -rw-r--r-- | LICENSE | 15 | ||||
| -rw-r--r-- | Makefile | 52 | ||||
| -rw-r--r-- | README.md | 208 | ||||
| -rw-r--r-- | dub.json | 40 | ||||
| -rw-r--r-- | source/cepis/client.d | 588 | ||||
| -rw-r--r-- | source/cepis/exception.d | 168 | ||||
| -rw-r--r-- | source/cepis/model.d | 379 | ||||
| -rw-r--r-- | source/cepis/package.d | 111 | ||||
| -rw-r--r-- | source/cepis/retry.d | 170 | ||||
| -rw-r--r-- | test/cepis_test.d | 637 | ||||
| -rw-r--r-- | test/runner.d | 12 |
12 files changed, 2390 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c2fbcb --- /dev/null +++ b/.gitignore @@ -0,0 +1,10 @@ +.dub/ +dub.selections.json +*.a +*.lib +*.o +*.obj +*.lst +__dummy.html +/cepis +/cepis-test-* @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2026 Murilo Ijanc <[email protected]> + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..091d007 --- /dev/null +++ b/Makefile @@ -0,0 +1,52 @@ +# Development tasks. Everything runs through `dub`, so there is nothing to +# install first. +# +# make list the targets +# make test run the suite +# make lint dscanner + a dfmt check, no files touched +# make build release build of the library +# +# `lint` and `format` shell out to dscanner and dfmt, which dub fetches and +# builds on first use. + +DUB ?= dub +DFMT_FLAGS := --max_line_length=100 +SOURCES := $(wildcard source/cepis/*.d) $(wildcard test/*.d) + +.DEFAULT_GOAL := help +.PHONY: help test lint format build check clean + +help: + @echo 'Targets:' + @echo ' test run the test suite (unit and integration)' + @echo ' lint dub lint + a dfmt formatting check (read-only)' + @echo ' format apply dfmt (writes)' + @echo ' build release build of the library' + @echo ' check lint + test + build, in that order' + @echo ' clean remove build artefacts' + +test: + $(DUB) test + +# Read-only by design, so it is safe in CI and in a pre-commit hook: it reports +# what `make format` would change without changing anything. +lint: + $(DUB) lint + @status=0; for file in $(SOURCES); do \ + $(DUB) run --quiet --yes dfmt -- $(DFMT_FLAGS) "$$file" \ + | diff -u "$$file" - || status=1; \ + done; exit $$status + +format: + $(DUB) run --quiet --yes dfmt -- $(DFMT_FLAGS) --inplace $(SOURCES) + +build: + $(DUB) build --build=release + +# The order matters: lint is the fastest to fail, build the slowest, so a broken +# run reports the cheapest problem first. +check: lint test build + +clean: + $(DUB) clean + rm -f libcepis.a cepis cepis-test-library diff --git a/README.md b/README.md new file mode 100644 index 0000000..9c76d3d --- /dev/null +++ b/README.md @@ -0,0 +1,208 @@ +# cepis + +D client for the [public CEP API at cep.is](https://cep.is). Official Correios +data (DNE), refreshed every two weeks. + +```d +import cepis; + +auto address = lookup("01001-000"); +writeln(address.logradouro, " ", address.localidade, " ", address.uf); +// Praça da Sé São Paulo SP +``` + +## Installation + +```sh +dub add cepis +``` + +Or in `dub.json`: + +```json +"dependencies": { + "cepis": "~>0.1.0" +} +``` + +No dub dependencies: HTTP comes from `std.net.curl`, JSON from `std.json`, CSV +from `std.csv`. `std.net.curl` needs **libcurl at runtime** (`libcurl.so.4` on +Linux, `libcurl.dll` on Windows) and `libcurl` at link time — most systems +already ship it; on Debian and derivatives that is `libcurl4-openssl-dev`. + +## Usage + +### A single lookup + +`lookup` accepts the CEP masked or unmasked and returns an `Address`: + +```d +auto a = lookup("01310-100"); + +a.logradouro; // "Avenida Paulista" +a.localidade; // "São Paulo" +a.uf; // "SP" +a.ibge; // "3550308" +a.cep; // "01310100" (always unmasked) +a.formattedCep; // "01310-100" +``` + +Fields the database does not have for that CEP come back as empty strings. +`a.raw` holds the response exactly as it arrived, so a field the API grows +before this library is updated is not lost. + +### Coordinates + +About 57% of CEPs have a coordinate. When they do not, `geo` is null: + +```d +if (!a.geo.isNull) +{ + auto geo = a.geo.get; + writeln(geo.lat, " ", geo.lon); // -23.567202 -46.649659 + writeln(geo.raio_m); // 1084 + writeln(geo.precisao); // "logradouro" +} +``` + +**Check `precisao` before computing distance.** There are two kinds of point, +and the difference between them is tens of kilometres: + +| `precisao` | typical radius | what the point is | +| --- | --- | --- | +| `logradouro` | 110–140 m | the stretch of street — good for maps and distance | +| `localidade` | 7–40 km | the municipality centroid: the CEP *is* the whole city | + +`geo.isPrecise` is the shorthand for `precisao == "logradouro"`: + +```d +auto cost = !a.geo.isNull && a.geo.get.isPrecise + ? shippingFrom(a.geo.get.lat, a.geo.get.lon) + : shippingFromMunicipality(a.ibge); +``` + +### Many CEPs at once + +Do not loop over `lookup`. Use `lookupMany`, which resolves up to 500 CEPs per +request: + +```d +auto ceps = ["01001000", "99999999", "20040020"]; + +foreach (index, address; lookupMany(ceps)) + writeln(ceps[index], ": ", address.isNull ? "not found" : address.get.localidade); +``` + +The returned array has **the same length and the same order** as the input. A +CEP that does not exist, or is malformed, is null at its position — it does not +vanish, and it does not throw. + +Arrays longer than 500 are chunked automatically. + +> **Batches carry no coordinate.** The endpoint behind them is CSV, and the CSV +> has no `geo` column. Every `Address` from `lookupMany` has a null `geo`, +> including for CEPs that do have one. Need `geo`? Use `lookup`, one at a time. + +### API health + +Useful for two things: checking the API answers, and knowing **which version of +the database** is live: + +```d +auto h = health(); + +h.ok; // true +h.dne; // "26071" identifier of the fortnightly database +h.ceps; // 1596168 +h.base_gerada_em; // "2026-07-24T03:48:28Z" +``` + +An API reporting itself degraded is not an error — that arrives in `h.status`. + +## Configuration + +For anything beyond the defaults, build a `Client` and reuse it. It keeps no +state between calls and is safe to copy and share across threads: + +```d +ClientConfig config; +config.baseUrl = "https://cep.is"; // point at another instance +config.timeout = dur!"seconds"(10); // per socket operation +config.maxAttempts = 3; // counting the first; 1 disables retrying +config.backoffBase = dur!"msecs"(500); // base of the exponential wait +config.backoffMax = dur!"seconds"(8); // cap on each wait +config.totalTimeout = dur!"seconds"(30); // cap on the whole call +config.userAgent = "my-app/1.0"; + +auto client = Client(config); +auto address = client.lookup("01001000"); +``` + +Retries cover 429, 5xx and network failures, with exponential backoff and +jitter; 400 and 404 are never retried. + +The module-level functions (`lookup`, `lookupMany`, `health`) share one default +client per thread, built on first use. Need a different configuration? Build +your own. + +### Pointing at a local instance + +`baseUrl` is the only setting that usually changes between environments. The +API repo brings a local instance up with `just up`: + +```d +ClientConfig config; +config.baseUrl = "http://localhost:8080"; + +auto client = Client(config); +``` + +## Errors + +Everything derives from `CepisException`, so one `catch` handles any failure. +Below that, the split is by **the decision you have to make**, not by the HTTP +status that caused it: + +| exception | when | does retrying help? | +| --- | --- | --- | +| `InvalidCepException` | not 8 digits | no | +| `CepNotFoundException` | well-formed, not in the database | no | +| `RateLimitedException` | 429; carries `retryAfter` | yes, later | +| `ServerException` | 5xx; carries `status` | yes, later | +| `NetworkException` | DNS, connection, timeout, TLS | yes, later | +| `InvalidResponseException` | unintelligible response | probably not | + +```d +try + render(lookup(cepFromForm)); +catch (CepNotFoundException) + askForAnotherCep(); +catch (CepisException error) +{ + if (error.retriable) + reschedule(); + else + stderr.writeln(error.msg); +} +``` + +`InvalidResponseException` almost always means something between you and the +API is answering instead of it — a captive Wi-Fi portal, a corporate proxy, an +HTML error page. Useful as a diagnosis: the problem is not in your code. + +## Development + +Everything runs through `make`, which shells out to `dub`: + +```sh +make # list the targets +make test # dub test +make lint # dscanner + a dfmt check, touches nothing +make format # apply dfmt (writes) +make build # release build +make check # lint, then test, then build +``` + +## Licence + +ISC. See [LICENSE](LICENSE). diff --git a/dub.json b/dub.json new file mode 100644 index 0000000..640f036 --- /dev/null +++ b/dub.json @@ -0,0 +1,40 @@ +{ + "name": "cepis", + "version": "0.1.0", + "description": "Client for the public CEP API at cep.is (official Correios/DNE data)", + "authors": [ + "Murilo Ijanc <[email protected]>" + ], + "homepage": "https://cep.is", + "license": "ISC", + "copyright": "Copyright (c) 2026 Murilo Ijanc", + "targetType": "library", + "sourcePaths": [ + "source" + ], + "importPaths": [ + "source" + ], + "libs": [ + "curl" + ], + "configurations": [ + { + "name": "library", + "targetType": "library" + }, + { + "name": "unittest", + "targetType": "executable", + "mainSourceFile": "test/runner.d", + "sourcePaths": [ + "source", + "test" + ], + "importPaths": [ + "source", + "test" + ] + } + ] +} diff --git a/source/cepis/client.d b/source/cepis/client.d new file mode 100644 index 0000000..a93af7f --- /dev/null +++ b/source/cepis/client.d @@ -0,0 +1,588 @@ +/** + * The HTTP client, built on `std.net.curl`. + * + * A fresh curl handle is opened per request and connections are not pooled. For + * a one-off lookup that is irrelevant. For thousands, $(B do not write a loop) — + * use $(D Client.lookupMany), which resolves up to 500 CEPs per request. A loop + * of 500 calls pays 500 TLS handshakes; the batch pays one. + * + * Retry policy: 429 and the transient 5xx in $(D RETRIABLE_STATUS), plus network + * failures, are retried with exponentially backed-off, jittered waits. 400 and + * 404 are deliberately absent from that set — both are definitive answers about + * the request, so retrying returns the same thing more slowly while burning + * someone else's quota. + */ +module cepis.client; + +import core.thread : Thread; +import core.time : Duration, MonoTime, dur; +import std.algorithm : all, canFind; +import std.array : appender, array, join; +import std.ascii : isDigit; +import std.conv : to; +import std.csv : Malformed, csvReader; +import std.encoding : sanitize; +import std.json : JSONType, JSONValue, parseJSON; +import std.net.curl : CurlException, HTTP; +import std.range : take; +import std.string : startsWith, strip, toLower; +import std.typecons : Nullable, nullable; +import std.uri : encodeComponent; +import std.utf : byDchar; + +import cepis.exception; +import cepis.model; +import cepis.retry; + +@safe: + +/// The version of this library, as sent in the `User-Agent`. +enum string VERSION = "0.1.0"; + +/// Where the public API lives. +enum string DEFAULT_URL = "https://cep.is"; + +/// How many CEPs the batch endpoint accepts in one request. +enum size_t MAX_BATCH = 500; + +/// The statuses worth trying again: rate limiting and the transient 5xx. +static immutable ushort[5] RETRIABLE_STATUS = [429, 500, 502, 503, 504]; + +private enum string ACCEPT = "application/json, text/csv;q=0.9, */*;q=0.1"; + +/** + * Return the CEP's 8 digits, accepting the mask. + * + * `"01001-000"`, `"01001000"` and `" 01001-000 "` all become `"01001000"`. + * + * Throws: $(D InvalidCepException) unless exactly 8 digits remain. This runs + * $(B before) any request: a 7-digit CEP is the caller's bug, and spending a + * round trip for the server to say so wastes both sides' time. + */ +string normalizeCep(scope const(char)[] value) +{ + auto cleaned = appender!string(); + foreach (character; value.strip()) + if (character != '-' && character != '.' && character != ' ') + cleaned.put(character); + + string result = cleaned.data; + if (result.length != 8 || !result.all!isDigit) + throw new InvalidCepException(`CEP must be 8 digits, got "` ~ value.idup ~ `"`); + return result; +} + +/// +unittest +{ + assert(normalizeCep("01001-000") == "01001000"); + assert(normalizeCep(" 01001.000 ") == "01001000"); +} + +/** + * Everything a $(D Client) can be told. Every field has a default; set only + * what differs. + */ +struct ClientConfig +{ + /** + * Where the API lives. + * + * Point it at a local instance (the API repo brings one up on + * `http://localhost:8080` with `just up`) or at a test server. It is the + * only setting that usually changes between environments. Trailing slashes + * are stripped, so it cannot produce doubled separators that some proxies + * normalise and others 404. + */ + string baseUrl = DEFAULT_URL; + + /** + * How long to wait $(B per socket operation) — connecting, and each stall + * while reading. + * + * This is not a cap on the whole call. That cap is $(D totalTimeout). + */ + Duration timeout = dur!"seconds"(10); + + /** + * How many attempts in total, counting the first. + * + * `1` disables retrying. The default of `3` covers the common case (one + * transient failure) without turning a real outage into a long wait. + */ + uint maxAttempts = 3; + + /** + * Base of the exponential wait. + * + * The actual wait is drawn from `[0, base * 2^^(n-1)]` — $(I full jitter). + * The randomness is not decoration: without it, N clients that fail + * together come back together and reproduce the spike that took the service + * down. + */ + Duration backoffBase = dur!"msecs"(500); + + /** + * Cap on each individual wait. + * + * It applies to a server-sent `Retry-After` too, or a `Retry-After: 3600` + * from a misconfigured proxy would park the thread for an hour. + */ + Duration backoffMax = dur!"seconds"(8); + + /** + * Cap on the whole call, attempts and waits included. + * + * Null leaves $(D maxAttempts) as the only bound. Set it in code serving a + * web request: without it the worst case is `maxAttempts * timeout` plus + * the sum of the backoffs. + */ + Nullable!Duration totalTimeout; + + /** + * Overrides the `User-Agent`. + * + * Left empty, the default identifies the library and its version, which is + * what lets the server diagnose an old client misbehaving. + */ + string userAgent; +} + +/** + * Client for the cep.is API. + * + * Build one and reuse it: the configuration is immutable after construction and + * no method keeps state between calls, so it is safe to copy and to share + * across threads. + * + * --- + * ClientConfig config; + * config.baseUrl = "http://localhost:8080"; + * config.totalTimeout = dur!"seconds"(15); + * + * auto client = Client(config); + * auto address = client.lookup("01001000"); + * --- + */ +struct Client +{ + private ClientConfig config; + + /** + * Build a client from a configuration. + * + * Params: + * config = the settings to use; $(D ClientConfig.init) is the default + * configuration, pointing at $(D DEFAULT_URL). + * + * Throws: $(D CepisException) for a configuration that can only produce a + * client which never completes a call. + */ + this(ClientConfig config) + { + if (config.maxAttempts < 1) + throw new CepisException("maxAttempts must be >= 1"); + if (config.timeout <= Duration.zero) + throw new CepisException("timeout must be > 0"); + + this.config = config; + while (this.config.baseUrl.length > 0 && this.config.baseUrl[$ - 1] == '/') + this.config.baseUrl = this.config.baseUrl[0 .. $ - 1]; + if (this.config.userAgent.length == 0) + this.config.userAgent = "cepis-d/" ~ VERSION ~ " (+https://cep.is)"; + } + + /// The API URL in use, without a trailing slash. + string baseUrl() const pure nothrow @nogc + { + return config.baseUrl; + } + + /// The `User-Agent` sent with every request. + string userAgent() const pure nothrow @nogc + { + return config.userAgent; + } + + /// The per-socket-operation timeout. + Duration timeout() const pure nothrow @nogc + { + return config.timeout; + } + + /// How many attempts each call makes at most, counting the first. + uint maxAttempts() const pure nothrow @nogc + { + return config.maxAttempts; + } + + /// The cap on the whole call, if one is set. + Nullable!Duration totalTimeout() const pure nothrow + { + return config.totalTimeout; + } + + /** + * Look up a single CEP. + * + * The CEP may be masked or not. This is the only way to obtain a + * coordinate — see $(D lookupMany) for why. + * + * A 400 from the API is surfaced as $(D InvalidCepException) carrying the + * API's own message: the format already passed local validation, so the + * server knows something this client does not. + * + * Throws: + * $(UL + * $(LI $(D InvalidCepException) — not 8 digits (no request is made).) + * $(LI $(D CepNotFoundException) — well-formed, but not in the + * database.) + * $(LI $(D RateLimitedException) — 429 after every attempt.) + * $(LI $(D ServerException) — 5xx after every attempt.) + * $(LI $(D NetworkException) — the request never completed.) + * $(LI $(D InvalidResponseException) — the body could not be read as + * JSON.) + * ) + */ + Address lookup(scope const(char)[] cep) const + { + string number = normalizeCep(cep); + const response = request("/" ~ number, [ushort(200)]); + + if (response.status == 404) + throw new CepNotFoundException(number); + if (response.status == 400) + throw new InvalidCepException(errorMessage(response.content, + "CEP " ~ number ~ " rejected")); + + return Address.fromJSON(jsonObject(response.content)); + } + + /** + * Look up many CEPs in one batch. + * + * Returns an array of $(B the same length and in the same order) as the + * input. A CEP that does not exist becomes null at its position — it does + * not vanish from the array, and it does not throw. That is the right + * difference from $(D lookup): in a batch of a thousand, one bad CEP must + * not cost the other nine hundred and ninety-nine. + * + * A malformed CEP also becomes null, for the same reason. If you need to + * know $(I which) one was malformed, validate first with $(D normalizeCep). + * + * Input is chunked into blocks of $(D MAX_BATCH) automatically. Do not loop + * over $(D lookup) instead: 500 calls pay 500 round trips, the batch pays + * one. + * + * $(B The batch carries no coordinate.) The endpoint behind it is CSV, and + * the CSV has no `geo` column. Every $(D Address) returned here has a null + * $(D geo), including for CEPs that do have one. Need `geo`? Use + * $(D lookup), one at a time. + * + * Throws: transport failures only — $(D RateLimitedException), + * $(D ServerException), $(D NetworkException). A bad CEP never fails a + * batch. + */ + Nullable!Address[] lookupMany(scope const(string)[] ceps) const + { + if (ceps.length == 0) + return null; + + Nullable!Address[] result; + result.reserve(ceps.length); + for (size_t start = 0; start < ceps.length; start += MAX_BATCH) + { + const end = start + MAX_BATCH < ceps.length ? start + MAX_BATCH : ceps.length; + result ~= lookupChunk(ceps[start .. end]); + } + return result; + } + + /** + * Query `GET /health`: does the API answer, and with which database. + * + * An API reporting itself degraded is not an error — that arrives in + * $(D Health.status), and checking health is exactly when you want the + * diagnosis rather than an exception. Network failures still throw. + */ + Health health() const + { + const response = request("/health", [ushort(200), ushort(503)]); + return Health.fromJSON(jsonObject(response.content)); + } + + /** + * One chunk of at most $(D MAX_BATCH) CEPs, in a single request. + * + * A blank CSV row means "not found": the API fills only the `cep` column + * and leaves the rest empty, precisely so it does not misalign whoever is + * consuming the batch in a spreadsheet. + * + * The API guarantees one row per requested CEP. Should that ever fail to + * hold — a proxy truncating, a cut response — the result is padded to the + * input length rather than returned short, because a shifted array silently + * pairs the wrong address with the wrong CEP. + */ + private Nullable!Address[] lookupChunk(scope const(string)[] ceps) const + { + const response = request("/csv?ceps=" ~ encodeComponent(ceps.join(",")), [ + ushort(200) + ]); + + string text = sanitize(response.content); + if (text.startsWith("")) + text = text[3 .. $]; + + Nullable!Address[] rows; + try + { + foreach (record; csvReader!(string[string], Malformed.ignore)(text, null)) + { + auto localidade = "localidade" in record; + if (localidade is null || (*localidade).strip().length == 0) + { + rows ~= Nullable!Address.init; + continue; + } + + JSONValue object = JSONValue.emptyObject; + foreach (column, value; record) + object[column] = JSONValue(value); + rows ~= nullable(Address.fromJSON(object)); + } + } + catch (Exception) + { + } + + if (rows.length > ceps.length) + rows = rows[0 .. ceps.length]; + while (rows.length < ceps.length) + rows ~= Nullable!Address.init; + return rows; + } + + /** + * Perform the GET, with retries. + * + * $(I Definitive) error statuses (400, 404) come back normally, for the + * caller to turn into the right exception — only transient failures become + * exceptions here, and only once the attempts are exhausted. + */ + private Response request(string path, scope const(ushort)[] accepted) const + { + string url = config.baseUrl ~ path; + const bounded = !config.totalTimeout.isNull; + MonoTime deadline; + if (bounded) + deadline = MonoTime.currTime + config.totalTimeout.get; + + ushort lastStatus; + Nullable!Duration lastRetryAfter; + string lastFailure; + + foreach (attempt; 1 .. config.maxAttempts + 1) + { + Duration cap; + if (bounded) + { + cap = deadline - MonoTime.currTime; + if (cap <= Duration.zero) + break; + } + + try + { + auto response = httpGet(url, config.timeout, cap, config.userAgent); + if (accepted.canFind(response.status) + || !RETRIABLE_STATUS[].canFind(response.status)) + return response; + lastStatus = response.status; + lastRetryAfter = parseRetryAfter(response.retryAfter); + lastFailure = null; + } + catch (CurlException failure) + { + lastStatus = 0; + lastRetryAfter.nullify(); + lastFailure = failure.msg; + } + + if (attempt == config.maxAttempts) + break; + + auto wait = backoff(attempt, lastRetryAfter); + if (bounded) + { + const left = deadline - MonoTime.currTime; + if (left <= Duration.zero) + break; + if (wait > left) + wait = left; + } + sleepFor(wait); + } + + const suffix = " (after " ~ config.maxAttempts.to!string ~ " attempt(s))"; + if (lastStatus == 429) + throw new RateLimitedException("rate limit reached at " ~ url ~ suffix, lastRetryAfter); + if (lastStatus != 0) + throw new ServerException( + "the API answered " ~ lastStatus.to!string ~ " at " ~ url ~ suffix, lastStatus); + throw new NetworkException("could not reach " ~ url ~ suffix ~ ": " ~ (lastFailure is null + ? "no attempt was made within the total timeout" : lastFailure)); + } + + /** + * How long to wait before the next attempt. + * + * The server's `Retry-After` wins over the local calculation when present: + * it knows when it will be back, we are guessing. It still goes through the + * $(D backoffMax) cap. Otherwise the wait is drawn from `[0, ceiling]` + * rather than being the ceiling. + */ + private Duration backoff(uint attempt, Nullable!Duration retryAfter) const + { + if (!retryAfter.isNull) + { + const asked = retryAfter.get; + return asked < config.backoffMax ? asked : config.backoffMax; + } + + Duration ceiling = config.backoffBase; + foreach (_; 1 .. attempt) + { + if (ceiling >= config.backoffMax) + break; + ceiling *= 2; + } + if (ceiling > config.backoffMax) + ceiling = config.backoffMax; + return jitter(ceiling); + } +} + +private struct Response +{ + ushort status; + string content; + string retryAfter; +} + +/** + * One GET through libcurl. + * + * Trusted because `std.net.curl` predates the safety annotations: the callbacks + * below only append to locals owned by this frame, and nothing escapes it. + */ +private Response httpGet(string url, Duration timeout, Duration cap, string userAgent) @trusted +{ + auto http = HTTP(url); + http.method = HTTP.Method.get; + http.connectTimeout = timeout; + http.dataTimeout = timeout; + if (cap > Duration.zero) + http.operationTimeout = cap; + http.addRequestHeader("User-Agent", userAgent); + http.addRequestHeader("Accept", ACCEPT); + http.addRequestHeader("Accept-Encoding", "identity"); + + auto content = appender!(ubyte[])(); + string retryAfter; + http.onReceive = (ubyte[] data) { content.put(data); return data.length; }; + http.onReceiveHeader = (in char[] key, in char[] value) { + if (key.toLower() == "retry-after") + retryAfter = value.idup; + }; + + http.perform(); + return Response(http.statusLine.code, cast(string) content.data.idup, retryAfter); +} + +private void sleepFor(Duration wait) @trusted +{ + if (wait > Duration.zero) + Thread.sleep(wait); +} + +/// Decode a JSON object body, or throw $(D InvalidResponseException). +private JSONValue jsonObject(string content) +{ + JSONValue value; + try + value = parseJSON(content); + catch (Exception) + throw new InvalidResponseException("response is not valid JSON — something between you " + ~ "and the API is answering instead of it: " ~ preview(content)); + + if (value.type != JSONType.object) + throw new InvalidResponseException("expected a JSON object, got " ~ preview(content)); + return value; +} + +/// Pull `mensagem` out of an error body, falling back to a default. +private string errorMessage(string content, string fallback) +{ + try + { + const value = parseJSON(content); + if (value.type == JSONType.object) + { + const object = value.objectNoRef; + if (auto found = "mensagem" in object) + if (found.type == JSONType.string && found.str.length > 0) + return found.str; + } + } + catch (Exception) + { + } + return fallback; +} + +private string preview(string content) +{ + return `"` ~ sanitize(content).byDchar.take(120).array.to!string ~ `"`; +} + +@("a trailing slash is not duplicated") +unittest +{ + ClientConfig config; + config.baseUrl = "http://x//"; + + assert(Client(config).baseUrl == "http://x"); +} + +@("the user agent identifies the library") +unittest +{ + assert(Client(ClientConfig.init).userAgent.startsWith("cepis-d/")); + assert(Client(ClientConfig.init).baseUrl == DEFAULT_URL); +} + +@("a bad configuration is refused") +unittest +{ + import std.exception : assertThrown; + + ClientConfig noAttempts; + noAttempts.maxAttempts = 0; + assertThrown!CepisException(Client(noAttempts)); + + ClientConfig noTimeout; + noTimeout.timeout = Duration.zero; + assertThrown!CepisException(Client(noTimeout)); +} + +@("a masked cep normalises and a short one does not") +unittest +{ + import std.exception : assertThrown; + + foreach (value; ["01001000", "01001-000", " 01001-000 ", "01001.000"]) + assert(normalizeCep(value) == "01001000"); + + foreach (value; ["", "123", "0100100a", "010010000", "abcdefgh"]) + assertThrown!InvalidCepException(normalizeCep(value)); +} diff --git a/source/cepis/exception.d b/source/cepis/exception.d new file mode 100644 index 0000000..9b77449 --- /dev/null +++ b/source/cepis/exception.d @@ -0,0 +1,168 @@ +/** + * Exception hierarchy. + * + * Everything derives from $(D CepisException), so callers who only care that + * $(I something) went wrong can catch one type. + * + * Below that, the split is by $(B the decision the caller has to make), not by + * the HTTP status that caused it: + * + * $(UL + * $(LI $(D InvalidCepException) and $(D CepNotFoundException) are problems + * with the $(I request). Retrying is pointless — the same CEP will fail + * again.) + * $(LI $(D NetworkException), $(D ServerException) and + * $(D RateLimitedException) are problems with $(I transport). Retrying + * may help, and the client already retried on its own before throwing + * any of them.) + * ) + * + * That split is why $(D CepNotFoundException) is not "the HTTP 404 exception": + * the client also throws $(D InvalidCepException) without making any request at + * all, when the CEP is malformed before it ever leaves the machine. + */ +module cepis.exception; + +import core.time : Duration; +import std.typecons : Nullable; + +@safe: + +/** + * Base of every error this library throws. Catch this to handle any failure. + * + * Also thrown directly for a configuration mistake — a $(D ClientConfig) that + * can only produce a client which never completes a call. + */ +class CepisException : Exception +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } + + /** + * Whether trying the same call again later could plausibly succeed. + * + * True for $(D RateLimitedException), $(D ServerException) and + * $(D NetworkException); false for a bad or unknown CEP, which fails the + * same way forever. + * + * The client has already retried internally by the time any of these are + * thrown, so "later" means later — not immediately. + */ + bool retriable() const pure nothrow @nogc + { + return false; + } +} + +/** + * The CEP is not 8 digits. + * + * Usually thrown $(B without) a request ever being made — the client checks the + * format before going to the network. It is also what a 400 from the API + * becomes, in which case the message is the API's own. + */ +class InvalidCepException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } +} + +/// The CEP is well-formed but does not exist in the DNE database. +class CepNotFoundException : CepisException +{ + /// The CEP that was looked up, unmasked. + const string cep; + + /// + this(string cep, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super("CEP " ~ cep ~ " does not exist in the DNE database", file, line); + this.cep = cep; + } +} + +/// The API (or the edge in front of it) answered 429 on every attempt. +class RateLimitedException : CepisException +{ + /** + * What the server asked for via $(D Retry-After), or null if it did not + * say. + * + * The client already honoured it across the attempts it made; it is exposed + * here for callers who want to reschedule the work. + */ + const Nullable!Duration retryAfter; + + /// + this(string message, Nullable!Duration retryAfter, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + this.retryAfter = retryAfter; + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/// The API answered 5xx on every attempt. +class ServerException : CepisException +{ + /// The HTTP status seen on the last attempt. + const ushort status; + + /// + this(string message, ushort status, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + this.status = status; + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/** + * The request never completed: DNS, refused connection, timeout, TLS. + * + * Thrown only after every configured attempt has failed. + */ +class NetworkException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/** + * A success status came back with a body that cannot be parsed. + * + * In practice this means something between you and the API is answering instead + * of it — a captive Wi-Fi portal, a corporate proxy, an HTML error page. Useful + * as a diagnosis: the problem is not in your code. + */ +class InvalidResponseException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } +} diff --git a/source/cepis/model.d b/source/cepis/model.d new file mode 100644 index 0000000..9ec3ffb --- /dev/null +++ b/source/cepis/model.d @@ -0,0 +1,379 @@ +/** + * The data the API returns, as values. + * + * $(B Field names are the API's own, in Portuguese) — $(D logradouro), + * $(D bairro), $(D localidade). Everything else in this library is English; the + * fields are the one deliberate exception. + * + * The reason is fidelity: what you read in the JSON at + * $(LINK https://cep.is/01001000) is exactly what you access on the value. + * Translating to $(D street)/$(D city) would mean maintaining a field-by-field + * mapping and forcing anyone reading the API docs alongside this library to + * translate back in their head. The values are Portuguese either way — they + * come from the Correios database. + */ +module cepis.model; + +import std.conv : to; +import std.json : JSONType, JSONValue; +import std.typecons : Nullable, nullable; + +@safe: + +/** + * The CEP's coordinate, when the database has one. + * + * The point is the $(B median) of the coordinates of that CEP's addresses, not + * the mean: one badly geocoded address 40 km away shifts a mean and does not + * move a median. + * + * Always check $(D precisao) before using $(D lat)/$(D lon) for distance: + * + * $(UL + * $(LI $(D "logradouro"): the point is a stretch of street. $(D raio_m) lands + * around 100–150 m. Good for maps and for distance.) + * $(LI $(D "localidade"): the CEP $(I is) the whole municipality (a city-wide + * general CEP), and the point is its centroid. $(D raio_m) runs from + * 7 km to 40 km. Computing shipping from this is wrong by tens of + * kilometres.) + * ) + * + * $(D raio_m) is the radius containing 90% of that CEP's addresses. It exists + * precisely so that error cannot pass unnoticed. + */ +struct Geo +{ + /// Latitude in decimal degrees, WGS 84. + double lat; + /// Longitude in decimal degrees, WGS 84. + double lon; + /// Radius, in metres, containing 90% of the CEP's addresses. + long raio_m; + /// `"logradouro"` (a street) or `"localidade"` (a municipality centroid). + string precisao; + + /** + * True when the coordinate points at a street, not a municipality. + * + * Shorthand for `precisao == "logradouro"`, which is the check to make + * before using the point to measure distance. + */ + bool isPrecise() const pure nothrow @nogc + { + return precisao == "logradouro"; + } + + /** + * Build from the `geo` block the API returned. + * + * Returns a null $(D Nullable) for anything that is not a complete, + * readable block: a partial or malformed coordinate is not worth losing the + * whole address over. + */ + static Nullable!Geo fromJSON(const JSONValue value) + { + if (value.type != JSONType.object) + return Nullable!Geo.init; + + const object = value.objectNoRef; + auto lat = "lat" in object; + auto lon = "lon" in object; + auto raio = "raio_m" in object; + auto precisao = "precisao" in object; + if (lat is null || lon is null || raio is null || precisao is null) + return Nullable!Geo.init; + if (precisao.type != JSONType.string) + return Nullable!Geo.init; + + const latitude = asNumber(*lat); + const longitude = asNumber(*lon); + const radius = asNumber(*raio); + if (latitude.isNull || longitude.isNull || radius.isNull) + return Nullable!Geo.init; + + return nullable(Geo(latitude.get, longitude.get, cast(long) radius.get, precisao.str)); + } +} + +/** + * A CEP and everything the database knows about it. + * + * Fields the database does not have for that CEP come back as empty strings, + * never absent — that is what the API returns, and changing it here would + * create a difference between reading the JSON and reading this library. + * + * $(D geo) is the exception: it is null when there is no coordinate. There the + * difference is real, because the API $(B omits) the whole field in that case + * (about 43% of CEPs) rather than sending null. + * + * $(D raw) holds the response exactly as it arrived, uninterpreted, so a field + * the API grows before this library is updated is not lost. + */ +struct Address +{ + /// The CEP, always unmasked: `"01001000"`. + string cep; + /// Street, avenue, square — the public way. + string logradouro; + /// Range or side of the street the CEP covers, when it covers only part. + string complemento; + /// Name of the large user or building the CEP belongs to, when it does. + string unidade; + /// Neighbourhood. + string bairro; + /// Municipality. + string localidade; + /// Federative unit, two letters: `"SP"`. + string uf; + /// Federative unit spelled out: `"São Paulo"`. + string estado; + /// Region: `"Sudeste"`, `"Nordeste"`, and so on. + string regiao; + /// IBGE municipality code — the key to join with public datasets. + string ibge; + /// GIA code, São Paulo state tax administration only. + string gia; + /// Area code for telephony: `"11"`. + string ddd; + /// SIAFI municipality code, federal administration. + string siafi; + + /** + * The coordinate, or null when the database has none for this CEP. + * + * Always null for an address that came from $(D Client.lookupMany) — + * batches carry no coordinate at all. + */ + Nullable!Geo geo; + + /// The response exactly as it arrived, so a new API field is never lost. + JSONValue raw; + + /** + * The CEP with its mask: `01001-000`. + * + * The native contract always returns it unmasked; formatting is + * presentation, and it lives here so it does not become a repeated + * $(D format) call in every project that uses this. + */ + string formattedCep() const pure nothrow + { + return cep.length == 8 ? cep[0 .. 5] ~ "-" ~ cep[5 .. $] : cep; + } + + /** + * Build from the JSON object the API returned. + * + * Unknown fields are ignored during construction but preserved in $(D raw). + * A new field in the API therefore cannot break this library. + * + * A malformed `geo` block yields a null $(D geo) rather than failing the + * whole value: it is not worth losing the address over, since the rest of + * the response is still useful. + */ + static Address fromJSON(JSONValue value) + { + Address address; + address.cep = jsonText(value, "cep"); + address.logradouro = jsonText(value, "logradouro"); + address.complemento = jsonText(value, "complemento"); + address.unidade = jsonText(value, "unidade"); + address.bairro = jsonText(value, "bairro"); + address.localidade = jsonText(value, "localidade"); + address.uf = jsonText(value, "uf"); + address.estado = jsonText(value, "estado"); + address.regiao = jsonText(value, "regiao"); + address.ibge = jsonText(value, "ibge"); + address.gia = jsonText(value, "gia"); + address.ddd = jsonText(value, "ddd"); + address.siafi = jsonText(value, "siafi"); + + if (value.type == JSONType.object) + { + const object = value.objectNoRef; + if (auto block = "geo" in object) + address.geo = Geo.fromJSON(*block); + } + + address.raw = value; + return address; + } +} + +/** + * The response of `GET /health`. + * + * Good for two things: checking that the API answers, and knowing $(B which + * version of the database) is live. The DNE database changes every two weeks; + * $(D dne) identifies it. + */ +struct Health +{ + /// `"ok"` when the API considers itself healthy. + string status; + /// Identifier of the fortnightly DNE database currently loaded. + string dne; + /// How many CEPs the database holds. + ulong ceps; + /// How many of those carry a coordinate. + ulong geo; + /// When the database was generated, RFC 3339 in UTC. + string base_gerada_em; + /// The response exactly as it arrived. + JSONValue raw; + + /// True when the API reports itself healthy. + bool ok() const pure nothrow @nogc + { + return status == "ok"; + } + + /// Build from the JSON object the API returned. + static Health fromJSON(JSONValue value) + { + Health health; + health.status = jsonText(value, "status"); + health.dne = jsonText(value, "dne"); + health.ceps = jsonCount(value, "ceps"); + health.geo = jsonCount(value, "geo"); + health.base_gerada_em = jsonText(value, "base_gerada_em"); + health.raw = value; + return health; + } +} + +private string jsonText(const JSONValue value, string key) +{ + if (value.type != JSONType.object) + return ""; + + const object = value.objectNoRef; + auto found = key in object; + if (found is null) + return ""; + + switch (found.type) + { + case JSONType.string: + return found.str; + case JSONType.null_: + return ""; + default: + return found.toString(); + } +} + +private ulong jsonCount(const JSONValue value, string key) +{ + if (value.type != JSONType.object) + return 0; + + const object = value.objectNoRef; + auto found = key in object; + if (found is null) + return 0; + + switch (found.type) + { + case JSONType.integer: + return found.integer < 0 ? 0 : cast(ulong) found.integer; + case JSONType.uinteger: + return found.uinteger; + case JSONType.float_: + return found.floating < 0 ? 0 : cast(ulong) found.floating; + case JSONType.string: + try + return found.str.to!ulong; + catch (Exception) + return 0; + default: + return 0; + } +} + +private Nullable!double asNumber(const JSONValue value) +{ + switch (value.type) + { + case JSONType.integer: + return nullable(cast(double) value.integer); + case JSONType.uinteger: + return nullable(cast(double) value.uinteger); + case JSONType.float_: + return nullable(value.floating); + case JSONType.string: + try + return nullable(value.str.to!double); + catch (Exception) + return Nullable!double.init; + default: + return Nullable!double.init; + } +} + +@("an address survives a malformed geo block") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON( + `{"cep":"01001000","localidade":"São Paulo","geo":{"lat":"nowhere"}}`)); + + assert(address.geo.isNull); + assert(address.localidade == "São Paulo"); +} + +@("a complete geo block is read") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000","geo":{"lat":-23.5,` + ~ `"lon":-46.6,"raio_m":1084,"precisao":"logradouro"}}`)); + + assert(!address.geo.isNull); + assert(address.geo.get.raio_m == 1084); + assert(address.geo.get.isPrecise); +} + +@("a municipality coordinate is not precise") +unittest +{ + const geo = Geo(-23.5, -46.6, 22_000, "localidade"); + + assert(!geo.isPrecise); +} + +@("missing fields become empty strings, not errors") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000"}`)); + + assert(address.logradouro == ""); + assert(address.formattedCep == "01001-000"); +} + +@("an unknown field is preserved in raw") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000","campo_do_futuro":"value"}`)); + + assert(address.raw["campo_do_futuro"].str == "value"); +} + +@("health reads counts as numbers") +unittest +{ + import std.json : parseJSON; + + const health = Health.fromJSON( + parseJSON(`{"status":"ok","dne":"26071","ceps":1596168,"geo":894489}`)); + + assert(health.ok); + assert(health.ceps == 1_596_168); + assert(health.geo == 894_489); +} diff --git a/source/cepis/package.d b/source/cepis/package.d new file mode 100644 index 0000000..073d377 --- /dev/null +++ b/source/cepis/package.d @@ -0,0 +1,111 @@ +/** + * D client for the public CEP API at $(LINK https://cep.is). Official Correios + * data (DNE), refreshed every two weeks. + * + * Quick use — for a script, the module-level functions are enough: + * + * --- + * import cepis; + * + * auto address = lookup("01001-000"); + * writeln(address.localidade, " ", address.uf); // São Paulo SP + * --- + * + * In an application — build a $(D Client) once and reuse it: + * + * --- + * ClientConfig config; + * config.timeout = dur!"seconds"(5); + * config.totalTimeout = dur!"seconds"(15); + * + * auto client = Client(config); + * auto address = client.lookup("01001000"); + * --- + * + * Pointing at a local instance (the API repo brings one up with `just up`): + * + * --- + * ClientConfig config; + * config.baseUrl = "http://localhost:8080"; + * auto client = Client(config); + * --- + * + * Errors are explicit and split by decision — whether retrying helps or not: + * + * --- + * try + * render(lookup(cepFromForm)); + * catch (CepNotFoundException) + * askForAnotherCep(); // the CEP does not exist + * catch (CepisException error) + * if (error.retriable) reschedule(); // network, server, rate limit + * --- + * + * Note on field names: the API surface is English, but the $(B data fields keep + * the API's own Portuguese names) ($(D logradouro), $(D bairro), + * $(D localidade)). What you read in the JSON is what you access on the value. + * + * The module-level functions share one default client per thread, built on + * first use. Anyone needing different settings should build their own + * $(D Client). + */ +module cepis; + +public import cepis.client; +public import cepis.exception; +public import cepis.model; + +import std.typecons : Nullable; + +@safe: + +/** + * Look up a CEP using the default configuration. + * + * Shorthand for `Client(ClientConfig.init).lookup(cep)`; see $(D Client.lookup). + */ +Address lookup(scope const(char)[] cep) +{ + return defaultClient().lookup(cep); +} + +/** + * Look up many CEPs in one batch, using the default configuration. + * + * Shorthand for $(D Client.lookupMany) — read that one, especially the warning + * that batches carry no coordinate. + */ +Nullable!Address[] lookupMany(scope const(string)[] ceps) +{ + return defaultClient().lookupMany(ceps); +} + +/** + * Query the API's health using the default configuration. + * + * Shorthand for $(D Client.health). + */ +Health health() +{ + return defaultClient().health(); +} + +private Client defaultClient() +{ + static Client instance; + static bool built; + + if (!built) + { + instance = Client(ClientConfig.init); + built = true; + } + return instance; +} + +@("the default client is built once and points at the public API") +unittest +{ + assert(defaultClient().baseUrl == DEFAULT_URL); + assert(defaultClient().maxAttempts == 3); +} diff --git a/source/cepis/retry.d b/source/cepis/retry.d new file mode 100644 index 0000000..dd3ac77 --- /dev/null +++ b/source/cepis/retry.d @@ -0,0 +1,170 @@ +/** + * Backoff arithmetic and `Retry-After` parsing. + * + * Internal to the package. + */ +module cepis.retry; + +import core.time : Duration, dur; +import std.array : split; +import std.conv : to; +import std.datetime.date : DateTime; +import std.datetime.systime : Clock, SysTime; +import std.datetime.timezone : UTC; +import std.math : isFinite; +import std.random : uniform01; +import std.string : indexOf, strip; +import std.typecons : Nullable, nullable; + +package @safe: + +/** + * A random $(D Duration) in `[0, ceiling]` — $(I full jitter). + * + * The randomness is not decoration: without it, N clients that fail together + * come back together and reproduce the spike that took the service down. + * + * The source is the default thread-local generator, which is plenty for + * spreading retries and is not suitable for anything cryptographic. + */ +Duration jitter(Duration ceiling) +{ + if (ceiling <= Duration.zero) + return Duration.zero; + + return dur!"hnsecs"(cast(long)(ceiling.total!"hnsecs" * uniform01())); +} + +/** + * Parse the `Retry-After` header in both RFC 9110 forms. + * + * It may arrive as a number of seconds (`"120"`) or as an HTTP date + * (`"Wed, 21 Oct 2026 07:28:00 GMT"`). Handling only the first is the common + * bug, and it makes a client ignore exactly the wait request that matters most. + * + * A date already in the past yields zero; anything unparseable yields null. + */ +Nullable!Duration parseRetryAfter(scope const(char)[] value) +{ + const text = value.strip(); + if (text.length == 0) + return Nullable!Duration.init; + + try + { + const seconds = text.to!double; + if (isFinite(seconds)) + return nullable(dur!"msecs"(seconds <= 0 ? 0 : cast(long)(seconds * 1000))); + } + catch (Exception) + { + } + + const target = httpDateToUnix(text); + if (target.isNull) + return Nullable!Duration.init; + + const delta = target.get - Clock.currTime(UTC()).toUnixTime(); + return nullable(dur!"seconds"(delta <= 0 ? 0 : delta)); +} + +/** + * Seconds since the Unix epoch for an IMF-fixdate, the form HTTP servers must + * send: `Wed, 21 Oct 2026 07:28:00 GMT`. The two obsolete formats RFC 9110 + * still mentions are not accepted. + */ +private Nullable!long httpDateToUnix(scope const(char)[] value) +{ + auto rest = value; + const comma = value.indexOf(','); + if (comma >= 0) + rest = value[comma + 1 .. $]; + + const parts = rest.strip().split(); + if (parts.length < 4) + return Nullable!long.init; + + const month = monthNumber(parts[1]); + if (month == 0) + return Nullable!long.init; + + const clock = parts[3].split(':'); + if (clock.length != 3) + return Nullable!long.init; + + try + { + const stamp = SysTime(DateTime(parts[2].to!int, month, + parts[0].to!int, clock[0].to!int, clock[1].to!int, clock[2].to!int), UTC()); + return nullable(stamp.toUnixTime()); + } + catch (Exception) + { + return Nullable!long.init; + } +} + +private int monthNumber(scope const(char)[] name) pure nothrow @nogc +{ + static immutable string[12] months = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", + "Nov", "Dec" + ]; + + foreach (index, month; months) + if (month == name) + return cast(int)(index + 1); + return 0; +} + +@("plain seconds are read") +unittest +{ + assert(parseRetryAfter(" 7 ").get == dur!"seconds"(7)); +} + +@("negative seconds become zero") +unittest +{ + assert(parseRetryAfter("-5").get == Duration.zero); +} + +@("an http date is understood") +unittest +{ + assert(httpDateToUnix("Wed, 21 Oct 2026 07:28:00 GMT").get == 1_792_567_680L); +} + +@("a date in the past is zero") +unittest +{ + assert(parseRetryAfter("Thu, 01 Jan 1970 00:00:00 GMT").get == Duration.zero); +} + +@("a date in the future is a real wait") +unittest +{ + assert(parseRetryAfter("Wed, 21 Oct 2099 07:28:00 GMT").get > dur!"days"(1)); +} + +@("nonsense is rejected") +unittest +{ + assert(parseRetryAfter("soon").isNull); + assert(parseRetryAfter("").isNull); + assert(parseRetryAfter("Wed, 99 Foo 2026 07:28:00 GMT").isNull); +} + +@("jitter stays within the ceiling") +unittest +{ + const ceiling = dur!"msecs"(50); + + foreach (_; 0 .. 200) + { + const wait = jitter(ceiling); + assert(wait >= Duration.zero && wait <= ceiling); + } + + assert(jitter(Duration.zero) == Duration.zero); +} diff --git a/test/cepis_test.d b/test/cepis_test.d new file mode 100644 index 0000000..3ced878 --- /dev/null +++ b/test/cepis_test.d @@ -0,0 +1,637 @@ +/** + * Integration tests. + * + * These bring up a real HTTP server on loopback and point the client at it via + * $(D ClientConfig.baseUrl). No test touches the network and no test fakes the + * HTTP layer: the code path exercised is the one that runs in production, + * request construction and error body reading included. + * + * It doubles as the test that $(D baseUrl) works — if it did not, none of this + * would run. + * + * The fixture field names stay Portuguese because they mirror the API's JSON. + */ +module cepis_test; + +import core.sync.mutex : Mutex; +import core.thread : Thread; +import core.time : Duration, MonoTime, dur; +import std.algorithm : filter, map, startsWith; +import std.array : array, join; +import std.conv : to; +import std.exception : assertThrown; +import std.format : format; +import std.socket : InternetAddress, Socket, SocketOption, SocketOptionLevel, + SocketShutdown, TcpSocket; +import std.string : indexOf, split, toLower; +import std.typecons : Nullable, nullable; + +import cepis; + +private enum string SE_ADDRESS = `{ + "cep": "01001000", "logradouro": "Praça da Sé", "complemento": "lado ímpar", + "unidade": "", "bairro": "Sé", "localidade": "São Paulo", "uf": "SP", + "estado": "São Paulo", "regiao": "Sudeste", "ibge": "3550308", "gia": "1004", + "ddd": "11", "siafi": "7107" +}`; + +private enum string CSV_HEADER = "cep,logradouro,complemento,unidade,bairro,localidade,uf," + ~ "estado,regiao,ibge,gia,ddd,siafi"; + +private enum string SE_ROW = "01001000,Praça da Sé,lado ímpar,,Sé,São Paulo,SP,São Paulo,Sudeste,3550308,1004,11,7107"; + +private enum string RJ_ROW = "20040020,Avenida Rio Branco,,,Centro,Rio de Janeiro,RJ," + ~ "Rio de Janeiro,Sudeste,3304557,,21,6001"; + +private enum string MISSING_ROW = "99999999,,,,,,,,,,,,"; + +/// One canned response: status, body and extra headers. +private struct Reply +{ + ushort status; + string content; + string[2][] headers; +} + +/** + * A throwaway HTTP server on a random loopback port. + * + * Routes are matched by longest prefix, so `/csv` catches `/csv?ceps=...`. A + * route may hold several replies: each request consumes the next one, which is + * how retrying is tested, and the last one repeats once the list runs out. + * + * The accept loop runs on a daemon thread that ends with the test process; + * nothing here outlives `dub test`. + */ +private final class TestServer +{ + private Socket listener; + private Mutex lock; + private Reply[][string] routes; + private string[] calls; + private string[][] headers; + private ushort port; + + this() + { + lock = new Mutex(); + listener = new TcpSocket(); + listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true); + listener.bind(new InternetAddress("127.0.0.1", 0)); + listener.listen(32); + port = listener.localAddress.toPortString.to!ushort; + + auto worker = new Thread(&run); + worker.isDaemon = true; + worker.start(); + } + + string url() const + { + return "http://127.0.0.1:" ~ port.to!string; + } + + Client client(uint maxAttempts = 3) + { + ClientConfig config; + config.baseUrl = url; + config.timeout = dur!"seconds"(5); + config.backoffBase = dur!"msecs"(1); + config.backoffMax = dur!"msecs"(10); + config.maxAttempts = maxAttempts; + return Client(config); + } + + void respond(string path, Reply[] replies...) + { + synchronized (lock) + routes[path] = replies.dup; + } + + size_t calledTimes() + { + synchronized (lock) + return calls.length; + } + + size_t calledTimes(string prefix) + { + synchronized (lock) + return calls.filter!(c => c.startsWith(prefix)).array.length; + } + + string[] lastHeaders() + { + synchronized (lock) + return headers.length ? headers[$ - 1].dup : null; + } + + private void run() + { + while (true) + { + Socket peer; + try + peer = listener.accept(); + catch (Exception) + return; + + try + serve(peer); + catch (Exception) + { + } + + try + { + peer.shutdown(SocketShutdown.BOTH); + peer.close(); + } + catch (Exception) + { + } + } + } + + private void serve(Socket peer) + { + char[] request; + ubyte[4096] buffer; + while (request.indexOf("\r\n\r\n") < 0) + { + const received = peer.receive(buffer[]); + if (received <= 0) + break; + request ~= cast(char[]) buffer[0 .. received]; + } + + auto lines = request.idup.split("\r\n"); + if (lines.length == 0) + return; + + auto fields = lines[0].split(" "); + const path = fields.length > 1 ? fields[1] : "/"; + + Reply reply; + synchronized (lock) + { + calls ~= path; + headers ~= lines.length > 1 ? lines[1 .. $].dup : null; + + string best; + bool matched; + foreach (route; routes.byKey) + if (path.startsWith(route) && (!matched || route.length > best.length)) + { + best = route; + matched = true; + } + + if (!matched) + reply = Reply(404, `{"erro":"cep_nao_encontrado"}`); + else + { + auto replies = best in routes; + reply = (*replies)[0]; + if (replies.length > 1) + *replies = (*replies)[1 .. $]; + } + } + + auto head = format!"HTTP/1.1 %d Status\r\n"(reply.status); + foreach (header; reply.headers) + head ~= header[0] ~ ": " ~ header[1] ~ "\r\n"; + head ~= format!"Content-Length: %d\r\nConnection: close\r\n\r\n"(reply.content.length); + + sendAll(peer, head); + sendAll(peer, reply.content); + } + + private static void sendAll(Socket peer, const(void)[] data) + { + while (data.length > 0) + { + const sent = peer.send(data); + if (sent <= 0) + return; + data = data[sent .. $]; + } + } +} + +private string csvBody(string[] rows...) +{ + return CSV_HEADER ~ "\n" ~ rows.join("\n") ~ "\n"; +} + +private string withGeo(string precisao, long raio_m) +{ + enum string template_ = `{"cep":"01310100","logradouro":"Avenida Paulista","complemento":"",` + ~ `"unidade":"","bairro":"Bela Vista","localidade":"São Paulo","uf":"SP",` + ~ `"estado":"São Paulo","regiao":"Sudeste","ibge":"3550308","gia":"1004",` + ~ `"ddd":"11","siafi":"7107","geo":{"lat":-23.5,"lon":-46.6,"raio_m":%d,` + ~ `"precisao":"%s"}}`; + + return format!template_(raio_m, precisao); +} + +@("a successful lookup reads every field") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(200, SE_ADDRESS)); + + const address = server.client.lookup("01001-000"); + + assert(address.localidade == "São Paulo"); + assert(address.uf == "SP"); + assert(address.logradouro == "Praça da Sé"); + assert(address.complemento == "lado ímpar"); + assert(address.ibge == "3550308"); + assert(address.formattedCep == "01001-000"); + assert(address.geo.isNull); +} + +@("the request carries the user agent and an accept header") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(200, SE_ADDRESS)); + server.client.lookup("01001000"); + + const sent = server.lastHeaders.map!(h => h.toLower).array; + + assert(sent.filter!(h => h.startsWith("user-agent: cepis-d/")).array.length == 1); + assert(sent.filter!(h => h.startsWith("accept: application/json")).array.length == 1); +} + +@("the coordinate is read when present") +unittest +{ + auto server = new TestServer(); + server.respond("/01310100", Reply(200, withGeo("logradouro", 120))); + + const geo = server.client.lookup("01310100").geo.get; + + assert(geo.lat > -23.6 && geo.lat < -23.4); + assert(geo.lon > -46.7 && geo.lon < -46.5); + assert(geo.raio_m == 120); + assert(geo.isPrecise); +} + +@("a municipality coordinate is flagged as imprecise") +unittest +{ + auto server = new TestServer(); + server.respond("/01310100", Reply(200, withGeo("localidade", 22_000))); + + const geo = server.client.lookup("01310100").geo.get; + + assert(!geo.isPrecise); + assert(geo.raio_m == 22_000); +} + +@("a malformed geo block does not lose the address") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(200, + `{"cep":"01001000","localidade":"São Paulo","uf":"SP","geo":{"lat":"nowhere"}}`)); + + const address = server.client.lookup("01001000"); + + assert(address.geo.isNull); + assert(address.localidade == "São Paulo"); +} + +@("a new api field does not break the client") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(200, + `{"cep":"01001000","localidade":"São Paulo","campo_do_futuro":"value"}`)); + + const address = server.client.lookup("01001000"); + + assert(address.raw["campo_do_futuro"].str == "value"); + assert(address.localidade == "São Paulo"); +} + +@("a missing cep is not found, and is not retried") +unittest +{ + auto server = new TestServer(); + server.respond("/99999999", Reply(404, + `{"erro":"cep_nao_encontrado","mensagem":"CEP não existe na base do DNE"}`)); + + auto client = server.client; + bool caught; + try + client.lookup("99999999"); + catch (CepNotFoundException error) + { + caught = true; + assert(error.cep == "99999999"); + assert(!error.retriable); + } + + assert(caught); + assert(server.calledTimes("/99999999") == 1); +} + +@("an invalid cep never leaves the machine") +unittest +{ + auto server = new TestServer(); + + assertThrown!InvalidCepException(server.client.lookup("123")); + assert(server.calledTimes == 0); +} + +@("a 400 from the api becomes an invalid cep carrying its message") +unittest +{ + auto server = new TestServer(); + server.respond("/00000000", Reply(400, + `{"erro":"cep_invalido","mensagem":"rejected by the API"}`)); + + auto client = server.client; + bool caught; + try + client.lookup("00000000"); + catch (InvalidCepException error) + { + caught = true; + assert(error.msg == "rejected by the API"); + } + + assert(caught); + assert(server.calledTimes("/00000000") == 1); +} + +@("a response that is not json is diagnosed") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(200, "<html>captive portal</html>")); + + assertThrown!InvalidResponseException(server.client.lookup("01001000")); +} + +@("a 500 is retried and then succeeds") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(500, `{"erro":"interno"}`), Reply(200, SE_ADDRESS)); + + const address = server.client.lookup("01001000"); + + assert(address.localidade == "São Paulo"); + assert(server.calledTimes("/01001000") == 2); +} + +@("the attempts run out and the server error surfaces") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(503, `{"erro":"interno"}`)); + + auto client = server.client; + bool caught; + try + client.lookup("01001000"); + catch (ServerException error) + { + caught = true; + assert(error.status == 503); + assert(error.retriable); + } + + assert(caught); + assert(server.calledTimes("/01001000") == 3); +} + +@("one attempt disables retrying") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(500, "{}")); + + assertThrown!ServerException(server.client(1).lookup("01001000")); + assert(server.calledTimes("/01001000") == 1); +} + +@("rate limiting exposes retry-after") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(429, "{}", [["Retry-After", "7"]])); + + auto client = server.client; + bool caught; + try + client.lookup("01001000"); + catch (RateLimitedException error) + { + caught = true; + assert(error.retryAfter.get == dur!"seconds"(7)); + assert(error.retriable); + } + + assert(caught); +} + +@("a wild retry-after is capped") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(429, "{}", [["Retry-After", "3600"]])); + + const started = MonoTime.currTime; + assertThrown!RateLimitedException(server.client.lookup("01001000")); + + assert(MonoTime.currTime - started < dur!"seconds"(5)); +} + +@("retry-after as an http date is understood and still capped") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(429, "{}", [ + ["Retry-After", "Wed, 21 Oct 2099 07:28:00 GMT"] + ])); + + const started = MonoTime.currTime; + auto client = server.client; + bool caught; + try + client.lookup("01001000"); + catch (RateLimitedException error) + { + caught = true; + assert(error.retryAfter.get > dur!"days"(1)); + } + + assert(caught); + assert(MonoTime.currTime - started < dur!"seconds"(5)); +} + +@("nothing listening is a network error") +unittest +{ + ClientConfig config; + config.baseUrl = "http://127.0.0.1:1"; + config.maxAttempts = 2; + config.timeout = dur!"seconds"(1); + config.backoffBase = dur!"msecs"(1); + config.backoffMax = dur!"msecs"(10); + + auto client = Client(config); + bool caught; + try + client.lookup("01001000"); + catch (NetworkException error) + { + caught = true; + assert(error.retriable); + } + + assert(caught); +} + +@("the total timeout bounds the whole call") +unittest +{ + auto server = new TestServer(); + server.respond("/01001000", Reply(500, "{}")); + + ClientConfig config; + config.baseUrl = server.url; + config.timeout = dur!"seconds"(5); + config.maxAttempts = 50; + config.backoffBase = dur!"msecs"(50); + config.backoffMax = dur!"msecs"(50); + config.totalTimeout = dur!"msecs"(300); + + const started = MonoTime.currTime; + assertThrown!CepisException(Client(config).lookup("01001000")); + + assert(MonoTime.currTime - started < dur!"seconds"(3)); + assert(server.calledTimes("/01001000") < 50); +} + +@("a batch keeps order and marks the missing entry") +unittest +{ + auto server = new TestServer(); + server.respond("/csv", Reply(200, csvBody(SE_ROW, MISSING_ROW, RJ_ROW))); + + auto result = server.client.lookupMany(["01001000", "99999999", "20040020"]); + + assert(result.length == 3); + assert(result[0].get.localidade == "São Paulo"); + assert(result[1].isNull); + assert(result[2].get.uf == "RJ"); + assert(result[2].get.logradouro == "Avenida Rio Branco"); +} + +@("a batch carries no coordinate") +unittest +{ + auto server = new TestServer(); + server.respond("/csv", Reply(200, csvBody( + "01001000,Praça da Sé,,,Sé,São Paulo,SP,São Paulo,Sudeste,3550308,1004,11,7107"))); + + auto result = server.client.lookupMany(["01001000"]); + + assert(result[0].get.geo.isNull); +} + +@("a quoted field containing a comma is parsed") +unittest +{ + auto server = new TestServer(); + server.respond("/csv", Reply(200, + csvBody(`01001000,"Rua Um, Dois",,,Centro,São Paulo,SP,São Paulo,Sudeste,1,,11,1`))); + + auto result = server.client.lookupMany(["01001000"]); + + assert(result[0].get.logradouro == "Rua Um, Dois"); +} + +@("an empty batch makes no request") +unittest +{ + auto server = new TestServer(); + + assert(server.client.lookupMany([]).length == 0); + assert(server.calledTimes == 0); +} + +@("a batch above the api limit is chunked") +unittest +{ + auto server = new TestServer(); + auto ceps = new string[501]; + foreach (index; 0 .. ceps.length) + ceps[index] = format!"%08d"(index); + + string row(string cep) + { + return cep ~ ",,,,,X,SP,São Paulo,Sudeste,1,,11,1"; + } + + server.respond("/csv", Reply(200, csvBody(ceps[0 .. 500].map!row.array)), + Reply(200, csvBody(ceps[500 .. $].map!row.array))); + + auto result = server.client.lookupMany(ceps); + + assert(result.length == 501); + assert(server.calledTimes("/csv") == 2); + foreach (address; result) + assert(!address.isNull); +} + +@("a truncated batch pads instead of shifting") +unittest +{ + auto server = new TestServer(); + server.respond("/csv", Reply(200, + csvBody("01001000,,,,,São Paulo,SP,São Paulo,Sudeste,1,,11,1"))); + + auto result = server.client.lookupMany(["01001000", "20040020"]); + + assert(result.length == 2); + assert(!result[0].isNull); + assert(result[1].isNull); +} + +@("health reports the live database") +unittest +{ + auto server = new TestServer(); + server.respond("/health", Reply(200, + `{"status":"ok","dne":"26071","ceps":1596168,"geo":894489,` + ~ `"base_gerada_em":"2026-07-24T03:48:28Z"}`)); + + const health = server.client.health(); + + assert(health.ok); + assert(health.dne == "26071"); + assert(health.ceps == 1_596_168); + assert(health.geo == 894_489); + assert(health.base_gerada_em == "2026-07-24T03:48:28Z"); +} + +@("a degraded api is not an error") +unittest +{ + auto server = new TestServer(); + server.respond("/health", Reply(503, `{"status":"degradado","dne":"","ceps":0,"geo":0}`)); + + const health = server.client.health(); + + assert(!health.ok); + assert(health.status == "degradado"); + assert(server.calledTimes("/health") == 1); +} diff --git a/test/runner.d b/test/runner.d new file mode 100644 index 0000000..b3c7a55 --- /dev/null +++ b/test/runner.d @@ -0,0 +1,12 @@ +/** + * Entry point for the `unittest` build configuration. + * + * The runtime runs every `unittest` block before deciding whether to call this, + * and reports the outcome itself, so there is nothing to do here — the module + * exists only because an executable has to have a `main`. + */ +module runner; + +void main() +{ +} |