From 794f28199a4dde9d18ec711864336c96c58bb422 Mon Sep 17 00:00:00 2001 From: Murilo Ijanc Date: Sun, 26 Jul 2026 12:47:38 -0300 Subject: Initial import --- source/cepis/client.d | 588 +++++++++++++++++++++++++++++++++++++++++++++++ source/cepis/exception.d | 168 ++++++++++++++ source/cepis/model.d | 379 ++++++++++++++++++++++++++++++ source/cepis/package.d | 111 +++++++++ source/cepis/retry.d | 170 ++++++++++++++ 5 files changed, 1416 insertions(+) create mode 100644 source/cepis/client.d create mode 100644 source/cepis/exception.d create mode 100644 source/cepis/model.d create mode 100644 source/cepis/package.d create mode 100644 source/cepis/retry.d (limited to 'source/cepis') diff --git a/source/cepis/client.d b/source/cepis/client.d new file mode 100644 index 0000000..a93af7f --- /dev/null +++ b/source/cepis/client.d @@ -0,0 +1,588 @@ +/** + * The HTTP client, built on `std.net.curl`. + * + * A fresh curl handle is opened per request and connections are not pooled. For + * a one-off lookup that is irrelevant. For thousands, $(B do not write a loop) — + * use $(D Client.lookupMany), which resolves up to 500 CEPs per request. A loop + * of 500 calls pays 500 TLS handshakes; the batch pays one. + * + * Retry policy: 429 and the transient 5xx in $(D RETRIABLE_STATUS), plus network + * failures, are retried with exponentially backed-off, jittered waits. 400 and + * 404 are deliberately absent from that set — both are definitive answers about + * the request, so retrying returns the same thing more slowly while burning + * someone else's quota. + */ +module cepis.client; + +import core.thread : Thread; +import core.time : Duration, MonoTime, dur; +import std.algorithm : all, canFind; +import std.array : appender, array, join; +import std.ascii : isDigit; +import std.conv : to; +import std.csv : Malformed, csvReader; +import std.encoding : sanitize; +import std.json : JSONType, JSONValue, parseJSON; +import std.net.curl : CurlException, HTTP; +import std.range : take; +import std.string : startsWith, strip, toLower; +import std.typecons : Nullable, nullable; +import std.uri : encodeComponent; +import std.utf : byDchar; + +import cepis.exception; +import cepis.model; +import cepis.retry; + +@safe: + +/// The version of this library, as sent in the `User-Agent`. +enum string VERSION = "0.1.0"; + +/// Where the public API lives. +enum string DEFAULT_URL = "https://cep.is"; + +/// How many CEPs the batch endpoint accepts in one request. +enum size_t MAX_BATCH = 500; + +/// The statuses worth trying again: rate limiting and the transient 5xx. +static immutable ushort[5] RETRIABLE_STATUS = [429, 500, 502, 503, 504]; + +private enum string ACCEPT = "application/json, text/csv;q=0.9, */*;q=0.1"; + +/** + * Return the CEP's 8 digits, accepting the mask. + * + * `"01001-000"`, `"01001000"` and `" 01001-000 "` all become `"01001000"`. + * + * Throws: $(D InvalidCepException) unless exactly 8 digits remain. This runs + * $(B before) any request: a 7-digit CEP is the caller's bug, and spending a + * round trip for the server to say so wastes both sides' time. + */ +string normalizeCep(scope const(char)[] value) +{ + auto cleaned = appender!string(); + foreach (character; value.strip()) + if (character != '-' && character != '.' && character != ' ') + cleaned.put(character); + + string result = cleaned.data; + if (result.length != 8 || !result.all!isDigit) + throw new InvalidCepException(`CEP must be 8 digits, got "` ~ value.idup ~ `"`); + return result; +} + +/// +unittest +{ + assert(normalizeCep("01001-000") == "01001000"); + assert(normalizeCep(" 01001.000 ") == "01001000"); +} + +/** + * Everything a $(D Client) can be told. Every field has a default; set only + * what differs. + */ +struct ClientConfig +{ + /** + * Where the API lives. + * + * Point it at a local instance (the API repo brings one up on + * `http://localhost:8080` with `just up`) or at a test server. It is the + * only setting that usually changes between environments. Trailing slashes + * are stripped, so it cannot produce doubled separators that some proxies + * normalise and others 404. + */ + string baseUrl = DEFAULT_URL; + + /** + * How long to wait $(B per socket operation) — connecting, and each stall + * while reading. + * + * This is not a cap on the whole call. That cap is $(D totalTimeout). + */ + Duration timeout = dur!"seconds"(10); + + /** + * How many attempts in total, counting the first. + * + * `1` disables retrying. The default of `3` covers the common case (one + * transient failure) without turning a real outage into a long wait. + */ + uint maxAttempts = 3; + + /** + * Base of the exponential wait. + * + * The actual wait is drawn from `[0, base * 2^^(n-1)]` — $(I full jitter). + * The randomness is not decoration: without it, N clients that fail + * together come back together and reproduce the spike that took the service + * down. + */ + Duration backoffBase = dur!"msecs"(500); + + /** + * Cap on each individual wait. + * + * It applies to a server-sent `Retry-After` too, or a `Retry-After: 3600` + * from a misconfigured proxy would park the thread for an hour. + */ + Duration backoffMax = dur!"seconds"(8); + + /** + * Cap on the whole call, attempts and waits included. + * + * Null leaves $(D maxAttempts) as the only bound. Set it in code serving a + * web request: without it the worst case is `maxAttempts * timeout` plus + * the sum of the backoffs. + */ + Nullable!Duration totalTimeout; + + /** + * Overrides the `User-Agent`. + * + * Left empty, the default identifies the library and its version, which is + * what lets the server diagnose an old client misbehaving. + */ + string userAgent; +} + +/** + * Client for the cep.is API. + * + * Build one and reuse it: the configuration is immutable after construction and + * no method keeps state between calls, so it is safe to copy and to share + * across threads. + * + * --- + * ClientConfig config; + * config.baseUrl = "http://localhost:8080"; + * config.totalTimeout = dur!"seconds"(15); + * + * auto client = Client(config); + * auto address = client.lookup("01001000"); + * --- + */ +struct Client +{ + private ClientConfig config; + + /** + * Build a client from a configuration. + * + * Params: + * config = the settings to use; $(D ClientConfig.init) is the default + * configuration, pointing at $(D DEFAULT_URL). + * + * Throws: $(D CepisException) for a configuration that can only produce a + * client which never completes a call. + */ + this(ClientConfig config) + { + if (config.maxAttempts < 1) + throw new CepisException("maxAttempts must be >= 1"); + if (config.timeout <= Duration.zero) + throw new CepisException("timeout must be > 0"); + + this.config = config; + while (this.config.baseUrl.length > 0 && this.config.baseUrl[$ - 1] == '/') + this.config.baseUrl = this.config.baseUrl[0 .. $ - 1]; + if (this.config.userAgent.length == 0) + this.config.userAgent = "cepis-d/" ~ VERSION ~ " (+https://cep.is)"; + } + + /// The API URL in use, without a trailing slash. + string baseUrl() const pure nothrow @nogc + { + return config.baseUrl; + } + + /// The `User-Agent` sent with every request. + string userAgent() const pure nothrow @nogc + { + return config.userAgent; + } + + /// The per-socket-operation timeout. + Duration timeout() const pure nothrow @nogc + { + return config.timeout; + } + + /// How many attempts each call makes at most, counting the first. + uint maxAttempts() const pure nothrow @nogc + { + return config.maxAttempts; + } + + /// The cap on the whole call, if one is set. + Nullable!Duration totalTimeout() const pure nothrow + { + return config.totalTimeout; + } + + /** + * Look up a single CEP. + * + * The CEP may be masked or not. This is the only way to obtain a + * coordinate — see $(D lookupMany) for why. + * + * A 400 from the API is surfaced as $(D InvalidCepException) carrying the + * API's own message: the format already passed local validation, so the + * server knows something this client does not. + * + * Throws: + * $(UL + * $(LI $(D InvalidCepException) — not 8 digits (no request is made).) + * $(LI $(D CepNotFoundException) — well-formed, but not in the + * database.) + * $(LI $(D RateLimitedException) — 429 after every attempt.) + * $(LI $(D ServerException) — 5xx after every attempt.) + * $(LI $(D NetworkException) — the request never completed.) + * $(LI $(D InvalidResponseException) — the body could not be read as + * JSON.) + * ) + */ + Address lookup(scope const(char)[] cep) const + { + string number = normalizeCep(cep); + const response = request("/" ~ number, [ushort(200)]); + + if (response.status == 404) + throw new CepNotFoundException(number); + if (response.status == 400) + throw new InvalidCepException(errorMessage(response.content, + "CEP " ~ number ~ " rejected")); + + return Address.fromJSON(jsonObject(response.content)); + } + + /** + * Look up many CEPs in one batch. + * + * Returns an array of $(B the same length and in the same order) as the + * input. A CEP that does not exist becomes null at its position — it does + * not vanish from the array, and it does not throw. That is the right + * difference from $(D lookup): in a batch of a thousand, one bad CEP must + * not cost the other nine hundred and ninety-nine. + * + * A malformed CEP also becomes null, for the same reason. If you need to + * know $(I which) one was malformed, validate first with $(D normalizeCep). + * + * Input is chunked into blocks of $(D MAX_BATCH) automatically. Do not loop + * over $(D lookup) instead: 500 calls pay 500 round trips, the batch pays + * one. + * + * $(B The batch carries no coordinate.) The endpoint behind it is CSV, and + * the CSV has no `geo` column. Every $(D Address) returned here has a null + * $(D geo), including for CEPs that do have one. Need `geo`? Use + * $(D lookup), one at a time. + * + * Throws: transport failures only — $(D RateLimitedException), + * $(D ServerException), $(D NetworkException). A bad CEP never fails a + * batch. + */ + Nullable!Address[] lookupMany(scope const(string)[] ceps) const + { + if (ceps.length == 0) + return null; + + Nullable!Address[] result; + result.reserve(ceps.length); + for (size_t start = 0; start < ceps.length; start += MAX_BATCH) + { + const end = start + MAX_BATCH < ceps.length ? start + MAX_BATCH : ceps.length; + result ~= lookupChunk(ceps[start .. end]); + } + return result; + } + + /** + * Query `GET /health`: does the API answer, and with which database. + * + * An API reporting itself degraded is not an error — that arrives in + * $(D Health.status), and checking health is exactly when you want the + * diagnosis rather than an exception. Network failures still throw. + */ + Health health() const + { + const response = request("/health", [ushort(200), ushort(503)]); + return Health.fromJSON(jsonObject(response.content)); + } + + /** + * One chunk of at most $(D MAX_BATCH) CEPs, in a single request. + * + * A blank CSV row means "not found": the API fills only the `cep` column + * and leaves the rest empty, precisely so it does not misalign whoever is + * consuming the batch in a spreadsheet. + * + * The API guarantees one row per requested CEP. Should that ever fail to + * hold — a proxy truncating, a cut response — the result is padded to the + * input length rather than returned short, because a shifted array silently + * pairs the wrong address with the wrong CEP. + */ + private Nullable!Address[] lookupChunk(scope const(string)[] ceps) const + { + const response = request("/csv?ceps=" ~ encodeComponent(ceps.join(",")), [ + ushort(200) + ]); + + string text = sanitize(response.content); + if (text.startsWith("")) + text = text[3 .. $]; + + Nullable!Address[] rows; + try + { + foreach (record; csvReader!(string[string], Malformed.ignore)(text, null)) + { + auto localidade = "localidade" in record; + if (localidade is null || (*localidade).strip().length == 0) + { + rows ~= Nullable!Address.init; + continue; + } + + JSONValue object = JSONValue.emptyObject; + foreach (column, value; record) + object[column] = JSONValue(value); + rows ~= nullable(Address.fromJSON(object)); + } + } + catch (Exception) + { + } + + if (rows.length > ceps.length) + rows = rows[0 .. ceps.length]; + while (rows.length < ceps.length) + rows ~= Nullable!Address.init; + return rows; + } + + /** + * Perform the GET, with retries. + * + * $(I Definitive) error statuses (400, 404) come back normally, for the + * caller to turn into the right exception — only transient failures become + * exceptions here, and only once the attempts are exhausted. + */ + private Response request(string path, scope const(ushort)[] accepted) const + { + string url = config.baseUrl ~ path; + const bounded = !config.totalTimeout.isNull; + MonoTime deadline; + if (bounded) + deadline = MonoTime.currTime + config.totalTimeout.get; + + ushort lastStatus; + Nullable!Duration lastRetryAfter; + string lastFailure; + + foreach (attempt; 1 .. config.maxAttempts + 1) + { + Duration cap; + if (bounded) + { + cap = deadline - MonoTime.currTime; + if (cap <= Duration.zero) + break; + } + + try + { + auto response = httpGet(url, config.timeout, cap, config.userAgent); + if (accepted.canFind(response.status) + || !RETRIABLE_STATUS[].canFind(response.status)) + return response; + lastStatus = response.status; + lastRetryAfter = parseRetryAfter(response.retryAfter); + lastFailure = null; + } + catch (CurlException failure) + { + lastStatus = 0; + lastRetryAfter.nullify(); + lastFailure = failure.msg; + } + + if (attempt == config.maxAttempts) + break; + + auto wait = backoff(attempt, lastRetryAfter); + if (bounded) + { + const left = deadline - MonoTime.currTime; + if (left <= Duration.zero) + break; + if (wait > left) + wait = left; + } + sleepFor(wait); + } + + const suffix = " (after " ~ config.maxAttempts.to!string ~ " attempt(s))"; + if (lastStatus == 429) + throw new RateLimitedException("rate limit reached at " ~ url ~ suffix, lastRetryAfter); + if (lastStatus != 0) + throw new ServerException( + "the API answered " ~ lastStatus.to!string ~ " at " ~ url ~ suffix, lastStatus); + throw new NetworkException("could not reach " ~ url ~ suffix ~ ": " ~ (lastFailure is null + ? "no attempt was made within the total timeout" : lastFailure)); + } + + /** + * How long to wait before the next attempt. + * + * The server's `Retry-After` wins over the local calculation when present: + * it knows when it will be back, we are guessing. It still goes through the + * $(D backoffMax) cap. Otherwise the wait is drawn from `[0, ceiling]` + * rather than being the ceiling. + */ + private Duration backoff(uint attempt, Nullable!Duration retryAfter) const + { + if (!retryAfter.isNull) + { + const asked = retryAfter.get; + return asked < config.backoffMax ? asked : config.backoffMax; + } + + Duration ceiling = config.backoffBase; + foreach (_; 1 .. attempt) + { + if (ceiling >= config.backoffMax) + break; + ceiling *= 2; + } + if (ceiling > config.backoffMax) + ceiling = config.backoffMax; + return jitter(ceiling); + } +} + +private struct Response +{ + ushort status; + string content; + string retryAfter; +} + +/** + * One GET through libcurl. + * + * Trusted because `std.net.curl` predates the safety annotations: the callbacks + * below only append to locals owned by this frame, and nothing escapes it. + */ +private Response httpGet(string url, Duration timeout, Duration cap, string userAgent) @trusted +{ + auto http = HTTP(url); + http.method = HTTP.Method.get; + http.connectTimeout = timeout; + http.dataTimeout = timeout; + if (cap > Duration.zero) + http.operationTimeout = cap; + http.addRequestHeader("User-Agent", userAgent); + http.addRequestHeader("Accept", ACCEPT); + http.addRequestHeader("Accept-Encoding", "identity"); + + auto content = appender!(ubyte[])(); + string retryAfter; + http.onReceive = (ubyte[] data) { content.put(data); return data.length; }; + http.onReceiveHeader = (in char[] key, in char[] value) { + if (key.toLower() == "retry-after") + retryAfter = value.idup; + }; + + http.perform(); + return Response(http.statusLine.code, cast(string) content.data.idup, retryAfter); +} + +private void sleepFor(Duration wait) @trusted +{ + if (wait > Duration.zero) + Thread.sleep(wait); +} + +/// Decode a JSON object body, or throw $(D InvalidResponseException). +private JSONValue jsonObject(string content) +{ + JSONValue value; + try + value = parseJSON(content); + catch (Exception) + throw new InvalidResponseException("response is not valid JSON — something between you " + ~ "and the API is answering instead of it: " ~ preview(content)); + + if (value.type != JSONType.object) + throw new InvalidResponseException("expected a JSON object, got " ~ preview(content)); + return value; +} + +/// Pull `mensagem` out of an error body, falling back to a default. +private string errorMessage(string content, string fallback) +{ + try + { + const value = parseJSON(content); + if (value.type == JSONType.object) + { + const object = value.objectNoRef; + if (auto found = "mensagem" in object) + if (found.type == JSONType.string && found.str.length > 0) + return found.str; + } + } + catch (Exception) + { + } + return fallback; +} + +private string preview(string content) +{ + return `"` ~ sanitize(content).byDchar.take(120).array.to!string ~ `"`; +} + +@("a trailing slash is not duplicated") +unittest +{ + ClientConfig config; + config.baseUrl = "http://x//"; + + assert(Client(config).baseUrl == "http://x"); +} + +@("the user agent identifies the library") +unittest +{ + assert(Client(ClientConfig.init).userAgent.startsWith("cepis-d/")); + assert(Client(ClientConfig.init).baseUrl == DEFAULT_URL); +} + +@("a bad configuration is refused") +unittest +{ + import std.exception : assertThrown; + + ClientConfig noAttempts; + noAttempts.maxAttempts = 0; + assertThrown!CepisException(Client(noAttempts)); + + ClientConfig noTimeout; + noTimeout.timeout = Duration.zero; + assertThrown!CepisException(Client(noTimeout)); +} + +@("a masked cep normalises and a short one does not") +unittest +{ + import std.exception : assertThrown; + + foreach (value; ["01001000", "01001-000", " 01001-000 ", "01001.000"]) + assert(normalizeCep(value) == "01001000"); + + foreach (value; ["", "123", "0100100a", "010010000", "abcdefgh"]) + assertThrown!InvalidCepException(normalizeCep(value)); +} diff --git a/source/cepis/exception.d b/source/cepis/exception.d new file mode 100644 index 0000000..9b77449 --- /dev/null +++ b/source/cepis/exception.d @@ -0,0 +1,168 @@ +/** + * Exception hierarchy. + * + * Everything derives from $(D CepisException), so callers who only care that + * $(I something) went wrong can catch one type. + * + * Below that, the split is by $(B the decision the caller has to make), not by + * the HTTP status that caused it: + * + * $(UL + * $(LI $(D InvalidCepException) and $(D CepNotFoundException) are problems + * with the $(I request). Retrying is pointless — the same CEP will fail + * again.) + * $(LI $(D NetworkException), $(D ServerException) and + * $(D RateLimitedException) are problems with $(I transport). Retrying + * may help, and the client already retried on its own before throwing + * any of them.) + * ) + * + * That split is why $(D CepNotFoundException) is not "the HTTP 404 exception": + * the client also throws $(D InvalidCepException) without making any request at + * all, when the CEP is malformed before it ever leaves the machine. + */ +module cepis.exception; + +import core.time : Duration; +import std.typecons : Nullable; + +@safe: + +/** + * Base of every error this library throws. Catch this to handle any failure. + * + * Also thrown directly for a configuration mistake — a $(D ClientConfig) that + * can only produce a client which never completes a call. + */ +class CepisException : Exception +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } + + /** + * Whether trying the same call again later could plausibly succeed. + * + * True for $(D RateLimitedException), $(D ServerException) and + * $(D NetworkException); false for a bad or unknown CEP, which fails the + * same way forever. + * + * The client has already retried internally by the time any of these are + * thrown, so "later" means later — not immediately. + */ + bool retriable() const pure nothrow @nogc + { + return false; + } +} + +/** + * The CEP is not 8 digits. + * + * Usually thrown $(B without) a request ever being made — the client checks the + * format before going to the network. It is also what a 400 from the API + * becomes, in which case the message is the API's own. + */ +class InvalidCepException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } +} + +/// The CEP is well-formed but does not exist in the DNE database. +class CepNotFoundException : CepisException +{ + /// The CEP that was looked up, unmasked. + const string cep; + + /// + this(string cep, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super("CEP " ~ cep ~ " does not exist in the DNE database", file, line); + this.cep = cep; + } +} + +/// The API (or the edge in front of it) answered 429 on every attempt. +class RateLimitedException : CepisException +{ + /** + * What the server asked for via $(D Retry-After), or null if it did not + * say. + * + * The client already honoured it across the attempts it made; it is exposed + * here for callers who want to reschedule the work. + */ + const Nullable!Duration retryAfter; + + /// + this(string message, Nullable!Duration retryAfter, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + this.retryAfter = retryAfter; + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/// The API answered 5xx on every attempt. +class ServerException : CepisException +{ + /// The HTTP status seen on the last attempt. + const ushort status; + + /// + this(string message, ushort status, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + this.status = status; + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/** + * The request never completed: DNS, refused connection, timeout, TLS. + * + * Thrown only after every configured attempt has failed. + */ +class NetworkException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } + + override bool retriable() const pure nothrow @nogc + { + return true; + } +} + +/** + * A success status came back with a body that cannot be parsed. + * + * In practice this means something between you and the API is answering instead + * of it — a captive Wi-Fi portal, a corporate proxy, an HTML error page. Useful + * as a diagnosis: the problem is not in your code. + */ +class InvalidResponseException : CepisException +{ + /// + this(string message, string file = __FILE__, size_t line = __LINE__) pure nothrow + { + super(message, file, line); + } +} diff --git a/source/cepis/model.d b/source/cepis/model.d new file mode 100644 index 0000000..9ec3ffb --- /dev/null +++ b/source/cepis/model.d @@ -0,0 +1,379 @@ +/** + * The data the API returns, as values. + * + * $(B Field names are the API's own, in Portuguese) — $(D logradouro), + * $(D bairro), $(D localidade). Everything else in this library is English; the + * fields are the one deliberate exception. + * + * The reason is fidelity: what you read in the JSON at + * $(LINK https://cep.is/01001000) is exactly what you access on the value. + * Translating to $(D street)/$(D city) would mean maintaining a field-by-field + * mapping and forcing anyone reading the API docs alongside this library to + * translate back in their head. The values are Portuguese either way — they + * come from the Correios database. + */ +module cepis.model; + +import std.conv : to; +import std.json : JSONType, JSONValue; +import std.typecons : Nullable, nullable; + +@safe: + +/** + * The CEP's coordinate, when the database has one. + * + * The point is the $(B median) of the coordinates of that CEP's addresses, not + * the mean: one badly geocoded address 40 km away shifts a mean and does not + * move a median. + * + * Always check $(D precisao) before using $(D lat)/$(D lon) for distance: + * + * $(UL + * $(LI $(D "logradouro"): the point is a stretch of street. $(D raio_m) lands + * around 100–150 m. Good for maps and for distance.) + * $(LI $(D "localidade"): the CEP $(I is) the whole municipality (a city-wide + * general CEP), and the point is its centroid. $(D raio_m) runs from + * 7 km to 40 km. Computing shipping from this is wrong by tens of + * kilometres.) + * ) + * + * $(D raio_m) is the radius containing 90% of that CEP's addresses. It exists + * precisely so that error cannot pass unnoticed. + */ +struct Geo +{ + /// Latitude in decimal degrees, WGS 84. + double lat; + /// Longitude in decimal degrees, WGS 84. + double lon; + /// Radius, in metres, containing 90% of the CEP's addresses. + long raio_m; + /// `"logradouro"` (a street) or `"localidade"` (a municipality centroid). + string precisao; + + /** + * True when the coordinate points at a street, not a municipality. + * + * Shorthand for `precisao == "logradouro"`, which is the check to make + * before using the point to measure distance. + */ + bool isPrecise() const pure nothrow @nogc + { + return precisao == "logradouro"; + } + + /** + * Build from the `geo` block the API returned. + * + * Returns a null $(D Nullable) for anything that is not a complete, + * readable block: a partial or malformed coordinate is not worth losing the + * whole address over. + */ + static Nullable!Geo fromJSON(const JSONValue value) + { + if (value.type != JSONType.object) + return Nullable!Geo.init; + + const object = value.objectNoRef; + auto lat = "lat" in object; + auto lon = "lon" in object; + auto raio = "raio_m" in object; + auto precisao = "precisao" in object; + if (lat is null || lon is null || raio is null || precisao is null) + return Nullable!Geo.init; + if (precisao.type != JSONType.string) + return Nullable!Geo.init; + + const latitude = asNumber(*lat); + const longitude = asNumber(*lon); + const radius = asNumber(*raio); + if (latitude.isNull || longitude.isNull || radius.isNull) + return Nullable!Geo.init; + + return nullable(Geo(latitude.get, longitude.get, cast(long) radius.get, precisao.str)); + } +} + +/** + * A CEP and everything the database knows about it. + * + * Fields the database does not have for that CEP come back as empty strings, + * never absent — that is what the API returns, and changing it here would + * create a difference between reading the JSON and reading this library. + * + * $(D geo) is the exception: it is null when there is no coordinate. There the + * difference is real, because the API $(B omits) the whole field in that case + * (about 43% of CEPs) rather than sending null. + * + * $(D raw) holds the response exactly as it arrived, uninterpreted, so a field + * the API grows before this library is updated is not lost. + */ +struct Address +{ + /// The CEP, always unmasked: `"01001000"`. + string cep; + /// Street, avenue, square — the public way. + string logradouro; + /// Range or side of the street the CEP covers, when it covers only part. + string complemento; + /// Name of the large user or building the CEP belongs to, when it does. + string unidade; + /// Neighbourhood. + string bairro; + /// Municipality. + string localidade; + /// Federative unit, two letters: `"SP"`. + string uf; + /// Federative unit spelled out: `"São Paulo"`. + string estado; + /// Region: `"Sudeste"`, `"Nordeste"`, and so on. + string regiao; + /// IBGE municipality code — the key to join with public datasets. + string ibge; + /// GIA code, São Paulo state tax administration only. + string gia; + /// Area code for telephony: `"11"`. + string ddd; + /// SIAFI municipality code, federal administration. + string siafi; + + /** + * The coordinate, or null when the database has none for this CEP. + * + * Always null for an address that came from $(D Client.lookupMany) — + * batches carry no coordinate at all. + */ + Nullable!Geo geo; + + /// The response exactly as it arrived, so a new API field is never lost. + JSONValue raw; + + /** + * The CEP with its mask: `01001-000`. + * + * The native contract always returns it unmasked; formatting is + * presentation, and it lives here so it does not become a repeated + * $(D format) call in every project that uses this. + */ + string formattedCep() const pure nothrow + { + return cep.length == 8 ? cep[0 .. 5] ~ "-" ~ cep[5 .. $] : cep; + } + + /** + * Build from the JSON object the API returned. + * + * Unknown fields are ignored during construction but preserved in $(D raw). + * A new field in the API therefore cannot break this library. + * + * A malformed `geo` block yields a null $(D geo) rather than failing the + * whole value: it is not worth losing the address over, since the rest of + * the response is still useful. + */ + static Address fromJSON(JSONValue value) + { + Address address; + address.cep = jsonText(value, "cep"); + address.logradouro = jsonText(value, "logradouro"); + address.complemento = jsonText(value, "complemento"); + address.unidade = jsonText(value, "unidade"); + address.bairro = jsonText(value, "bairro"); + address.localidade = jsonText(value, "localidade"); + address.uf = jsonText(value, "uf"); + address.estado = jsonText(value, "estado"); + address.regiao = jsonText(value, "regiao"); + address.ibge = jsonText(value, "ibge"); + address.gia = jsonText(value, "gia"); + address.ddd = jsonText(value, "ddd"); + address.siafi = jsonText(value, "siafi"); + + if (value.type == JSONType.object) + { + const object = value.objectNoRef; + if (auto block = "geo" in object) + address.geo = Geo.fromJSON(*block); + } + + address.raw = value; + return address; + } +} + +/** + * The response of `GET /health`. + * + * Good for two things: checking that the API answers, and knowing $(B which + * version of the database) is live. The DNE database changes every two weeks; + * $(D dne) identifies it. + */ +struct Health +{ + /// `"ok"` when the API considers itself healthy. + string status; + /// Identifier of the fortnightly DNE database currently loaded. + string dne; + /// How many CEPs the database holds. + ulong ceps; + /// How many of those carry a coordinate. + ulong geo; + /// When the database was generated, RFC 3339 in UTC. + string base_gerada_em; + /// The response exactly as it arrived. + JSONValue raw; + + /// True when the API reports itself healthy. + bool ok() const pure nothrow @nogc + { + return status == "ok"; + } + + /// Build from the JSON object the API returned. + static Health fromJSON(JSONValue value) + { + Health health; + health.status = jsonText(value, "status"); + health.dne = jsonText(value, "dne"); + health.ceps = jsonCount(value, "ceps"); + health.geo = jsonCount(value, "geo"); + health.base_gerada_em = jsonText(value, "base_gerada_em"); + health.raw = value; + return health; + } +} + +private string jsonText(const JSONValue value, string key) +{ + if (value.type != JSONType.object) + return ""; + + const object = value.objectNoRef; + auto found = key in object; + if (found is null) + return ""; + + switch (found.type) + { + case JSONType.string: + return found.str; + case JSONType.null_: + return ""; + default: + return found.toString(); + } +} + +private ulong jsonCount(const JSONValue value, string key) +{ + if (value.type != JSONType.object) + return 0; + + const object = value.objectNoRef; + auto found = key in object; + if (found is null) + return 0; + + switch (found.type) + { + case JSONType.integer: + return found.integer < 0 ? 0 : cast(ulong) found.integer; + case JSONType.uinteger: + return found.uinteger; + case JSONType.float_: + return found.floating < 0 ? 0 : cast(ulong) found.floating; + case JSONType.string: + try + return found.str.to!ulong; + catch (Exception) + return 0; + default: + return 0; + } +} + +private Nullable!double asNumber(const JSONValue value) +{ + switch (value.type) + { + case JSONType.integer: + return nullable(cast(double) value.integer); + case JSONType.uinteger: + return nullable(cast(double) value.uinteger); + case JSONType.float_: + return nullable(value.floating); + case JSONType.string: + try + return nullable(value.str.to!double); + catch (Exception) + return Nullable!double.init; + default: + return Nullable!double.init; + } +} + +@("an address survives a malformed geo block") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON( + `{"cep":"01001000","localidade":"São Paulo","geo":{"lat":"nowhere"}}`)); + + assert(address.geo.isNull); + assert(address.localidade == "São Paulo"); +} + +@("a complete geo block is read") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000","geo":{"lat":-23.5,` + ~ `"lon":-46.6,"raio_m":1084,"precisao":"logradouro"}}`)); + + assert(!address.geo.isNull); + assert(address.geo.get.raio_m == 1084); + assert(address.geo.get.isPrecise); +} + +@("a municipality coordinate is not precise") +unittest +{ + const geo = Geo(-23.5, -46.6, 22_000, "localidade"); + + assert(!geo.isPrecise); +} + +@("missing fields become empty strings, not errors") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000"}`)); + + assert(address.logradouro == ""); + assert(address.formattedCep == "01001-000"); +} + +@("an unknown field is preserved in raw") +unittest +{ + import std.json : parseJSON; + + const address = Address.fromJSON(parseJSON(`{"cep":"01001000","campo_do_futuro":"value"}`)); + + assert(address.raw["campo_do_futuro"].str == "value"); +} + +@("health reads counts as numbers") +unittest +{ + import std.json : parseJSON; + + const health = Health.fromJSON( + parseJSON(`{"status":"ok","dne":"26071","ceps":1596168,"geo":894489}`)); + + assert(health.ok); + assert(health.ceps == 1_596_168); + assert(health.geo == 894_489); +} diff --git a/source/cepis/package.d b/source/cepis/package.d new file mode 100644 index 0000000..073d377 --- /dev/null +++ b/source/cepis/package.d @@ -0,0 +1,111 @@ +/** + * D client for the public CEP API at $(LINK https://cep.is). Official Correios + * data (DNE), refreshed every two weeks. + * + * Quick use — for a script, the module-level functions are enough: + * + * --- + * import cepis; + * + * auto address = lookup("01001-000"); + * writeln(address.localidade, " ", address.uf); // São Paulo SP + * --- + * + * In an application — build a $(D Client) once and reuse it: + * + * --- + * ClientConfig config; + * config.timeout = dur!"seconds"(5); + * config.totalTimeout = dur!"seconds"(15); + * + * auto client = Client(config); + * auto address = client.lookup("01001000"); + * --- + * + * Pointing at a local instance (the API repo brings one up with `just up`): + * + * --- + * ClientConfig config; + * config.baseUrl = "http://localhost:8080"; + * auto client = Client(config); + * --- + * + * Errors are explicit and split by decision — whether retrying helps or not: + * + * --- + * try + * render(lookup(cepFromForm)); + * catch (CepNotFoundException) + * askForAnotherCep(); // the CEP does not exist + * catch (CepisException error) + * if (error.retriable) reschedule(); // network, server, rate limit + * --- + * + * Note on field names: the API surface is English, but the $(B data fields keep + * the API's own Portuguese names) ($(D logradouro), $(D bairro), + * $(D localidade)). What you read in the JSON is what you access on the value. + * + * The module-level functions share one default client per thread, built on + * first use. Anyone needing different settings should build their own + * $(D Client). + */ +module cepis; + +public import cepis.client; +public import cepis.exception; +public import cepis.model; + +import std.typecons : Nullable; + +@safe: + +/** + * Look up a CEP using the default configuration. + * + * Shorthand for `Client(ClientConfig.init).lookup(cep)`; see $(D Client.lookup). + */ +Address lookup(scope const(char)[] cep) +{ + return defaultClient().lookup(cep); +} + +/** + * Look up many CEPs in one batch, using the default configuration. + * + * Shorthand for $(D Client.lookupMany) — read that one, especially the warning + * that batches carry no coordinate. + */ +Nullable!Address[] lookupMany(scope const(string)[] ceps) +{ + return defaultClient().lookupMany(ceps); +} + +/** + * Query the API's health using the default configuration. + * + * Shorthand for $(D Client.health). + */ +Health health() +{ + return defaultClient().health(); +} + +private Client defaultClient() +{ + static Client instance; + static bool built; + + if (!built) + { + instance = Client(ClientConfig.init); + built = true; + } + return instance; +} + +@("the default client is built once and points at the public API") +unittest +{ + assert(defaultClient().baseUrl == DEFAULT_URL); + assert(defaultClient().maxAttempts == 3); +} diff --git a/source/cepis/retry.d b/source/cepis/retry.d new file mode 100644 index 0000000..dd3ac77 --- /dev/null +++ b/source/cepis/retry.d @@ -0,0 +1,170 @@ +/** + * Backoff arithmetic and `Retry-After` parsing. + * + * Internal to the package. + */ +module cepis.retry; + +import core.time : Duration, dur; +import std.array : split; +import std.conv : to; +import std.datetime.date : DateTime; +import std.datetime.systime : Clock, SysTime; +import std.datetime.timezone : UTC; +import std.math : isFinite; +import std.random : uniform01; +import std.string : indexOf, strip; +import std.typecons : Nullable, nullable; + +package @safe: + +/** + * A random $(D Duration) in `[0, ceiling]` — $(I full jitter). + * + * The randomness is not decoration: without it, N clients that fail together + * come back together and reproduce the spike that took the service down. + * + * The source is the default thread-local generator, which is plenty for + * spreading retries and is not suitable for anything cryptographic. + */ +Duration jitter(Duration ceiling) +{ + if (ceiling <= Duration.zero) + return Duration.zero; + + return dur!"hnsecs"(cast(long)(ceiling.total!"hnsecs" * uniform01())); +} + +/** + * Parse the `Retry-After` header in both RFC 9110 forms. + * + * It may arrive as a number of seconds (`"120"`) or as an HTTP date + * (`"Wed, 21 Oct 2026 07:28:00 GMT"`). Handling only the first is the common + * bug, and it makes a client ignore exactly the wait request that matters most. + * + * A date already in the past yields zero; anything unparseable yields null. + */ +Nullable!Duration parseRetryAfter(scope const(char)[] value) +{ + const text = value.strip(); + if (text.length == 0) + return Nullable!Duration.init; + + try + { + const seconds = text.to!double; + if (isFinite(seconds)) + return nullable(dur!"msecs"(seconds <= 0 ? 0 : cast(long)(seconds * 1000))); + } + catch (Exception) + { + } + + const target = httpDateToUnix(text); + if (target.isNull) + return Nullable!Duration.init; + + const delta = target.get - Clock.currTime(UTC()).toUnixTime(); + return nullable(dur!"seconds"(delta <= 0 ? 0 : delta)); +} + +/** + * Seconds since the Unix epoch for an IMF-fixdate, the form HTTP servers must + * send: `Wed, 21 Oct 2026 07:28:00 GMT`. The two obsolete formats RFC 9110 + * still mentions are not accepted. + */ +private Nullable!long httpDateToUnix(scope const(char)[] value) +{ + auto rest = value; + const comma = value.indexOf(','); + if (comma >= 0) + rest = value[comma + 1 .. $]; + + const parts = rest.strip().split(); + if (parts.length < 4) + return Nullable!long.init; + + const month = monthNumber(parts[1]); + if (month == 0) + return Nullable!long.init; + + const clock = parts[3].split(':'); + if (clock.length != 3) + return Nullable!long.init; + + try + { + const stamp = SysTime(DateTime(parts[2].to!int, month, + parts[0].to!int, clock[0].to!int, clock[1].to!int, clock[2].to!int), UTC()); + return nullable(stamp.toUnixTime()); + } + catch (Exception) + { + return Nullable!long.init; + } +} + +private int monthNumber(scope const(char)[] name) pure nothrow @nogc +{ + static immutable string[12] months = [ + "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", + "Nov", "Dec" + ]; + + foreach (index, month; months) + if (month == name) + return cast(int)(index + 1); + return 0; +} + +@("plain seconds are read") +unittest +{ + assert(parseRetryAfter(" 7 ").get == dur!"seconds"(7)); +} + +@("negative seconds become zero") +unittest +{ + assert(parseRetryAfter("-5").get == Duration.zero); +} + +@("an http date is understood") +unittest +{ + assert(httpDateToUnix("Wed, 21 Oct 2026 07:28:00 GMT").get == 1_792_567_680L); +} + +@("a date in the past is zero") +unittest +{ + assert(parseRetryAfter("Thu, 01 Jan 1970 00:00:00 GMT").get == Duration.zero); +} + +@("a date in the future is a real wait") +unittest +{ + assert(parseRetryAfter("Wed, 21 Oct 2099 07:28:00 GMT").get > dur!"days"(1)); +} + +@("nonsense is rejected") +unittest +{ + assert(parseRetryAfter("soon").isNull); + assert(parseRetryAfter("").isNull); + assert(parseRetryAfter("Wed, 99 Foo 2026 07:28:00 GMT").isNull); +} + +@("jitter stays within the ceiling") +unittest +{ + const ceiling = dur!"msecs"(50); + + foreach (_; 0 .. 200) + { + const wait = jitter(ceiling); + assert(wait >= Duration.zero && wait <= ceiling); + } + + assert(jitter(Duration.zero) == Duration.zero); +} -- cgit v1.3