path: root/test/cepis_test.d

/**
 * Integration tests.
 *
 * These bring up a real HTTP server on loopback and point the client at it via
 * $(D ClientConfig.baseUrl). No test touches the network and no test fakes the
 * HTTP layer: the code path exercised is the one that runs in production,
 * request construction and error body reading included.
 *
 * It doubles as the test that $(D baseUrl) works — if it did not, none of this
 * would run.
 *
 * The fixture field names stay Portuguese because they mirror the API's JSON.
 */
module cepis_test;

import core.sync.mutex : Mutex;
import core.thread : Thread;
import core.time : Duration, MonoTime, dur;
import std.algorithm : canFind, filter, map, startsWith;
import std.array : array, join;
import std.conv : to;
import std.exception : assertThrown;
import std.format : format;
import std.socket : InternetAddress, Socket, SocketOption, SocketOptionLevel,
    SocketShutdown, TcpSocket;
import std.string : indexOf, split, toLower;
import std.typecons : Nullable, nullable;

import cepis;

private enum string SE_ADDRESS = `{
  "cep": "01001000", "logradouro": "Praça da Sé", "complemento": "lado ímpar",
  "unidade": "", "bairro": "Sé", "localidade": "São Paulo", "uf": "SP",
  "estado": "São Paulo", "regiao": "Sudeste", "ibge": "3550308", "gia": "1004",
  "ddd": "11", "siafi": "7107"
}`;

/**
 * A 404 that knows which municipality the CEP would have belonged to.
 *
 * `25845000` is the DNE's own start for Areal and does not exist; the smallest
 * one that does is `25845001`. Fixed here rather than fetched: production does
 * not serve `faixa` yet, so a live call would pass for the wrong reason.
 */
private enum string AREAL_NOT_FOUND = `{
  "erro": "cep_nao_encontrado", "mensagem": "CEP não existe na base do DNE",
  "faixa": {
    "localidade": "Areal", "uf": "RJ", "estado": "Rio de Janeiro",
    "regiao": "Sudeste", "ibge": "3300225", "cep_ini": "25845001",
    "cep_fim": "25849899", "ceps": 118
  }
}`;

private enum string CSV_HEADER = "cep,logradouro,complemento,unidade,bairro,localidade,uf,"
    ~ "estado,regiao,ibge,gia,ddd,siafi";

private enum string SE_ROW = "01001000,Praça da Sé,lado ímpar,,Sé,São Paulo,SP,São Paulo,Sudeste,3550308,1004,11,7107";

private enum string RJ_ROW = "20040020,Avenida Rio Branco,,,Centro,Rio de Janeiro,RJ,"
    ~ "Rio de Janeiro,Sudeste,3304557,,21,6001";

private enum string MISSING_ROW = "99999999,,,,,,,,,,,,";

/// One canned response: status, body and extra headers.
private struct Reply
{
    ushort status;
    string content;
    string[2][] headers;
}

/**
 * A throwaway HTTP server on a random loopback port.
 *
 * Routes are matched by longest prefix, so `/csv` catches `/csv?ceps=...`. A
 * route may hold several replies: each request consumes the next one, which is
 * how retrying is tested, and the last one repeats once the list runs out.
 *
 * The accept loop runs on a daemon thread that ends with the test process;
 * nothing here outlives `dub test`.
 */
private final class TestServer
{
    private Socket listener;
    private Mutex lock;
    private Reply[][string] routes;
    private string[] calls;
    private string[][] headers;
    private ushort port;

    this()
    {
        lock = new Mutex();
        listener = new TcpSocket();
        listener.setOption(SocketOptionLevel.SOCKET, SocketOption.REUSEADDR, true);
        listener.bind(new InternetAddress("127.0.0.1", 0));
        listener.listen(32);
        port = listener.localAddress.toPortString.to!ushort;

        auto worker = new Thread(&run);
        worker.isDaemon = true;
        worker.start();
    }

    string url() const
    {
        return "http://127.0.0.1:" ~ port.to!string;
    }

    Client client(uint maxAttempts = 3)
    {
        ClientConfig config;
        config.baseUrl = url;
        config.timeout = dur!"seconds"(5);
        config.backoffBase = dur!"msecs"(1);
        config.backoffMax = dur!"msecs"(10);
        config.maxAttempts = maxAttempts;
        return Client(config);
    }

    void respond(string path, Reply[] replies...)
    {
        synchronized (lock)
            routes[path] = replies.dup;
    }

    size_t calledTimes()
    {
        synchronized (lock)
            return calls.length;
    }

    size_t calledTimes(string prefix)
    {
        synchronized (lock)
            return calls.filter!(c => c.startsWith(prefix)).array.length;
    }

    string[] lastHeaders()
    {
        synchronized (lock)
            return headers.length ? headers[$ - 1].dup : null;
    }

    private void run()
    {
        while (true)
        {
            Socket peer;
            try
                peer = listener.accept();
            catch (Exception)
                return;

            try
                serve(peer);
            catch (Exception)
            {
            }

            try
            {
                peer.shutdown(SocketShutdown.BOTH);
                peer.close();
            }
            catch (Exception)
            {
            }
        }
    }

    private void serve(Socket peer)
    {
        char[] request;
        ubyte[4096] buffer;
        while (request.indexOf("\r\n\r\n") < 0)
        {
            const received = peer.receive(buffer[]);
            if (received <= 0)
                break;
            request ~= cast(char[]) buffer[0 .. received];
        }

        auto lines = request.idup.split("\r\n");
        if (lines.length == 0)
            return;

        auto fields = lines[0].split(" ");
        const path = fields.length > 1 ? fields[1] : "/";

        Reply reply;
        synchronized (lock)
        {
            calls ~= path;
            headers ~= lines.length > 1 ? lines[1 .. $].dup : null;

            string best;
            bool matched;
            foreach (route; routes.byKey)
                if (path.startsWith(route) && (!matched || route.length > best.length))
                {
                    best = route;
                    matched = true;
                }

            if (!matched)
                reply = Reply(404, `{"erro":"cep_nao_encontrado"}`);
            else
            {
                auto replies = best in routes;
                reply = (*replies)[0];
                if (replies.length > 1)
                    *replies = (*replies)[1 .. $];
            }
        }

        auto head = format!"HTTP/1.1 %d Status\r\n"(reply.status);
        foreach (header; reply.headers)
            head ~= header[0] ~ ": " ~ header[1] ~ "\r\n";
        head ~= format!"Content-Length: %d\r\nConnection: close\r\n\r\n"(reply.content.length);

        sendAll(peer, head);
        sendAll(peer, reply.content);
    }

    private static void sendAll(Socket peer, const(void)[] data)
    {
        while (data.length > 0)
        {
            const sent = peer.send(data);
            if (sent <= 0)
                return;
            data = data[sent .. $];
        }
    }
}

private string csvBody(string[] rows...)
{
    return CSV_HEADER ~ "\n" ~ rows.join("\n") ~ "\n";
}

private string withGeo(string precisao, long raio_m)
{
    enum string template_ = `{"cep":"01310100","logradouro":"Avenida Paulista","complemento":"",`
        ~ `"unidade":"","bairro":"Bela Vista","localidade":"São Paulo","uf":"SP",`
        ~ `"estado":"São Paulo","regiao":"Sudeste","ibge":"3550308","gia":"1004",`
        ~ `"ddd":"11","siafi":"7107","geo":{"lat":-23.5,"lon":-46.6,"raio_m":%d,`
        ~ `"precisao":"%s"}}`;

    return format!template_(raio_m, precisao);
}

@("a successful lookup reads every field")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(200, SE_ADDRESS));

    const address = server.client.lookup("01001-000");

    assert(address.localidade == "São Paulo");
    assert(address.uf == "SP");
    assert(address.logradouro == "Praça da Sé");
    assert(address.complemento == "lado ímpar");
    assert(address.ibge == "3550308");
    assert(address.formattedCep == "01001-000");
    assert(address.geo.isNull);
}

@("the request carries the user agent and an accept header")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(200, SE_ADDRESS));
    server.client.lookup("01001000");

    const sent = server.lastHeaders.map!(h => h.toLower).array;

    assert(sent.filter!(h => h.startsWith("user-agent: cepis-d/")).array.length == 1);
    assert(sent.filter!(h => h.startsWith("accept: application/json")).array.length == 1);
}

@("the coordinate is read when present")
unittest
{
    auto server = new TestServer();
    server.respond("/01310100", Reply(200, withGeo("logradouro", 120)));

    const geo = server.client.lookup("01310100").geo.get;

    assert(geo.lat > -23.6 && geo.lat < -23.4);
    assert(geo.lon > -46.7 && geo.lon < -46.5);
    assert(geo.raio_m == 120);
    assert(geo.isPrecise);
}

@("a municipality coordinate is flagged as imprecise")
unittest
{
    auto server = new TestServer();
    server.respond("/01310100", Reply(200, withGeo("localidade", 22_000)));

    const geo = server.client.lookup("01310100").geo.get;

    assert(!geo.isPrecise);
    assert(geo.raio_m == 22_000);
}

@("a malformed geo block does not lose the address")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(200,
            `{"cep":"01001000","localidade":"São Paulo","uf":"SP","geo":{"lat":"nowhere"}}`));

    const address = server.client.lookup("01001000");

    assert(address.geo.isNull);
    assert(address.localidade == "São Paulo");
}

@("a new api field does not break the client")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(200,
            `{"cep":"01001000","localidade":"São Paulo","campo_do_futuro":"value"}`));

    const address = server.client.lookup("01001000");

    assert(address.raw["campo_do_futuro"].str == "value");
    assert(address.localidade == "São Paulo");
}

@("a missing cep is not found, and is not retried")
unittest
{
    auto server = new TestServer();
    server.respond("/99999999", Reply(404,
            `{"erro":"cep_nao_encontrado","mensagem":"CEP não existe na base do DNE"}`));

    auto client = server.client;
    bool caught;
    try
        client.lookup("99999999");
    catch (CepNotFoundException error)
    {
        caught = true;
        assert(error.cep == "99999999");
        assert(!error.retriable);
    }

    assert(caught);
    assert(server.calledTimes("/99999999") == 1);
}

@("a not-found says which municipality the cep would have belonged to")
unittest
{
    auto server = new TestServer();
    server.respond("/25845000", Reply(404, AREAL_NOT_FOUND));

    auto client = server.client;
    bool caught;
    try
        client.lookup("25845-000");
    catch (CepNotFoundException error)
    {
        caught = true;
        assert(error.cep == "25845000");
        assert(!error.retriable);
        assert(!error.faixa.isNull);

        const faixa = error.faixa.get;
        assert(faixa.localidade == "Areal");
        assert(faixa.uf == "RJ");
        assert(faixa.estado == "Rio de Janeiro");
        assert(faixa.regiao == "Sudeste");
        assert(faixa.ibge == "3300225");
        assert(faixa.cep_ini == "25845001");
        assert(faixa.cep_fim == "25849899");
        assert(faixa.ceps == 118);
        assert(error.msg.canFind("Areal/RJ"));
    }

    assert(caught);
    assert(server.calledTimes("/25845000") == 1);
}

@("a not-found in a band nobody owns carries no faixa")
unittest
{
    auto server = new TestServer();
    server.respond("/99999999", Reply(404,
            `{"erro":"cep_nao_encontrado","mensagem":"CEP não existe na base do DNE"}`));

    auto client = server.client;
    bool caught;
    try
        client.lookup("99999999");
    catch (CepNotFoundException error)
    {
        caught = true;
        assert(error.faixa.isNull);
        assert(error.msg.canFind("99999999"));
    }

    assert(caught);
}

@("a malformed faixa is still an ordinary not-found")
unittest
{
    foreach (content; [
        `{"erro":"cep_nao_encontrado","faixa":{}}`,
        `{"erro":"cep_nao_encontrado","faixa":{"uf":"RJ"}}`,
        `{"erro":"cep_nao_encontrado","faixa":"Areal"}`,
        `{"erro":"cep_nao_encontrado","faixa":null}`, `<html>gateway</html>`, ``
    ])
    {
        auto server = new TestServer();
        server.respond("/99999999", Reply(404, content));

        auto client = server.client;
        bool caught;
        try
            client.lookup("99999999");
        catch (CepNotFoundException error)
        {
            caught = true;
            assert(error.faixa.isNull);
        }

        assert(caught);
    }
}

@("an invalid cep never leaves the machine")
unittest
{
    auto server = new TestServer();

    assertThrown!InvalidCepException(server.client.lookup("123"));
    assert(server.calledTimes == 0);
}

@("a 400 from the api becomes an invalid cep carrying its message")
unittest
{
    auto server = new TestServer();
    server.respond("/00000000", Reply(400,
            `{"erro":"cep_invalido","mensagem":"rejected by the API"}`));

    auto client = server.client;
    bool caught;
    try
        client.lookup("00000000");
    catch (InvalidCepException error)
    {
        caught = true;
        assert(error.msg == "rejected by the API");
    }

    assert(caught);
    assert(server.calledTimes("/00000000") == 1);
}

@("a response that is not json is diagnosed")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(200, "<html>captive portal</html>"));

    assertThrown!InvalidResponseException(server.client.lookup("01001000"));
}

@("a 500 is retried and then succeeds")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(500, `{"erro":"interno"}`), Reply(200, SE_ADDRESS));

    const address = server.client.lookup("01001000");

    assert(address.localidade == "São Paulo");
    assert(server.calledTimes("/01001000") == 2);
}

@("the attempts run out and the server error surfaces")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(503, `{"erro":"interno"}`));

    auto client = server.client;
    bool caught;
    try
        client.lookup("01001000");
    catch (ServerException error)
    {
        caught = true;
        assert(error.status == 503);
        assert(error.retriable);
    }

    assert(caught);
    assert(server.calledTimes("/01001000") == 3);
}

@("one attempt disables retrying")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(500, "{}"));

    assertThrown!ServerException(server.client(1).lookup("01001000"));
    assert(server.calledTimes("/01001000") == 1);
}

@("rate limiting exposes retry-after")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(429, "{}", [["Retry-After", "7"]]));

    auto client = server.client;
    bool caught;
    try
        client.lookup("01001000");
    catch (RateLimitedException error)
    {
        caught = true;
        assert(error.retryAfter.get == dur!"seconds"(7));
        assert(error.retriable);
    }

    assert(caught);
}

@("a wild retry-after is capped")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(429, "{}", [["Retry-After", "3600"]]));

    const started = MonoTime.currTime;
    assertThrown!RateLimitedException(server.client.lookup("01001000"));

    assert(MonoTime.currTime - started < dur!"seconds"(5));
}

@("retry-after as an http date is understood and still capped")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(429, "{}", [
            ["Retry-After", "Wed, 21 Oct 2099 07:28:00 GMT"]
    ]));

    const started = MonoTime.currTime;
    auto client = server.client;
    bool caught;
    try
        client.lookup("01001000");
    catch (RateLimitedException error)
    {
        caught = true;
        assert(error.retryAfter.get > dur!"days"(1));
    }

    assert(caught);
    assert(MonoTime.currTime - started < dur!"seconds"(5));
}

@("nothing listening is a network error")
unittest
{
    ClientConfig config;
    config.baseUrl = "http://127.0.0.1:1";
    config.maxAttempts = 2;
    config.timeout = dur!"seconds"(1);
    config.backoffBase = dur!"msecs"(1);
    config.backoffMax = dur!"msecs"(10);

    auto client = Client(config);
    bool caught;
    try
        client.lookup("01001000");
    catch (NetworkException error)
    {
        caught = true;
        assert(error.retriable);
    }

    assert(caught);
}

@("the total timeout bounds the whole call")
unittest
{
    auto server = new TestServer();
    server.respond("/01001000", Reply(500, "{}"));

    ClientConfig config;
    config.baseUrl = server.url;
    config.timeout = dur!"seconds"(5);
    config.maxAttempts = 50;
    config.backoffBase = dur!"msecs"(50);
    config.backoffMax = dur!"msecs"(50);
    config.totalTimeout = dur!"msecs"(300);

    const started = MonoTime.currTime;
    assertThrown!CepisException(Client(config).lookup("01001000"));

    assert(MonoTime.currTime - started < dur!"seconds"(3));
    assert(server.calledTimes("/01001000") < 50);
}

@("a batch keeps order and marks the missing entry")
unittest
{
    auto server = new TestServer();
    server.respond("/csv", Reply(200, csvBody(SE_ROW, MISSING_ROW, RJ_ROW)));

    auto result = server.client.lookupMany(["01001000", "99999999", "20040020"]);

    assert(result.length == 3);
    assert(result[0].get.localidade == "São Paulo");
    assert(result[1].isNull);
    assert(result[2].get.uf == "RJ");
    assert(result[2].get.logradouro == "Avenida Rio Branco");
}

@("a batch carries no coordinate")
unittest
{
    auto server = new TestServer();
    server.respond("/csv", Reply(200, csvBody(
            "01001000,Praça da Sé,,,Sé,São Paulo,SP,São Paulo,Sudeste,3550308,1004,11,7107")));

    auto result = server.client.lookupMany(["01001000"]);

    assert(result[0].get.geo.isNull);
}

@("a quoted field containing a comma is parsed")
unittest
{
    auto server = new TestServer();
    server.respond("/csv", Reply(200,
            csvBody(`01001000,"Rua Um, Dois",,,Centro,São Paulo,SP,São Paulo,Sudeste,1,,11,1`)));

    auto result = server.client.lookupMany(["01001000"]);

    assert(result[0].get.logradouro == "Rua Um, Dois");
}

@("an empty batch makes no request")
unittest
{
    auto server = new TestServer();

    assert(server.client.lookupMany([]).length == 0);
    assert(server.calledTimes == 0);
}

@("a batch above the api limit is chunked")
unittest
{
    auto server = new TestServer();
    auto ceps = new string[501];
    foreach (index; 0 .. ceps.length)
        ceps[index] = format!"%08d"(index);

    string row(string cep)
    {
        return cep ~ ",,,,,X,SP,São Paulo,Sudeste,1,,11,1";
    }

    server.respond("/csv", Reply(200, csvBody(ceps[0 .. 500].map!row.array)),
            Reply(200, csvBody(ceps[500 .. $].map!row.array)));

    auto result = server.client.lookupMany(ceps);

    assert(result.length == 501);
    assert(server.calledTimes("/csv") == 2);
    foreach (address; result)
        assert(!address.isNull);
}

@("a truncated batch pads instead of shifting")
unittest
{
    auto server = new TestServer();
    server.respond("/csv", Reply(200,
            csvBody("01001000,,,,,São Paulo,SP,São Paulo,Sudeste,1,,11,1")));

    auto result = server.client.lookupMany(["01001000", "20040020"]);

    assert(result.length == 2);
    assert(!result[0].isNull);
    assert(result[1].isNull);
}

@("health reports the live database")
unittest
{
    auto server = new TestServer();
    server.respond("/health", Reply(200,
            `{"status":"ok","dne":"26071","ceps":1596168,"geo":894489,`
            ~ `"base_gerada_em":"2026-07-24T03:48:28Z"}`));

    const health = server.client.health();

    assert(health.ok);
    assert(health.dne == "26071");
    assert(health.ceps == 1_596_168);
    assert(health.geo == 894_489);
    assert(health.base_gerada_em == "2026-07-24T03:48:28Z");
}

@("a degraded api is not an error")
unittest
{
    auto server = new TestServer();
    server.respond("/health", Reply(503, `{"status":"degradado","dne":"","ceps":0,"geo":0}`));

    const health = server.client.health();

    assert(!health.ok);
    assert(health.status == "degradado");
    assert(server.calledTimes("/health") == 1);
}