From f979d07d23098b96e758b3feb555cd06d85d9b2d Mon Sep 17 00:00:00 2001 From: Murilo Ijanc Date: Sun, 26 Jul 2026 12:49:48 -0300 Subject: Initial import --- .gitignore | 8 + LICENSE | 15 + Makefile | 87 +++++ README.md | 264 +++++++++++++++ copy/cepis-address.cpy | 41 +++ copy/cepis-csv.cpy | 14 + copy/cepis-http.cpy | 17 + copy/cepis-json.cpy | 24 ++ copy/cepis-status.cpy | 23 ++ copy/cepis.cpy | 81 +++++ src/cepis-config.cob | 38 +++ src/cepis-csv.cob | 124 ++++++++ src/cepis-health.cob | 93 ++++++ src/cepis-http.c | 263 +++++++++++++++ src/cepis-json.cob | 409 ++++++++++++++++++++++++ src/cepis-lookup-many.cob | 346 ++++++++++++++++++++ src/cepis-lookup.cob | 324 +++++++++++++++++++ src/cepis-normalize-cep.cob | 74 +++++ src/cepis-request.cob | 363 +++++++++++++++++++++ tests/cepis-tests.cob | 761 ++++++++++++++++++++++++++++++++++++++++++++ tests/server.py | 254 +++++++++++++++ 21 files changed, 3623 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 copy/cepis-address.cpy create mode 100644 copy/cepis-csv.cpy create mode 100644 copy/cepis-http.cpy create mode 100644 copy/cepis-json.cpy create mode 100644 copy/cepis-status.cpy create mode 100644 copy/cepis.cpy create mode 100644 src/cepis-config.cob create mode 100644 src/cepis-csv.cob create mode 100644 src/cepis-health.cob create mode 100644 src/cepis-http.c create mode 100644 src/cepis-json.cob create mode 100644 src/cepis-lookup-many.cob create mode 100644 src/cepis-lookup.cob create mode 100644 src/cepis-normalize-cep.cob create mode 100644 src/cepis-request.cob create mode 100644 tests/cepis-tests.cob create mode 100644 tests/server.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7127d96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +build/ +*.o +*.a +*.so +*.i +*.c.l.h +*.c.h +libcob.* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5e9104d --- /dev/null +++ b/LICENSE @@ -0,0 +1,15 @@ +ISC License + +Copyright (c) 2026 Murilo Ijanc + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1de6116 --- /dev/null +++ b/Makefile @@ -0,0 +1,87 @@ +# Development tasks. +# +# make list the targets +# make build compile the library into build/libcepis.a +# make test run the suite against the local fake API +# make lint compile every unit for syntax and warnings, no output +# make check lint, then test, then build +# +# Requires GnuCOBOL 3.x (cobc), a C compiler, libcurl with its headers and +# python3 for the test server. Nothing else. + +COBC ?= cobc +CC ?= cc +PYTHON ?= python3 +AR ?= ar + +COPYDIR := copy +BUILDDIR := build + +# Free source format, GnuCOBOL's default dialect. -fstatic-call resolves +# CALL "literal" at link time, which is what lets the archive below work +# without a runtime module search path. +COBFLAGS ?= -free -Wall -I $(COPYDIR) -fstatic-call +CFLAGS ?= -O2 -Wall -Wextra +LDLIBS ?= -lcurl + +MODULES := \ + cepis-config \ + cepis-normalize-cep \ + cepis-json \ + cepis-csv \ + cepis-request \ + cepis-lookup \ + cepis-lookup-many \ + cepis-health + +COBSRC := $(addprefix src/,$(addsuffix .cob,$(MODULES))) +COBOBJ := $(addprefix $(BUILDDIR)/,$(addsuffix .o,$(MODULES))) +COBJ := $(BUILDDIR)/cepis-http.o +LIB := $(BUILDDIR)/libcepis.a +TESTBIN := $(BUILDDIR)/cepis-tests +COPYBOOKS := $(wildcard $(COPYDIR)/*.cpy) + +.DEFAULT_GOAL := help +.PHONY: help build test lint check clean + +help: + @echo 'Targets:' + @echo ' build compile the library into $(LIB)' + @echo ' test run the suite against the local fake API' + @echo ' lint syntax and warning check only, writes nothing' + @echo ' check lint + test + build, in that order' + @echo ' clean remove build artefacts' + @echo + @echo 'COBOL has no standard formatter, so there is no format target.' + +build: $(LIB) + +$(LIB): $(COBOBJ) $(COBJ) + $(AR) rcs $@ $^ + +$(BUILDDIR)/%.o: src/%.cob $(COPYBOOKS) + @mkdir -p $(BUILDDIR) + $(COBC) -c $(COBFLAGS) -o $@ $< + +$(COBJ): src/cepis-http.c + @mkdir -p $(BUILDDIR) + $(CC) $(CFLAGS) -c -o $@ $< + +$(TESTBIN): tests/cepis-tests.cob $(LIB) + $(COBC) -x $(COBFLAGS) -o $@ $< $(LIB) $(LDLIBS) + +test: $(TESTBIN) + $(PYTHON) tests/server.py ./$(TESTBIN) + +# Read-only by design, so it is safe in CI and in a pre-commit hook. +lint: + $(CC) $(CFLAGS) -fsyntax-only src/cepis-http.c + $(COBC) -fsyntax-only $(COBFLAGS) $(COBSRC) + $(COBC) -fsyntax-only $(COBFLAGS) tests/cepis-tests.cob + +# The order matters: lint is the fastest to fail, so a broken run reports +# the cheapest problem first. +check: lint test build + +clean: + rm -rf $(BUILDDIR) diff --git a/README.md b/README.md new file mode 100644 index 0000000..021448c --- /dev/null +++ b/README.md @@ -0,0 +1,264 @@ +# cepis + +COBOL client for the [public CEP API at cep.is](https://cep.is). Official +Correios data (DNE), refreshed every two weeks. + +```cobol +WORKING-STORAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION. + CALL "CEPIS-DEFAULT-CONFIG" USING CEPIS-CONFIG + MOVE "01001-000" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + IF CEPIS-OK + DISPLAY ADDR-LOGRADOURO " " ADDR-LOCALIDADE " " ADDR-UF + END-IF +``` + +Targets **GnuCOBOL 3.x**, default dialect, **free source format** +(`cobc -free`). HTTP goes through **libcurl**, reached from a small C shim +in `src/cepis-http.c` rather than from COBOL directly, because +`curl_easy_setopt` is variadic and a COBOL `CALL` cannot express that +portably. + +## Installation + +```sh +git clone https://git.cep.is/cobol.git cepis-cobol +cd cepis-cobol +make build +``` + +That produces `build/libcepis.a`. Link it into your program and add the +copybook directory to the search path: + +```sh +cobc -x -free -I /path/to/cepis-cobol/copy \ + -o myprog myprog.cob /path/to/cepis-cobol/build/libcepis.a -lcurl +``` + +Requires GnuCOBOL 3.x, a C compiler and libcurl with its development +headers. `python3` is needed only to run the test suite. + +## Usage + +Everything is a `CALL` to a subprogram, and everything shares two records +from `cepis.cpy`: `CEPIS-CONFIG` going in and `CEPIS-STATUS` coming back. +`CALL "CEPIS-DEFAULT-CONFIG" USING CEPIS-CONFIG` fills the configuration +with its defaults; override afterwards what you need. + +### A single lookup + +`CEPIS-LOOKUP` accepts the CEP masked or unmasked: + +```cobol +MOVE "01310-100" TO CEPIS-CEP-INPUT +CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + +ADDR-LOGRADOURO *> 'Avenida Paulista' +ADDR-LOCALIDADE *> 'São Paulo' +ADDR-UF *> 'SP' +ADDR-IBGE *> '3550308' +ADDR-CEP *> '01310100' always unmasked +ADDR-CEP-MASKED *> '01310-100' +``` + +Fields the database does not have for that CEP come back as spaces. +`CEPIS-ADDRESS` is cleared before every call, so a failed lookup never +leaves the previous CEP's street behind. + +### Coordinates + +About 57% of CEPs have a coordinate. There is an explicit flag rather than +a marker value: + +```cobol +IF ADDR-GEO-KNOWN + DISPLAY ADDR-GEO-LAT " " ADDR-GEO-LON + DISPLAY ADDR-GEO-RAIO-M " " ADDR-GEO-PRECISAO +END-IF +``` + +**Check `ADDR-GEO-PRECISAO` before computing distance.** There are two +kinds of point, and the difference between them is tens of kilometres: + +| `precisao` | condition name | typical radius | what the point is | +| --- | --- | --- | --- | +| `logradouro` | `ADDR-GEO-PRECISE` | 110–140 m | the stretch of street — good for maps and distance | +| `localidade` | `ADDR-GEO-CITY-WIDE` | 7–40 km | the municipality centroid: the CEP *is* the whole city | + +```cobol +IF ADDR-GEO-KNOWN AND ADDR-GEO-PRECISE + PERFORM SHIPPING-FROM-POINT +ELSE + PERFORM SHIPPING-FROM-MUNICIPALITY +END-IF +``` + +`ADDR-GEO-LAT` and `ADDR-GEO-LON` are fixed-point `S9(3)V9(6)`, six +decimal places — about 11 cm, well past what the data supports. + +### Many CEPs at once + +Do not loop over `CEPIS-LOOKUP`. `CEPIS-LOOKUP-MANY` resolves up to 500 +CEPs per request: + +```cobol +MOVE 3 TO BATCH-COUNT +MOVE "01001000" TO BATCH-REQUESTED (1) +MOVE "99999999" TO BATCH-REQUESTED (2) +MOVE "20040020" TO BATCH-REQUESTED (3) + +CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH CEPIS-STATUS + +PERFORM VARYING I FROM 1 BY 1 UNTIL I > BATCH-COUNT + IF BATCH-ENTRY-FOUND (I) + DISPLAY BATCH-REQUESTED (I) ": " BAT-LOCALIDADE (I) + ELSE + DISPLAY BATCH-REQUESTED (I) ": not found" + END-IF +END-PERFORM +``` + +Every slot comes back in place. A CEP that does not exist, or that is not +eight digits, leaves its slot `BATCH-ENTRY-MISSING` — it is never removed, +never shifted, and never fails the call. + +The table holds 1000 slots and requests are chunked at 500 automatically. +A `BATCH-COUNT` above 1000 is refused with `CEPIS-INVALID-CEP` rather than +silently truncated. + +> **Batches carry no coordinate.** The endpoint behind them is CSV, and the +> CSV has no `geo` column. Every slot has `BAT-GEO-ABSENT` set, including +> CEPs that do have a coordinate. Need `geo`? Use `CEPIS-LOOKUP`, one at a +> time. + +### API health + +```cobol +CALL "CEPIS-HEALTH" USING CEPIS-CONFIG CEPIS-HEALTH CEPIS-STATUS + +HEALTH-OK *> condition name, true when status is "ok" +HEALTH-DNE *> '26071' identifier of the fortnightly database +HEALTH-CEPS *> 1596168 +HEALTH-BASE-GERADA-EM *> '2026-07-24T03:48:28Z' +``` + +An API reporting itself degraded does not fail the call — that arrives in +`HEALTH-STATUS` with `CEPIS-OK` set. + +### Validating a CEP on its own + +```cobol +MOVE "01001-000" TO CEPIS-CEP-INPUT +CALL "CEPIS-NORMALIZE-CEP" USING CEPIS-CEP-INPUT CEPIS-CEP CEPIS-STATUS +*> CEPIS-CEP is now '01001000' +``` + +## Configuration + +| field | default | meaning | +| --- | --- | --- | +| `CFG-BASE-URL` | `https://cep.is` | where the API lives; point it at a local instance | +| `CFG-TIMEOUT-MS` | `10000` | per request: bounds the connect and the transfer | +| `CFG-MAX-ATTEMPTS` | `3` | attempts in total, counting the first; `1` disables retrying | +| `CFG-BACKOFF-BASE-MS` | `500` | base of the exponential wait | +| `CFG-BACKOFF-MAX-MS` | `8000` | cap on each wait, `Retry-After` included | +| `CFG-TOTAL-TIMEOUT-MS` | `0` | cap on the whole call, attempts and waits included; `0` means none | +| `CFG-USER-AGENT` | `cepis-cobol/0.1.0 (+https://cep.is)` | identifies the library and version | + +Retries cover 429, 5xx and transport failures, with exponential backoff and +full jitter; 400 and 404 are never retried. + +`CFG-BASE-URL` is the only field that usually changes between environments. +The API repo brings a local instance up with `just up`: + +```cobol +MOVE "http://localhost:8080" TO CFG-BASE-URL +``` + +## Errors + +`CEPIS-STATUS` carries the outcome. Test `CEPIS-OK` first; `CEPIS-FAILED` +covers every failure at once, and `CEPIS-RETRIABLE` the three the client +already retried on its own before giving up. + +| condition | when | does retrying help? | +| --- | --- | --- | +| `CEPIS-INVALID-CEP` | not 8 digits | no | +| `CEPIS-NOT-FOUND` | well-formed, not in the database | no | +| `CEPIS-RATE-LIMITED` | 429; see `CEPIS-STATUS-RETRY-AFTER-MS` | yes, later | +| `CEPIS-SERVER-ERROR` | 5xx; see `CEPIS-STATUS-HTTP` | yes, later | +| `CEPIS-NETWORK-ERROR` | DNS, connection, timeout, TLS | yes, later | +| `CEPIS-INVALID-RESPONSE` | success status, unintelligible body | probably not | + +```cobol +EVALUATE TRUE + WHEN CEPIS-OK PERFORM USE-THE-ADDRESS + WHEN CEPIS-NOT-FOUND PERFORM ASK-FOR-ANOTHER-CEP + WHEN CEPIS-RETRIABLE PERFORM RESCHEDULE + WHEN OTHER DISPLAY CEPIS-STATUS-MESSAGE +END-EVALUATE +``` + +`CEPIS-STATUS-MESSAGE` holds a description, `CEPIS-STATUS-HTTP` the last +status seen and `CEPIS-STATUS-ATTEMPTS` how many attempts were made. + +`CEPIS-INVALID-RESPONSE` almost always means something between you and the +API is answering instead of it — a captive Wi-Fi portal, a corporate proxy, +an HTML error page. Useful as a diagnosis: the problem is not in your code. + +## Field widths + +COBOL records are fixed width. A value longer than its field is truncated +on the right and `ADDR-TRUNCATED` (or `BAT-TRUNCATED`, per slot) is set, so +truncation is visible rather than silent. Lengths are in bytes and the data +is UTF-8, so an accented character costs two. + +| field | bytes | | field | bytes | +| --- | --- | --- | --- | --- | +| `cep` | 8 | | `uf` | 2 | +| `cep-masked` | 9 | | `estado` | 60 | +| `logradouro` | 120 | | `regiao` | 30 | +| `complemento` | 120 | | `ibge` | 10 | +| `unidade` | 80 | | `gia` | 10 | +| `bairro` | 100 | | `ddd` | 4 | +| `localidade` | 100 | | `siafi` | 10 | + +A response body larger than 256 KiB is reported as +`CEPIS-INVALID-RESPONSE`; 500 CSV rows come to roughly 60 KiB. + +## Limitations + +- **One call at a time per process.** A GnuCOBOL subprogram's + `WORKING-STORAGE` is static, so the subprograms here are not reentrant. +- The raw response body is not kept. Fields the client does not know are + ignored without error, but they are not readable afterwards. +- `CFG-TIMEOUT-MS` maps onto libcurl's connect and transfer timeouts, not + onto each individual socket read. + +## Development + +```sh +make # list the targets +make build # compile the library into build/libcepis.a +make test # run the suite against a local fake API +make lint # syntax and warning check, writes nothing +make check # lint, then test, then build +make clean # remove build artefacts +``` + +`make test` stands a fake cep.is up on a loopback port with +`tests/server.py`, points `CFG-BASE-URL` at it and runs the driver in +`tests/cepis-tests.cob`. No test touches the network. The run exits +non-zero if any check fails. + +COBOL has no standard formatter, so there is no `format` target; `lint` +compiles every unit with `-Wall -fsyntax-only`. + +## Licence + +ISC. See [LICENSE](LICENSE). diff --git a/copy/cepis-address.cpy b/copy/cepis-address.cpy new file mode 100644 index 0000000..b1e6c35 --- /dev/null +++ b/copy/cepis-address.cpy @@ -0,0 +1,41 @@ +*> --------------------------------------------------------------------- +*> Address layout without its 01 level, so the same definition serves +*> both the standalone record and the elements of the batch table. +*> +*> Include it under an 01 (or under a table entry) and replace the three +*> tags: the field-name prefix and the two level numbers. cepis.cpy does +*> exactly that, twice. +*> +*> Every text field is a fixed-width PIC X. A value longer than its field +*> is truncated on the right and the record is flagged; the sizes below +*> are the documented limits and are repeated in the README. +*> --------------------------------------------------------------------- +:L1: :P:-CEP PIC X(8). +:L1: :P:-CEP-MASKED PIC X(9). +:L1: :P:-LOGRADOURO PIC X(120). +:L1: :P:-COMPLEMENTO PIC X(120). +:L1: :P:-UNIDADE PIC X(80). +:L1: :P:-BAIRRO PIC X(100). +:L1: :P:-LOCALIDADE PIC X(100). +:L1: :P:-UF PIC X(2). +:L1: :P:-ESTADO PIC X(60). +:L1: :P:-REGIAO PIC X(30). +:L1: :P:-IBGE PIC X(10). +:L1: :P:-GIA PIC X(10). +:L1: :P:-DDD PIC X(4). +:L1: :P:-SIAFI PIC X(10). +:L1: :P:-TRUNCATED PIC X. + 88 :P:-WAS-TRUNCATED VALUE "Y". + 88 :P:-NOT-TRUNCATED VALUE "N". +:L1: :P:-GEO. + :L2: :P:-GEO-PRESENT PIC X. + 88 :P:-GEO-KNOWN VALUE "Y". + 88 :P:-GEO-ABSENT VALUE "N". + :L2: :P:-GEO-LAT PIC S9(3)V9(6) + SIGN IS LEADING SEPARATE. + :L2: :P:-GEO-LON PIC S9(3)V9(6) + SIGN IS LEADING SEPARATE. + :L2: :P:-GEO-RAIO-M PIC 9(9). + :L2: :P:-GEO-PRECISAO PIC X(16). + 88 :P:-GEO-PRECISE VALUE "logradouro". + 88 :P:-GEO-CITY-WIDE VALUE "localidade". diff --git a/copy/cepis-csv.cpy b/copy/cepis-csv.cpy new file mode 100644 index 0000000..8ef9599 --- /dev/null +++ b/copy/cepis-csv.cpy @@ -0,0 +1,14 @@ +*> --------------------------------------------------------------------- +*> Control block for CEPIS-CSV-NEXT. Internal. +*> +*> Set CSV-LEN once and CSV-POS to 1, then call repeatedly: each call +*> yields the next field. CSV-ROW-END marks the field that closed the +*> record, CSV-EXHAUSTED that there was nothing left to read. +*> --------------------------------------------------------------------- +01 CEPIS-CSV-SCAN. + 05 CSV-POS PIC 9(9). + 05 CSV-LEN PIC 9(9). + 05 CSV-FIELD PIC X(512). + 05 CSV-FIELD-LEN PIC 9(9). + 05 CSV-ROW-END PIC X. + 05 CSV-EXHAUSTED PIC X. diff --git a/copy/cepis-http.cpy b/copy/cepis-http.cpy new file mode 100644 index 0000000..5868ed2 --- /dev/null +++ b/copy/cepis-http.cpy @@ -0,0 +1,17 @@ +*> --------------------------------------------------------------------- +*> Internal layouts shared between CEPIS-REQUEST and its callers. Not +*> part of the public surface; nothing outside this library needs it. +*> +*> HTTP-BODY is fixed at 256 KiB. A response larger than that is reported +*> as CEPIS-INVALID-RESPONSE rather than being silently cut: 500 CSV rows +*> come to roughly 60 KiB, so the margin is wide. +*> --------------------------------------------------------------------- +01 CEPIS-HTTP-REQUEST. + 05 HTTPQ-PATH PIC X(8192). + 05 HTTPQ-PATH-LEN PIC 9(9). + 05 HTTPQ-ALLOW-503 PIC X. + +01 CEPIS-HTTP-RESPONSE. + 05 HTTP-STATUS PIC 9(3). + 05 HTTP-BODY-LEN PIC 9(9). + 05 HTTP-BODY PIC X(262144). diff --git a/copy/cepis-json.cpy b/copy/cepis-json.cpy new file mode 100644 index 0000000..05e43ea --- /dev/null +++ b/copy/cepis-json.cpy @@ -0,0 +1,24 @@ +*> --------------------------------------------------------------------- +*> Control block for CEPIS-JSON-GET. Internal. +*> +*> Set JS-FROM and JS-TO to the span of the object to search (JS-FROM on +*> its opening brace) and JS-KEY to the member wanted. On return JS-TYPE +*> says what was found: +*> +*> S string JS-VALUE holds it, escapes already decoded +*> N number JS-VALUE holds the literal text, for FUNCTION NUMVAL +*> O object JS-VSTART and JS-VEND span it; search it in turn +*> A array likewise +*> B boolean, Z null +*> space the key is not in this object +*> E the text is not a JSON object +*> --------------------------------------------------------------------- +01 CEPIS-JSON-SCAN. + 05 JS-FROM PIC 9(9). + 05 JS-TO PIC 9(9). + 05 JS-KEY PIC X(32). + 05 JS-TYPE PIC X. + 05 JS-VALUE PIC X(1024). + 05 JS-VALUE-LEN PIC 9(9). + 05 JS-VSTART PIC 9(9). + 05 JS-VEND PIC 9(9). diff --git a/copy/cepis-status.cpy b/copy/cepis-status.cpy new file mode 100644 index 0000000..75632c8 --- /dev/null +++ b/copy/cepis-status.cpy @@ -0,0 +1,23 @@ +*> --------------------------------------------------------------------- +*> Status layout without its 01 level, so the library can keep a private +*> scratch status alongside the caller's without duplicating the +*> definition. Replace the field-name tag and the level number. +*> +*> The six failure categories are split by the decision the caller has to +*> make, not by the HTTP status that produced them. +*> --------------------------------------------------------------------- +:L1: :P:-STATUS-CODE PIC 9(2). + 88 :P:-OK VALUE 0. + 88 :P:-INVALID-CEP VALUE 1. + 88 :P:-NOT-FOUND VALUE 2. + 88 :P:-RATE-LIMITED VALUE 3. + 88 :P:-SERVER-ERROR VALUE 4. + 88 :P:-NETWORK-ERROR VALUE 5. + 88 :P:-INVALID-RESPONSE VALUE 6. + 88 :P:-FAILED VALUE 1 THRU 6. + 88 :P:-RETRIABLE VALUE 3 THRU 5. +:L1: :P:-STATUS-MESSAGE PIC X(200). +:L1: :P:-STATUS-HTTP PIC 9(3). +:L1: :P:-STATUS-RETRY-AFTER-MS PIC S9(9) + SIGN IS LEADING SEPARATE. +:L1: :P:-STATUS-ATTEMPTS PIC 9(4). diff --git a/copy/cepis.cpy b/copy/cepis.cpy new file mode 100644 index 0000000..59d643c --- /dev/null +++ b/copy/cepis.cpy @@ -0,0 +1,81 @@ +*> --------------------------------------------------------------------- +*> cepis 0.1.0 -- public data layouts for the cep.is client. +*> +*> Copy this into WORKING-STORAGE (callers) or LINKAGE (subprograms). +*> It defines, in order: +*> +*> CEPIS-CONFIG client configuration; fill it with +*> CALL "CEPIS-DEFAULT-CONFIG" before overriding +*> CEPIS-STATUS outcome of the last call, with the six error +*> categories as 88-levels +*> CEPIS-CEP-INPUT a CEP as the user typed it, masked or not +*> CEPIS-CEP the same CEP, 8 digits +*> CEPIS-ADDRESS one address +*> CEPIS-HEALTH the answer of GET /health +*> CEPIS-BATCH the batch table, one slot per requested CEP +*> +*> Data field names are the API's own, in Portuguese. Everything else in +*> this library is English. +*> --------------------------------------------------------------------- + +*> Client configuration. CEPIS-DEFAULT-CONFIG fills every field; change +*> what you need afterwards. CFG-BASE-URL is the one that usually differs +*> between environments. +01 CEPIS-CONFIG. + 05 CFG-BASE-URL PIC X(200). + 05 CFG-TIMEOUT-MS PIC 9(9). + 05 CFG-MAX-ATTEMPTS PIC 9(4). + 05 CFG-BACKOFF-BASE-MS PIC 9(9). + 05 CFG-BACKOFF-MAX-MS PIC 9(9). + 05 CFG-TOTAL-TIMEOUT-MS PIC 9(9). + 05 CFG-USER-AGENT PIC X(120). + +*> Outcome of the last call. Test CEPIS-OK first; the six failure +*> categories are split by the decision the caller has to make, not by +*> the HTTP status that produced them. CEPIS-RETRIABLE covers the three +*> the client already retried on its own before giving up. +01 CEPIS-STATUS. + COPY "cepis-status.cpy" REPLACING + ==:P:== BY ==CEPIS== ==:L1:== BY ==05==. + +*> A CEP as the caller has it: "01001-000", "01001000" or "01001.000". +01 CEPIS-CEP-INPUT PIC X(20). + +*> The same CEP reduced to its 8 digits by CEPIS-NORMALIZE-CEP. +01 CEPIS-CEP PIC X(8). + +*> One address. Fields the database lacks come back as spaces, never as +*> a marker value. ADDR-GEO-PRESENT says whether the coordinate exists; +*> ADDR-GEO-LAT and ADDR-GEO-LON are meaningless when it does not. +01 CEPIS-ADDRESS. + COPY "cepis-address.cpy" REPLACING + ==:P:== BY ==ADDR== ==:L1:== BY ==05== ==:L2:== BY ==10==. + +*> The answer of GET /health. HEALTH-DEGRADED is not an error: the API +*> reporting itself unwell is the diagnosis you asked for. +01 CEPIS-HEALTH. + 05 HEALTH-STATUS PIC X(20). + 88 HEALTH-OK VALUE "ok". + 05 HEALTH-DNE PIC X(20). + 05 HEALTH-CEPS PIC 9(12). + 05 HEALTH-GEO PIC 9(12). + 05 HEALTH-BASE-GERADA-EM PIC X(32). + +*> The batch table. Fill BATCH-COUNT and BATCH-REQUESTED for each slot, +*> then CALL "CEPIS-LOOKUP-MANY". Every slot comes back in place: a CEP +*> that does not exist, or that is not 8 digits, leaves its slot +*> BATCH-ENTRY-MISSING. Slots are never removed or shifted. +*> +*> Batches carry no coordinate: the endpoint behind them is CSV and the +*> CSV has no geo column, so BAT-GEO-PRESENT is always "N" here even for +*> CEPs that do have one. Need the coordinate? Use CEPIS-LOOKUP. +01 CEPIS-BATCH. + 05 BATCH-COUNT PIC 9(4). + 05 BATCH-ENTRY OCCURS 1000 TIMES. + 10 BATCH-REQUESTED PIC X(20). + 10 BATCH-FOUND PIC X. + 88 BATCH-ENTRY-FOUND VALUE "Y". + 88 BATCH-ENTRY-MISSING VALUE "N". + 10 BATCH-ADDRESS. + COPY "cepis-address.cpy" REPLACING + ==:P:== BY ==BAT== ==:L1:== BY ==15== ==:L2:== BY ==20==. diff --git a/src/cepis-config.cob b/src/cepis-config.cob new file mode 100644 index 0000000..0f17fd1 --- /dev/null +++ b/src/cepis-config.cob @@ -0,0 +1,38 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-DEFAULT-CONFIG. + +*> --------------------------------------------------------------------- +*> Fill CEPIS-CONFIG with the shipped defaults. +*> +*> CALL "CEPIS-DEFAULT-CONFIG" USING CEPIS-CONFIG +*> +*> Call this before anything else, then override what you need. Every +*> field is written, so a record left over from another use is safe to +*> pass in. CFG-BASE-URL is the one that usually differs between +*> environments: point it at a local instance and the whole library +*> follows. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +01 CEPIS-VERSION PIC X(5) VALUE "0.1.0". + +LINKAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION USING CEPIS-CONFIG. + +MAIN-START. + MOVE "https://cep.is" TO CFG-BASE-URL + MOVE 10000 TO CFG-TIMEOUT-MS + MOVE 3 TO CFG-MAX-ATTEMPTS + MOVE 500 TO CFG-BACKOFF-BASE-MS + MOVE 8000 TO CFG-BACKOFF-MAX-MS + MOVE 0 TO CFG-TOTAL-TIMEOUT-MS + MOVE SPACES TO CFG-USER-AGENT + STRING "cepis-cobol/" CEPIS-VERSION " (+https://cep.is)" + DELIMITED BY SIZE INTO CFG-USER-AGENT + END-STRING + GOBACK. + +END PROGRAM CEPIS-DEFAULT-CONFIG. diff --git a/src/cepis-csv.cob b/src/cepis-csv.cob new file mode 100644 index 0000000..29cc6d2 --- /dev/null +++ b/src/cepis-csv.cob @@ -0,0 +1,124 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-CSV-NEXT. + +*> --------------------------------------------------------------------- +*> Read one field of an RFC 4180 record. +*> +*> CALL "CEPIS-CSV-NEXT" USING CSV-TEXT CEPIS-CSV-SCAN +*> +*> Values from the batch endpoint contain commas -- "Rua Um, Dois" is an +*> ordinary street name -- and are then double-quoted with "" for an +*> embedded quote. Splitting on commas would shear those rows in two and +*> shift every column after them, so this walks the quoting properly, +*> newlines inside a quoted value included. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +01 WS-C PIC X. +01 WS-DOUBLED PIC X. +01 CONST-QUOTE PIC X VALUE '"'. + +LINKAGE SECTION. +01 CSV-TEXT PIC X(262144). +COPY "cepis-csv.cpy". + +PROCEDURE DIVISION USING CSV-TEXT CEPIS-CSV-SCAN. + +MAIN-START. + MOVE SPACES TO CSV-FIELD + MOVE 0 TO CSV-FIELD-LEN + MOVE "N" TO CSV-ROW-END + MOVE "N" TO CSV-EXHAUSTED + + IF CSV-POS > CSV-LEN + MOVE "Y" TO CSV-EXHAUSTED + MOVE "Y" TO CSV-ROW-END + GOBACK + END-IF + + IF CSV-TEXT(CSV-POS:1) = CONST-QUOTE + PERFORM READ-QUOTED + ELSE + PERFORM READ-BARE + END-IF + + PERFORM CLOSE-FIELD + GOBACK. + +*> ------------------------------------------------------------------- +READ-QUOTED. + ADD 1 TO CSV-POS + PERFORM UNTIL CSV-POS > CSV-LEN + MOVE CSV-TEXT(CSV-POS:1) TO WS-C + IF WS-C = CONST-QUOTE + MOVE "N" TO WS-DOUBLED + IF CSV-POS < CSV-LEN + IF CSV-TEXT(CSV-POS + 1:1) = CONST-QUOTE + MOVE "Y" TO WS-DOUBLED + END-IF + END-IF + IF WS-DOUBLED = "Y" + PERFORM APPEND-CHAR + ADD 2 TO CSV-POS + ELSE + ADD 1 TO CSV-POS + EXIT PERFORM + END-IF + ELSE + PERFORM APPEND-CHAR + ADD 1 TO CSV-POS + END-IF + END-PERFORM + PERFORM UNTIL CSV-POS > CSV-LEN + MOVE CSV-TEXT(CSV-POS:1) TO WS-C + IF WS-C = "," OR WS-C = X"0D" OR WS-C = X"0A" + EXIT PERFORM + END-IF + ADD 1 TO CSV-POS + END-PERFORM. + +*> ------------------------------------------------------------------- +READ-BARE. + PERFORM UNTIL CSV-POS > CSV-LEN + MOVE CSV-TEXT(CSV-POS:1) TO WS-C + IF WS-C = "," OR WS-C = X"0D" OR WS-C = X"0A" + EXIT PERFORM + END-IF + PERFORM APPEND-CHAR + ADD 1 TO CSV-POS + END-PERFORM. + +*> ------------------------------------------------------------------- +CLOSE-FIELD. + IF CSV-POS > CSV-LEN + MOVE "Y" TO CSV-ROW-END + EXIT PARAGRAPH + END-IF + MOVE CSV-TEXT(CSV-POS:1) TO WS-C + EVALUATE TRUE + WHEN WS-C = "," + ADD 1 TO CSV-POS + WHEN WS-C = X"0D" + ADD 1 TO CSV-POS + IF CSV-POS <= CSV-LEN + IF CSV-TEXT(CSV-POS:1) = X"0A" + ADD 1 TO CSV-POS + END-IF + END-IF + MOVE "Y" TO CSV-ROW-END + WHEN WS-C = X"0A" + ADD 1 TO CSV-POS + MOVE "Y" TO CSV-ROW-END + WHEN OTHER + ADD 1 TO CSV-POS + END-EVALUATE. + +*> ------------------------------------------------------------------- +APPEND-CHAR. + IF CSV-FIELD-LEN < 512 + ADD 1 TO CSV-FIELD-LEN + MOVE WS-C TO CSV-FIELD(CSV-FIELD-LEN:1) + END-IF. + +END PROGRAM CEPIS-CSV-NEXT. diff --git a/src/cepis-health.cob b/src/cepis-health.cob new file mode 100644 index 0000000..86da407 --- /dev/null +++ b/src/cepis-health.cob @@ -0,0 +1,93 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-HEALTH. + +*> --------------------------------------------------------------------- +*> Ask the API how it is. +*> +*> CALL "CEPIS-HEALTH" USING CEPIS-CONFIG CEPIS-HEALTH CEPIS-STATUS +*> +*> Good for two things: confirming the API answers at all, and knowing +*> which version of the database is live. The DNE database is rebuilt +*> every two weeks and HEALTH-DNE identifies the edition in use. +*> +*> An API reporting itself degraded is not a failure of this call. It +*> answers 503 with a body, and that arrives in HEALTH-STATUS with +*> CEPIS-OK set -- checking health is exactly when you want the +*> diagnosis rather than an error. A transport failure still fails. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +COPY "cepis-http.cpy". +COPY "cepis-json.cpy". + +LINKAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION USING CEPIS-CONFIG CEPIS-HEALTH CEPIS-STATUS. + +MAIN-START. + INITIALIZE CEPIS-HEALTH + + MOVE SPACES TO HTTPQ-PATH + MOVE "/health" TO HTTPQ-PATH + MOVE 7 TO HTTPQ-PATH-LEN + MOVE "Y" TO HTTPQ-ALLOW-503 + + CALL "CEPIS-REQUEST" USING CEPIS-CONFIG CEPIS-HTTP-REQUEST + CEPIS-HTTP-RESPONSE CEPIS-STATUS + END-CALL + IF CEPIS-FAILED + GOBACK + END-IF + + IF HTTP-STATUS NOT = 200 AND HTTP-STATUS NOT = 503 + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "unexpected status " HTTP-STATUS " from /health" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + GOBACK + END-IF + + MOVE "status" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE = "E" + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE "response to /health is not a JSON object" + TO CEPIS-STATUS-MESSAGE + GOBACK + END-IF + MOVE JS-VALUE TO HEALTH-STATUS + + MOVE "dne" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO HEALTH-DNE + + MOVE "ceps" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE = "N" + COMPUTE HEALTH-CEPS ROUNDED = FUNCTION NUMVAL (JS-VALUE) + END-IF + + MOVE "geo" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE = "N" + COMPUTE HEALTH-GEO ROUNDED = FUNCTION NUMVAL (JS-VALUE) + END-IF + + MOVE "base_gerada_em" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO HEALTH-BASE-GERADA-EM + + SET CEPIS-OK TO TRUE + GOBACK. + +*> ------------------------------------------------------------------- +GET-MEMBER. + MOVE 1 TO JS-FROM + MOVE HTTP-BODY-LEN TO JS-TO + CALL "CEPIS-JSON-GET" USING HTTP-BODY CEPIS-JSON-SCAN + END-CALL. + +END PROGRAM CEPIS-HEALTH. diff --git a/src/cepis-http.c b/src/cepis-http.c new file mode 100644 index 0000000..10fa023 --- /dev/null +++ b/src/cepis-http.c @@ -0,0 +1,263 @@ +/* + * Transport primitives for the cep.is COBOL client. + * + * COBOL has no HTTP, and `curl_easy_setopt` is variadic, which a COBOL + * CALL cannot express portably. Everything variadic therefore lives here, + * behind four fixed-signature entry points that a COBOL CALL can reach: + * + * cepis_http_get one GET, no retrying -- the policy stays in COBOL + * cepis_sleep_ms the backoff wait + * cepis_monotonic_ms the clock CFG-TOTAL-TIMEOUT-MS is measured on + * cepis_http_date_delta the RFC 9110 HTTP-date form of Retry-After + * + * All scalar arguments are `int` or `long long` passed BY REFERENCE, which + * COBOL declares as PIC S9(9) COMP-5 and PIC S9(18) COMP-5. All string + * arguments in are NUL-terminated by the caller. All string arguments out + * are blank-filled to their capacity, so the COBOL field they land in is a + * normal space-padded PIC X. + */ + +#include +#include +#include +#include +#include + +#include + +#define CEPIS_OK_TRANSPORT 0 +#define CEPIS_ERR_TRANSPORT 1 +#define CEPIS_ERR_TOO_LARGE 2 + +struct cepis_body { + char *buf; + long cap; + long len; + int overflow; +}; + +struct cepis_header { + char *buf; + long cap; + int seen; +}; + +static int cepis_curl_ready; + +static void cepis_blank(char *dst, long cap) +{ + if (dst && cap > 0) { + memset(dst, ' ', (size_t)cap); + } +} + +static void cepis_put(char *dst, long cap, const char *src) +{ + size_t n; + + cepis_blank(dst, cap); + if (!dst || cap <= 0 || !src) { + return; + } + n = strlen(src); + if (n > (size_t)cap) { + n = (size_t)cap; + } + memcpy(dst, src, n); +} + +static size_t cepis_on_body(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + struct cepis_body *sink = (struct cepis_body *)userdata; + size_t n = size * nmemb; + + if (sink->len + (long)n > sink->cap) { + sink->overflow = 1; + return 0; + } + memcpy(sink->buf + sink->len, ptr, n); + sink->len += (long)n; + return n; +} + +static size_t cepis_on_header(char *ptr, size_t size, size_t nmemb, void *userdata) +{ + struct cepis_header *hdr = (struct cepis_header *)userdata; + size_t n = size * nmemb; + static const char name[] = "retry-after:"; + size_t name_len = sizeof(name) - 1; + size_t start, end, len; + + if (hdr->seen || n <= name_len || strncasecmp(ptr, name, name_len) != 0) { + return n; + } + start = name_len; + end = n; + while (start < end && (ptr[start] == ' ' || ptr[start] == '\t')) { + start++; + } + while (end > start && (ptr[end - 1] == '\r' || ptr[end - 1] == '\n' || + ptr[end - 1] == ' ' || ptr[end - 1] == '\t')) { + end--; + } + len = end - start; + if (len > (size_t)hdr->cap) { + len = (size_t)hdr->cap; + } + memcpy(hdr->buf, ptr + start, len); + hdr->seen = 1; + return n; +} + +/* + * Perform one GET. Returns 0 when the exchange completed (whatever the + * status), 1 when it never completed, 2 when the body did not fit. + */ +int cepis_http_get(const char *url, + const char *user_agent, + const int *timeout_ms, + int *http_status, + char *retry_after, + const int *retry_after_cap, + char *body, + const int *body_cap, + int *body_len, + char *error_msg, + const int *error_cap) +{ + CURL *curl; + CURLcode rc; + struct curl_slist *headers = NULL; + struct cepis_body sink; + struct cepis_header hdr; + char errbuf[CURL_ERROR_SIZE]; + long status = 0; + int result = CEPIS_ERR_TRANSPORT; + + *http_status = 0; + *body_len = 0; + cepis_blank(retry_after, *retry_after_cap); + cepis_blank(error_msg, *error_cap); + + if (!cepis_curl_ready) { + if (curl_global_init(CURL_GLOBAL_DEFAULT) != CURLE_OK) { + cepis_put(error_msg, *error_cap, "curl_global_init failed"); + return CEPIS_ERR_TRANSPORT; + } + cepis_curl_ready = 1; + } + + curl = curl_easy_init(); + if (!curl) { + cepis_put(error_msg, *error_cap, "curl_easy_init failed"); + return CEPIS_ERR_TRANSPORT; + } + + sink.buf = body; + sink.cap = *body_cap; + sink.len = 0; + sink.overflow = 0; + + hdr.buf = retry_after; + hdr.cap = *retry_after_cap; + hdr.seen = 0; + + errbuf[0] = '\0'; + + headers = curl_slist_append(headers, + "Accept: application/json, text/csv;q=0.9, */*;q=0.1"); + headers = curl_slist_append(headers, "Accept-Encoding: identity"); + + curl_easy_setopt(curl, CURLOPT_URL, url); + curl_easy_setopt(curl, CURLOPT_HTTPGET, 1L); + curl_easy_setopt(curl, CURLOPT_USERAGENT, user_agent); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(curl, CURLOPT_MAXREDIRS, 5L); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(curl, CURLOPT_CONNECTTIMEOUT_MS, (long)*timeout_ms); + curl_easy_setopt(curl, CURLOPT_TIMEOUT_MS, (long)*timeout_ms); + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, cepis_on_body); + curl_easy_setopt(curl, CURLOPT_WRITEDATA, &sink); + curl_easy_setopt(curl, CURLOPT_HEADERFUNCTION, cepis_on_header); + curl_easy_setopt(curl, CURLOPT_HEADERDATA, &hdr); + curl_easy_setopt(curl, CURLOPT_ERRORBUFFER, errbuf); + + rc = curl_easy_perform(curl); + + if (rc == CURLE_OK) { + curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &status); + *http_status = (int)status; + *body_len = (int)sink.len; + result = CEPIS_OK_TRANSPORT; + } else if (sink.overflow) { + cepis_put(error_msg, *error_cap, "response body exceeds the client buffer"); + result = CEPIS_ERR_TOO_LARGE; + } else { + cepis_put(error_msg, *error_cap, + errbuf[0] ? errbuf : curl_easy_strerror(rc)); + result = CEPIS_ERR_TRANSPORT; + } + + curl_slist_free_all(headers); + curl_easy_cleanup(curl); + return result; +} + +/* Sleep for the backoff. Negative and zero waits return at once. */ +void cepis_sleep_ms(const int *ms) +{ + struct timespec req; + + if (*ms <= 0) { + return; + } + req.tv_sec = *ms / 1000; + req.tv_nsec = (long)(*ms % 1000) * 1000000L; + while (nanosleep(&req, &req) == -1) { + break; + } +} + +/* + * Milliseconds off a monotonic clock. Wall time would let a clock step + * during an NTP correction shorten or extend CFG-TOTAL-TIMEOUT-MS. + */ +void cepis_monotonic_ms(long long *out) +{ + struct timespec now; + + if (clock_gettime(CLOCK_MONOTONIC, &now) != 0) { + *out = 0; + return; + } + *out = (long long)now.tv_sec * 1000LL + (long long)(now.tv_nsec / 1000000L); +} + +/* + * Seconds from now until an HTTP-date, the second form Retry-After may + * take. Returns 0 and sets *seconds when the date parses, 1 otherwise. + * The numeric form is handled in COBOL, where it belongs. + */ +int cepis_http_date_delta(const char *value, int *seconds) +{ + time_t target; + time_t now; + double delta; + + *seconds = 0; + target = curl_getdate(value, NULL); + if (target == (time_t)-1) { + return 1; + } + now = time(NULL); + delta = difftime(target, now); + if (delta < 0) { + delta = 0; + } + if (delta > 2147483000.0) { + delta = 2147483000.0; + } + *seconds = (int)delta; + return 0; +} diff --git a/src/cepis-json.cob b/src/cepis-json.cob new file mode 100644 index 0000000..389d488 --- /dev/null +++ b/src/cepis-json.cob @@ -0,0 +1,409 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-JSON-GET. + +*> --------------------------------------------------------------------- +*> Read one member out of a JSON object. +*> +*> CALL "CEPIS-JSON-GET" USING JSON-TEXT CEPIS-JSON-SCAN +*> +*> Only the members of the object delimited by JS-FROM and JS-TO are +*> considered; nested objects and arrays are skipped whole, so a member +*> named "lat" inside "geo" is not mistaken for a top-level one. Members +*> the caller does not ask about are ignored, which is what lets a new +*> field appear in the API without breaking this client. +*> +*> String values come back with their escapes decoded, including \uXXXX +*> for the whole of Unicode, encoded as UTF-8. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +01 WS-P PIC 9(9). +01 WS-Q PIC 9(9). +01 WS-C PIC X. +01 WS-D PIC X. +01 WS-DEPTH PIC S9(9). +01 WS-IN-STRING PIC X. +01 WS-LEN PIC 9(9). +01 WS-STR PIC X(1024). +01 WS-STR-LEN PIC 9(9). +01 WS-KEY-TEXT PIC X(32). +01 WS-KEY-LEN PIC 9(9). +01 WS-VAL-START PIC 9(9). +01 WS-VAL-END PIC 9(9). +01 WS-VAL-TYPE PIC X. +01 WS-DONE PIC X. +01 WS-CODE PIC 9(9). +01 WS-LOW PIC 9(9). +01 WS-NIB PIC 9(9). +01 WS-B1 PIC 9(9). +01 WS-B2 PIC 9(9). +01 WS-B3 PIC 9(9). +01 WS-B4 PIC 9(9). +01 WS-REST PIC 9(9). +01 WS-HEX-OK PIC X. +01 WS-I PIC 9(9). +01 CONST-QUOTE PIC X VALUE '"'. +01 CONST-BACKSLASH PIC X VALUE "\". + +LINKAGE SECTION. +01 JSON-TEXT PIC X(262144). +COPY "cepis-json.cpy". + +PROCEDURE DIVISION USING JSON-TEXT CEPIS-JSON-SCAN. + +MAIN-START. + MOVE SPACE TO JS-TYPE + MOVE SPACES TO JS-VALUE + MOVE 0 TO JS-VALUE-LEN + MOVE 0 TO JS-VSTART + MOVE 0 TO JS-VEND + MOVE "N" TO WS-DONE + + MOVE JS-FROM TO WS-P + IF WS-P < 1 + MOVE 1 TO WS-P + END-IF + PERFORM SKIP-BLANKS + IF WS-P > JS-TO + MOVE "E" TO JS-TYPE + GOBACK + END-IF + IF JSON-TEXT(WS-P:1) NOT = "{" + MOVE "E" TO JS-TYPE + GOBACK + END-IF + ADD 1 TO WS-P + + PERFORM UNTIL WS-DONE = "Y" OR WS-P > JS-TO + PERFORM SKIP-BLANKS + IF WS-P > JS-TO + MOVE "Y" TO WS-DONE + EXIT PERFORM + END-IF + IF JSON-TEXT(WS-P:1) = "}" + MOVE "Y" TO WS-DONE + EXIT PERFORM + END-IF + IF JSON-TEXT(WS-P:1) NOT = CONST-QUOTE + MOVE "E" TO JS-TYPE + MOVE "Y" TO WS-DONE + EXIT PERFORM + END-IF + + PERFORM READ-STRING + MOVE WS-STR(1:32) TO WS-KEY-TEXT + MOVE WS-STR-LEN TO WS-KEY-LEN + + PERFORM SKIP-BLANKS + IF WS-P > JS-TO OR JSON-TEXT(WS-P:1) NOT = ":" + MOVE "E" TO JS-TYPE + MOVE "Y" TO WS-DONE + EXIT PERFORM + END-IF + ADD 1 TO WS-P + PERFORM SKIP-BLANKS + + MOVE WS-P TO WS-VAL-START + PERFORM SCAN-VALUE + COMPUTE WS-VAL-END = WS-P - 1 + + IF WS-KEY-TEXT = JS-KEY AND WS-KEY-LEN <= 32 + PERFORM TAKE-VALUE + MOVE "Y" TO WS-DONE + EXIT PERFORM + END-IF + + PERFORM SKIP-BLANKS + IF WS-P > JS-TO + MOVE "Y" TO WS-DONE + ELSE + IF JSON-TEXT(WS-P:1) = "," + ADD 1 TO WS-P + ELSE + MOVE "Y" TO WS-DONE + END-IF + END-IF + END-PERFORM + GOBACK. + +*> ------------------------------------------------------------------- +SKIP-BLANKS. + PERFORM UNTIL WS-P > JS-TO + MOVE JSON-TEXT(WS-P:1) TO WS-C + IF WS-C = SPACE OR WS-C = X"09" OR WS-C = X"0A" OR WS-C = X"0D" + ADD 1 TO WS-P + ELSE + EXIT PERFORM + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +TAKE-VALUE. + MOVE WS-VAL-TYPE TO JS-TYPE + MOVE WS-VAL-START TO JS-VSTART + MOVE WS-VAL-END TO JS-VEND + IF WS-VAL-TYPE = "S" + MOVE WS-VAL-START TO WS-P + PERFORM READ-STRING + MOVE WS-STR TO JS-VALUE + MOVE WS-STR-LEN TO JS-VALUE-LEN + ELSE + COMPUTE WS-LEN = WS-VAL-END - WS-VAL-START + 1 + IF WS-LEN > 1024 + MOVE 1024 TO WS-LEN + END-IF + IF WS-LEN > 0 + MOVE JSON-TEXT(WS-VAL-START:WS-LEN) TO JS-VALUE + MOVE WS-LEN TO JS-VALUE-LEN + END-IF + END-IF. + +*> ------------------------------------------------------------------- +*> Classify the value at WS-P and step past it. Objects and arrays are +*> walked with a depth counter that is blind inside string literals, so +*> a brace in a street name cannot close the object early. +SCAN-VALUE. + IF WS-P > JS-TO + MOVE "E" TO WS-VAL-TYPE + EXIT PARAGRAPH + END-IF + MOVE JSON-TEXT(WS-P:1) TO WS-C + EVALUATE TRUE + WHEN WS-C = CONST-QUOTE + MOVE "S" TO WS-VAL-TYPE + PERFORM SKIP-STRING + WHEN WS-C = "{" + MOVE "O" TO WS-VAL-TYPE + PERFORM SKIP-NESTED + WHEN WS-C = "[" + MOVE "A" TO WS-VAL-TYPE + PERFORM SKIP-NESTED + WHEN WS-C = "t" OR WS-C = "f" + MOVE "B" TO WS-VAL-TYPE + PERFORM SKIP-BARE + WHEN WS-C = "n" + MOVE "Z" TO WS-VAL-TYPE + PERFORM SKIP-BARE + WHEN OTHER + MOVE "N" TO WS-VAL-TYPE + PERFORM SKIP-BARE + END-EVALUATE. + +*> ------------------------------------------------------------------- +SKIP-BARE. + PERFORM UNTIL WS-P > JS-TO + MOVE JSON-TEXT(WS-P:1) TO WS-C + IF WS-C = "," OR WS-C = "}" OR WS-C = "]" + OR WS-C = SPACE OR WS-C = X"09" OR WS-C = X"0A" OR WS-C = X"0D" + EXIT PERFORM + END-IF + ADD 1 TO WS-P + END-PERFORM. + +*> ------------------------------------------------------------------- +SKIP-STRING. + ADD 1 TO WS-P + PERFORM UNTIL WS-P > JS-TO + MOVE JSON-TEXT(WS-P:1) TO WS-C + IF WS-C = CONST-BACKSLASH + ADD 2 TO WS-P + ELSE + ADD 1 TO WS-P + IF WS-C = CONST-QUOTE + EXIT PERFORM + END-IF + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +SKIP-NESTED. + MOVE 0 TO WS-DEPTH + MOVE "N" TO WS-IN-STRING + PERFORM UNTIL WS-P > JS-TO + MOVE JSON-TEXT(WS-P:1) TO WS-C + IF WS-IN-STRING = "Y" + IF WS-C = CONST-BACKSLASH + ADD 2 TO WS-P + ELSE + IF WS-C = CONST-QUOTE + MOVE "N" TO WS-IN-STRING + END-IF + ADD 1 TO WS-P + END-IF + ELSE + EVALUATE TRUE + WHEN WS-C = CONST-QUOTE + MOVE "Y" TO WS-IN-STRING + ADD 1 TO WS-P + WHEN WS-C = "{" OR WS-C = "[" + ADD 1 TO WS-DEPTH + ADD 1 TO WS-P + WHEN WS-C = "}" OR WS-C = "]" + SUBTRACT 1 FROM WS-DEPTH + ADD 1 TO WS-P + WHEN OTHER + ADD 1 TO WS-P + END-EVALUATE + IF WS-DEPTH <= 0 + EXIT PERFORM + END-IF + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +*> Read the string literal starting at WS-P into WS-STR, decoding the +*> escapes, and leave WS-P just past the closing quote. +READ-STRING. + MOVE SPACES TO WS-STR + MOVE 0 TO WS-STR-LEN + ADD 1 TO WS-P + PERFORM UNTIL WS-P > JS-TO + MOVE JSON-TEXT(WS-P:1) TO WS-C + IF WS-C = CONST-QUOTE + ADD 1 TO WS-P + EXIT PERFORM + END-IF + IF WS-C = CONST-BACKSLASH + ADD 1 TO WS-P + IF WS-P > JS-TO + EXIT PERFORM + END-IF + MOVE JSON-TEXT(WS-P:1) TO WS-D + EVALUATE WS-D + WHEN "n" + MOVE X"0A" TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + WHEN "t" + MOVE X"09" TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + WHEN "r" + MOVE X"0D" TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + WHEN "b" + MOVE X"08" TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + WHEN "f" + MOVE X"0C" TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + WHEN "u" + PERFORM APPEND-UNICODE + WHEN OTHER + MOVE WS-D TO WS-C + PERFORM APPEND-CHAR + ADD 1 TO WS-P + END-EVALUATE + ELSE + PERFORM APPEND-CHAR + ADD 1 TO WS-P + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +APPEND-CHAR. + IF WS-STR-LEN < 1024 + ADD 1 TO WS-STR-LEN + MOVE WS-C TO WS-STR(WS-STR-LEN:1) + END-IF. + +*> ------------------------------------------------------------------- +*> WS-P sits on the "u" of a \uXXXX escape. Decode it, join it with the +*> escape that follows when the two form a surrogate pair, and append the +*> UTF-8 bytes. +APPEND-UNICODE. + ADD 1 TO WS-P + MOVE 0 TO WS-CODE + PERFORM READ-HEX-4 + IF WS-HEX-OK NOT = "Y" + EXIT PARAGRAPH + END-IF + IF WS-CODE >= 55296 AND WS-CODE <= 56319 + IF WS-P + 5 <= JS-TO + AND JSON-TEXT(WS-P:1) = CONST-BACKSLASH + AND JSON-TEXT(WS-P + 1:1) = "u" + MOVE WS-CODE TO WS-LOW + ADD 2 TO WS-P + MOVE 0 TO WS-CODE + PERFORM READ-HEX-4 + IF WS-HEX-OK = "Y" AND WS-CODE >= 56320 AND WS-CODE <= 57343 + COMPUTE WS-CODE = + 65536 + (WS-LOW - 55296) * 1024 + (WS-CODE - 56320) + END-IF + END-IF + END-IF + PERFORM EMIT-UTF8. + +*> ------------------------------------------------------------------- +READ-HEX-4. + MOVE "Y" TO WS-HEX-OK + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > 4 + IF WS-P > JS-TO + MOVE "N" TO WS-HEX-OK + EXIT PERFORM + END-IF + MOVE JSON-TEXT(WS-P:1) TO WS-D + EVALUATE TRUE + WHEN WS-D >= "0" AND WS-D <= "9" + COMPUTE WS-NIB = FUNCTION ORD(WS-D) - FUNCTION ORD("0") + WHEN WS-D >= "a" AND WS-D <= "f" + COMPUTE WS-NIB = FUNCTION ORD(WS-D) - FUNCTION ORD("a") + 10 + WHEN WS-D >= "A" AND WS-D <= "F" + COMPUTE WS-NIB = FUNCTION ORD(WS-D) - FUNCTION ORD("A") + 10 + WHEN OTHER + MOVE "N" TO WS-HEX-OK + EXIT PERFORM + END-EVALUATE + COMPUTE WS-CODE = WS-CODE * 16 + WS-NIB + ADD 1 TO WS-P + END-PERFORM. + +*> ------------------------------------------------------------------- +EMIT-UTF8. + EVALUATE TRUE + WHEN WS-CODE < 128 + MOVE FUNCTION CHAR(WS-CODE + 1) TO WS-C + PERFORM APPEND-CHAR + WHEN WS-CODE < 2048 + DIVIDE WS-CODE BY 64 GIVING WS-B1 REMAINDER WS-B2 + COMPUTE WS-B1 = 192 + WS-B1 + COMPUTE WS-B2 = 128 + WS-B2 + MOVE FUNCTION CHAR(WS-B1 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B2 + 1) TO WS-C + PERFORM APPEND-CHAR + WHEN WS-CODE < 65536 + DIVIDE WS-CODE BY 4096 GIVING WS-B1 REMAINDER WS-REST + DIVIDE WS-REST BY 64 GIVING WS-B2 REMAINDER WS-B3 + COMPUTE WS-B1 = 224 + WS-B1 + COMPUTE WS-B2 = 128 + WS-B2 + COMPUTE WS-B3 = 128 + WS-B3 + MOVE FUNCTION CHAR(WS-B1 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B2 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B3 + 1) TO WS-C + PERFORM APPEND-CHAR + WHEN OTHER + DIVIDE WS-CODE BY 262144 GIVING WS-B1 REMAINDER WS-REST + DIVIDE WS-REST BY 4096 GIVING WS-B2 REMAINDER WS-Q + DIVIDE WS-Q BY 64 GIVING WS-B3 REMAINDER WS-B4 + COMPUTE WS-B1 = 240 + WS-B1 + COMPUTE WS-B2 = 128 + WS-B2 + COMPUTE WS-B3 = 128 + WS-B3 + COMPUTE WS-B4 = 128 + WS-B4 + MOVE FUNCTION CHAR(WS-B1 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B2 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B3 + 1) TO WS-C + PERFORM APPEND-CHAR + MOVE FUNCTION CHAR(WS-B4 + 1) TO WS-C + PERFORM APPEND-CHAR + END-EVALUATE. + +END PROGRAM CEPIS-JSON-GET. diff --git a/src/cepis-lookup-many.cob b/src/cepis-lookup-many.cob new file mode 100644 index 0000000..d76fa72 --- /dev/null +++ b/src/cepis-lookup-many.cob @@ -0,0 +1,346 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-LOOKUP-MANY. + +*> --------------------------------------------------------------------- +*> Resolve a whole table of CEPs. +*> +*> CALL "CEPIS-LOOKUP-MANY" +*> USING CEPIS-CONFIG CEPIS-BATCH CEPIS-STATUS +*> +*> Fill BATCH-COUNT and BATCH-REQUESTED for each slot; every slot comes +*> back in place. A CEP that does not exist, or that is not eight +*> digits, leaves its slot BATCH-ENTRY-MISSING -- it is never removed, +*> never shifted, and never raises. In a batch of a thousand, one bad +*> CEP must not cost the other nine hundred and ninety-nine. To find out +*> which one was malformed, call CEPIS-NORMALIZE-CEP on it first. +*> +*> Do not loop over CEPIS-LOOKUP instead. This resolves up to 500 CEPs +*> per request and chunks longer tables automatically; a loop of 500 +*> calls pays 500 TLS handshakes where the batch pays one. +*> +*> The batch carries no coordinate. The endpoint behind it is CSV and +*> the CSV has no geo column, so BAT-GEO-PRESENT is "N" on every slot, +*> including CEPs that do have a coordinate. Need it? Use CEPIS-LOOKUP. +*> +*> The table holds 1000 slots. A BATCH-COUNT above that is rejected with +*> CEPIS-INVALID-CEP rather than silently truncated. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +COPY "cepis-http.cpy". +COPY "cepis-csv.cpy". + +01 WS-NORM. + COPY "cepis-status.cpy" REPLACING + ==:P:== BY ==WSN== ==:L1:== BY ==05==. + +01 WS-CHUNK-MAX PIC 9(4) VALUE 500. +01 WS-BATCH-MAX PIC 9(4) VALUE 1000. +01 WS-COLUMNS PIC 9(4) VALUE 13. + +01 WS-INPUT PIC X(20). +01 WS-CEP PIC X(8). +01 WS-ROW-CEP PIC X(8). +01 WS-NEXT PIC 9(4). +01 WS-CHUNK-N PIC 9(4). +01 WS-ROW PIC 9(4). +01 WS-SLOTX PIC 9(4). +01 WS-I PIC 9(4). +01 WS-K PIC 9(4). +01 WS-ID PIC 9(4). +01 WS-NCOLS PIC 9(4). +01 WS-QLEN PIC 9(9). +01 WS-QUERY PIC X(5000). + +01 WS-CHUNK-MAP. + 05 WS-CHUNK-ROW OCCURS 500 TIMES. + 10 WS-SLOT PIC 9(4). + 10 WS-SENT PIC X(8). + +01 WS-HEADER-MAP. + 05 WS-COL-ID OCCURS 64 TIMES PIC 9(4). + +01 WS-ROW-VALUES. + 05 WS-ROW-FIELD OCCURS 13 TIMES. + 10 WS-VAL PIC X(512). + 10 WS-VAL-LEN PIC 9(9). + +01 WS-NAME-TABLE. + 05 FILLER PIC X(16) VALUE "cep". + 05 FILLER PIC X(16) VALUE "logradouro". + 05 FILLER PIC X(16) VALUE "complemento". + 05 FILLER PIC X(16) VALUE "unidade". + 05 FILLER PIC X(16) VALUE "bairro". + 05 FILLER PIC X(16) VALUE "localidade". + 05 FILLER PIC X(16) VALUE "uf". + 05 FILLER PIC X(16) VALUE "estado". + 05 FILLER PIC X(16) VALUE "regiao". + 05 FILLER PIC X(16) VALUE "ibge". + 05 FILLER PIC X(16) VALUE "gia". + 05 FILLER PIC X(16) VALUE "ddd". + 05 FILLER PIC X(16) VALUE "siafi". +01 WS-NAMES REDEFINES WS-NAME-TABLE. + 05 WS-NAME OCCURS 13 TIMES PIC X(16). + +LINKAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION USING CEPIS-CONFIG CEPIS-BATCH CEPIS-STATUS. + +MAIN-START. + SET CEPIS-OK TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + MOVE 0 TO CEPIS-STATUS-HTTP + MOVE -1 TO CEPIS-STATUS-RETRY-AFTER-MS + MOVE 0 TO CEPIS-STATUS-ATTEMPTS + + IF BATCH-COUNT = 0 + GOBACK + END-IF + IF BATCH-COUNT > WS-BATCH-MAX + SET CEPIS-INVALID-CEP TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "BATCH-COUNT is " BATCH-COUNT + " but the table holds only " WS-BATCH-MAX " slots" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + GOBACK + END-IF + + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > BATCH-COUNT + PERFORM CLEAR-SLOT + END-PERFORM + + MOVE 1 TO WS-NEXT + PERFORM UNTIL WS-NEXT > BATCH-COUNT + PERFORM BUILD-CHUNK + IF WS-CHUNK-N > 0 + PERFORM FETCH-CHUNK + IF CEPIS-FAILED + GOBACK + END-IF + PERFORM PARSE-CHUNK + IF CEPIS-FAILED + GOBACK + END-IF + END-IF + END-PERFORM + GOBACK. + +*> ------------------------------------------------------------------- +CLEAR-SLOT. + INITIALIZE BATCH-ADDRESS (WS-I) + SET BATCH-ENTRY-MISSING (WS-I) TO TRUE + SET BAT-NOT-TRUNCATED (WS-I) TO TRUE + SET BAT-GEO-ABSENT (WS-I) TO TRUE + MOVE SPACES TO BAT-GEO-PRECISAO (WS-I). + +*> ------------------------------------------------------------------- +*> Collect the next run of at most 500 well-formed CEPs and remember, +*> for each, which slot it came from. Malformed entries are skipped +*> here: they keep their slot but cost no round trip, because a CEP that +*> is not eight digits is the caller's bug and the server can only say +*> so more slowly. +BUILD-CHUNK. + MOVE 0 TO WS-CHUNK-N + MOVE 0 TO WS-QLEN + MOVE SPACES TO WS-QUERY + PERFORM UNTIL WS-NEXT > BATCH-COUNT + OR WS-CHUNK-N >= WS-CHUNK-MAX + MOVE BATCH-REQUESTED (WS-NEXT) TO WS-INPUT + CALL "CEPIS-NORMALIZE-CEP" USING WS-INPUT WS-CEP WS-NORM + END-CALL + IF WSN-OK + ADD 1 TO WS-CHUNK-N + MOVE WS-NEXT TO WS-SLOT (WS-CHUNK-N) + MOVE WS-CEP TO WS-SENT (WS-CHUNK-N) + IF WS-QLEN > 0 + ADD 1 TO WS-QLEN + MOVE "," TO WS-QUERY (WS-QLEN:1) + END-IF + MOVE WS-CEP TO WS-QUERY (WS-QLEN + 1:8) + ADD 8 TO WS-QLEN + END-IF + ADD 1 TO WS-NEXT + END-PERFORM. + +*> ------------------------------------------------------------------- +FETCH-CHUNK. + MOVE SPACES TO HTTPQ-PATH + STRING "/csv?ceps=" WS-QUERY (1:WS-QLEN) + DELIMITED BY SIZE INTO HTTPQ-PATH + END-STRING + COMPUTE HTTPQ-PATH-LEN = 10 + WS-QLEN + MOVE "N" TO HTTPQ-ALLOW-503 + + CALL "CEPIS-REQUEST" USING CEPIS-CONFIG CEPIS-HTTP-REQUEST + CEPIS-HTTP-RESPONSE CEPIS-STATUS + END-CALL + IF CEPIS-FAILED + EXIT PARAGRAPH + END-IF + + IF HTTP-STATUS = 400 + SET CEPIS-INVALID-CEP TO TRUE + MOVE "the API rejected the batch request" TO + CEPIS-STATUS-MESSAGE + EXIT PARAGRAPH + END-IF + IF HTTP-STATUS NOT = 200 + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "unexpected status " HTTP-STATUS " from /csv" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + END-IF. + +*> ------------------------------------------------------------------- +*> The API answers one row per requested CEP, in the requested order. +*> Should that ever fail to hold -- a proxy truncating, a cut response +*> -- the rows that did arrive are placed and the rest of the slots stay +*> missing. Padding beats shifting: a shifted list silently pairs the +*> wrong address with the wrong CEP, and nothing downstream can tell. +PARSE-CHUNK. + MOVE 1 TO CSV-POS + MOVE HTTP-BODY-LEN TO CSV-LEN + IF CSV-LEN >= 3 + IF HTTP-BODY (1:3) = X"EFBBBF" + MOVE 4 TO CSV-POS + END-IF + END-IF + + PERFORM READ-HEADER + IF WS-NCOLS = 0 + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE "the batch response carries no CSV header row" + TO CEPIS-STATUS-MESSAGE + EXIT PARAGRAPH + END-IF + + MOVE 0 TO WS-ROW + PERFORM UNTIL CSV-POS > CSV-LEN OR WS-ROW >= WS-CHUNK-N + ADD 1 TO WS-ROW + PERFORM READ-ROW + PERFORM ASSIGN-ROW + END-PERFORM. + +*> ------------------------------------------------------------------- +*> Map the header's column order onto the field identifiers, so a column +*> the API adds, moves or drops cannot shift the others out of place. +READ-HEADER. + MOVE 0 TO WS-NCOLS + PERFORM VARYING WS-K FROM 1 BY 1 UNTIL WS-K > 64 + MOVE 0 TO WS-COL-ID (WS-K) + END-PERFORM + MOVE 0 TO WS-K + PERFORM UNTIL WS-K >= 64 + ADD 1 TO WS-K + CALL "CEPIS-CSV-NEXT" USING HTTP-BODY CEPIS-CSV-SCAN + END-CALL + IF CSV-EXHAUSTED = "Y" + SUBTRACT 1 FROM WS-K + EXIT PERFORM + END-IF + MOVE WS-K TO WS-NCOLS + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > WS-COLUMNS + IF CSV-FIELD (1:16) = WS-NAME (WS-I) + MOVE WS-I TO WS-COL-ID (WS-K) + EXIT PERFORM + END-IF + END-PERFORM + IF CSV-ROW-END = "Y" + EXIT PERFORM + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +READ-ROW. + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > WS-COLUMNS + MOVE SPACES TO WS-VAL (WS-I) + MOVE 0 TO WS-VAL-LEN (WS-I) + END-PERFORM + MOVE 0 TO WS-K + PERFORM UNTIL WS-K >= 64 + ADD 1 TO WS-K + CALL "CEPIS-CSV-NEXT" USING HTTP-BODY CEPIS-CSV-SCAN + END-CALL + IF WS-K <= WS-NCOLS + MOVE WS-COL-ID (WS-K) TO WS-ID + IF WS-ID > 0 + MOVE CSV-FIELD TO WS-VAL (WS-ID) + MOVE CSV-FIELD-LEN TO WS-VAL-LEN (WS-ID) + END-IF + END-IF + IF CSV-ROW-END = "Y" + EXIT PERFORM + END-IF + END-PERFORM. + +*> ------------------------------------------------------------------- +*> A row the API filled only in its cep column means "not in the DNE +*> database"; it keeps its slot as missing. The cep column is also +*> matched against the CEP that was actually sent, so a response that +*> came back in the wrong order marks slots missing instead of pairing +*> them with someone else's street. +ASSIGN-ROW. + MOVE WS-SLOT (WS-ROW) TO WS-SLOTX + IF WS-VAL (6) = SPACES + EXIT PARAGRAPH + END-IF + MOVE WS-VAL (1) (1:20) TO WS-INPUT + CALL "CEPIS-NORMALIZE-CEP" USING WS-INPUT WS-ROW-CEP WS-NORM + END-CALL + IF NOT WSN-OK OR WS-ROW-CEP NOT = WS-SENT (WS-ROW) + EXIT PARAGRAPH + END-IF + + SET BATCH-ENTRY-FOUND (WS-SLOTX) TO TRUE + MOVE WS-ROW-CEP TO BAT-CEP (WS-SLOTX) + STRING WS-ROW-CEP (1:5) "-" WS-ROW-CEP (6:3) + DELIMITED BY SIZE INTO BAT-CEP-MASKED (WS-SLOTX) + END-STRING + + IF WS-VAL-LEN (2) > 120 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (2) TO BAT-LOGRADOURO (WS-SLOTX) + + IF WS-VAL-LEN (3) > 120 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (3) TO BAT-COMPLEMENTO (WS-SLOTX) + + IF WS-VAL-LEN (4) > 80 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (4) TO BAT-UNIDADE (WS-SLOTX) + + IF WS-VAL-LEN (5) > 100 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (5) TO BAT-BAIRRO (WS-SLOTX) + + IF WS-VAL-LEN (6) > 100 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (6) TO BAT-LOCALIDADE (WS-SLOTX) + + MOVE WS-VAL (7) TO BAT-UF (WS-SLOTX) + + IF WS-VAL-LEN (8) > 60 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (8) TO BAT-ESTADO (WS-SLOTX) + + IF WS-VAL-LEN (9) > 30 + SET BAT-WAS-TRUNCATED (WS-SLOTX) TO TRUE + END-IF + MOVE WS-VAL (9) TO BAT-REGIAO (WS-SLOTX) + + MOVE WS-VAL (10) TO BAT-IBGE (WS-SLOTX) + MOVE WS-VAL (11) TO BAT-GIA (WS-SLOTX) + MOVE WS-VAL (12) TO BAT-DDD (WS-SLOTX) + MOVE WS-VAL (13) TO BAT-SIAFI (WS-SLOTX). + +END PROGRAM CEPIS-LOOKUP-MANY. diff --git a/src/cepis-lookup.cob b/src/cepis-lookup.cob new file mode 100644 index 0000000..cc6e412 --- /dev/null +++ b/src/cepis-lookup.cob @@ -0,0 +1,324 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-LOOKUP. + +*> --------------------------------------------------------------------- +*> Look up one CEP. +*> +*> CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT +*> CEPIS-ADDRESS CEPIS-STATUS +*> +*> The CEP may be masked or not. It is validated locally first, so a +*> seven-digit CEP costs no round trip: CEPIS-INVALID-CEP comes back +*> without a socket ever being opened. +*> +*> On CEPIS-OK the address is filled. On anything else CEPIS-ADDRESS is +*> cleared, so a caller that forgets to test the status reads spaces +*> rather than the previous CEP's street. +*> +*> ADDR-GEO-PRESENT says whether the coordinate exists -- about 57% of +*> CEPs have one. When it does, read ADDR-GEO-PRECISAO before measuring +*> any distance with it: "logradouro" means the point is a stretch of +*> street, with ADDR-GEO-RAIO-M around 100-150 m, while "localidade" +*> means the CEP is a whole municipality and the point is its centroid, +*> with a radius of 7 to 40 km. Computing shipping from the second is +*> wrong by tens of kilometres. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +COPY "cepis-http.cpy". +COPY "cepis-json.cpy". + +01 WS-CEP PIC X(8). +01 WS-GEO-FROM PIC 9(9). +01 WS-GEO-TO PIC 9(9). +01 WS-GEO-OK PIC X. +01 WS-NUM-OK PIC X. +01 WS-LAT PIC S9(3)V9(6). +01 WS-LON PIC S9(3)V9(6). +01 WS-RAIO PIC 9(9). +01 WS-PRECISAO PIC X(16). +01 WS-I PIC 9(4). +01 WS-C PIC X. +01 WS-DOTS PIC 9(4). +01 WS-JUNK PIC 9(4). +01 WS-DIGITS PIC 9(4). + +LINKAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS. + +MAIN-START. + PERFORM CLEAR-ADDRESS + + CALL "CEPIS-NORMALIZE-CEP" + USING CEPIS-CEP-INPUT WS-CEP CEPIS-STATUS + END-CALL + IF CEPIS-FAILED + GOBACK + END-IF + + MOVE SPACES TO HTTPQ-PATH + STRING "/" WS-CEP DELIMITED BY SIZE INTO HTTPQ-PATH + END-STRING + MOVE 9 TO HTTPQ-PATH-LEN + MOVE "N" TO HTTPQ-ALLOW-503 + + CALL "CEPIS-REQUEST" USING CEPIS-CONFIG CEPIS-HTTP-REQUEST + CEPIS-HTTP-RESPONSE CEPIS-STATUS + END-CALL + IF CEPIS-FAILED + GOBACK + END-IF + + EVALUATE HTTP-STATUS + WHEN 200 + PERFORM PARSE-ADDRESS + WHEN 404 + SET CEPIS-NOT-FOUND TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "CEP " WS-CEP " does not exist in the DNE database" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + WHEN 400 + SET CEPIS-INVALID-CEP TO TRUE + PERFORM API-MESSAGE + WHEN OTHER + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "unexpected status " HTTP-STATUS + " for CEP " WS-CEP + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + END-EVALUATE + GOBACK. + +*> ------------------------------------------------------------------- +CLEAR-ADDRESS. + INITIALIZE CEPIS-ADDRESS + SET ADDR-NOT-TRUNCATED TO TRUE + SET ADDR-GEO-ABSENT TO TRUE + MOVE SPACES TO ADDR-GEO-PRECISAO. + +*> ------------------------------------------------------------------- +*> The API's own 400 carries a message this client does not have: the +*> format already passed local validation, so the server knows something +*> we do not. Surface its "mensagem" when there is one. +API-MESSAGE. + MOVE SPACES TO CEPIS-STATUS-MESSAGE + MOVE "mensagem" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE = "S" AND JS-VALUE-LEN > 0 + MOVE JS-VALUE TO CEPIS-STATUS-MESSAGE + ELSE + STRING "CEP " WS-CEP " rejected by the API" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + END-IF. + +*> ------------------------------------------------------------------- +GET-MEMBER. + MOVE 1 TO JS-FROM + MOVE HTTP-BODY-LEN TO JS-TO + CALL "CEPIS-JSON-GET" USING HTTP-BODY CEPIS-JSON-SCAN + END-CALL. + +*> ------------------------------------------------------------------- +GET-GEO-MEMBER. + MOVE WS-GEO-FROM TO JS-FROM + MOVE WS-GEO-TO TO JS-TO + CALL "CEPIS-JSON-GET" USING HTTP-BODY CEPIS-JSON-SCAN + END-CALL. + +*> ------------------------------------------------------------------- +PARSE-ADDRESS. + MOVE "cep" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE = "E" + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE "response is not a JSON object -- something between you " + & "and the API is answering instead of it" + TO CEPIS-STATUS-MESSAGE + EXIT PARAGRAPH + END-IF + + IF JS-VALUE-LEN > 8 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-CEP + IF ADDR-CEP = SPACES + MOVE WS-CEP TO ADDR-CEP + END-IF + STRING ADDR-CEP(1:5) "-" ADDR-CEP(6:3) + DELIMITED BY SIZE INTO ADDR-CEP-MASKED + END-STRING + + MOVE "logradouro" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 120 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-LOGRADOURO + + MOVE "complemento" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 120 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-COMPLEMENTO + + MOVE "unidade" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 80 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-UNIDADE + + MOVE "bairro" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 100 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-BAIRRO + + MOVE "localidade" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 100 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-LOCALIDADE + + MOVE "uf" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 2 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-UF + + MOVE "estado" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 60 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-ESTADO + + MOVE "regiao" TO JS-KEY + PERFORM GET-MEMBER + IF JS-VALUE-LEN > 30 + SET ADDR-WAS-TRUNCATED TO TRUE + END-IF + MOVE JS-VALUE TO ADDR-REGIAO + + MOVE "ibge" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO ADDR-IBGE + + MOVE "gia" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO ADDR-GIA + + MOVE "ddd" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO ADDR-DDD + + MOVE "siafi" TO JS-KEY + PERFORM GET-MEMBER + MOVE JS-VALUE TO ADDR-SIAFI + + PERFORM PARSE-GEO + SET CEPIS-OK TO TRUE. + +*> ------------------------------------------------------------------- +*> The coordinate is omitted entirely when the database has none, so its +*> absence is normal, not an error. A geo block that is present but +*> malformed is dropped the same way: the rest of the address is still +*> useful and losing it over a bad float would be a poor trade. +PARSE-GEO. + SET ADDR-GEO-ABSENT TO TRUE + MOVE "geo" TO JS-KEY + PERFORM GET-MEMBER + IF JS-TYPE NOT = "O" + EXIT PARAGRAPH + END-IF + MOVE JS-VSTART TO WS-GEO-FROM + MOVE JS-VEND TO WS-GEO-TO + MOVE "Y" TO WS-GEO-OK + + MOVE "lat" TO JS-KEY + PERFORM GET-GEO-MEMBER + PERFORM CHECK-NUMERIC + IF WS-NUM-OK = "Y" + COMPUTE WS-LAT = FUNCTION NUMVAL (JS-VALUE) + ELSE + MOVE "N" TO WS-GEO-OK + END-IF + + MOVE "lon" TO JS-KEY + PERFORM GET-GEO-MEMBER + PERFORM CHECK-NUMERIC + IF WS-NUM-OK = "Y" + COMPUTE WS-LON = FUNCTION NUMVAL (JS-VALUE) + ELSE + MOVE "N" TO WS-GEO-OK + END-IF + + MOVE "raio_m" TO JS-KEY + PERFORM GET-GEO-MEMBER + PERFORM CHECK-NUMERIC + IF WS-NUM-OK = "Y" + COMPUTE WS-RAIO ROUNDED = FUNCTION NUMVAL (JS-VALUE) + ELSE + MOVE "N" TO WS-GEO-OK + END-IF + + MOVE "precisao" TO JS-KEY + PERFORM GET-GEO-MEMBER + IF JS-TYPE = "S" AND JS-VALUE-LEN > 0 + MOVE JS-VALUE TO WS-PRECISAO + ELSE + MOVE "N" TO WS-GEO-OK + END-IF + + IF WS-GEO-OK = "Y" + SET ADDR-GEO-KNOWN TO TRUE + MOVE WS-LAT TO ADDR-GEO-LAT + MOVE WS-LON TO ADDR-GEO-LON + MOVE WS-RAIO TO ADDR-GEO-RAIO-M + MOVE WS-PRECISAO TO ADDR-GEO-PRECISAO + END-IF. + +*> ------------------------------------------------------------------- +*> FUNCTION NUMVAL answers 0 for text it cannot read, which would turn +*> "lat": "not a number" into a coordinate off the coast of Africa. The +*> text is therefore checked before it is converted. +CHECK-NUMERIC. + MOVE "N" TO WS-NUM-OK + IF JS-TYPE NOT = "N" AND JS-TYPE NOT = "S" + EXIT PARAGRAPH + END-IF + IF JS-VALUE-LEN = 0 OR JS-VALUE-LEN > 32 + EXIT PARAGRAPH + END-IF + MOVE 0 TO WS-DIGITS + MOVE 0 TO WS-DOTS + MOVE 0 TO WS-JUNK + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > JS-VALUE-LEN + MOVE JS-VALUE(WS-I:1) TO WS-C + EVALUATE TRUE + WHEN WS-C >= "0" AND WS-C <= "9" + ADD 1 TO WS-DIGITS + WHEN WS-C = "." + ADD 1 TO WS-DOTS + WHEN (WS-C = "-" OR WS-C = "+") AND WS-I = 1 + CONTINUE + WHEN OTHER + ADD 1 TO WS-JUNK + END-EVALUATE + END-PERFORM + IF WS-DIGITS > 0 AND WS-DOTS <= 1 AND WS-JUNK = 0 + MOVE "Y" TO WS-NUM-OK + END-IF. + +END PROGRAM CEPIS-LOOKUP. diff --git a/src/cepis-normalize-cep.cob b/src/cepis-normalize-cep.cob new file mode 100644 index 0000000..41615ad --- /dev/null +++ b/src/cepis-normalize-cep.cob @@ -0,0 +1,74 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-NORMALIZE-CEP. + +*> --------------------------------------------------------------------- +*> Reduce a CEP to its eight digits. +*> +*> CALL "CEPIS-NORMALIZE-CEP" +*> USING CEPIS-CEP-INPUT CEPIS-CEP CEPIS-STATUS +*> +*> "01001-000", "01001000", " 01001-000 " and "01001.000" all give +*> "01001000". Anything that does not leave exactly eight digits sets +*> CEPIS-INVALID-CEP and leaves CEPIS-CEP as spaces. +*> +*> CEPIS-LOOKUP runs this before opening a socket. A seven-digit CEP is +*> the caller's bug, and a round trip for the server to say so wastes +*> both sides' time. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +01 WS-I PIC 9(4). +01 WS-N PIC 9(4). +01 WS-C PIC X. +01 WS-CLEAN PIC X(20). + +LINKAGE SECTION. +COPY "cepis.cpy". + +PROCEDURE DIVISION USING CEPIS-CEP-INPUT CEPIS-CEP CEPIS-STATUS. + +MAIN-START. + MOVE SPACES TO CEPIS-CEP + MOVE SPACES TO WS-CLEAN + MOVE 0 TO WS-N + MOVE 0 TO CEPIS-STATUS-HTTP + MOVE 0 TO CEPIS-STATUS-ATTEMPTS + MOVE -1 TO CEPIS-STATUS-RETRY-AFTER-MS + MOVE SPACES TO CEPIS-STATUS-MESSAGE + + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > 20 + MOVE CEPIS-CEP-INPUT(WS-I:1) TO WS-C + EVALUATE TRUE + WHEN WS-C = SPACE OR WS-C = "-" OR WS-C = "." + CONTINUE + WHEN WS-C >= "0" AND WS-C <= "9" + ADD 1 TO WS-N + IF WS-N <= 20 + MOVE WS-C TO WS-CLEAN(WS-N:1) + END-IF + WHEN OTHER + PERFORM REJECT + GOBACK + END-EVALUATE + END-PERFORM + + IF WS-N NOT = 8 + PERFORM REJECT + GOBACK + END-IF + + MOVE WS-CLEAN(1:8) TO CEPIS-CEP + SET CEPIS-OK TO TRUE + GOBACK. + +*> ------------------------------------------------------------------- +REJECT. + SET CEPIS-INVALID-CEP TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "CEP must be 8 digits, got '" + FUNCTION TRIM(CEPIS-CEP-INPUT) "'" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING. + +END PROGRAM CEPIS-NORMALIZE-CEP. diff --git a/src/cepis-request.cob b/src/cepis-request.cob new file mode 100644 index 0000000..b45f5da --- /dev/null +++ b/src/cepis-request.cob @@ -0,0 +1,363 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-REQUEST. + +*> --------------------------------------------------------------------- +*> One GET against the API, retried according to CEPIS-CONFIG. Internal: +*> callers of this library use CEPIS-LOOKUP and friends. +*> +*> CALL "CEPIS-REQUEST" USING CEPIS-CONFIG CEPIS-HTTP-REQUEST +*> CEPIS-HTTP-RESPONSE CEPIS-STATUS +*> +*> 429 and the transient 5xx are retried, and so are transport failures. +*> 400 and 404 are handed back to the caller untouched: both are +*> definitive answers about the request, so retrying returns the same +*> thing more slowly while burning someone else's quota. +*> +*> Waits are drawn from [0, min(base * 2^(n-1), max)] -- full jitter. The +*> randomness is not decoration: without it, N clients that failed +*> together come back together and reproduce the spike. +*> +*> A Retry-After from the server wins over that calculation, in both the +*> RFC 9110 forms, and is still capped at CFG-BACKOFF-MAX-MS so that a +*> misconfigured proxy answering "Retry-After: 3600" cannot park the +*> caller for an hour. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +01 WS-SEEDED PIC X VALUE "N". +01 WS-SEED PIC 9(9). +01 WS-RANDOM PIC 9V9(9). + +01 WS-BASE PIC X(200). +01 WS-BASE-LEN PIC S9(9). +01 WS-URL PIC X(8500). +01 WS-URL-TEXT PIC X(8500). +01 WS-USER-AGENT PIC X(140). + +01 WS-TIMEOUT-MS PIC S9(9) COMP-5. +01 WS-CURL-STATUS PIC S9(9) COMP-5. +01 WS-BODY-LEN PIC S9(9) COMP-5. +01 WS-BODY-CAP PIC S9(9) COMP-5 VALUE 262144. +01 WS-RETRY-AFTER-CAP PIC S9(9) COMP-5 VALUE 128. +01 WS-ERROR-CAP PIC S9(9) COMP-5 VALUE 256. +01 WS-SECONDS PIC S9(9) COMP-5. +01 WS-SLEEP-MS PIC S9(9) COMP-5. +01 WS-RC PIC S9(9) COMP-5. +01 WS-DATE-RC PIC S9(9) COMP-5. + +01 WS-RETRY-AFTER-TEXT PIC X(128). +01 WS-RETRY-AFTER-TRIM PIC X(128). +01 WS-RETRY-AFTER-Z PIC X(129). +01 WS-ERROR-TEXT PIC X(256). + +01 WS-NOW PIC S9(18) COMP-5. +01 WS-DEADLINE PIC S9(18) COMP-5. +01 WS-REMAINING PIC S9(18). +01 WS-HAS-DEADLINE PIC X. + +01 WS-MAX-ATTEMPTS PIC 9(4). +01 WS-ATTEMPT PIC 9(4). +01 WS-EXPONENT PIC 9(4). +01 WS-CEILING PIC 9(18). +01 WS-WAIT-MS PIC 9(18). +01 WS-RETRY-AFTER-MS PIC S9(18). +01 WS-LAST-STATUS PIC 9(3). +01 WS-DONE PIC X. +01 WS-KEEP PIC X. +01 WS-DIGITS PIC 9(4). +01 WS-JUNK PIC 9(4). +01 WS-I PIC 9(4). +01 WS-C PIC X. +01 WS-TRIM-LEN PIC 9(4). + +LINKAGE SECTION. +COPY "cepis.cpy". +COPY "cepis-http.cpy". + +PROCEDURE DIVISION USING CEPIS-CONFIG CEPIS-HTTP-REQUEST + CEPIS-HTTP-RESPONSE CEPIS-STATUS. + +MAIN-START. + PERFORM SEED-JITTER + PERFORM RESET-STATE + PERFORM BUILD-URL + PERFORM ARM-DEADLINE + + PERFORM VARYING WS-ATTEMPT FROM 1 BY 1 + UNTIL WS-ATTEMPT > WS-MAX-ATTEMPTS + + IF WS-HAS-DEADLINE = "Y" + CALL "cepis_monotonic_ms" USING WS-NOW + IF WS-NOW >= WS-DEADLINE + EXIT PERFORM + END-IF + END-IF + + MOVE WS-ATTEMPT TO CEPIS-STATUS-ATTEMPTS + PERFORM ONE-ATTEMPT + + IF WS-DONE = "Y" + EXIT PERFORM + END-IF + IF WS-ATTEMPT >= WS-MAX-ATTEMPTS + EXIT PERFORM + END-IF + + PERFORM COMPUTE-WAIT + IF WS-HAS-DEADLINE = "Y" + CALL "cepis_monotonic_ms" USING WS-NOW + COMPUTE WS-REMAINING = WS-DEADLINE - WS-NOW + IF WS-REMAINING <= 0 + EXIT PERFORM + END-IF + IF WS-WAIT-MS > WS-REMAINING + MOVE WS-REMAINING TO WS-WAIT-MS + END-IF + END-IF + MOVE WS-WAIT-MS TO WS-SLEEP-MS + CALL "cepis_sleep_ms" USING WS-SLEEP-MS + END-PERFORM + + IF WS-DONE NOT = "Y" + PERFORM CLASSIFY-FAILURE + END-IF + GOBACK. + +*> ------------------------------------------------------------------- +*> The jitter is seeded once per process, off the monotonic clock. An +*> unseeded FUNCTION RANDOM gives every process the same sequence, which +*> is the one thing full jitter exists to avoid. +SEED-JITTER. + IF WS-SEEDED = "N" + CALL "cepis_monotonic_ms" USING WS-NOW + COMPUTE WS-SEED = FUNCTION MOD (WS-NOW, 1000000) + COMPUTE WS-RANDOM = FUNCTION RANDOM (WS-SEED) + MOVE "Y" TO WS-SEEDED + END-IF. + +*> ------------------------------------------------------------------- +RESET-STATE. + SET CEPIS-OK TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + MOVE 0 TO CEPIS-STATUS-HTTP + MOVE -1 TO CEPIS-STATUS-RETRY-AFTER-MS + MOVE 0 TO CEPIS-STATUS-ATTEMPTS + MOVE 0 TO HTTP-STATUS + MOVE 0 TO HTTP-BODY-LEN + MOVE "N" TO WS-DONE + MOVE 0 TO WS-LAST-STATUS + MOVE -1 TO WS-RETRY-AFTER-MS + MOVE SPACES TO WS-ERROR-TEXT + MOVE CFG-MAX-ATTEMPTS TO WS-MAX-ATTEMPTS + IF WS-MAX-ATTEMPTS < 1 + MOVE 1 TO WS-MAX-ATTEMPTS + END-IF. + +*> ------------------------------------------------------------------- +*> A trailing slash on CFG-BASE-URL is dropped so it cannot produce the +*> doubled separator that some proxies normalise and others answer 404. +BUILD-URL. + MOVE CFG-BASE-URL TO WS-BASE + COMPUTE WS-BASE-LEN = FUNCTION LENGTH (FUNCTION TRIM (WS-BASE)) + IF WS-BASE = SPACES + MOVE 0 TO WS-BASE-LEN + END-IF + PERFORM UNTIL WS-BASE-LEN <= 0 + OR WS-BASE (WS-BASE-LEN:1) NOT = "/" + SUBTRACT 1 FROM WS-BASE-LEN + END-PERFORM + + MOVE SPACES TO WS-URL-TEXT + IF WS-BASE-LEN > 0 + STRING WS-BASE (1:WS-BASE-LEN) + HTTPQ-PATH (1:HTTPQ-PATH-LEN) + DELIMITED BY SIZE INTO WS-URL-TEXT + END-STRING + ELSE + STRING HTTPQ-PATH (1:HTTPQ-PATH-LEN) + DELIMITED BY SIZE INTO WS-URL-TEXT + END-STRING + END-IF + + MOVE SPACES TO WS-URL + STRING FUNCTION TRIM (WS-URL-TEXT) X"00" + DELIMITED BY SIZE INTO WS-URL + END-STRING + + MOVE SPACES TO WS-USER-AGENT + STRING FUNCTION TRIM (CFG-USER-AGENT) X"00" + DELIMITED BY SIZE INTO WS-USER-AGENT + END-STRING + + MOVE CFG-TIMEOUT-MS TO WS-TIMEOUT-MS + IF WS-TIMEOUT-MS <= 0 + MOVE 10000 TO WS-TIMEOUT-MS + END-IF. + +*> ------------------------------------------------------------------- +ARM-DEADLINE. + MOVE "N" TO WS-HAS-DEADLINE + IF CFG-TOTAL-TIMEOUT-MS > 0 + CALL "cepis_monotonic_ms" USING WS-NOW + COMPUTE WS-DEADLINE = WS-NOW + CFG-TOTAL-TIMEOUT-MS + MOVE "Y" TO WS-HAS-DEADLINE + END-IF. + +*> ------------------------------------------------------------------- +ONE-ATTEMPT. + CALL "cepis_http_get" USING + WS-URL WS-USER-AGENT WS-TIMEOUT-MS + WS-CURL-STATUS + WS-RETRY-AFTER-TEXT WS-RETRY-AFTER-CAP + HTTP-BODY WS-BODY-CAP WS-BODY-LEN + WS-ERROR-TEXT WS-ERROR-CAP + RETURNING WS-RC + END-CALL + + EVALUATE WS-RC + WHEN 0 + MOVE WS-CURL-STATUS TO HTTP-STATUS + MOVE WS-BODY-LEN TO HTTP-BODY-LEN + MOVE HTTP-STATUS TO CEPIS-STATUS-HTTP + PERFORM DECIDE-KEEP + IF WS-KEEP = "Y" + SET CEPIS-OK TO TRUE + MOVE "Y" TO WS-DONE + ELSE + MOVE HTTP-STATUS TO WS-LAST-STATUS + PERFORM PARSE-RETRY-AFTER + END-IF + WHEN 2 + SET CEPIS-INVALID-RESPONSE TO TRUE + MOVE SPACES TO CEPIS-STATUS-MESSAGE + STRING "response from " FUNCTION TRIM (WS-URL-TEXT) + " is larger than the 256 KiB client buffer" + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + MOVE "Y" TO WS-DONE + WHEN OTHER + MOVE 0 TO WS-LAST-STATUS + MOVE -1 TO WS-RETRY-AFTER-MS + END-EVALUATE. + +*> ------------------------------------------------------------------- +*> Keep this answer, or try again? 200 always; 503 too when the caller +*> asked for it, which is how /health reports itself degraded without +*> being retried three times first. Everything outside the retriable set +*> -- 400 and 404 above all -- is kept for the caller to interpret. +DECIDE-KEEP. + MOVE "Y" TO WS-KEEP + IF HTTP-STATUS = 200 + EXIT PARAGRAPH + END-IF + IF HTTP-STATUS = 503 AND HTTPQ-ALLOW-503 = "Y" + EXIT PARAGRAPH + END-IF + IF HTTP-STATUS = 429 OR HTTP-STATUS = 500 OR HTTP-STATUS = 502 + OR HTTP-STATUS = 503 OR HTTP-STATUS = 504 + MOVE "N" TO WS-KEEP + END-IF. + +*> ------------------------------------------------------------------- +*> Retry-After arrives either as a number of seconds or as an HTTP date. +*> Handling only the first is the common bug, and it makes a client +*> ignore exactly the wait request that matters most. The date form goes +*> to curl_getdate through the shim. +PARSE-RETRY-AFTER. + MOVE -1 TO WS-RETRY-AFTER-MS + IF WS-RETRY-AFTER-TEXT = SPACES + EXIT PARAGRAPH + END-IF + MOVE FUNCTION TRIM (WS-RETRY-AFTER-TEXT) TO WS-RETRY-AFTER-TRIM + COMPUTE WS-TRIM-LEN = + FUNCTION LENGTH (FUNCTION TRIM (WS-RETRY-AFTER-TEXT)) + + MOVE 0 TO WS-DIGITS + MOVE 0 TO WS-JUNK + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > WS-TRIM-LEN + MOVE WS-RETRY-AFTER-TRIM (WS-I:1) TO WS-C + EVALUATE TRUE + WHEN WS-C >= "0" AND WS-C <= "9" + ADD 1 TO WS-DIGITS + WHEN WS-C = "." + CONTINUE + WHEN OTHER + ADD 1 TO WS-JUNK + END-EVALUATE + END-PERFORM + + IF WS-DIGITS > 0 AND WS-JUNK = 0 + COMPUTE WS-RETRY-AFTER-MS ROUNDED = + FUNCTION NUMVAL (WS-RETRY-AFTER-TRIM) * 1000 + IF WS-RETRY-AFTER-MS < 0 + MOVE 0 TO WS-RETRY-AFTER-MS + END-IF + EXIT PARAGRAPH + END-IF + + MOVE SPACES TO WS-RETRY-AFTER-Z + STRING WS-RETRY-AFTER-TRIM (1:WS-TRIM-LEN) X"00" + DELIMITED BY SIZE INTO WS-RETRY-AFTER-Z + END-STRING + CALL "cepis_http_date_delta" USING WS-RETRY-AFTER-Z WS-SECONDS + RETURNING WS-DATE-RC + END-CALL + IF WS-DATE-RC = 0 + COMPUTE WS-RETRY-AFTER-MS = WS-SECONDS * 1000 + END-IF. + +*> ------------------------------------------------------------------- +COMPUTE-WAIT. + IF WS-RETRY-AFTER-MS >= 0 + MOVE WS-RETRY-AFTER-MS TO WS-WAIT-MS + IF WS-WAIT-MS > CFG-BACKOFF-MAX-MS + MOVE CFG-BACKOFF-MAX-MS TO WS-WAIT-MS + END-IF + EXIT PARAGRAPH + END-IF + + MOVE WS-ATTEMPT TO WS-EXPONENT + IF WS-EXPONENT > 20 + MOVE 20 TO WS-EXPONENT + END-IF + COMPUTE WS-CEILING = + CFG-BACKOFF-BASE-MS * (2 ** (WS-EXPONENT - 1)) + IF WS-CEILING > CFG-BACKOFF-MAX-MS + MOVE CFG-BACKOFF-MAX-MS TO WS-CEILING + END-IF + COMPUTE WS-RANDOM = FUNCTION RANDOM + COMPUTE WS-WAIT-MS ROUNDED = WS-RANDOM * WS-CEILING. + +*> ------------------------------------------------------------------- +CLASSIFY-FAILURE. + MOVE SPACES TO CEPIS-STATUS-MESSAGE + EVALUATE TRUE + WHEN WS-LAST-STATUS = 429 + SET CEPIS-RATE-LIMITED TO TRUE + MOVE WS-LAST-STATUS TO CEPIS-STATUS-HTTP + IF WS-RETRY-AFTER-MS > 999999999 + MOVE 999999999 TO CEPIS-STATUS-RETRY-AFTER-MS + ELSE + MOVE WS-RETRY-AFTER-MS TO CEPIS-STATUS-RETRY-AFTER-MS + END-IF + STRING "rate limit reached at " + FUNCTION TRIM (WS-URL-TEXT) + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + WHEN WS-LAST-STATUS > 0 + SET CEPIS-SERVER-ERROR TO TRUE + MOVE WS-LAST-STATUS TO CEPIS-STATUS-HTTP + STRING "the API answered " WS-LAST-STATUS " at " + FUNCTION TRIM (WS-URL-TEXT) + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + WHEN OTHER + SET CEPIS-NETWORK-ERROR TO TRUE + STRING "could not reach " FUNCTION TRIM (WS-URL-TEXT) + ": " FUNCTION TRIM (WS-ERROR-TEXT) + DELIMITED BY SIZE INTO CEPIS-STATUS-MESSAGE + END-STRING + END-EVALUATE. + +END PROGRAM CEPIS-REQUEST. diff --git a/tests/cepis-tests.cob b/tests/cepis-tests.cob new file mode 100644 index 0000000..4bc1a11 --- /dev/null +++ b/tests/cepis-tests.cob @@ -0,0 +1,761 @@ +IDENTIFICATION DIVISION. +PROGRAM-ID. CEPIS-TESTS. + +*> --------------------------------------------------------------------- +*> The suite. Run it through tests/server.py, which binds a loopback +*> port, exports CEPIS_TEST_URL and runs this program: +*> +*> python3 tests/server.py ./build/cepis-tests +*> +*> Every test goes through the real transport against that server, so +*> what is exercised is the code that runs in production -- request +*> construction, header handling and error-body reading included. It +*> doubles as the test that CFG-BASE-URL works: without it none of this +*> would reach anything. +*> +*> The process exits non-zero if any check fails. +*> --------------------------------------------------------------------- + +DATA DIVISION. +WORKING-STORAGE SECTION. +COPY "cepis.cpy". +COPY "cepis-http.cpy". + +01 WS-BASE PIC X(200). +01 WS-CHECKS PIC 9(4) VALUE 0. +01 WS-FAILURES PIC 9(4) VALUE 0. +01 WS-LABEL PIC X(56). +01 WS-DETAIL PIC X(200). +01 WS-OK PIC X. +01 WS-TALLY PIC 9(4). +01 WS-I PIC 9(4). +01 WS-COUNT PIC 9(9). +01 WS-COUNT-BEFORE PIC 9(9). +01 WS-COUNT-PATH PIC X(40). +01 WS-NUM8 PIC 9(8). +01 WS-T0 PIC S9(18) COMP-5. +01 WS-T1 PIC S9(18) COMP-5. +01 WS-ELAPSED PIC S9(18). +01 WS-SAVED-ATTEMPTS PIC 9(4). +01 WS-SAVED-RETRY PIC S9(9). + +01 WS-GOOD-TABLE. + 05 FILLER PIC X(20) VALUE "01001000". + 05 FILLER PIC X(20) VALUE "01001-000". + 05 FILLER PIC X(20) VALUE " 01001-000 ". + 05 FILLER PIC X(20) VALUE "01001.000". +01 WS-GOODS REDEFINES WS-GOOD-TABLE. + 05 WS-GOOD OCCURS 4 TIMES PIC X(20). + +01 WS-BAD-TABLE. + 05 FILLER PIC X(20) VALUE SPACES. + 05 FILLER PIC X(20) VALUE "123". + 05 FILLER PIC X(20) VALUE "0100100a". + 05 FILLER PIC X(20) VALUE "010010000". + 05 FILLER PIC X(20) VALUE "abcdefgh". +01 WS-BADS REDEFINES WS-BAD-TABLE. + 05 WS-BAD OCCURS 5 TIMES PIC X(20). + +PROCEDURE DIVISION. + +MAIN-START. + MOVE SPACES TO WS-BASE + ACCEPT WS-BASE FROM ENVIRONMENT "CEPIS_TEST_URL" + IF WS-BASE = SPACES + DISPLAY "CEPIS_TEST_URL is not set -- run through " + "tests/server.py" + MOVE 2 TO RETURN-CODE + STOP RUN + END-IF + DISPLAY "cepis 0.1.0 test suite against " FUNCTION TRIM (WS-BASE) + DISPLAY " " + + PERFORM T-NORMALIZE-ACCEPTS + PERFORM T-NORMALIZE-REJECTS + PERFORM T-LOOKUP-SUCCESS + PERFORM T-GEO-PRECISE + PERFORM T-GEO-CITY-WIDE + PERFORM T-GEO-MALFORMED + PERFORM T-UNKNOWN-FIELDS + PERFORM T-LONG-VALUE-FLAGGED + PERFORM T-NOT-FOUND + PERFORM T-NOT-FOUND-NOT-RETRIED + PERFORM T-INVALID-NEVER-LEAVES + PERFORM T-API-400 + PERFORM T-NOT-JSON + PERFORM T-RETRY-THEN-SUCCESS + PERFORM T-RETRY-EXHAUSTED + PERFORM T-ONE-ATTEMPT-NO-RETRY + PERFORM T-RATE-LIMITED + PERFORM T-RETRY-AFTER-CAPPED + PERFORM T-RETRY-AFTER-HTTP-DATE + PERFORM T-NETWORK-ERROR + PERFORM T-TOTAL-TIMEOUT + PERFORM T-BATCH-ORDER + PERFORM T-BATCH-QUOTING + PERFORM T-BATCH-MALFORMED-ENTRY + PERFORM T-BATCH-EMPTY + PERFORM T-BATCH-TRUNCATED + PERFORM T-BATCH-CHUNKING + PERFORM T-BATCH-OVER-CAPACITY + PERFORM T-HEALTH + PERFORM T-HEALTH-DEGRADED + PERFORM T-TRAILING-SLASH + PERFORM T-DEFAULT-USER-AGENT + + DISPLAY " " + DISPLAY WS-CHECKS " checks, " WS-FAILURES " failed" + IF WS-FAILURES > 0 + MOVE 1 TO RETURN-CODE + ELSE + MOVE 0 TO RETURN-CODE + END-IF + STOP RUN. + +*> =================================================================== +*> Fixtures +*> =================================================================== +FRESH-CONFIG. + CALL "CEPIS-DEFAULT-CONFIG" USING CEPIS-CONFIG + MOVE WS-BASE TO CFG-BASE-URL + MOVE 5000 TO CFG-TIMEOUT-MS + MOVE 3 TO CFG-MAX-ATTEMPTS + MOVE 1 TO CFG-BACKOFF-BASE-MS + MOVE 10 TO CFG-BACKOFF-MAX-MS + MOVE 0 TO CFG-TOTAL-TIMEOUT-MS. + +CHECK. + ADD 1 TO WS-CHECKS + IF WS-OK = "Y" + DISPLAY "ok " WS-LABEL + ELSE + ADD 1 TO WS-FAILURES + DISPLAY "FAIL " WS-LABEL " -- " FUNCTION TRIM (WS-DETAIL) + END-IF + MOVE SPACES TO WS-DETAIL + MOVE "N" TO WS-OK. + +*> The fake server counts requests per path; that is how a 404 not being +*> retried, or 501 CEPs becoming two requests, is proved rather than +*> assumed. +COUNT-FOR. + PERFORM FRESH-CONFIG + MOVE 1 TO CFG-MAX-ATTEMPTS + MOVE SPACES TO HTTPQ-PATH + STRING "/count?path=" FUNCTION TRIM (WS-COUNT-PATH) + DELIMITED BY SIZE INTO HTTPQ-PATH + END-STRING + COMPUTE HTTPQ-PATH-LEN = + 12 + FUNCTION LENGTH (FUNCTION TRIM (WS-COUNT-PATH)) + MOVE "N" TO HTTPQ-ALLOW-503 + MOVE 0 TO WS-COUNT + CALL "CEPIS-REQUEST" USING CEPIS-CONFIG CEPIS-HTTP-REQUEST + CEPIS-HTTP-RESPONSE CEPIS-STATUS + END-CALL + IF CEPIS-OK AND HTTP-BODY-LEN > 0 + COMPUTE WS-COUNT = + FUNCTION NUMVAL (HTTP-BODY (1:HTTP-BODY-LEN)) + END-IF. + +*> =================================================================== +*> Local validation +*> =================================================================== +T-NORMALIZE-ACCEPTS. + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > 4 + MOVE WS-GOOD (WS-I) TO CEPIS-CEP-INPUT + CALL "CEPIS-NORMALIZE-CEP" + USING CEPIS-CEP-INPUT CEPIS-CEP CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND CEPIS-CEP = "01001000" + MOVE "Y" TO WS-OK + END-IF + MOVE SPACES TO WS-LABEL + STRING "normalize accepts " WS-GOOD (WS-I) + DELIMITED BY SIZE INTO WS-LABEL + END-STRING + MOVE CEPIS-CEP TO WS-DETAIL + PERFORM CHECK + END-PERFORM. + +T-NORMALIZE-REJECTS. + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > 5 + MOVE WS-BAD (WS-I) TO CEPIS-CEP-INPUT + CALL "CEPIS-NORMALIZE-CEP" + USING CEPIS-CEP-INPUT CEPIS-CEP CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-INVALID-CEP AND CEPIS-CEP = SPACES + MOVE "Y" TO WS-OK + END-IF + MOVE SPACES TO WS-LABEL + STRING "normalize rejects '" WS-BAD (WS-I) "'" + DELIMITED BY SIZE INTO WS-LABEL + END-STRING + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK + END-PERFORM. + +*> =================================================================== +*> Single lookup +*> =================================================================== +T-LOOKUP-SUCCESS. + PERFORM FRESH-CONFIG + MOVE "01001-000" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK + AND ADDR-LOCALIDADE = "São Paulo" + AND ADDR-UF = "SP" + AND ADDR-LOGRADOURO = "Praça da Sé" + AND ADDR-IBGE = "3550308" + AND ADDR-CEP-MASKED = "01001-000" + AND ADDR-GEO-ABSENT + AND ADDR-NOT-TRUNCATED + MOVE "Y" TO WS-OK + END-IF + MOVE "lookup returns the address, masked cep, no geo" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-GEO-PRECISE. + PERFORM FRESH-CONFIG + MOVE "01310100" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-GEO-KNOWN AND ADDR-GEO-PRECISE + AND ADDR-GEO-RAIO-M = 120 + AND ADDR-GEO-LAT < -23.5 AND ADDR-GEO-LAT > -23.6 + AND ADDR-GEO-LON < -46.6 AND ADDR-GEO-LON > -46.7 + MOVE "Y" TO WS-OK + END-IF + MOVE "geo present is read and reports precisao logradouro" + TO WS-LABEL + PERFORM CHECK. + +T-GEO-CITY-WIDE. + PERFORM FRESH-CONFIG + MOVE "01001001" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-GEO-KNOWN AND ADDR-GEO-CITY-WIDE + AND NOT ADDR-GEO-PRECISE AND ADDR-GEO-RAIO-M = 22000 + MOVE "Y" TO WS-OK + END-IF + MOVE "a municipality centroid is not precise" TO WS-LABEL + PERFORM CHECK. + +T-GEO-MALFORMED. + PERFORM FRESH-CONFIG + MOVE "01001002" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-GEO-ABSENT + AND ADDR-LOCALIDADE = "São Paulo" + MOVE "Y" TO WS-OK + END-IF + MOVE "a malformed geo does not lose the address" TO WS-LABEL + PERFORM CHECK. + +T-UNKNOWN-FIELDS. + PERFORM FRESH-CONFIG + MOVE "01001003" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-LOCALIDADE = "São Paulo" + AND ADDR-GEO-ABSENT + MOVE "Y" TO WS-OK + END-IF + MOVE "fields the client does not know are ignored" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-LONG-VALUE-FLAGGED. + PERFORM FRESH-CONFIG + MOVE "01001010" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-WAS-TRUNCATED + MOVE "Y" TO WS-OK + END-IF + MOVE "a value past the field width sets ADDR-TRUNCATED" + TO WS-LABEL + PERFORM CHECK. + +T-NOT-FOUND. + PERFORM FRESH-CONFIG + MOVE "99999999" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-NOT-FOUND AND ADDR-LOCALIDADE = SPACES + MOVE "Y" TO WS-OK + END-IF + MOVE "a missing CEP is CEPIS-NOT-FOUND" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-NOT-FOUND-NOT-RETRIED. + MOVE "/99999999" TO WS-COUNT-PATH + PERFORM COUNT-FOR + MOVE "N" TO WS-OK + IF WS-COUNT = 1 + MOVE "Y" TO WS-OK + END-IF + MOVE "404 is definitive and is not retried" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "requests: " WS-COUNT + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-INVALID-NEVER-LEAVES. + PERFORM FRESH-CONFIG + MOVE "123" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-INVALID-CEP + MOVE "Y" TO WS-OK + END-IF + MOVE "/123" TO WS-COUNT-PATH + PERFORM COUNT-FOR + IF WS-COUNT NOT = 0 + MOVE "N" TO WS-OK + END-IF + MOVE "a malformed CEP never leaves the machine" TO WS-LABEL + PERFORM CHECK. + +T-API-400. + PERFORM FRESH-CONFIG + MOVE "00000000" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE 0 TO WS-TALLY + INSPECT CEPIS-STATUS-MESSAGE + TALLYING WS-TALLY FOR ALL "rejected by the API" + MOVE "N" TO WS-OK + IF CEPIS-INVALID-CEP AND WS-TALLY = 1 + MOVE "Y" TO WS-OK + END-IF + MOVE "an API 400 carries the API's own message" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-NOT-JSON. + PERFORM FRESH-CONFIG + MOVE "01001004" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-INVALID-RESPONSE + MOVE "Y" TO WS-OK + END-IF + MOVE "a body that is not JSON is CEPIS-INVALID-RESPONSE" + TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +*> =================================================================== +*> Retrying +*> =================================================================== +T-RETRY-THEN-SUCCESS. + PERFORM FRESH-CONFIG + MOVE "01001005" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-LOCALIDADE = "São Paulo" + MOVE "Y" TO WS-OK + END-IF + MOVE "/01001005" TO WS-COUNT-PATH + PERFORM COUNT-FOR + IF WS-COUNT NOT = 2 + MOVE "N" TO WS-OK + END-IF + MOVE "500 is retried and the second attempt succeeds" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "requests: " WS-COUNT + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-RETRY-EXHAUSTED. + PERFORM FRESH-CONFIG + MOVE "01001006" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-SERVER-ERROR AND CEPIS-STATUS-HTTP = 503 + AND CEPIS-STATUS-ATTEMPTS = 3 + MOVE "Y" TO WS-OK + END-IF + MOVE "/01001006" TO WS-COUNT-PATH + PERFORM COUNT-FOR + IF WS-COUNT NOT = 3 + MOVE "N" TO WS-OK + END-IF + MOVE "three attempts, then CEPIS-SERVER-ERROR" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "requests: " WS-COUNT + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-ONE-ATTEMPT-NO-RETRY. + PERFORM FRESH-CONFIG + MOVE 1 TO CFG-MAX-ATTEMPTS + MOVE "01001012" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-SERVER-ERROR + MOVE "Y" TO WS-OK + END-IF + MOVE "/01001012" TO WS-COUNT-PATH + PERFORM COUNT-FOR + IF WS-COUNT NOT = 1 + MOVE "N" TO WS-OK + END-IF + MOVE "CFG-MAX-ATTEMPTS of 1 disables retrying" TO WS-LABEL + PERFORM CHECK. + +T-RATE-LIMITED. + PERFORM FRESH-CONFIG + MOVE "01001007" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-RATE-LIMITED AND CEPIS-STATUS-RETRY-AFTER-MS = 7000 + MOVE "Y" TO WS-OK + END-IF + MOVE "429 exposes Retry-After in seconds" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "retry-after ms: " CEPIS-STATUS-RETRY-AFTER-MS + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-RETRY-AFTER-CAPPED. + PERFORM FRESH-CONFIG + MOVE "01001008" TO CEPIS-CEP-INPUT + CALL "cepis_monotonic_ms" USING WS-T0 + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + CALL "cepis_monotonic_ms" USING WS-T1 + COMPUTE WS-ELAPSED = WS-T1 - WS-T0 + MOVE "N" TO WS-OK + IF CEPIS-RATE-LIMITED AND WS-ELAPSED < 3000 + MOVE "Y" TO WS-OK + END-IF + MOVE "an absurd Retry-After is capped at the backoff maximum" + TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "elapsed ms: " WS-ELAPSED + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-RETRY-AFTER-HTTP-DATE. + PERFORM FRESH-CONFIG + MOVE "01001009" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-RATE-LIMITED + AND CEPIS-STATUS-RETRY-AFTER-MS >= 1000 + AND CEPIS-STATUS-RETRY-AFTER-MS <= 8000 + MOVE "Y" TO WS-OK + END-IF + MOVE "Retry-After is also read as an HTTP date" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "retry-after ms: " CEPIS-STATUS-RETRY-AFTER-MS + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-NETWORK-ERROR. + PERFORM FRESH-CONFIG + MOVE "http://127.0.0.1:1" TO CFG-BASE-URL + MOVE 2 TO CFG-MAX-ATTEMPTS + MOVE 1000 TO CFG-TIMEOUT-MS + MOVE "01001000" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-NETWORK-ERROR AND CEPIS-STATUS-ATTEMPTS = 2 + MOVE "Y" TO WS-OK + END-IF + MOVE "a refused connection is CEPIS-NETWORK-ERROR" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-TOTAL-TIMEOUT. + PERFORM FRESH-CONFIG + MOVE 50 TO CFG-MAX-ATTEMPTS + MOVE 50 TO CFG-BACKOFF-BASE-MS + MOVE 50 TO CFG-BACKOFF-MAX-MS + MOVE 300 TO CFG-TOTAL-TIMEOUT-MS + MOVE "01001011" TO CEPIS-CEP-INPUT + CALL "cepis_monotonic_ms" USING WS-T0 + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + CALL "cepis_monotonic_ms" USING WS-T1 + COMPUTE WS-ELAPSED = WS-T1 - WS-T0 + MOVE CEPIS-STATUS-ATTEMPTS TO WS-SAVED-ATTEMPTS + MOVE "N" TO WS-OK + IF CEPIS-SERVER-ERROR AND WS-ELAPSED < 3000 + AND WS-SAVED-ATTEMPTS < 50 + MOVE "Y" TO WS-OK + END-IF + MOVE "CFG-TOTAL-TIMEOUT-MS bounds the whole call" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "elapsed ms: " WS-ELAPSED + " attempts: " WS-SAVED-ATTEMPTS + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +*> =================================================================== +*> Batch +*> =================================================================== +T-BATCH-ORDER. + PERFORM FRESH-CONFIG + MOVE 3 TO BATCH-COUNT + MOVE "01001000" TO BATCH-REQUESTED (1) + MOVE "99999999" TO BATCH-REQUESTED (2) + MOVE "20040020" TO BATCH-REQUESTED (3) + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK + AND BATCH-ENTRY-FOUND (1) + AND BAT-LOCALIDADE (1) = "São Paulo" + AND BATCH-ENTRY-MISSING (2) + AND BATCH-ENTRY-FOUND (3) + AND BAT-UF (3) = "RJ" + AND BAT-GEO-ABSENT (1) + MOVE "Y" TO WS-OK + END-IF + MOVE "a missing CEP keeps its slot and nothing shifts" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-BATCH-QUOTING. + PERFORM FRESH-CONFIG + MOVE 1 TO BATCH-COUNT + MOVE "01001111" TO BATCH-REQUESTED (1) + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND BATCH-ENTRY-FOUND (1) + AND BAT-LOGRADOURO (1) = "Rua Um, Dois" + AND BAT-COMPLEMENTO (1) = 'aspas "aqui"' + MOVE "Y" TO WS-OK + END-IF + MOVE "quoted CSV fields with commas and quotes are parsed" + TO WS-LABEL + MOVE BAT-LOGRADOURO (1) TO WS-DETAIL + PERFORM CHECK. + +T-BATCH-MALFORMED-ENTRY. + PERFORM FRESH-CONFIG + MOVE 3 TO BATCH-COUNT + MOVE "01001000" TO BATCH-REQUESTED (1) + MOVE "nonsense" TO BATCH-REQUESTED (2) + MOVE "20040020" TO BATCH-REQUESTED (3) + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK + AND BATCH-ENTRY-FOUND (1) + AND BATCH-ENTRY-MISSING (2) + AND BATCH-ENTRY-FOUND (3) + AND BAT-UF (3) = "RJ" + MOVE "Y" TO WS-OK + END-IF + MOVE "a malformed CEP is missing, not an error" TO WS-LABEL + PERFORM CHECK. + +T-BATCH-EMPTY. + MOVE "/csv" TO WS-COUNT-PATH + PERFORM COUNT-FOR + MOVE WS-COUNT TO WS-COUNT-BEFORE + PERFORM FRESH-CONFIG + MOVE 0 TO BATCH-COUNT + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK + MOVE "Y" TO WS-OK + END-IF + MOVE "/csv" TO WS-COUNT-PATH + PERFORM COUNT-FOR + IF WS-COUNT NOT = WS-COUNT-BEFORE + MOVE "N" TO WS-OK + END-IF + MOVE "an empty batch makes no request" TO WS-LABEL + PERFORM CHECK. + +T-BATCH-TRUNCATED. + PERFORM FRESH-CONFIG + MOVE 2 TO BATCH-COUNT + MOVE "01001000" TO BATCH-REQUESTED (1) + MOVE "00000099" TO BATCH-REQUESTED (2) + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND BATCH-ENTRY-FOUND (1) + AND BATCH-ENTRY-MISSING (2) + MOVE "Y" TO WS-OK + END-IF + MOVE "a short response pads rather than shifts" TO WS-LABEL + PERFORM CHECK. + +T-BATCH-CHUNKING. + MOVE "/csv" TO WS-COUNT-PATH + PERFORM COUNT-FOR + MOVE WS-COUNT TO WS-COUNT-BEFORE + PERFORM FRESH-CONFIG + MOVE 501 TO BATCH-COUNT + PERFORM VARYING WS-I FROM 1 BY 1 UNTIL WS-I > 501 + COMPUTE WS-NUM8 = 70000000 + WS-I + MOVE WS-NUM8 TO BATCH-REQUESTED (WS-I) + END-PERFORM + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND BATCH-ENTRY-FOUND (1) + AND BATCH-ENTRY-FOUND (500) + AND BATCH-ENTRY-FOUND (501) + AND BAT-LOCALIDADE (501) = "Cidade Sintetica" + MOVE "Y" TO WS-OK + END-IF + MOVE "/csv" TO WS-COUNT-PATH + PERFORM COUNT-FOR + COMPUTE WS-COUNT = WS-COUNT - WS-COUNT-BEFORE + IF WS-COUNT NOT = 2 + MOVE "N" TO WS-OK + END-IF + MOVE "501 CEPs become two requests, not one 400" TO WS-LABEL + MOVE SPACES TO WS-DETAIL + STRING "requests: " WS-COUNT " status: " CEPIS-STATUS-MESSAGE + DELIMITED BY SIZE INTO WS-DETAIL + END-STRING + PERFORM CHECK. + +T-BATCH-OVER-CAPACITY. + PERFORM FRESH-CONFIG + MOVE 1001 TO BATCH-COUNT + CALL "CEPIS-LOOKUP-MANY" USING CEPIS-CONFIG CEPIS-BATCH + CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-INVALID-CEP + MOVE "Y" TO WS-OK + END-IF + MOVE 0 TO BATCH-COUNT + MOVE "a BATCH-COUNT past the table is refused, not truncated" + TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +*> =================================================================== +*> Health and configuration +*> =================================================================== +T-HEALTH. + PERFORM FRESH-CONFIG + CALL "CEPIS-HEALTH" USING CEPIS-CONFIG CEPIS-HEALTH CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND HEALTH-OK AND HEALTH-DNE = "26071" + AND HEALTH-CEPS = 1596168 AND HEALTH-GEO = 894489 + AND HEALTH-BASE-GERADA-EM = "2026-07-24T03:48:28Z" + MOVE "Y" TO WS-OK + END-IF + MOVE "health reports the live database" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-HEALTH-DEGRADED. + PERFORM FRESH-CONFIG + MOVE SPACES TO CFG-BASE-URL + STRING FUNCTION TRIM (WS-BASE) "/degraded" + DELIMITED BY SIZE INTO CFG-BASE-URL + END-STRING + CALL "CEPIS-HEALTH" USING CEPIS-CONFIG CEPIS-HEALTH CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND NOT HEALTH-OK + AND HEALTH-STATUS = "degradado" + AND CEPIS-STATUS-ATTEMPTS = 1 + MOVE "Y" TO WS-OK + END-IF + MOVE "a degraded API is diagnosed, not retried" TO WS-LABEL + MOVE HEALTH-STATUS TO WS-DETAIL + PERFORM CHECK. + +T-TRAILING-SLASH. + PERFORM FRESH-CONFIG + MOVE SPACES TO CFG-BASE-URL + STRING FUNCTION TRIM (WS-BASE) "/" + DELIMITED BY SIZE INTO CFG-BASE-URL + END-STRING + MOVE "01001000" TO CEPIS-CEP-INPUT + CALL "CEPIS-LOOKUP" USING CEPIS-CONFIG CEPIS-CEP-INPUT + CEPIS-ADDRESS CEPIS-STATUS + END-CALL + MOVE "N" TO WS-OK + IF CEPIS-OK AND ADDR-UF = "SP" + MOVE "Y" TO WS-OK + END-IF + MOVE "a trailing slash on the base URL is dropped" TO WS-LABEL + MOVE CEPIS-STATUS-MESSAGE TO WS-DETAIL + PERFORM CHECK. + +T-DEFAULT-USER-AGENT. + CALL "CEPIS-DEFAULT-CONFIG" USING CEPIS-CONFIG + MOVE 0 TO WS-TALLY + INSPECT CFG-USER-AGENT TALLYING WS-TALLY FOR ALL "cepis-cobol/0.1.0" + MOVE "N" TO WS-OK + IF WS-TALLY = 1 AND CFG-BASE-URL = "https://cep.is" + AND CFG-MAX-ATTEMPTS = 3 AND CFG-BACKOFF-BASE-MS = 500 + AND CFG-BACKOFF-MAX-MS = 8000 + MOVE "Y" TO WS-OK + END-IF + MOVE "the defaults identify the library and its version" + TO WS-LABEL + MOVE CFG-USER-AGENT TO WS-DETAIL + PERFORM CHECK. + +END PROGRAM CEPIS-TESTS. diff --git a/tests/server.py b/tests/server.py new file mode 100644 index 0000000..46185a0 --- /dev/null +++ b/tests/server.py @@ -0,0 +1,254 @@ +#!/usr/bin/env python3 +"""Fake cep.is for the COBOL test suite. + +Binds a loopback port, points CEPIS_TEST_URL at it, runs the command given +on the argv and exits with that command's status. Standing an HTTP server up +from COBOL is not practical; every route below is a fixture the driver in +`cepis-tests.cob` asserts against, and no test touches the network. + + python3 tests/server.py ./build/cepis-tests + +The counters behind `/count` are what let the driver prove that a 404 is not +retried, that a 500 is, and that 501 CEPs become two batch requests. +""" + +import email.utils +import json +import os +import subprocess +import sys +import threading +import time +import urllib.parse +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +SE = { + "cep": "01001000", + "logradouro": "Praça da Sé", + "complemento": "lado ímpar", + "unidade": "", + "bairro": "Sé", + "localidade": "São Paulo", + "uf": "SP", + "estado": "São Paulo", + "regiao": "Sudeste", + "ibge": "3550308", + "gia": "1004", + "ddd": "11", + "siafi": "7107", +} + +CSV_COLUMNS = [ + "cep", + "logradouro", + "complemento", + "unidade", + "bairro", + "localidade", + "uf", + "estado", + "regiao", + "ibge", + "gia", + "ddd", + "siafi", +] + +CSV_ROWS = { + "01001000": [ + "Praça da Sé", "lado ímpar", "", "Sé", "São Paulo", "SP", + "São Paulo", "Sudeste", "3550308", "1004", "11", "7107", + ], + "20040020": [ + "Avenida Rio Branco", "", "", "Centro", "Rio de Janeiro", "RJ", + "Rio de Janeiro", "Sudeste", "3304557", "", "21", "6001", + ], + "01001111": [ + "Rua Um, Dois", 'aspas "aqui"', "", "Centro", "São Paulo", "SP", + "São Paulo", "Sudeste", "3550308", "", "11", "7107", + ], +} + +SYNTHETIC = [ + "Rua Sintetica", "", "", "Centro", "Cidade Sintetica", "SP", + "São Paulo", "Sudeste", "3550308", "", "11", "7107", +] + +DROPPED_CEP = "00000099" + +LONG_STREET = "Avenida " + ("Muito Comprida " * 20) + +COUNTS = {} +COUNT_LOCK = threading.Lock() + + +def with_geo(precisao, raio): + body = dict(SE) + body["geo"] = { + "lat": -23.567202, + "lon": -46.649659, + "raio_m": raio, + "precisao": precisao, + } + return body + + +def csv_escape(value): + if any(c in value for c in (",", '"', "\n", "\r")): + return '"' + value.replace('"', '""') + '"' + return value + + +def csv_body(ceps): + lines = [",".join(CSV_COLUMNS)] + for cep in ceps: + if cep == DROPPED_CEP: + continue + if cep in CSV_ROWS: + values = [cep] + CSV_ROWS[cep] + elif cep.startswith("7") and len(cep) == 8: + values = [cep] + SYNTHETIC + else: + values = [cep] + [""] * 12 + lines.append(",".join(csv_escape(v) for v in values)) + return "\n".join(lines) + "\n" + + +def json_body(payload): + return json.dumps(payload, ensure_ascii=False) + + +def bump(path): + with COUNT_LOCK: + COUNTS[path] = COUNTS.get(path, 0) + 1 + return COUNTS[path] + + +def route(path, query, hits): + """Return (status, body, headers) for one request.""" + if path == "/01001000": + return 200, json_body(SE), {} + if path == "/01310100": + return 200, json_body(with_geo("logradouro", 120)), {} + if path == "/01001001": + return 200, json_body(with_geo("localidade", 22000)), {} + if path == "/01001002": + body = dict(SE) + body["geo"] = {"lat": "not a number"} + return 200, json_body(body), {} + if path == "/01001003": + body = dict(SE) + body["campo_do_futuro"] = "value" + body["objeto_do_futuro"] = {"a": [1, 2, {"lat": 9}]} + return 200, json_body(body), {} + if path == "/01001004": + return 200, "captive portal", {} + if path == "/01001010": + body = dict(SE) + body["logradouro"] = LONG_STREET + return 200, json_body(body), {} + if path == "/01001005": + if hits == 1: + return 500, json_body({"erro": "interno"}), {} + return 200, json_body(SE), {} + if path in ("/01001006", "/01001011"): + return 503, json_body({"erro": "interno"}), {} + if path == "/01001012": + return 500, json_body({"erro": "interno"}), {} + if path == "/01001007": + return 429, json_body({"erro": "rate"}), {"Retry-After": "7"} + if path == "/01001008": + return 429, json_body({"erro": "rate"}), {"Retry-After": "3600"} + if path == "/01001009": + when = email.utils.formatdate(time.time() + 4, usegmt=True) + return 429, json_body({"erro": "rate"}), {"Retry-After": when} + if path == "/99999999": + return 404, json_body( + {"erro": "cep_nao_encontrado", "mensagem": "gone"} + ), {} + if path == "/00000000": + return 400, json_body( + {"erro": "cep_invalido", "mensagem": "rejected by the API"} + ), {} + if path == "/health": + return 200, json_body( + { + "status": "ok", + "dne": "26071", + "ceps": 1596168, + "geo": 894489, + "base_gerada_em": "2026-07-24T03:48:28Z", + } + ), {} + if path == "/degraded/health": + return 503, json_body( + {"status": "degradado", "dne": "", "ceps": 0, "geo": 0} + ), {} + if path == "/csv": + raw = urllib.parse.parse_qs(query).get("ceps", [""])[0] + ceps = [c for c in raw.split(",") if c] + if len(ceps) > 500: + return 400, json_body( + {"erro": "muitos_ceps", "mensagem": "no máximo 500"} + ), {} + return 200, csv_body(ceps), {} + return 404, json_body({"erro": "cep_nao_encontrado"}), {} + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self): + parsed = urllib.parse.urlsplit(self.path) + path = parsed.path + + if path == "/count": + wanted = urllib.parse.parse_qs(parsed.query).get("path", [""])[0] + with COUNT_LOCK: + total = sum( + n for p, n in COUNTS.items() if p.startswith(wanted) + ) + self.reply(200, str(total), {}) + return + + hits = bump(path) + status, body, headers = route(path, parsed.query, hits) + self.reply(status, body, headers) + + def reply(self, status, body, headers): + data = body.encode("utf-8") + self.send_response(status) + for key, value in headers.items(): + self.send_header(key, value) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *args): + pass + + +def main(): + if len(sys.argv) < 2: + print("usage: server.py [args...]", file=sys.stderr) + return 2 + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + host, port = server.server_address[:2] + thread = threading.Thread( + target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True + ) + thread.start() + + env = dict(os.environ) + env["CEPIS_TEST_URL"] = "http://{}:{}".format(host, port) + try: + return subprocess.call(sys.argv[1:], env=env) + finally: + server.shutdown() + server.server_close() + + +if __name__ == "__main__": + sys.exit(main()) -- cgit v1.3