defmodule Cepis.HTTP do @moduledoc false alias Cepis.Client alias Cepis.{NetworkError, RateLimitError, ServerError} @retriable_status [429, 500, 502, 503, 504] @accept ~c"application/json, text/csv;q=0.9, */*;q=0.1" @epoch_offset 62_167_219_200 @spec request(Client.t(), String.t(), [pos_integer()]) :: {:ok, pos_integer(), binary(), list()} | {:error, Cepis.Error.t()} def request(%Client{} = client, path, accepted \\ [200]) do url = client.base_url <> path deadline = if client.total_timeout, do: monotonic() + client.total_timeout attempt(client, url, accepted, 1, deadline, nil) end defp attempt(client, url, accepted, number, deadline, last) do if expired?(deadline) do give_up(client, url, last) else case get(client, url) do {:ok, status, body, headers} -> if status in accepted or status not in @retriable_status do {:ok, status, body, headers} else next(client, url, accepted, number, deadline, {:status, status, retry_after(headers)}) end {:error, reason} -> next(client, url, accepted, number, deadline, {:network, reason}) end end end defp next(client, url, accepted, number, deadline, last) do if number >= client.max_attempts do give_up(client, url, last) else case wait_for(client, number, deadline, last) do :expired -> give_up(client, url, last) milliseconds -> Process.sleep(milliseconds) attempt(client, url, accepted, number + 1, deadline, last) end end end defp wait_for(client, number, deadline, last) do wait = backoff(client, number, requested_wait(last)) case deadline do nil -> wait _ -> remaining = deadline - monotonic() if remaining <= 0, do: :expired, else: min(wait, remaining) end end defp requested_wait({:status, _status, retry_after}), do: retry_after defp requested_wait(_), do: nil defp backoff(client, _number, retry_after) when is_integer(retry_after) do min(retry_after, client.backoff_max) end defp backoff(client, number, _retry_after) do ceiling = min(client.backoff_base * Integer.pow(2, min(number - 1, 30)), client.backoff_max) trunc(:rand.uniform() * ceiling) end defp get(client, url) do ensure_started() request = {String.to_charlist(url), headers(client)} http_options = [ timeout: client.timeout, connect_timeout: client.timeout ] ++ tls_options(url) case :httpc.request(:get, request, http_options, body_format: :binary) do {:ok, {{_version, status, _reason}, headers, body}} -> {:ok, status, body, headers} {:ok, {status, body}} -> {:ok, status, body, []} {:error, reason} -> {:error, reason} end end defp headers(client) do [ {~c"user-agent", String.to_charlist(client.user_agent)}, {~c"accept", @accept}, {~c"accept-encoding", ~c"identity"} ] end defp tls_options("https://" <> _rest) do [ ssl: [ verify: :verify_peer, cacerts: :public_key.cacerts_get(), depth: 3, customize_hostname_check: [ match_fun: :public_key.pkix_verify_hostname_match_fun(:https) ] ] ] rescue _ -> [] end defp tls_options(_url), do: [] defp ensure_started do {:ok, _} = Application.ensure_all_started(:inets) {:ok, _} = Application.ensure_all_started(:ssl) :ok end defp retry_after(headers) do case header(headers, "retry-after") do nil -> nil value -> parse_retry_after(String.trim(value)) end end defp parse_retry_after(value) do case Float.parse(value) do {seconds, ""} -> max(0, trunc(seconds * 1000)) _ -> parse_http_date(value) end end defp parse_http_date(value) do case :httpd_util.convert_request_date(String.to_charlist(value)) do :bad_date -> nil datetime -> target = :calendar.datetime_to_gregorian_seconds(datetime) - @epoch_offset max(0, (target - System.os_time(:second)) * 1000) end end defp header(headers, name) do Enum.find_value(headers, fn {key, value} -> if key |> to_string() |> String.downcase() == name, do: to_string(value) end) end defp give_up(client, url, {:status, 429, retry_after}) do {:error, %RateLimitError{ message: "rate limit reached at #{url} (after #{client.max_attempts} attempt(s))", retry_after: retry_after }} end defp give_up(client, url, {:status, status, _retry_after}) do {:error, %ServerError{ message: "the API answered #{status} at #{url} (after #{client.max_attempts} attempt(s))", status: status }} end defp give_up(client, url, {:network, reason}) do {:error, %NetworkError{ message: "could not reach #{url} (after #{client.max_attempts} attempt(s)): #{inspect(reason)}", reason: reason }} end defp give_up(_client, url, nil) do {:error, %NetworkError{ message: "gave up on #{url}: total_timeout elapsed before any attempt completed", reason: :timeout }} end defp expired?(nil), do: false defp expired?(deadline), do: monotonic() >= deadline defp monotonic, do: System.monotonic_time(:millisecond) end