#!/usr/bin/env bash # overcast source provider: edgar (SEC EDGAR corporate filings — turn a company's # filing history into scan records, one per filing). NO key: the SEC APIs are # public, but they 403 a blank/bot User-Agent — AND, empirically, any UA bearing a # URL or parentheses. So edgar sends a plain `product/version email` UA and, unlike # overpass/chain, does NOT trust the shared OVERCAST_HTTP_UA blindly: if that knob # is set to the URL/parens form those sources document, it is rejected here and the # compliant default is used instead (an explicit SEC-safe OVERCAST_HTTP_UA is kept). # # Bind with: overcast source add 'edgar:320193' # by CIK (Apple) -> submissions API # overcast source add 'edgar:Tesla Inc' # by name/query -> full-text search # overcast scan --source edgar --limit 20 # overcast scan --source edgar --since 2024-01-01 # Refs / queries (enumerate --query — the ref after `edgar:`): # — a bare 1–10 digit CIK (optional leading `CIK`) → # the company's recent filings (submissions API) # "" — anything else → EDGAR full-text search (efts) # Each filing becomes one hit: payload.created = the FILING date, media.ref = the # sec.gov/Archives filing document (an HTML/txt page → fetch kind:"page"), plus # form / accession / cik / company in the payload. NO gps. # Implements: enumerate --query [--limit N] [--since S] | fetch --url --out

| init | describe set -uo pipefail # shared outbound-fetch guard (scheme pinning, bounded redirects, private-address # refusal on the FINAL hop) — see providers/engines/net/guarded-fetch.sh here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=../../engines/net/guarded-fetch.sh . "$here/../../engines/net/guarded-fetch.sh" SUB_API="https://data.sec.gov/submissions" FTS_API="https://efts.sec.gov/LATEST/search-index" # SEC REQUIRES a descriptive User-Agent carrying a CONTACT EMAIL (their fair-access # policy 403s blank/bot UAs — AND, empirically, UAs bearing a URL or parentheses; # only the plain `product/version email` shape passes). Take OVERCAST_HTTP_UA if # set, but fall back to a compliant default when the resolved UA looks SEC-unsafe # (contains a URL or parentheses) so the shared knob can't silently 403 EDGAR. UA="${OVERCAST_HTTP_UA:-overcast-osint/0.0.8 research@overcast.video}" case "$UA" in *"("*|*")"*|*"http"*) UA="overcast-osint/0.0.8 research@overcast.video" ;; esac op="${1:-enumerate}"; shift || true case "$op" in init) exit 0 ;; # keyless (public SEC EDGAR API) describe) echo '{"source":"edgar","emits":"scan.hit","needs":[]}'; exit 0 ;; enumerate) query=""; limit=25; since="" while [ "$#" -gt 0 ]; do case "$1" in --query) query="${2:-}"; shift 2 2>/dev/null || shift ;; --limit) limit="${2:-}"; shift 2 2>/dev/null || shift ;; --since) since="${2:-}"; shift 2 2>/dev/null || shift ;; *) shift ;; esac; done # trim surrounding whitespace so a padded ref still parses (a whitespace-only # query then trips the empty check) — consistent with the other sources. query="${query#"${query%%[![:space:]]*}"}"; query="${query%"${query##*[![:space:]]}"}" [ -n "$query" ] || { echo "edgar enumerate needs a CIK or query: bind edgar: or edgar:\"\"" >&2; exit 1; } case "$limit" in ''|*[!0-9]*) limit=25 ;; esac [ "$limit" -gt 200 ] 2>/dev/null && limit=200 [ "$limit" -lt 1 ] 2>/dev/null && limit=1 # --since -> a client-side floor on the FILING date (drop older filings), sort # newest-first, THEN --limit — the firms/overpass pattern. Portable epoch math; # fail CLOSED on an unparseable window. now="$(date -u +%s)"; cutdate="" if [ -n "$since" ]; then cutepoch="" case "$since" in *[0-9]s) cutepoch=$(( now - 10#${since%s} )) ;; *[0-9]m) cutepoch=$(( now - 10#${since%m} * 60 )) ;; *[0-9]h) cutepoch=$(( now - 10#${since%h} * 3600 )) ;; *[0-9]d) cutepoch=$(( now - 10#${since%d} * 86400 )) ;; *[0-9]w) cutepoch=$(( now - 10#${since%w} * 604800 )) ;; [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) cutepoch="$(date -u -d "$since" +%s 2>/dev/null || date -u -j -f '%Y-%m-%d %H:%M:%S' "$since 00:00:00" +%s 2>/dev/null || echo '')" ;; *) echo "edgar: could not parse --since '$since' (use Ns/Nm/Nh/Nd/Nw or YYYY-MM-DD)" >&2; exit 1 ;; esac [ -n "$cutepoch" ] || { echo "edgar: could not parse --since '$since'" >&2; exit 1; } # EDGAR filing dates are day-granular (YYYY-MM-DD), so compare on the DATE. cutdate="$(date -u -r "$cutepoch" +%Y-%m-%d 2>/dev/null || date -u -d "@$cutepoch" +%Y-%m-%d 2>/dev/null || echo '')" [ -n "$cutdate" ] || { echo "edgar: could not format --since '$since' into a date" >&2; exit 1; } fi # bare CIK (optional leading `CIK`) → submissions API; anything else → full-text. qup="$(printf '%s' "$query" | tr '[:lower:]' '[:upper:]')" cikcand="$query" case "$qup" in CIK*) cikcand="${query:3}" ;; esac if printf '%s' "$cikcand" | grep -qE '^[0-9]{1,10}$'; then # ---- submissions API: CIK zero-padded to 10 digits; Archives path uses the # NON-padded integer CIK ---- cikpad="$(printf '%010d' "$((10#$cikcand))")" cikint="$((10#$cikcand))" if ! resp="$(curl -fsS -m 45 -H "User-Agent: $UA" "$SUB_API/CIK$cikpad.json")"; then echo "edgar submissions request failed for CIK $cikpad (check the CIK)" >&2; exit 1 fi # A valid response is a JSON object whose filings.recent carries the parallel # accessionNumber array; a non-JSON body (rate-limit HTML) that slips past # curl -f is a HARD error, never a fake-empty [] (mirrors firms/overpass). if ! printf '%s' "$resp" | jq -e 'type == "object" and (.filings.recent.accessionNumber | type == "array")' >/dev/null 2>&1; then echo "edgar submissions: unexpected response for CIK $cikpad: $(printf '%s' "$resp" | head -c 200)" >&2; exit 1 fi printf '%s' "$resp" | jq -c --arg cik "$cikint" --argjson n "$limit" --arg cut "$cutdate" ' (.name // "") as $company | (.filings.recent) as $r | [ range(0; ($r.accessionNumber | length)) as $i | { form: ($r.form[$i] // ""), date: ($r.filingDate[$i] // null), acc: ($r.accessionNumber[$i] // ""), doc: ($r.primaryDocument[$i] // ""), report: ($r.reportDate[$i] // null) } | (.acc | gsub("-"; "")) as $accnd | ("https://www.sec.gov/Archives/edgar/data/" + $cik + "/" + $accnd + (if (.doc // "") != "" then "/" + .doc else "/" end)) as $url | { title: ((.form // "filing") + " filed " + (.date // "?") + (if $company != "" then " — " + $company else "" end)), url: $url, source: "edgar", # graph/brief rank + --since-filter by payload.created = the FILING # date, not scan ingest — so an old filing scanned today sorts old. created: .date, published: .date, snippet: ("form " + (.form // "?") + " · accession " + (.acc // "?") + (if (.report // "") != "" then " · report period " + .report else "" end)), form: .form, accession: .acc, cik: $cik, company: (if $company != "" then $company else null end), report_date: .report, media: { ref: $url } } ] | map(select($cut == "" or (.created != null and .created >= $cut))) | sort_by(.created // "") | reverse | .[0:$n]' else # ---- full-text search (efts): hits.hits[]._source carries adsh / ciks / # file_type / file_date / display_names; _id = ":". # EFTS is Elasticsearch-backed and returns only ~10 hits/page, so a single # request silently caps well under --limit. Paginate with &from=, # accumulating hits until we have --limit (or a short page ends results), # bounded to ≤20 pages. Then hand the merged hits to the mapper below. # NOTE: efts orders by RELEVANCE and exposes no server-side date sort, so # --limit caps the most-RELEVANT filings (the mapper date-sorts that set for # display); for exhaustive newest-first chronology use edgar:, and pass # --since to bound the window server-side (startdt below). ---- qenc="$(jq -rn --arg v "$query" '$v|@uri')" # Push --since to the SERVER (efts startdt) so pagination retrieves in-window # filings: efts orders by relevance, so stopping at --limit RAW hits and # filtering client-side could miss in-window filings ranked lower / on later # pages. With startdt every returned hit is already in-window. dateparam="" [ -n "$cutdate" ] && dateparam="&dateRange=custom&startdt=$cutdate&enddt=$(date -u +%Y-%m-%d)" allhits="[]"; from=0; ftpages=0 while : ; do if ! resp="$(curl -fsS -m 20 -H "User-Agent: $UA" "$FTS_API?q=$qenc&from=$from$dateparam")"; then echo "edgar full-text request failed for '$query' (from=$from)" >&2; exit 1 fi # A valid response is a JSON object with a hits.hits array (zero matches → # []); a non-JSON body is a HARD error, never a fake-empty scan. if ! printf '%s' "$resp" | jq -e 'type == "object" and (.hits.hits | type == "array")' >/dev/null 2>&1; then echo "edgar full-text: unexpected response for '$query': $(printf '%s' "$resp" | head -c 200)" >&2; exit 1 fi phits="$(printf '%s' "$resp" | jq -c '.hits.hits')" pcount="$(printf '%s' "$phits" | jq 'length')" allhits="$(printf '%s\n%s' "$allhits" "$phits" | jq -sc 'add')" acount="$(printf '%s' "$allhits" | jq 'length')" ftpages=$((ftpages + 1)) [ "$acount" -ge "$limit" ] && break # have enough to satisfy --limit [ "$pcount" -lt 10 ] && break # short page → end of results [ "$ftpages" -ge 20 ] && break # safety bound from=$((from + 10)) done # wrap the merged hits back into the {hits:{hits:…}} shape the mapper reads resp="$(printf '%s' "$allhits" | jq -c '{hits:{hits:.}}')" printf '%s' "$resp" | jq -c --argjson n "$limit" --arg cut "$cutdate" ' [ (.hits.hits // [])[] | (._source // {}) as $s | (._id // "") as $id | ($s.adsh // ($id | split(":")[0]) // "") as $acc | ($acc | gsub("-"; "")) as $accnd | (($id | split(":")[1]) // "") as $doc | (($s.ciks // [])[0] // "") as $cikraw | (if $cikraw != "" then ($cikraw | sub("^0+"; "")) else "" end) as $cik | ($s.file_type // ($s.root_forms[0] // "")) as $form | ($s.file_date // null) as $date | (($s.display_names // [])[0] // "") as $company | (if ($cik != "" and $accnd != "") then ("https://www.sec.gov/Archives/edgar/data/" + $cik + "/" + $accnd + (if $doc != "" then "/" + $doc else "/" end)) # No CIK/accession to build an Archives link — fall back to a UNIQUE # EDGAR full-text deep link keyed on the accession (or the hit _id), # so distinct incomplete filings do not collapse to one monitor dedup # key (a single static browse-edgar URL would). else ("https://efts.sec.gov/LATEST/search-index?q=%22" + ((if $acc != "" then $acc else $id end) | @uri) + "%22") end) as $url | { title: (($form | if . == "" then "filing" else . end) + " filed " + ($date // "?") + (if $company != "" then " — " + $company else "" end)), url: $url, source: "edgar", created: $date, published: $date, snippet: ("form " + ($form | if . == "" then "?" else . end) + " · accession " + ($acc | if . == "" then "?" else . end) + (if $company != "" then " · " + $company else "" end)), form: (if $form == "" then null else $form end), accession: (if $acc == "" then null else $acc end), cik: (if $cik == "" then null else $cik end), company: (if $company == "" then null else $company end), media: { ref: $url } } ] | map(select($cut == "" or (.created != null and .created >= $cut))) | sort_by(.created // "") | reverse | .[0:$n]' fi ;; fetch) url=""; out="" while [ "$#" -gt 0 ]; do case "$1" in --url) url="${2:-}"; shift 2 2>/dev/null || shift ;; --out) out="${2:-}"; shift 2 2>/dev/null || shift ;; *) shift ;; esac; done [ -n "$url" ] || { echo "edgar fetch needs --url" >&2; exit 1; } # a hit's ref is a filing document (HTML/txt) — curl it as evidence and report # the kind by content type (default page). SEC needs the descriptive UA here too. if ! ct="$(oc_guarded_fetch "$url" "$out" -m 60 -H "User-Agent: $UA")" || [ ! -s "$out" ]; then echo "edgar fetch failed for $url" >&2; rm -f "$out"; exit 1 fi case "$ct" in image/*) kind="image" ;; application/pdf*) kind="file" ;; *) kind="page" ;; # HTML / txt filing documents esac jq -nc --arg p "$out" --arg k "$kind" --arg u "$url" '{kind:$k,path:$p,source:"edgar",url:$u}' ;; *) echo "edgar source: unknown op (expected enumerate|fetch|init|describe)" >&2; exit 2 ;; esac