#!/usr/bin/env python3 import sys import io import re import argparse import html import json from pathlib import Path from datetime import datetime # Force UTF-8 output to handle non-ASCII paths (e.g. Cyrillic usernames) sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace') sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace') HEADER_RE = re.compile(r"^=== Results for (.+?) ===\s*$") SECRET_LINE_RE = re.compile(r"^\[secret(/low)?\]") def parse_results_streaming(filepath): """ Generator that yields blocks one at a time by reading the file line-by-line. Each block is: {"url": "...", "secret_lines": [...], "has_secrets": bool} """ current_url = None current_lines = [] def flush(): nonlocal current_url, current_lines if current_url is None: return None secret_lines = [l.strip() for l in current_lines if SECRET_LINE_RE.match(l.strip())] block = { "url": current_url.strip(), "secret_lines": secret_lines, "has_secrets": bool(secret_lines), } current_url = None current_lines = [] return block with open(filepath, "r", encoding="utf-8", errors="replace") as fh: for line in fh: line = line.rstrip("\n\r") m = HEADER_RE.match(line) if m: block = flush() if block is not None: yield block current_url = m.group(1) current_lines = [] else: if current_url is not None: current_lines.append(line) block = flush() if block is not None: yield block def write_html_streaming(blocks_iter, out_fh, input_name="input"): """ Two-pass approach for large files: Pass 1: iterate blocks, collect them in a list (only metadata, not raw text). Pass 2: stream JSON array into the HTML file piece by piece. This avoids holding the full JSON string in memory — we write each element individually instead of json.dumps(entire_list). """ # Collect parsed blocks — this is the essential data we need and is much # smaller than the raw input file (we already discarded non-secret lines). blocks = list(blocks_iter) total_blocks = len(blocks) blocks_with_secrets = sum(1 for b in blocks if b["has_secrets"]) total_secret_lines = sum(len(b["secret_lines"]) for b in blocks if b["has_secrets"]) now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") # We'll write the HTML in parts, streaming the JSON array element-by-element # so we never build the full JSON string in memory. w = out_fh.write # --- Part 1: HTML head + CSS + body header (with dynamic stats) --- w(f""" Secrets Report

Secrets Report

Source: {html.escape(input_name)}  •  Generated: {now}
Total result blocks found
{total_blocks}
Blocks with secrets
{blocks_with_secrets}
Total secret lines
{total_secret_lines}
Ready
' will break the HTML parser out of the script tag. for idx, block in enumerate(blocks): if idx > 0: w(",") chunk = json.dumps(block, ensure_ascii=False) w(chunk.replace(" { const url = card.dataset.url; const box = card.querySelector('.visited-box'); if (!box) return; box.checked = visitedSet.has(url); }); } // ---------------- Helpers ---------------- function escapeHtml(str) { return str.replace(/[&<>"']/g, s => { switch (s) { case '&': return '&'; case '<': return '<'; case '>': return '>'; case '"': return '"'; case "'": return '''; default: return s; } }); } function parseSecretLine(line, sourceUrl) { // Strip trailing [https://...] or [http://...] bracketed URLs const cleaned = line.trim().replace(/\s*\[https?:\/\/[^\]]*\]\s*$/, ''); const escaped = escapeHtml(cleaned); const urlEsc = escapeHtml(sourceUrl); return `
  • ${escaped} source
    • `; } function buildCardHTML(item, index) { const urlEsc = escapeHtml(item.url); const badge = `#${index + 1}`; const count = item.secret_lines.length; let bodyHtml = ''; if (item.has_secrets) { const secrets = item.secret_lines.map(l => parseSecretLine(l, item.url)).join(''); bodyHtml = ``; } else { bodyHtml = `
    No secrets detected.
    `; } const openAttr = item.has_secrets ? 'open' : ''; return `
    ${badge} ${urlEsc} ${count} secret(s)
    ${bodyHtml}
    `; } // Smart filter: regex-based false-positive patterns const FALSE_POSITIVE_PATTERNS = [ // Cloudflare challenge tokens /\/cdn-cgi\/challenge-platform\//, /\d+\.\d+:\d+:[a-zA-Z0-9_-]+ivzz/, // i18n/localization keys /["']secret:[a-z]+_[a-z]+_[a-z]+["']/, // Code patterns /[a-zA-Z]+Key\s*=\s*[a-z]\.define\(/, /[a-zA-Z]+Key\s*=\s*function/, /[a-zA-Z]+Secret\s*=\s*function/, /PrivateKey\s*=\s*[a-z]\.define\(/, /PublicKey\s*=\s*[a-z]\.define\(/, // Common test/placeholder values /["']?password["']?\s*[:=]\s*["']?(test|demo|example|placeholder|changeme|password|123456)["']?/i, /["']?secret["']?\s*[:=]\s*["']?(test|demo|example|placeholder)["']?/i, // SDK/Library code /tokenize\s*:\s*function/, /getToken\s*:\s*[a-z]$/, /setPassword\s*:\s*function/, ]; // JS noise values — if the extracted value is *only* one of these (or starts // with them followed by typical JS syntax), the finding is worthless. const JS_NOISE_VALUES = new Set([ 'function', 'function(', 'function (', 'async function', 'undefined', 'null', 'true', 'false', 'NaN', 'Infinity', 'this', 'window', 'document', 'self', 'globalThis', 'void 0', 'void(0)', '{}', '[]', 'new ', 'class ', 'typeof ', 'instanceof ', 'return ', 'throw ', 'delete ', 'Object', 'Array', 'String', 'Number', 'Boolean', 'Promise', 'Symbol', 'Map', 'Set', 'WeakMap', 'WeakSet', 'console', 'module', 'exports', 'require', 'import', 'prototype', '__proto__', ]); // Email detection regex const EMAIL_RE = /[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}/; // Regex to strip the [secret] or [secret/low] prefix and grab key = value const SECRET_KV_RE = /^\[secret(?:\/low)?\]\s*(.+)/i; function extractKeyValue(line) { const m = line.match(SECRET_KV_RE); if (!m) return null; // Strip trailing [https://...] or [http://...] bracketed URL first let rest = m[1].trim().replace(/\s*\[https?:\/\/[^\]]*\]\s*$/, ''); // Strip outer quotes from the whole payload: 'key:value' -> key:value rest = rest.replace(/^["']+|["']+$/g, ''); // Try key=value, key:value, key: value, key =value const sepIdx = rest.search(/\s*[:=]\s*/); if (sepIdx === -1) return { key: rest, value: '' }; const key = rest.slice(0, sepIdx).trim(); const value = rest.slice(sepIdx).replace(/^[\s:=]+/, '').trim(); return { key, value }; } function isFalsePositive(line) { // 1. Regex-based patterns for (const pat of FALSE_POSITIVE_PATTERNS) { if (pat.test(line)) return true; } // 2. Key/value based checks const kv = extractKeyValue(line); if (kv) { // Remove quotes around key for length check const bareKey = kv.key.replace(/^["']+|["']+$/g, ''); // Filter out keys shorter than 3 characters if (bareKey.length < 3) return true; // Remove surrounding quotes from value const bareVal = kv.value.replace(/^["']+|["']+$/g, '').trim(); // Filter out values shorter than 3 characters (single letters, flags, etc.) if (bareVal.length < 3) return true; // Filter out values that are JS noise const valLower = bareVal.toLowerCase(); for (const noise of JS_NOISE_VALUES) { if (valLower === noise.toLowerCase()) return true; // Also catch values that start with JS keywords like "function(" or "function foo()" if (valLower.startsWith(noise.toLowerCase() + '(') || valLower.startsWith(noise.toLowerCase() + ' ')) return true; } // Catch anonymous/arrow functions: (a,b)=>{...}, ()=>{}, (a)=>... if (/^\(.*\)\s*=>/.test(bareVal) || /^\(.*\)\s*\{/.test(bareVal)) return true; // Catch: function keyword anywhere at the start if (/^(async\s+)?function\b/.test(bareVal)) return true; } return false; } function getFilteredData() { const q = searchEl.value.toLowerCase().trim(); const exRaw = excludeEl.value.toLowerCase().trim(); const exTerms = exRaw ? exRaw.split(',').map(t => t.trim()).filter(Boolean) : []; const onlySecrets = onlySecretsEl.checked; const hideVisited = hideVisitedEl.checked; const smartFilter = smartFilterEl.checked; const emailsOnly = emailsOnlyEl.checked; const results = []; for (const item of DATA) { const urlLower = item.url.toLowerCase(); if (onlySecrets && !item.has_secrets) continue; if (hideVisited && visitedSet.has(item.url)) continue; if (q) { const matchesSearch = urlLower.includes(q) || (item.has_secrets && item.secret_lines.some(s => s.toLowerCase().includes(q))); if (!matchesSearch) continue; } // Apply filters at the finding level let lines = item.secret_lines; if (exTerms.length > 0 && item.has_secrets) { lines = lines.filter(s => { const lower = s.toLowerCase(); return !exTerms.some(term => lower.includes(term)); }); } if (smartFilter && item.has_secrets) { lines = lines.filter(s => !isFalsePositive(s)); } if (emailsOnly) { lines = lines.filter(s => EMAIL_RE.test(s)); if (lines.length === 0) continue; } if ((exTerms.length > 0 || smartFilter || emailsOnly) && item.has_secrets) { if (lines.length === 0) continue; results.push({ url: item.url, secret_lines: lines, has_secrets: true, }); } else { results.push(item); } } return results; } function clearCards() { const nodes = Array.from(cardsContainer.querySelectorAll('details.card')); nodes.forEach(n => n.remove()); } function setStatus(msg) { statusText.textContent = msg; } let scheduleRender = null; function wireVisitedForRenderedCards() { cardsContainer.querySelectorAll('details.card').forEach(card => { if (card.dataset.visitedWired) return; card.dataset.visitedWired = "1"; const url = card.dataset.url; const box = card.querySelector('.visited-box'); const link = card.querySelector('a.url'); if (box) { box.addEventListener('change', () => { if (box.checked) visitedSet.add(url); else visitedSet.delete(url); saveVisited(visitedSet); if (hideVisitedEl.checked && scheduleRender) { scheduleRender(); } }); } if (link) { link.addEventListener('click', () => { visitedSet.add(url); saveVisited(visitedSet); if (box) box.checked = true; if (hideVisitedEl.checked && scheduleRender) { scheduleRender(); } }); } }); } function renderIncrementally(items) { const myToken = ++renderToken; clearCards(); if (items.length === 0) { emptyBox.style.display = ''; setStatus('No matches'); return; } emptyBox.style.display = 'none'; const chunkSize = 200; let i = 0; setStatus(`Rendering 0 / ${items.length}`); function step() { if (myToken !== renderToken) return; const end = Math.min(i + chunkSize, items.length); let htmlChunk = ''; for (; i < end; i++) { htmlChunk += buildCardHTML(items[i], i); } cardsContainer.insertAdjacentHTML('beforeend', htmlChunk); applyVisitedUI(cardsContainer); wireVisitedForRenderedCards(); setStatus(`Rendering ${end} / ${items.length}`); if (i < items.length) { requestAnimationFrame(step); } else { const totalEntries = DATA.filter(d => d.has_secrets).length; const suffix = items.length < totalEntries ? ` (${totalEntries} total, ${totalEntries - items.length} filtered out)` : ''; setStatus(`Done \u2022 Showing ${items.length} entries${suffix}`); } } requestAnimationFrame(step); } // ---------------- Debounce search/render ---------------- let searchTimer = null; scheduleRender = function() { if (searchTimer) clearTimeout(searchTimer); searchTimer = setTimeout(() => { renderIncrementally(getFilteredData()); }, 120); }; searchEl.addEventListener('input', scheduleRender); excludeEl.addEventListener('input', scheduleRender); onlySecretsEl.addEventListener('change', scheduleRender); hideVisitedEl.addEventListener('change', scheduleRender); smartFilterEl.addEventListener('change', scheduleRender); emailsOnlyEl.addEventListener('change', scheduleRender); expandAllBtn.addEventListener('click', () => { document.querySelectorAll('details.card').forEach(d => d.open = true); }); collapseAllBtn.addEventListener('click', () => { document.querySelectorAll('details.card').forEach(d => d.open = false); }); clearVisitedBtn.addEventListener('click', () => { visitedSet = new Set(); saveVisited(visitedSet); applyVisitedUI(cardsContainer); if (hideVisitedEl.checked) { scheduleRender(); } else { setStatus("Visited cleared"); } }); // Initial render renderIncrementally(getFilteredData()); """) return total_blocks, blocks_with_secrets def main(): ap = argparse.ArgumentParser( description="Re-organize secret scanner results into a neat HTML report (fast, visited tracking + hide visited filter)." ) ap.add_argument("-i", "--input", required=True, help="Input text file with results.") ap.add_argument("-o", "--output", required=True, help="Output HTML file.") ap.add_argument( "--max-blocks", type=int, default=2000, help="Max blocks per HTML file before splitting into chunks. Default: 2000.", ) args = ap.parse_args() in_path = Path(args.input) out_path = Path(args.output) max_blocks = args.max_blocks # Collect all blocks first to decide if we need chunking all_blocks = list(parse_results_streaming(in_path)) total_blocks = len(all_blocks) if total_blocks <= max_blocks: # Single file — no chunking needed with open(out_path, "w", encoding="utf-8") as out_fh: _, blocks_with_secrets = write_html_streaming( iter(all_blocks), out_fh, input_name=in_path.name ) print(f"[+] Parsed blocks: {total_blocks}") print(f"[+] Blocks with secrets: {blocks_with_secrets}") print(f"[+] Wrote HTML: {out_path}") else: # Split into multiple chunk files stem = out_path.stem suffix = out_path.suffix or ".html" parent = out_path.parent chunk_num = 0 written_files = [] for i in range(0, total_blocks, max_blocks): chunk_num += 1 chunk_blocks = all_blocks[i : i + max_blocks] chunk_path = parent / f"{stem}_part{chunk_num}{suffix}" chunk_label = f"{in_path.name} (part {chunk_num}, blocks {i+1}-{i+len(chunk_blocks)})" with open(chunk_path, "w", encoding="utf-8") as out_fh: write_html_streaming(iter(chunk_blocks), out_fh, input_name=chunk_label) written_files.append(chunk_path) print(f"[+] Wrote chunk: {chunk_path} ({len(chunk_blocks)} blocks)") total_secrets = sum(1 for b in all_blocks if b["has_secrets"]) print(f"[+] Parsed blocks: {total_blocks}") print(f"[+] Blocks with secrets: {total_secrets}") print(f"[+] Split into {chunk_num} HTML files") if __name__ == "__main__": main()