#!/usr/bin/env bash

set -e
export LC_CTYPE=C

tmp=$(mktemp -t freemailXXX)
cat ./data/blacklist.txt \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq > $tmp
mv $tmp ./data/blacklist.txt

new=$(mktemp -t freemailXXX)

curl --silent https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json | jq -r .[] > $new

tmp=$(mktemp -t freemailXXX)
cat $new ./data/disposable.txt \
    | sed '/^$/d' \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/blacklist.txt > $tmp
mv $tmp ./data/disposable.txt

sources=$(cat ./data/sources.txt)
new=$(mktemp -t freemailXXX)
for source in $sources; do
    echo "$(curl --silent $source)" >> $new
done;

tmp=$(mktemp -t freemailXXX)
cat $new ./data/free.txt \
    | sed '/^$/d' \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/blacklist.txt \
    | comm -23 - ./data/disposable.txt > $tmp
mv $tmp ./data/free.txt

echo 'Complete!'