freemail/update

#!/usr/bin/env bash

set -e

tmp=$(mktemp -t freemailXXX)
cat ./data/blacklist.txt \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq > $tmp
mv $tmp ./data/blacklist.txt

tmp=$(mktemp -t freemailXXX)
cat ./data/free.txt \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/blacklist.txt > $tmp
mv $tmp ./data/free.txt

tmp=$(mktemp -t freemailXXX)
cat ./data/disposable.txt \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/blacklist.txt \
    | comm -23 - ./data/free.txt > $tmp
mv $tmp ./data/disposable.txt

sources=$(cat ./data/sources.txt)
new=$(mktemp -t freemailXXX)
for source in $sources; do
    echo "$(curl --silent $source)" >> $new
done;

curl --silent https://raw.githubusercontent.com/ivolo/disposable-email-domains/master/index.json | jq -r .[] >> $new

tmp=$(mktemp -t freemailXXX)
cat $new \
    | sed '/./,$!d' \
    | sed -e 's/^ *//' -e 's/ *$//' \
    | awk '{print tolower($0)}' \
    | sort \
    | uniq \
    | comm -23 - ./data/blacklist.txt \
    | comm -23 - ./data/free.txt \
    | comm -23 - ./data/disposable.txt > $tmp

confirmed=$(mktemp -t freemailXXX)
for domain in $(cat $tmp); do
    result=`dig +short mx $domain`
    if [ -n "$result" ]; then
        echo $domain >> $confirmed
    fi
done

tmp=$(mktemp -t freemailXXX)
cat $confirmed ./data/free.txt \
    | sort \
    | uniq > $tmp
mv $tmp ./data/free.txt

echo 'Complete!'