19 lines
326 B
Plaintext
19 lines
326 B
Plaintext
|
#!/usr/bin/env bash
|
||
|
|
|
||
|
|
set -e
|
||
|
|
|
||
|
|
sources=$(cat ./data/sources.txt)
|
||
|
|
|
||
|
|
for source in $sources; do
|
||
|
|
curl --silent $source >> tmp
|
||
|
|
done;
|
||
|
|
|
||
|
|
sort tmp \
|
||
|
|
| uniq \
|
||
|
|
| sed '/./,$!d' \
|
||
|
|
| awk '{print tolower($0)}' \
|
||
|
|
| comm -23 - ./data/blacklist.txt \
|
||
|
|
| comm -23 - ./data/disposable.txt > ./data/free.txt
|
||
|
|
|
||
|
|
echo 'Complete!'
|