21 lines
390 B
Bash
Executable File
21 lines
390 B
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
set -e
|
|
|
|
sources=$(cat ./data/sources.txt)
|
|
|
|
for source in $sources; do
|
|
curl --silent $source >> tmp
|
|
done;
|
|
|
|
cat data/free.txt tmp \
|
|
| sort \
|
|
| uniq \
|
|
| sed '/./,$!d' \
|
|
| sed -e 's/^ *//' -e 's/ *$//' \
|
|
| awk '{print tolower($0)}' \
|
|
| comm -23 - ./data/blacklist.txt \
|
|
| comm -23 - ./data/disposable.txt > ./data/free.txt
|
|
|
|
echo 'Complete!'
|