Initial commit

This commit is contained in:
Will White
2015-02-06 22:15:25 -08:00
commit dda0e499ac
12 changed files with 5459 additions and 0 deletions

18
update Executable file
View File

@@ -0,0 +1,18 @@
#!/usr/bin/env bash
set -e
sources=$(cat ./data/sources.txt)
for source in $sources; do
curl --silent $source >> tmp
done;
sort tmp \
| uniq \
| sed '/./,$!d' \
| awk '{print tolower($0)}' \
| comm -23 - ./data/blacklist.txt \
| comm -23 - ./data/disposable.txt > ./data/free.txt
echo 'Complete!'