diff --git a/blacklist.sh b/blacklist.sh index 4e3c705..c4e8b3b 100755 --- a/blacklist.sh +++ b/blacklist.sh @@ -12,9 +12,13 @@ urls="http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt" # Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt" + # badips.com, from score 2 up urls="$urls http://www.badips.com/get/list/ssh/2" +# iblocklist.com is also supported +# urls="$urls http://list.iblocklist.com/?list=srzondksmjuwsvmgdbhi&fileformat=p2p&archiveformat=gz&username=USERNAMEx$&pin=PIN" + # This is how it will look like on the server # Chain blocklists (2 references) @@ -91,18 +95,37 @@ for url in $urls; do set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}" - # this is required for blocklist.de that sends compressed content if asked for it or not + # this is required for blocklist.de that sends compressed content regardless of asked or not if [ -z "$COMPRESS_OPT" ]; then if grep -qi 'content-encoding: gzip' "${headers}"; then mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz" gzip -d "${unsorted_blocklist}.gz" fi fi + # autodetect iblocklist.com format as it needs additional conversion + if echo "${url}" | grep -q 'iblocklist.com'; then + if [ -f /etc/range2cidr.awk ]; then + mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz" + gzip -d "${unsorted_blocklist}.gz" + awk_tmp=$(mktemp) + awk -f /etc/range2cidr.awk <"${unsorted_blocklist}" >"${awk_tmp}" + mv "${awk_tmp}" "${unsorted_blocklist}" + else + echo "range2cidr.awk script not found, cannot process ${unsorted_blocklist}, skipping" + continue + fi + fi sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}" # calculate performance parameters for the new set - tmp_set_name="tmp_${RANDOM}" + if "${RANDOM}"; then + # bash + tmp_set_name="tmp_${RANDOM}" + else + # non-bash + tmp_set_name="tmp_$$" + fi new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' ) hash_size=$(expr $new_list_size / 2) diff --git a/range2cidr.awk b/range2cidr.awk new file mode 100644 index 0000000..d88ec76 --- /dev/null +++ b/range2cidr.awk @@ -0,0 +1,71 @@ +# AWK script to convert iblocklist.com ranges into CIDR format +# usable with ipset + +# based on scripts posted at +# http://www.unix.com/shell-programming-and-scripting/233825-convert-ip-ranges-cidr-netblocks.html + +function bit_or(a, b, r, i, c) { + for (r=i=0;i<32;i++) { + c = 2 ^ i + if ((int(a/c) % 2) || (int(b/c) % 2)) r += c + } + return r +} +function bit_lshift(var, x) { + while(x--) var*=2; + return var; +} +function bit_rshift(var, x) { + while(x--) var=int(var/2); + return var; +} +function range2cidr(ipStart, ipEnd, bits, mask, newip) { + bits = 1 + mask = 1 + while (bits < 32) { + newip = bit_or(ipStart, mask) + if ((newip>ipEnd) || ((bit_lshift(bit_rshift(ipStart,bits),bits)) != ipStart)) { + bits-- + mask = bit_rshift(mask,1) + break + } + bits++ + mask = bit_lshift(mask,1)+1 + } + newip = bit_or(ipStart, mask) + bits = 32 - bits + result = dec2ip(ipStart) "/" bits + if (newip < ipEnd) result = result "\n" range2cidr(newip + 1, ipEnd) + return result +} +function ip2dec(ip, slice) { + split(ip, slice, ".") + return (slice[1] * 2^24) + (slice[2] * 2^16) + (slice[3] * 2^8) + slice[4] +} +function dec2ip(dec, ip, quad) { + for (i=3; i>=1; i--) { + quad = 256^i + ip = ip int(dec/quad) "." + dec = dec%quad + } + return ip dec +} + +# example iblocklist.com format +# TOT Public Company/Irdeto:1.0.128.0-1.0.255.255 +BEGIN { FS = ":"; } + +$2 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ { + n = split($2, array, "-"); + if (n == 2) { + ip1 = array[1]; + ip2 = array[2]; + if (ip1 == ip2) { + # some records are just single IPs listed as range + print ip1; + } else { + # and some are really ranges + print range2cidr(ip2dec(ip1), ip2dec(ip2)); + } + } +}