add support for iblocklist.com

blacklist.sh

@@ -12,9 +12,13 @@ urls="http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
 
 # Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders
 urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt"
+
 # badips.com, from score 2 up
 urls="$urls http://www.badips.com/get/list/ssh/2"
 
+# iblocklist.com is also supported
+# urls="$urls http://list.iblocklist.com/?list=srzondksmjuwsvmgdbhi&fileformat=p2p&archiveformat=gz&username=USERNAMEx$&pin=PIN"
+
 # This is how it will look like on the server
 
 # Chain blocklists (2 references)
@@ -91,18 +95,37 @@ for url in $urls; do
     set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname
     curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}"
 
-    # this is required for blocklist.de that sends compressed content if asked for it or not
+    # this is required for blocklist.de that sends compressed content regardless of asked or not
     if [ -z "$COMPRESS_OPT" ]; then
         if grep -qi 'content-encoding: gzip' "${headers}"; then
             mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
             gzip -d "${unsorted_blocklist}.gz"
         fi
     fi
+    # autodetect iblocklist.com format as it needs additional conversion
+    if echo "${url}" | grep -q 'iblocklist.com'; then
+        if [ -f /etc/range2cidr.awk ]; then
+            mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
+            gzip -d "${unsorted_blocklist}.gz"
+            awk_tmp=$(mktemp)
+            awk -f /etc/range2cidr.awk <"${unsorted_blocklist}" >"${awk_tmp}"
+            mv "${awk_tmp}" "${unsorted_blocklist}"
+        else
+            echo "range2cidr.awk script not found, cannot process ${unsorted_blocklist}, skipping"
+            continue
+        fi
+    fi
 
     sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}"
 
     # calculate performance parameters for the new set
+    if "${RANDOM}"; then
+        # bash
         tmp_set_name="tmp_${RANDOM}"
+    else
+        # non-bash
+        tmp_set_name="tmp_$$"
+    fi
     new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' )
     hash_size=$(expr $new_list_size / 2)
 

range2cidr.awk

@@ -0,0 +1,71 @@
+# AWK script to convert iblocklist.com ranges into CIDR format
+# usable with ipset
+
+# based on scripts posted at 
+# http://www.unix.com/shell-programming-and-scripting/233825-convert-ip-ranges-cidr-netblocks.html
+
+function bit_or(a, b, r, i, c) {
+    for (r=i=0;i<32;i++) {
+        c = 2 ^ i
+        if ((int(a/c) % 2) || (int(b/c) % 2)) r += c
+    }
+    return r
+}
+function bit_lshift(var, x) {
+  while(x--) var*=2;
+  return var;
+}
+function bit_rshift(var, x) {
+  while(x--) var=int(var/2);
+  return var;
+}
+function range2cidr(ipStart, ipEnd,  bits, mask, newip) {
+    bits = 1
+    mask = 1
+    while (bits < 32) {
+        newip = bit_or(ipStart, mask)
+        if ((newip>ipEnd) || ((bit_lshift(bit_rshift(ipStart,bits),bits)) != ipStart)) {
+           bits--
+           mask = bit_rshift(mask,1)
+           break
+        }
+        bits++
+        mask = bit_lshift(mask,1)+1
+    }
+    newip = bit_or(ipStart, mask)
+    bits = 32 - bits
+    result = dec2ip(ipStart) "/" bits
+    if (newip < ipEnd) result = result "\n" range2cidr(newip + 1, ipEnd)
+    return result
+}
+function ip2dec(ip,   slice) {
+    split(ip, slice, ".")
+    return (slice[1] * 2^24) + (slice[2] * 2^16) + (slice[3] * 2^8) + slice[4]
+}
+function dec2ip(dec,    ip, quad) {
+	for (i=3; i>=1; i--) {
+		quad = 256^i
+		ip = ip int(dec/quad) "."
+		dec = dec%quad
+	}
+	return ip dec
+}
+
+# example iblocklist.com format
+# TOT Public Company/Irdeto:1.0.128.0-1.0.255.255
+BEGIN { FS = ":"; }
+
+$2 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {
+    n = split($2, array, "-");
+    if (n == 2) {
+        ip1 = array[1];
+        ip2 = array[2];
+        if (ip1 == ip2) {
+            # some records are just single IPs listed as range
+            print ip1;
+        } else {
+            # and some are really ranges
+            print range2cidr(ip2dec(ip1), ip2dec(ip2));
+        }
+    }
+}