add support for iblocklist.com

This commit is contained in:
Pawel Krawczyk 2015-07-09 15:43:22 +01:00
parent f31e9e35eb
commit 048ecb0e3b
2 changed files with 96 additions and 2 deletions

View File

@ -12,9 +12,13 @@ urls="http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
# Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders # Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders
urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt" urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt"
# badips.com, from score 2 up # badips.com, from score 2 up
urls="$urls http://www.badips.com/get/list/ssh/2" urls="$urls http://www.badips.com/get/list/ssh/2"
# iblocklist.com is also supported
# urls="$urls http://list.iblocklist.com/?list=srzondksmjuwsvmgdbhi&fileformat=p2p&archiveformat=gz&username=USERNAMEx$&pin=PIN"
# This is how it will look like on the server # This is how it will look like on the server
# Chain blocklists (2 references) # Chain blocklists (2 references)
@ -91,18 +95,37 @@ for url in $urls; do
set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname
curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}" curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}"
# this is required for blocklist.de that sends compressed content if asked for it or not # this is required for blocklist.de that sends compressed content regardless of asked or not
if [ -z "$COMPRESS_OPT" ]; then if [ -z "$COMPRESS_OPT" ]; then
if grep -qi 'content-encoding: gzip' "${headers}"; then if grep -qi 'content-encoding: gzip' "${headers}"; then
mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz" mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
gzip -d "${unsorted_blocklist}.gz" gzip -d "${unsorted_blocklist}.gz"
fi fi
fi fi
# autodetect iblocklist.com format as it needs additional conversion
if echo "${url}" | grep -q 'iblocklist.com'; then
if [ -f /etc/range2cidr.awk ]; then
mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
gzip -d "${unsorted_blocklist}.gz"
awk_tmp=$(mktemp)
awk -f /etc/range2cidr.awk <"${unsorted_blocklist}" >"${awk_tmp}"
mv "${awk_tmp}" "${unsorted_blocklist}"
else
echo "range2cidr.awk script not found, cannot process ${unsorted_blocklist}, skipping"
continue
fi
fi
sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}" sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}"
# calculate performance parameters for the new set # calculate performance parameters for the new set
tmp_set_name="tmp_${RANDOM}" if "${RANDOM}"; then
# bash
tmp_set_name="tmp_${RANDOM}"
else
# non-bash
tmp_set_name="tmp_$$"
fi
new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' ) new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' )
hash_size=$(expr $new_list_size / 2) hash_size=$(expr $new_list_size / 2)

71
range2cidr.awk Normal file
View File

@ -0,0 +1,71 @@
# AWK script to convert iblocklist.com ranges into CIDR format
# usable with ipset
# based on scripts posted at
# http://www.unix.com/shell-programming-and-scripting/233825-convert-ip-ranges-cidr-netblocks.html
function bit_or(a, b, r, i, c) {
for (r=i=0;i<32;i++) {
c = 2 ^ i
if ((int(a/c) % 2) || (int(b/c) % 2)) r += c
}
return r
}
function bit_lshift(var, x) {
while(x--) var*=2;
return var;
}
function bit_rshift(var, x) {
while(x--) var=int(var/2);
return var;
}
function range2cidr(ipStart, ipEnd, bits, mask, newip) {
bits = 1
mask = 1
while (bits < 32) {
newip = bit_or(ipStart, mask)
if ((newip>ipEnd) || ((bit_lshift(bit_rshift(ipStart,bits),bits)) != ipStart)) {
bits--
mask = bit_rshift(mask,1)
break
}
bits++
mask = bit_lshift(mask,1)+1
}
newip = bit_or(ipStart, mask)
bits = 32 - bits
result = dec2ip(ipStart) "/" bits
if (newip < ipEnd) result = result "\n" range2cidr(newip + 1, ipEnd)
return result
}
function ip2dec(ip, slice) {
split(ip, slice, ".")
return (slice[1] * 2^24) + (slice[2] * 2^16) + (slice[3] * 2^8) + slice[4]
}
function dec2ip(dec, ip, quad) {
for (i=3; i>=1; i--) {
quad = 256^i
ip = ip int(dec/quad) "."
dec = dec%quad
}
return ip dec
}
# example iblocklist.com format
# TOT Public Company/Irdeto:1.0.128.0-1.0.255.255
BEGIN { FS = ":"; }
$2 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {
n = split($2, array, "-");
if (n == 2) {
ip1 = array[1];
ip2 = array[2];
if (ip1 == ip2) {
# some records are just single IPs listed as range
print ip1;
} else {
# and some are really ranges
print range2cidr(ip2dec(ip1), ip2dec(ip2));
}
}
}