add support for iblocklist.com
This commit is contained in:
parent
f31e9e35eb
commit
048ecb0e3b
25
blacklist.sh
25
blacklist.sh
@ -12,9 +12,13 @@ urls="http://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt"
|
|||||||
|
|
||||||
# Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders
|
# Blocklist.de collects reports from fail2ban probes, listing password brute-forces, scanners and other offenders
|
||||||
urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt"
|
urls="$urls https://www.blocklist.de/downloads/export-ips_all.txt"
|
||||||
|
|
||||||
# badips.com, from score 2 up
|
# badips.com, from score 2 up
|
||||||
urls="$urls http://www.badips.com/get/list/ssh/2"
|
urls="$urls http://www.badips.com/get/list/ssh/2"
|
||||||
|
|
||||||
|
# iblocklist.com is also supported
|
||||||
|
# urls="$urls http://list.iblocklist.com/?list=srzondksmjuwsvmgdbhi&fileformat=p2p&archiveformat=gz&username=USERNAMEx$&pin=PIN"
|
||||||
|
|
||||||
# This is how it will look like on the server
|
# This is how it will look like on the server
|
||||||
|
|
||||||
# Chain blocklists (2 references)
|
# Chain blocklists (2 references)
|
||||||
@ -91,18 +95,37 @@ for url in $urls; do
|
|||||||
set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname
|
set_name=$(echo "$url" | awk -F/ '{print substr($3,0,21);}') # set name is derived from source URL hostname
|
||||||
curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}"
|
curl -v -s ${COMPRESS_OPT} -k "$url" >"${unsorted_blocklist}" 2>"${headers}"
|
||||||
|
|
||||||
# this is required for blocklist.de that sends compressed content if asked for it or not
|
# this is required for blocklist.de that sends compressed content regardless of asked or not
|
||||||
if [ -z "$COMPRESS_OPT" ]; then
|
if [ -z "$COMPRESS_OPT" ]; then
|
||||||
if grep -qi 'content-encoding: gzip' "${headers}"; then
|
if grep -qi 'content-encoding: gzip' "${headers}"; then
|
||||||
mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
|
mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
|
||||||
gzip -d "${unsorted_blocklist}.gz"
|
gzip -d "${unsorted_blocklist}.gz"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
# autodetect iblocklist.com format as it needs additional conversion
|
||||||
|
if echo "${url}" | grep -q 'iblocklist.com'; then
|
||||||
|
if [ -f /etc/range2cidr.awk ]; then
|
||||||
|
mv "${unsorted_blocklist}" "${unsorted_blocklist}.gz"
|
||||||
|
gzip -d "${unsorted_blocklist}.gz"
|
||||||
|
awk_tmp=$(mktemp)
|
||||||
|
awk -f /etc/range2cidr.awk <"${unsorted_blocklist}" >"${awk_tmp}"
|
||||||
|
mv "${awk_tmp}" "${unsorted_blocklist}"
|
||||||
|
else
|
||||||
|
echo "range2cidr.awk script not found, cannot process ${unsorted_blocklist}, skipping"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}"
|
sort -u <"${unsorted_blocklist}" | egrep "^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]{1,2})?$" >"${sorted_blocklist}"
|
||||||
|
|
||||||
# calculate performance parameters for the new set
|
# calculate performance parameters for the new set
|
||||||
|
if "${RANDOM}"; then
|
||||||
|
# bash
|
||||||
tmp_set_name="tmp_${RANDOM}"
|
tmp_set_name="tmp_${RANDOM}"
|
||||||
|
else
|
||||||
|
# non-bash
|
||||||
|
tmp_set_name="tmp_$$"
|
||||||
|
fi
|
||||||
new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' )
|
new_list_size=$(wc -l "${sorted_blocklist}" | awk '{print $1;}' )
|
||||||
hash_size=$(expr $new_list_size / 2)
|
hash_size=$(expr $new_list_size / 2)
|
||||||
|
|
||||||
|
71
range2cidr.awk
Normal file
71
range2cidr.awk
Normal file
@ -0,0 +1,71 @@
|
|||||||
|
# AWK script to convert iblocklist.com ranges into CIDR format
|
||||||
|
# usable with ipset
|
||||||
|
|
||||||
|
# based on scripts posted at
|
||||||
|
# http://www.unix.com/shell-programming-and-scripting/233825-convert-ip-ranges-cidr-netblocks.html
|
||||||
|
|
||||||
|
function bit_or(a, b, r, i, c) {
|
||||||
|
for (r=i=0;i<32;i++) {
|
||||||
|
c = 2 ^ i
|
||||||
|
if ((int(a/c) % 2) || (int(b/c) % 2)) r += c
|
||||||
|
}
|
||||||
|
return r
|
||||||
|
}
|
||||||
|
function bit_lshift(var, x) {
|
||||||
|
while(x--) var*=2;
|
||||||
|
return var;
|
||||||
|
}
|
||||||
|
function bit_rshift(var, x) {
|
||||||
|
while(x--) var=int(var/2);
|
||||||
|
return var;
|
||||||
|
}
|
||||||
|
function range2cidr(ipStart, ipEnd, bits, mask, newip) {
|
||||||
|
bits = 1
|
||||||
|
mask = 1
|
||||||
|
while (bits < 32) {
|
||||||
|
newip = bit_or(ipStart, mask)
|
||||||
|
if ((newip>ipEnd) || ((bit_lshift(bit_rshift(ipStart,bits),bits)) != ipStart)) {
|
||||||
|
bits--
|
||||||
|
mask = bit_rshift(mask,1)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
bits++
|
||||||
|
mask = bit_lshift(mask,1)+1
|
||||||
|
}
|
||||||
|
newip = bit_or(ipStart, mask)
|
||||||
|
bits = 32 - bits
|
||||||
|
result = dec2ip(ipStart) "/" bits
|
||||||
|
if (newip < ipEnd) result = result "\n" range2cidr(newip + 1, ipEnd)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
function ip2dec(ip, slice) {
|
||||||
|
split(ip, slice, ".")
|
||||||
|
return (slice[1] * 2^24) + (slice[2] * 2^16) + (slice[3] * 2^8) + slice[4]
|
||||||
|
}
|
||||||
|
function dec2ip(dec, ip, quad) {
|
||||||
|
for (i=3; i>=1; i--) {
|
||||||
|
quad = 256^i
|
||||||
|
ip = ip int(dec/quad) "."
|
||||||
|
dec = dec%quad
|
||||||
|
}
|
||||||
|
return ip dec
|
||||||
|
}
|
||||||
|
|
||||||
|
# example iblocklist.com format
|
||||||
|
# TOT Public Company/Irdeto:1.0.128.0-1.0.255.255
|
||||||
|
BEGIN { FS = ":"; }
|
||||||
|
|
||||||
|
$2 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {
|
||||||
|
n = split($2, array, "-");
|
||||||
|
if (n == 2) {
|
||||||
|
ip1 = array[1];
|
||||||
|
ip2 = array[2];
|
||||||
|
if (ip1 == ip2) {
|
||||||
|
# some records are just single IPs listed as range
|
||||||
|
print ip1;
|
||||||
|
} else {
|
||||||
|
# and some are really ranges
|
||||||
|
print range2cidr(ip2dec(ip1), ip2dec(ip2));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user