diff --git a/README.md b/README.md index 9aabf8e..52890d5 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,27 @@ ipset-blacklist =============== -A tiny Bash shell script which uses ipset and iptables to ban a large number of IP addresses published in IP blacklists. ipset uses a hashtable to store/fetch IP addresses and thus the IP lookup is a lot (!) faster than thousands of sequentially parsed iptables ban rules. However, the limit of an ipset list is 2^16 entries. +A set of shell scripts which use ipset and iptables to ban a large number of IP addresses published in IP blacklists. `ipset` uses a hashtable to store/fetch IP addresses and thus the IP lookup is a lot (!) faster than thousands of sequentially parsed iptables ban rules. However, the limit of an ipset list is 2^16 entries. -The ipset command doesn't work under OpenVZ. It works fine on dedicated and fully virtualized servers like KVM though. +Note: Updating the blacklists takes a long time (8m 20s on a TPLink TL-WDR3600), and running at a low priority (with `nice` and `ionice`) is recommended to avoid impacting packet routing & firewall performance. -## Quick start for Debian/Ubuntu based installations -1. Copy update-blacklist.sh into /usr/local/bin -2. chmod +x /usr/local/bin/update-blacklist.sh -2. Modify update-blacklist.sh according to your needs. Per default, the blacklisted IP addresses will be saved to /etc/ip-blacklist.conf -3. apt-get install ipset -4. Create the ipset blacklist and insert it into your iptables input filter (see below). After proper testing, make sure to persist it in your firewall script or similar or the rules will be lost after the next reboot. -5. Auto-update the blacklist using a cron job +## Quick start for OpenWRT +1. Copy the scripts to a temporary directory on your OpenWRT router +2. Run `sh install.sh` # iptables filter rule ``` -ipset create blacklist hash:net -iptables -I INPUT -m set --match-set blacklist src -j DROP +ipset create blacklist_net hash:net +ipset create blacklist_ip hash:ip +iptables -I INPUT -m set --match-set blacklist_net src -j DROP +iptables -I INPUT -m set --match-set blacklist_ip src -j DROP ``` Make sure to run this snippet in your firewall script. If you don't, the ipset blacklist and the iptables rule to ban the blacklisted ip addresses will be missing! # Cron job In order to auto-update the blacklist, copy the following code into /etc/cron.d/update-blacklist. Don't update the list too often or some blacklist providers will ban your IP address. Once a day should be OK though. ``` -MAILTO=root -33 23 * * * root /usr/local/bin/update-blacklist.sh +33 23 * * * /bin/nice /usr/local/bin/update-blacklist.sh > /tmp/update-blacklist.log 2>&1 ``` ## Check for dropped packets @@ -36,13 +33,3 @@ Chain INPUT (policy DROP 3064 packets, 177K bytes) pkts bytes target prot opt in out source destination 43 2498 DROP all -- * * 0.0.0.0/0 0.0.0.0/0 match-set blacklist src ``` - -## Modify the blacklists you want to use -Edit the BLACKLIST array to add or remove blacklists, or use it to add your own blacklists. -``` -BLACKLISTS=( -"http://www.mysite.me/files/mycustomblacklist.txt" # Your personal blacklist -"http://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1" # Project Honey Pot Directory of Dictionary Attacker IPs -# I don't want this: "http://www.openbl.org/lists/base.txt" # OpenBL.org 30 day List -) -``` diff --git a/bindechex.lua b/bindechex.lua new file mode 100755 index 0000000..741c653 --- /dev/null +++ b/bindechex.lua @@ -0,0 +1,544 @@ +--[[ +/* + * Copyright (c) 2007 Tim Kelly/Dialectronics + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT + * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +--]] + +--[[ +/* + * Copyright (c) 2007 Tim Kelly/Dialectronics + * + * Permission is hereby granted, free of charge, to any person obtaining + * a copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY + * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT + * OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR + * THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +--]] + +module(..., package.seeall); + +local hex2bin = { + ["0"] = "0000", + ["1"] = "0001", + ["2"] = "0010", + ["3"] = "0011", + ["4"] = "0100", + ["5"] = "0101", + ["6"] = "0110", + ["7"] = "0111", + ["8"] = "1000", + ["9"] = "1001", + ["a"] = "1010", + ["b"] = "1011", + ["c"] = "1100", + ["d"] = "1101", + ["e"] = "1110", + ["f"] = "1111" + } + + + +local bin2hex = { + ["0000"] = "0", + ["0001"] = "1", + ["0010"] = "2", + ["0011"] = "3", + ["0100"] = "4", + ["0101"] = "5", + ["0110"] = "6", + ["0111"] = "7", + ["1000"] = "8", + ["1001"] = "9", + ["1010"] = "A", + ["1011"] = "B", + ["1100"] = "C", + ["1101"] = "D", + ["1110"] = "E", + ["1111"] = "F" + } + +--[[ +local dec2hex = { + ["0"] = "0", + ["1"] = "1", + ["2"] = "2", + ["3"] = "3", + ["4"] = "4", + ["5"] = "5", + ["6"] = "6", + ["7"] = "7", + ["8"] = "8", + ["9"] = "9", + ["10"] = "A", + ["11"] = "B", + ["12"] = "C", + ["13"] = "D", + ["14"] = "E", + ["15"] = "F" + } +--]] + + +-- These functions are big-endian and take up to 32 bits + +-- Hex2Bin +-- Bin2Hex +-- Hex2Dec +-- Dec2Hex +-- Bin2Dec +-- Dec2Bin + + +function Hex2Bin(s) + +-- s -> hexadecimal string + +local ret = "" +local i = 0 + + + for i in string.gfind(s, ".") do + i = string.lower(i) + + ret = ret..hex2bin[i] + + end + + return ret +end + + +function Bin2Hex(s) + +-- s -> binary string + +local l = 0 +local h = "" +local b = "" +local rem + +l = string.len(s) +rem = l % 4 +l = l-1 +h = "" + + -- need to prepend zeros to eliminate mod 4 + if (rem > 0) then + s = string.rep("0", 4 - rem)..s + end + + for i = 1, l, 4 do + b = string.sub(s, i, i+3) + h = h..bin2hex[b] + end + + return h + +end + + +function Bin2Dec(s) + +-- s -> binary string + +local num = 0 +local ex = string.len(s) - 1 +local l = 0 + + l = ex + 1 + for i = 1, l do + b = string.sub(s, i, i) + if b == "1" then + num = num + 2^ex + end + ex = ex - 1 + end + + return string.format("%u", num) + +end + + + +function Dec2Bin(s, num) + +-- s -> Base10 string +-- num -> string length to extend to + +local n + + if (num == nil) then + n = 0 + else + n = num + end + + s = string.format("%x", s) + + s = Hex2Bin(s) + + while string.len(s) < n do + s = "0"..s + end + + return s + +end + + + + +function Hex2Dec(s) + +-- s -> hexadecimal string + +local s = Hex2Bin(s) + + return Bin2Dec(s) + +end + + + +function Dec2Hex(s) + +-- s -> Base10 string + + s = string.format("%x", s) + + return s + +end + + + + +-- These functions are big-endian and will extend to 32 bits + +-- BMAnd +-- BMNAnd +-- BMOr +-- BMXOr +-- BMNot + + +function BMAnd(v, m) + +-- v -> hex string to be masked +-- m -> hex string mask + +-- s -> hex string as masked + +-- bv -> binary string of v +-- bm -> binary string mask + +local bv = Hex2Bin(v) +local bm = Hex2Bin(m) + +local i = 0 +local s = "" + + while (string.len(bv) < 32) do + bv = "0000"..bv + end + + while (string.len(bm) < 32) do + bm = "0000"..bm + end + + + for i = 1, 32 do + cv = string.sub(bv, i, i) + cm = string.sub(bm, i, i) + if cv == cm then + if cv == "1" then + s = s.."1" + else + s = s.."0" + end + else + s = s.."0" + + end + end + + return Bin2Hex(s) + +end + + +function BMNAnd(v, m) + +-- v -> hex string to be masked +-- m -> hex string mask + +-- s -> hex string as masked + +-- bv -> binary string of v +-- bm -> binary string mask + +local bv = Hex2Bin(v) +local bm = Hex2Bin(m) + +local i = 0 +local s = "" + + while (string.len(bv) < 32) do + bv = "0000"..bv + end + + while (string.len(bm) < 32) do + bm = "0000"..bm + end + + + for i = 1, 32 do + cv = string.sub(bv, i, i) + cm = string.sub(bm, i, i) + if cv == cm then + if cv == "1" then + s = s.."0" + else + s = s.."1" + end + else + s = s.."1" + + end + end + + return Bin2Hex(s) + +end + + + +function BMOr(v, m) + +-- v -> hex string to be masked +-- m -> hex string mask + +-- s -> hex string as masked + +-- bv -> binary string of v +-- bm -> binary string mask + +local bv = Hex2Bin(v) +local bm = Hex2Bin(m) + +local i = 0 +local s = "" + + while (string.len(bv) < 32) do + bv = "0000"..bv + end + + while (string.len(bm) < 32) do + bm = "0000"..bm + end + + + for i = 1, 32 do + cv = string.sub(bv, i, i) + cm = string.sub(bm, i, i) + if cv == "1" then + s = s.."1" + elseif cm == "1" then + s = s.."1" + else + s = s.."0" + end + end + + return Bin2Hex(s) + +end + +function BMXOr(v, m) + +-- v -> hex string to be masked +-- m -> hex string mask + +-- s -> hex string as masked + +-- bv -> binary string of v +-- bm -> binary string mask + +local bv = Hex2Bin(v) +local bm = Hex2Bin(m) + +local i = 0 +local s = "" + + while (string.len(bv) < 32) do + bv = "0000"..bv + end + + while (string.len(bm) < 32) do + bm = "0000"..bm + end + + + for i = 1, 32 do + cv = string.sub(bv, i, i) + cm = string.sub(bm, i, i) + if cv == "1" then + if cm == "0" then + s = s.."1" + else + s = s.."0" + end + elseif cm == "1" then + if cv == "0" then + s = s.."1" + else + s = s.."0" + end + else + -- cv and cm == "0" + s = s.."0" + end + end + + return Bin2Hex(s) + +end + + +function BMNot(v, m) + +-- v -> hex string to be masked +-- m -> hex string mask + +-- s -> hex string as masked + +-- bv -> binary string of v +-- bm -> binary string mask + +local bv = Hex2Bin(v) +local bm = Hex2Bin(m) + +local i = 0 +local s = "" + + while (string.len(bv) < 32) do + bv = "0000"..bv + end + + while (string.len(bm) < 32) do + bm = "0000"..bm + end + + + for i = 1, 32 do + cv = string.sub(bv, i, i) + cm = string.sub(bm, i, i) + if cm == "1" then + if cv == "1" then + -- turn off + s = s.."0" + else + -- turn on + s = s.."1" + end + else + -- leave untouched + s = s..cv + + end + end + + return Bin2Hex(s) + +end + + +-- these functions shift right and left, adding zeros to lost or gained bits +-- returned values are 32 bits long + +-- BShRight(v, nb) +-- BShLeft(v, nb) + + +function BShRight(v, nb) + +-- v -> hexstring value to be shifted +-- nb -> number of bits to shift to the right + +-- s -> binary string of v + +local s = Hex2Bin(v) + + while (string.len(s) < 32) do + s = "0000"..s + end + + s = string.sub(s, 1, 32 - nb) + + while (string.len(s) < 32) do + s = "0"..s + end + + return Bin2Hex(s) + +end + +function BShLeft(v, nb) + +-- v -> hexstring value to be shifted +-- nb -> number of bits to shift to the right + +-- s -> binary string of v + +local s = Hex2Bin(v) + + while (string.len(s) < 32) do + s = "0000"..s + end + + s = string.sub(s, nb + 1, 32) + + while (string.len(s) < 32) do + s = s.."0" + end + + return Bin2Hex(s) + +end + + diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..1055351 --- /dev/null +++ b/install.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Install ipset blacklist scripts + +REQUIRED_PACKAGES="ipset wget lua" + +opkg install $REQUIRED_PACKAGES || (opkg update && opkg install $REQUIRED_PACKAGES) || exit + +# copy the scripts to the PATH +mkdir -p /usr/local/bin +if ! grep /usr/local/bin /etc/profile; then echo "PATH=$PATH:/usr/local/bin" >> /etc/profile; fi +cp update_blacklist.sh load_blacklist.sh normalize_ip.lua uniq_cidr.lua /usr/local/bin +cp bindechex.lua /usr/share/lua + +# load blacklists with firewall rules at boot +if ! grep load_blacklist.sh /etc/firewall.user +then + echo "/bin/nice /usr/local/bin/load_blacklist.sh ip /etc/blacklist.ip.conf" >> /etc/firewall.user + echo "/bin/nice /usr/local/bin/load_blacklist.sh net /etc/blacklist.net.conf" >> /etc/firewall.user +fi + +# add cron job to update the blacklists (don't set this to update too often or you might get banned) +if ! grep update_blacklist.sh /etc/crontabs/root +then + echo "17 10 * * * sleep 5 && nice update-blacklist.sh > /tmp/update_blacklist.log 2>&1" >> /etc/crontabs/root +fi diff --git a/load_blacklist.sh b/load_blacklist.sh new file mode 100755 index 0000000..8da2a12 --- /dev/null +++ b/load_blacklist.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# load a blacklist into an ipset +# Usage: load_blacklist.sh [name] [file] + +FW_RULE="INPUT -m set --match-set blacklist_$1 src -j DROP" + +ipset -exist create blacklist_$1 hash:$1 maxelem `wc -l $2 | cut -f1 -d" "` +if ! iptables -C $FW_RULE; then iptables -I $FW_RULE; fi + +cat $2 | while IFS= read -r ip +do + ipset -exist add blacklist_$1 $ip +done + diff --git a/normalize_ip.lua b/normalize_ip.lua new file mode 100755 index 0000000..988726f --- /dev/null +++ b/normalize_ip.lua @@ -0,0 +1,34 @@ +#!/usr/bin/lua +-- normalize IP addresses (eg. convert 001.002.003.004 to 1.2.3.4, etc) + +-- get list from stdin +function get_list() + raw_list = {} + for line in io.stdin:lines() + do + table.insert(raw_list, line) + end + return raw_list +end + +function normalize_ip(ip) + norm_parts = {} + for part in string.gfind(ip, "[0-9]+") + do + table.insert(norm_parts, tostring(tonumber(part))) + end + + ip = "" + for i in pairs(norm_parts) + do + ip = ip .. "." .. norm_parts[i] + end + + return string.sub(ip, 2) +end + +list=get_list() +for i in pairs(list) +do + print(normalize_ip(list[i])) +end diff --git a/uniq_cidr.lua b/uniq_cidr.lua new file mode 100755 index 0000000..76f3314 --- /dev/null +++ b/uniq_cidr.lua @@ -0,0 +1,115 @@ +#!/usr/bin/lua +-- takes a list of CIDR ranges from stdin and removes duplicates and merges overlapping ranges and prints the list to stdout + +require "bindechex" + +function split(str, delim, maxNb) + -- Eliminate bad cases... + if string.find(str, delim) == nil then + return { str } + end + if maxNb == nil or maxNb < 1 then + maxNb = 0 -- No limit + end + local result = {} + local pat = "(.-)" .. delim .. "()" + local nb = 0 + local lastPos + for part, pos in string.gfind(str, pat) do + nb = nb + 1 + result[nb] = part + lastPos = pos + if nb == maxNb then break end + end + -- Handle the last field + if nb ~= maxNb then + result[nb + 1] = string.sub(str, lastPos) + end + return result +end + +-- compare 2 cidr ranges, return true if a comes before b +function cmp_cidr(a, b) + a_parts = split(a, "[\\/]+") + b_parts = split(b, "[\\/]+") + + -- if cidr netmasks are the same, compare actual ranges + if a_parts[2] == b_parts[2] then + a_parts[1] = split(a_parts[1], "[\\.]+") + b_parts[1] = split(b_parts[1], "[\\.]+") + + for i in pairs(a_parts[1]) do + if a_parts[1][i] ~= b_parts[1][i] then + return tonumber(a_parts[1][i]) > tonumber(b_parts[1][i]) + end + end + else + return tonumber(a_parts[2]) < tonumber(b_parts[2]) + end +end + +-- get list of ranges from stdin +function get_list() + raw_list = {} + for line in io.stdin:lines() + do + -- table.insert(raw_list, split(line, "[\\/]+")) + table.insert(raw_list, line) + end + return raw_list +end + +-- convert an IP address string & cidr range (eg. 192.168.1.0/24) to a 32bit binary string +function ip_to_bits(ip) + bits = "" + + parts = split(ip, "[\\/\\.]+") + for i = 1,4,1 do + byte = bindechex.Dec2Bin(parts[i]) + bits_needed = 8 - (byte:len()%8) + if bits_needed < 8 then + for i = 1,(8-(byte:len()%8)),1 do + byte = "0" .. byte + end + end + bits = bits .. byte + end + + return bits:sub(1, parts[5]) +end + +-- return true if cidr range 'outer' (eg "192.168.1.1/24") contains 'inner' +function range_contains(outer, inner) + inner = ip_to_bits(inner) + outer = ip_to_bits(outer) + if outer:len() > inner:len() then + print("ERROR: outer > inner") + return false + end + + for i = 1,#outer,1 do + if outer:sub(i,i) ~= inner:sub(i,i) then + return false + end + end + + return true +end + +raw_list = get_list() + +table.sort(raw_list, cmp_cidr) + +for i in pairs(raw_list) +do + for j=#raw_list,i+1,-1 do + if range_contains(raw_list[i], raw_list[j]) then + table.remove(raw_list, j) + end + end +end + +for i in pairs(raw_list) +do + print(raw_list[i]) +end diff --git a/update-blacklist.sh b/update-blacklist.sh deleted file mode 100644 index 91f8024..0000000 --- a/update-blacklist.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash -IP_TMP=/tmp/ip.tmp -IP_BLACKLIST=/etc/ip-blacklist.conf -IP_BLACKLIST_TMP=/tmp/ip-blacklist.tmp -IP_BLACKLIST_CUSTOM=/etc/ip-blacklist-custom.conf # optional -BLACKLISTS=( -"http://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1" # Project Honey Pot Directory of Dictionary Attacker IPs -"http://check.torproject.org/cgi-bin/TorBulkExitList.py?ip=1.1.1.1" # TOR Exit Nodes -"http://www.maxmind.com/en/anonymous_proxies" # MaxMind GeoIP Anonymous Proxies -"http://danger.rulez.sk/projects/bruteforceblocker/blist.php" # BruteForceBlocker IP List -"http://rules.emergingthreats.net/blockrules/rbn-ips.txt" # Emerging Threats - Russian Business Networks List -"http://www.spamhaus.org/drop/drop.lasso" # Spamhaus Don't Route Or Peer List (DROP) -"http://cinsscore.com/list/ci-badguys.txt" # C.I. Army Malicious IP List -"http://www.openbl.org/lists/base.txt" # OpenBL.org 30 day List -"http://www.autoshun.org/files/shunlist.csv" # Autoshun Shun List -"http://lists.blocklist.de/lists/all.txt" # blocklist.de attackers -) -for i in "${BLACKLISTS[@]}" -do - curl "$i" > $IP_TMP - grep -Po '(?:\d{1,3}\.){3}\d{1,3}(?:/\d{1,2})?' $IP_TMP >> $IP_BLACKLIST_TMP -done -sort $IP_BLACKLIST_TMP -n | uniq > $IP_BLACKLIST -rm $IP_BLACKLIST_TMP -wc -l $IP_BLACKLIST - -ipset flush blacklist -egrep -v "^#|^$" $IP_BLACKLIST | while IFS= read -r ip -do - ipset add blacklist $ip -done - -if [ -f $IP_BLACKLIST_CUSTOM ]; then - egrep -v "^#|^$" $IP_BLACKLIST_CUSTOM | while IFS= read -r ip - do - ipset add blacklist $ip - done -fi diff --git a/update_blacklist.sh b/update_blacklist.sh new file mode 100755 index 0000000..f892fce --- /dev/null +++ b/update_blacklist.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# script to create blacklists with ipset for suspicious IPv4 addresses (eg infected hosts) + +IP_BLACKLIST=/etc/blacklist.ip.conf +NET_BLACKLIST=/etc/blacklist.net.conf + +TEMP_DIR=/tmp/blacklist_temp +WGET_LOG=/tmp/update_blacklist.log +BLACKLISTS=" + http://www.projecthoneypot.org/list_of_ips.php?t=d&rss=1 + http://check.torproject.org/exit-addresses + http://www.maxmind.com/en/anonymous_proxies + http://www.spamhaus.org/drop/drop.lasso + http://danger.rulez.sk/projects/bruteforceblocker/blist.php + http://www.openbl.org/lists/base.txt + http://cinsscore.com/list/ci-badguys.txt + http://www.autoshun.org/files/shunlist.csv + http://lists.blocklist.de/lists/all.txt + http://rules.emergingthreats.net/blockrules/emerging-compromised.rules + http://malc0de.com/bl/IP_Blacklist.ttx + http://www.dshield.org/ipsascii.html?limit=10000 + http://www.blocklist.de/downloads/export-ips_all.txt + " +IP_REGEX="([0-9]{1,3}\.){3}[0-9]{1,3}" +RANGE_REGEX="$IP_REGEX ?- ?$IP_REGEX" +CIDR_REGEX="$IP_REGEX/[0-9]{1,2}" +NET_REGEX="($RANGE_REGEX)|($CIDR_REGEX)" + +# download multiple URLs and output to stdout (log errors to logfile) +download_lists(){ + # clear the wget log + echo -n > $WGET_LOG + + # download blacklists + for i in $* + do + (wget --no-check-certificate -O - "$i" 2>> $WGET_LOG) || echo Error downloading $i 1>&2 + done + + return 0 +} + +# make sure each IP/net is on a separate line and drop unneeded lines +process_raw_list(){ + # make sure each IP/net is on a seperate line + sed -r 's/,/\n/g' |\ + + # drop the lines without IP addresses + grep -E $IP_REGEX || return 1 +} + +# take a list of IPs or nets on stdin (1 per line) and create & apply an ipset +# arg 1: ipset type +# arg 2: ipset name +create_ipset(){ + # remove duplicate IPs/nets, save to the temp file, and count them + NUM_ELEMS=`tee $TEMP_DIR/temp_list | wc -l` + + # create the temporary ipset + if ! ipset create $2_tmp $1 maxelem $NUM_ELEMS; then return 1; fi + + # fill the ipset + cat $TEMP_DIR/temp_list | while IFS= read -r elem + do + #echo "adding $elem to $2" + if ! ipset add $2_tmp $elem ; then return 1; fi + done + + # try to swap the temp ipset with the final ipset. + if !(ipset swap $2_tmp $2 >/dev/null 2>&1) + then + # if swapping fails, try to rename it + if ! (ipset rename $2_tmp $2 >/dev/null 2>&1) + then + echo Error renaming ipset >&2 + ipset destroy $2_tmp + return 1 + fi + fi + + # destroy the old ipset + if ! ipset destroy $2_tmp; then return 1; fi + + echo Created ipset $2 with $NUM_ELEMS elements +} + +# make the temp dir +if ! mkdir -p $TEMP_DIR; then echo Error making temp dir && exit; fi + +# download the raw list of suspicious hosts & nets +if ! (download_lists $BLACKLISTS | process_raw_list > $TEMP_DIR/raw); then echo Error downloading blacklists 1>&2 && exit; fi + +# create blacklist for IP nets +if ! ((grep -oE "$CIDR_REGEX" $TEMP_DIR/raw | uniq_cidr.lua ; +grep -oE "$RANGE_REGEX" $TEMP_DIR/raw) |\ +tee $NET_BLACKLIST |\ +create_ipset hash:net blacklist_net); then echo Error creating hash:net blacklist 1>&2 && exit; fi + + +# create blacklist for individual IP addrs +if ! (grep -vE "$NET_REGEX" $TEMP_DIR/raw |\ +grep -oE "$IP_REGEX" |\ +normalize_ip.lua |\ +sort -u |\ +tee $IP_BLACKLIST |\ +create_ipset hash:ip blacklist_ip); then echo Error creating hash:ip blacklist 1>&2 && exit; fi + + +# delete temp dir +rm -rf $TEMP_DIR