net/banip/files/banip-functions.sh

   1 # banIP shared function library/include
   2 # Copyright (c) 2018-2023 Dirk Brenken (dev@brenken.org)
   3 # This is free software, licensed under the GNU General Public License v3.
   4
   5 # (s)hellcheck exceptions
   6 # shellcheck disable=all
   7
   8 # set initial defaults
   9 #
  10 export LC_ALL=C
  11 export PATH="/usr/sbin:/usr/bin:/sbin:/bin"
  12
  13 ban_basedir="/tmp"
  14 ban_backupdir="${ban_basedir}/banIP-backup"
  15 ban_reportdir="${ban_basedir}/banIP-report"
  16 ban_feedfile="/etc/banip/banip.feeds"
  17 ban_pidfile="/var/run/banip.pid"
  18 ban_lock="/var/run/banip.lock"
  19 ban_blocklist="/etc/banip/banip.blocklist"
  20 ban_allowlist="/etc/banip/banip.allowlist"
  21 ban_fetchcmd=""
  22 ban_logreadcmd="$(command -v logread)"
  23 ban_logcmd="$(command -v logger)"
  24 ban_ubuscmd="$(command -v ubus)"
  25 ban_nftcmd="$(command -v nft)"
  26 ban_fw4cmd="$(command -v fw4)"
  27 ban_awkcmd="$(command -v awk)"
  28 ban_grepcmd="$(command -v grep)"
  29 ban_lookupcmd="$(command -v nslookup)"
  30 ban_mailcmd="$(command -v msmtp)"
  31 ban_mailsender="no-reply@banIP"
  32 ban_mailreceiver=""
  33 ban_mailtopic="banIP notification"
  34 ban_mailprofile="ban_notify"
  35 ban_mailtemplate="/etc/banip/banip.tpl"
  36 ban_nftpriority="-200"
  37 ban_nftexpiry=""
  38 ban_loglevel="warn"
  39 ban_loglimit="100"
  40 ban_logcount="1"
  41 ban_logterm=""
  42 ban_country=""
  43 ban_asn=""
  44 ban_loginput="1"
  45 ban_logforwardwan="1"
  46 ban_logforwardlan="0"
  47 ban_allowlistonly="0"
  48 ban_autoallowlist="1"
  49 ban_autoblocklist="1"
  50 ban_deduplicate="1"
  51 ban_splitsize="0"
  52 ban_autodetect=""
  53 ban_feed=""
  54 ban_blockinput=""
  55 ban_blockforwardwan=""
  56 ban_blockforwardlan=""
  57 ban_protov4="0"
  58 ban_protov6="0"
  59 ban_ifv4=""
  60 ban_ifv6=""
  61 ban_dev=""
  62 ban_sub=""
  63 ban_fetchinsecure=""
  64 ban_cores=""
  65 ban_memory=""
  66 ban_trigger=""
  67 ban_triggerdelay="10"
  68 ban_resolver=""
  69 ban_enabled="0"
  70 ban_debug="0"
  71
  72 # gather system information
  73 #
  74 f_system() {
  75         local cpu core
  76
  77         ban_memory="$("${ban_awkcmd}" '/^MemAvailable/{printf "%s",int($2/1000)}' "/proc/meminfo" 2>/dev/null)"
  78         ban_ver="$(${ban_ubuscmd} -S call rpc-sys packagelist 2>/dev/null | jsonfilter -ql1 -e '@.packages.banip')"
  79         ban_sysver="$(${ban_ubuscmd} -S call system board 2>/dev/null | jsonfilter -ql1 -e '@.model' -e '@.release.description' |
  80                 "${ban_awkcmd}" 'BEGIN{RS="";FS="\n"}{printf "%s, %s",$1,$2}')"
  81         if [ -z "${ban_cores}" ]; then
  82                 cpu="$("${ban_grepcmd}" -c '^processor' /proc/cpuinfo 2>/dev/null)"
  83                 core="$("${ban_grepcmd}" -cm1 '^core id' /proc/cpuinfo 2>/dev/null)"
  84                 [ "${cpu}" = "0" ] && cpu="1"
  85                 [ "${core}" = "0" ] && core="1"
  86                 ban_cores="$((cpu * core))"
  87         fi
  88
  89         f_log "debug" "f_system  ::: system: ${ban_sysver:-"n/a"}, version: ${ban_ver:-"n/a"}, memory: ${ban_memory:-"0"}, cpu_cores: ${ban_cores}"
  90 }
  91
  92 # create directories
  93 #
  94 f_mkdir() {
  95         local dir="${1}"
  96
  97         if [ ! -d "${dir}" ]; then
  98                 rm -f "${dir}"
  99                 mkdir -p "${dir}"
 100                 f_log "debug" "f_mkdir   ::: created directory: ${dir}"
 101         fi
 102 }
 103
 104 # create files
 105 #
 106 f_mkfile() {
 107         local file="${1}"
 108
 109         if [ ! -f "${file}" ]; then
 110                 : >"${file}"
 111                 f_log "debug" "f_mkfile  ::: created file: ${file}"
 112         fi
 113 }
 114
 115 # create temporary files and directories
 116 #
 117 f_tmp() {
 118         f_mkdir "${ban_basedir}"
 119         ban_tmpdir="$(mktemp -p "${ban_basedir}" -d)"
 120         ban_tmpfile="$(mktemp -p "${ban_tmpdir}" -tu)"
 121
 122         f_log "debug" "f_tmp     ::: base_dir: ${ban_basedir:-"-"}, tmp_dir: ${ban_tmpdir:-"-"}"
 123 }
 124
 125 # remove directories
 126 #
 127 f_rmdir() {
 128         local dir="${1}"
 129
 130         if [ -d "${dir}" ]; then
 131                 rm -rf "${dir}"
 132                 f_log "debug" "f_rmdir   ::: deleted directory: ${dir}"
 133         fi
 134 }
 135
 136 # convert chars
 137 #
 138 f_char() {
 139         local char="${1}"
 140
 141         [ "${char}" = "1" ] && printf "%s" "✔" || printf "%s" "✘"
 142 }
 143
 144 # trim strings
 145 #
 146 f_trim() {
 147         local string="${1}"
 148
 149         string="${string#"${string%%[![:space:]]*}"}"
 150         string="${string%"${string##*[![:space:]]}"}"
 151         printf "%s" "${string}"
 152 }
 153
 154 # write log messages
 155 #
 156 f_log() {
 157         local class="${1}" log_msg="${2}"
 158
 159         if [ -n "${log_msg}" ] && { [ "${class}" != "debug" ] || [ "${ban_debug}" = "1" ]; }; then
 160                 if [ -x "${ban_logcmd}" ]; then
 161                         "${ban_logcmd}" -p "${class}" -t "banIP-${ban_ver}[${$}]" "${log_msg}"
 162                 else
 163                         printf "%s %s %s\n" "${class}" "banIP-${ban_ver}[${$}]" "${log_msg}"
 164                 fi
 165         fi
 166         if [ "${class}" = "err" ]; then
 167                 f_genstatus "error"
 168                 f_rmdir "${ban_tmpdir}"
 169                 rm -rf "${ban_lock}"
 170                 exit 1
 171         fi
 172 }
 173
 174 # load config
 175 #
 176 f_conf() {
 177         unset ban_dev ban_ifv4 ban_ifv6 ban_feed ban_blockinput ban_blockforwardwan ban_blockforwardlan ban_logterm ban_country ban_asn
 178         config_cb() {
 179                 option_cb() {
 180                         local option="${1}"
 181                         local value="${2}"
 182                         eval "${option}=\"${value}\""
 183                 }
 184                 list_cb() {
 185                         local option="${1}"
 186                         local value="${2}"
 187                         case "${option}" in
 188                                 "ban_dev")
 189                                         eval "${option}=\"$(printf "%s" "${ban_dev}")${value} \""
 190                                         ;;
 191                                 "ban_ifv4")
 192                                         eval "${option}=\"$(printf "%s" "${ban_ifv4}")${value} \""
 193                                         ;;
 194                                 "ban_ifv6")
 195                                         eval "${option}=\"$(printf "%s" "${ban_ifv6}")${value} \""
 196                                         ;;
 197                                 "ban_feed")
 198                                         eval "${option}=\"$(printf "%s" "${ban_feed}")${value} \""
 199                                         ;;
 200                                 "ban_blockinput")
 201                                         eval "${option}=\"$(printf "%s" "${ban_blockinput}")${value} \""
 202                                         ;;
 203                                 "ban_blockforwardwan")
 204                                         eval "${option}=\"$(printf "%s" "${ban_blockforwardwan}")${value} \""
 205                                         ;;
 206                                 "ban_blockforwardlan")
 207                                         eval "${option}=\"$(printf "%s" "${ban_blockforwardlan}")${value} \""
 208                                         ;;
 209                                 "ban_logterm")
 210                                         eval "${option}=\"$(printf "%s" "${ban_logterm}")${value}\\|\""
 211                                         ;;
 212                                 "ban_country")
 213                                         eval "${option}=\"$(printf "%s" "${ban_country}")${value} \""
 214                                         ;;
 215                                 "ban_asn")
 216                                         eval "${option}=\"$(printf "%s" "${ban_asn}")${value} \""
 217                                         ;;
 218                         esac
 219                 }
 220         }
 221         config_load banip
 222
 223         [ "${ban_action}" = "boot" ] && [ -z "${ban_trigger}" ] && sleep ${ban_triggerdelay}
 224 }
 225
 226 # prepare fetch utility
 227 #
 228 f_fetch() {
 229         local ut utils packages insecure
 230
 231         if [ -z "${ban_fetchcmd}" ] || [ ! -x "${ban_fetchcmd}" ]; then
 232                 packages="$(${ban_ubuscmd} -S call rpc-sys packagelist 2>/dev/null)"
 233                 [ -z "${packages}" ] && f_log "err" "local opkg package repository is not available, please set the download utility 'ban_fetchcmd' manually"
 234                 utils="aria2c curl wget uclient-fetch"
 235                 for ut in ${utils}; do
 236                         if { [ "${ut}" = "uclient-fetch" ] && printf "%s" "${packages}" | "${ban_grepcmd}" -q '"libustream-'; } ||
 237                                 { [ "${ut}" = "wget" ] && printf "%s" "${packages}" | "${ban_grepcmd}" -q '"wget-ssl'; } ||
 238                                 [ "${ut}" = "curl" ] || [ "${ut}" = "aria2c" ]; then
 239                                 ban_fetchcmd="$(command -v "${ut}")"
 240                                 if [ -x "${ban_fetchcmd}" ]; then
 241                                         uci_set banip global ban_fetchcmd "${ban_fetchcmd##*/}"
 242                                         uci_commit "banip"
 243                                         break
 244                                 fi
 245                         fi
 246                 done
 247         fi
 248         [ ! -x "${ban_fetchcmd}" ] && f_log "err" "download utility with SSL support not found"
 249         case "${ban_fetchcmd##*/}" in
 250                 "aria2c")
 251                         [ "${ban_fetchinsecure}" = "1" ] && insecure="--check-certificate=false"
 252                         ban_fetchparm="${ban_fetchparm:-"${insecure} --timeout=20 --allow-overwrite=true --auto-file-renaming=false --log-level=warn --dir=/ -o"}"
 253                         ;;
 254                 "curl")
 255                         [ "${ban_fetchinsecure}" = "1" ] && insecure="--insecure"
 256                         ban_fetchparm="${ban_fetchparm:-"${insecure} --connect-timeout 20 --fail --silent --show-error --location -o"}"
 257                         ;;
 258                 "uclient-fetch")
 259                         [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate"
 260                         ban_fetchparm="${ban_fetchparm:-"${insecure} --timeout=20 -O"}"
 261                         ;;
 262                 "wget")
 263                         [ "${ban_fetchinsecure}" = "1" ] && insecure="--no-check-certificate"
 264                         ban_fetchparm="${ban_fetchparm:-"${insecure} --no-cache --no-cookies --max-redirect=0 --timeout=20 -O"}"
 265                         ;;
 266         esac
 267
 268         f_log "debug" "f_fetch   ::: fetch_cmd: ${ban_fetchcmd:-"-"}, fetch_parm: ${ban_fetchparm:-"-"}"
 269 }
 270
 271 # remove logservice
 272 #
 273 f_rmpid() {
 274         local ppid pid pids
 275
 276         ppid="$(cat "${ban_pidfile}" 2>/dev/null)"
 277         [ -n "${ppid}" ] && pids="$(pgrep -P "${ppid}" 2>/dev/null)" || return 0
 278         for pid in ${pids}; do
 279                 kill -INT "${pid}" >/dev/null 2>&1
 280         done
 281         : >"${ban_pidfile}"
 282 }
 283
 284 # get wan interfaces
 285 #
 286 f_getif() {
 287         local iface
 288
 289         "${ban_ubuscmd}" -t 5 wait_for network.device network.interface 2>/dev/null
 290         if [ "${ban_autodetect}" = "1" ]; then
 291                 if [ -z "${ban_ifv4}" ]; then
 292                         network_find_wan iface
 293                         if [ -n "${iface}" ] && ! printf "%s" "${ban_ifv4}" | "${ban_grepcmd}" -q "${iface}"; then
 294                                 ban_protov4="1"
 295                                 ban_ifv4="${ban_ifv4}${iface} "
 296                                 uci_set banip global ban_protov4 "1"
 297                                 uci_add_list banip global ban_ifv4 "${iface}"
 298                         fi
 299                 fi
 300                 if [ -z "${ban_ifv6}" ]; then
 301                         network_find_wan6 iface
 302                         if [ -n "${iface}" ] && ! printf "%s" "${ban_ifv6}" | "${ban_grepcmd}" -q "${iface}"; then
 303                                 ban_protov6="1"
 304                                 ban_ifv6="${ban_ifv6}${iface} "
 305                                 uci_set banip global ban_protov6 "1"
 306                                 uci_add_list banip global ban_ifv6 "${iface}"
 307                         fi
 308                 fi
 309                 ban_ifv4="${ban_ifv4%%?}"
 310                 ban_ifv6="${ban_ifv6%%?}"
 311                 [ -n "$(uci -q changes "banip")" ] && uci_commit "banip"
 312         fi
 313         [ -z "${ban_ifv4}" ] && [ -z "${ban_ifv6}" ] && f_log "err" "wan interfaces not found, please check your configuration"
 314
 315         f_log "debug" "f_getif   ::: auto_detect: ${ban_autodetect}, interfaces (4/6): ${ban_ifv4}/${ban_ifv6}, protocols (4/6): ${ban_protov4}/${ban_protov6}"
 316 }
 317
 318 # get wan devices
 319 #
 320 f_getdev() {
 321         local dev iface
 322
 323         if [ "${ban_autodetect}" = "1" ] && [ -z "${ban_dev}" ]; then
 324                 for iface in ${ban_ifv4} ${ban_ifv6}; do
 325                         network_get_device dev "${iface}"
 326                         if [ -n "${dev}" ] && ! printf "%s" "${ban_dev}" | "${ban_grepcmd}" -q "${dev}"; then
 327                                 ban_dev="${ban_dev}${dev} "
 328                                 uci_add_list banip global ban_dev "${dev}"
 329                         else
 330                                 network_get_physdev dev "${iface}"
 331                                 if [ -n "${dev}" ] && ! printf "%s" "${ban_dev}" | "${ban_grepcmd}" -q "${dev}"; then
 332                                         ban_dev="${ban_dev}${dev} "
 333                                         uci_add_list banip global ban_dev "${dev}"
 334                                 fi
 335                         fi
 336                 done
 337                 ban_dev="${ban_dev%%?}"
 338                 [ -n "$(uci -q changes "banip")" ] && uci_commit "banip"
 339         fi
 340         [ -z "${ban_dev}" ] && f_log "err" "wan devices not found, please check your configuration"
 341
 342         f_log "debug" "f_getdev  ::: auto_detect: ${ban_autodetect}, devices: ${ban_dev}"
 343 }
 344
 345 # get local subnets
 346 #
 347 f_getsub() {
 348         local sub iface ip
 349
 350         for iface in ${ban_ifv4} ${ban_ifv6}; do
 351                 network_get_subnet sub "${iface}"
 352                 if [ -n "${sub}" ] && ! printf "%s" "${ban_sub}" | "${ban_grepcmd}" -q "${sub}"; then
 353                         ban_sub="${ban_sub} ${sub}"
 354                 fi
 355                 network_get_subnet6 sub "${iface}"
 356                 if [ -n "${sub}" ] && ! printf "%s" "${ban_sub}" | "${ban_grepcmd}" -q "${sub}"; then
 357                         ban_sub="${ban_sub} ${sub}"
 358                 fi
 359         done
 360         if [ "${ban_autoallowlist}" = "1" ]; then
 361                 for ip in ${ban_sub}; do
 362                         if ! "${ban_grepcmd}" -q "${ip}" "${ban_allowlist}"; then
 363                                 printf "%-42s%s\n" "${ip}" "added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}"
 364                                 f_log "info" "add subnet '${ip}' to local allowlist"
 365                         fi
 366                 done
 367         fi
 368         [ -z "${ban_sub}" ] && f_log "err" "wan subnet(s) not found, please check your configuration"
 369
 370         f_log "debug" "f_getsub  ::: auto_allowlist: ${ban_autoallowlist}, subnet(s): ${ban_sub:-"-"}"
 371 }
 372
 373 # get set elements
 374 #
 375 f_getelements() {
 376         local file="${1}"
 377
 378         [ -s "${file}" ] && printf "%s" "elements={ $(cat "${file}") };"
 379 }
 380
 381 # build initial nft file with base table, chains and rules
 382 #
 383 f_nftinit() {
 384         local feed_log feed_rc file="${1}"
 385
 386         {
 387                 # nft header (tables and chains)
 388                 #
 389                 printf "%s\n\n" "#!/usr/sbin/nft -f"
 390                 if "${ban_nftcmd}" -t list table inet banIP >/dev/null 2>&1; then
 391                         printf "%s\n" "delete table inet banIP"
 392                 fi
 393                 printf "%s\n" "add table inet banIP"
 394                 printf "%s\n" "add chain inet banIP wan-input { type filter hook input priority ${ban_nftpriority}; policy accept; }"
 395                 printf "%s\n" "add chain inet banIP wan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }"
 396                 printf "%s\n" "add chain inet banIP lan-forward { type filter hook forward priority ${ban_nftpriority}; policy accept; }"
 397
 398                 # default wan-input rules
 399                 #
 400                 printf "%s\n" "add rule inet banIP wan-input ct state established,related counter accept"
 401                 printf "%s\n" "add rule inet banIP wan-input iifname != { ${ban_dev// /, } } counter accept"
 402                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv4 udp sport 67-68 udp dport 67-68 counter accept"
 403                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 udp sport 547 udp dport 546 counter accept"
 404                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv4 icmp type { echo-request } limit rate 1000/second counter accept"
 405                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { echo-request } limit rate 1000/second counter accept"
 406                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} limit rate 1000/second ip6 hoplimit 1 counter accept"
 407                 printf "%s\n" "add rule inet banIP wan-input meta nfproto ipv6 icmpv6 type { nd-neighbor-advert, nd-neighbor-solicit, nd-router-advert} limit rate 1000/second ip6 hoplimit 255 counter accept"
 408
 409                 # default wan-forward rules
 410                 #
 411                 printf "%s\n" "add rule inet banIP wan-forward ct state established,related counter accept"
 412                 printf "%s\n" "add rule inet banIP wan-forward iifname != { ${ban_dev// /, } } counter accept"
 413
 414                 # default lan-forward rules
 415                 #
 416                 printf "%s\n" "add rule inet banIP lan-forward ct state established,related counter accept"
 417                 printf "%s\n" "add rule inet banIP lan-forward oifname != { ${ban_dev// /, } } counter accept"
 418         } >"${file}"
 419
 420         # load initial banIP table within nft (atomic load)
 421         #
 422         feed_log="$("${ban_nftcmd}" -f "${file}" 2>&1)"
 423         feed_rc="${?}"
 424
 425         f_log "debug" "f_nftinit ::: devices: ${ban_dev}, priority: ${ban_nftpriority}, rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}"
 426         return ${feed_rc}
 427 }
 428
 429 f_down() {
 430         local log_input log_forwardwan log_forwardlan start_ts end_ts tmp_raw tmp_load tmp_file split_file input_handles forwardwan_handles forwardlan_handles handle
 431         local cnt_set cnt_dl restore_rc feed_direction feed_rc feed_log feed="${1}" proto="${2}" feed_url="${3}" feed_rule="${4}" feed_flag="${5}"
 432
 433         start_ts="$(date +%s)"
 434         feed="${feed}v${proto}"
 435         tmp_load="${ban_tmpfile}.${feed}.load"
 436         tmp_raw="${ban_tmpfile}.${feed}.raw"
 437         tmp_split="${ban_tmpfile}.${feed}.split"
 438         tmp_file="${ban_tmpfile}.${feed}.file"
 439         tmp_flush="${ban_tmpfile}.${feed}.flush"
 440         tmp_nft="${ban_tmpfile}.${feed}.nft"
 441
 442         [ "${ban_loginput}" = "1" ] && log_input="log level ${ban_loglevel} prefix \"banIP/inp-wan/drp/${feed}: \""
 443         [ "${ban_logforwardwan}" = "1" ] && log_forwardwan="log level ${ban_loglevel} prefix \"banIP/fwd-wan/drp/${feed}: \""
 444         [ "${ban_logforwardlan}" = "1" ] && log_forwardlan="log level ${ban_loglevel} prefix \"banIP/fwd-lan/rej/${feed}: \""
 445
 446         # set source block direction
 447         #
 448         if printf "%s" "${ban_blockinput}" | "${ban_grepcmd}" -q "${feed%v*}"; then
 449                 feed_direction="input"
 450         fi
 451         if printf "%s" "${ban_blockforwardwan}" | "${ban_grepcmd}" -q "${feed%v*}"; then
 452                 feed_direction="${feed_direction} forwardwan"
 453         fi
 454         if printf "%s" "${ban_blockforwardlan}" | "${ban_grepcmd}" -q "${feed%v*}"; then
 455                 feed_direction="${feed_direction} forwardlan"
 456         fi
 457
 458         # chain/rule maintenance
 459         #
 460         if [ "${ban_action}" = "reload" ] && "${ban_nftcmd}" -t list set inet banIP "${feed}" >/dev/null 2>&1; then
 461                 input_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP wan-input 2>/dev/null)"
 462                 forwardwan_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP wan-forward 2>/dev/null)"
 463                 forwardlan_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP lan-forward 2>/dev/null)"
 464                 {
 465                         printf "%s\n" "flush set inet banIP ${feed}"
 466                         handle="$(printf "%s\n" "${input_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}")"
 467                         [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-input handle ${handle}"
 468                         handle="$(printf "%s\n" "${forwardwan_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}")"
 469                         [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-forward handle ${handle}"
 470                         handle="$(printf "%s\n" "${forwardlan_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}")"
 471                         [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP lan-forward handle ${handle}"
 472                 } >"${tmp_flush}"
 473         fi
 474
 475         # restore local backups during init
 476         #
 477         if { [ "${ban_action}" != "reload" ] || [ "${feed_url}" = "local" ]; } && [ "${feed%v*}" != "allowlist" ] && [ "${feed%v*}" != "blocklist" ]; then
 478                 f_restore "${feed}" "${feed_url}" "${tmp_load}"
 479                 restore_rc="${?}"
 480                 feed_rc="${restore_rc}"
 481         fi
 482
 483         # handle local lists
 484         #
 485         if [ "${feed%v*}" = "allowlist" ]; then
 486                 {
 487                         printf "%s\n\n" "#!/usr/sbin/nft -f"
 488                         [ -s "${tmp_flush}" ] && cat "${tmp_flush}"
 489                         if [ "${proto}" = "MAC" ]; then
 490                                 "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}([[:space:]]|$)/{printf "%s, ",tolower($1)}' "${ban_allowlist}" >"${tmp_file}"
 491                                 printf "%s\n" "add set inet banIP ${feed} { type ether_addr; policy memory; $(f_getelements "${tmp_file}") }"
 492                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr @${feed} counter accept"
 493                         elif [ "${proto}" = "4" ]; then
 494                                 "${ban_awkcmd}" '/^(([0-9]{1,3}\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]]|$)/{printf "%s, ",$1}' "${ban_allowlist}" >"${tmp_file}"
 495                                 printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy memory; $(f_getelements "${tmp_file}") }"
 496                                 if [ -z "${feed_direction##*input*}" ]; then
 497                                         if [ "${ban_allowlistonly}" = "1" ]; then
 498                                                 printf "%s\n" "add rule inet banIP wan-input ip saddr != @${feed} ${log_input} counter drop"
 499                                         else
 500                                                 printf "%s\n" "add rule inet banIP wan-input ip saddr @${feed} counter accept"
 501                                         fi
 502                                 fi
 503                                 if [ -z "${feed_direction##*forwardwan*}" ]; then
 504                                         if [ "${ban_allowlistonly}" = "1" ]; then
 505                                                 printf "%s\n" "add rule inet banIP wan-forward ip saddr != @${feed} ${log_forwardwan} counter drop"
 506                                         else
 507                                                 printf "%s\n" "add rule inet banIP wan-forward ip saddr @${feed} counter accept"
 508                                         fi
 509                                 fi
 510                                 if [ -z "${feed_direction##*forwardlan*}" ]; then
 511                                         if [ "${ban_allowlistonly}" = "1" ]; then
 512                                                 printf "%s\n" "add rule inet banIP lan-forward ip daddr != @${feed} ${log_forwardlan} counter reject with icmp type admin-prohibited"
 513                                         else
 514                                                 printf "%s\n" "add rule inet banIP lan-forward ip daddr @${feed} counter accept"
 515                                         fi
 516                                 fi
 517                         elif [ "${proto}" = "6" ]; then
 518                                 "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}([[:space:]]|$)/{printf "%s\n",$1}' "${ban_allowlist}" |
 519                                         "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]]|$)/{printf "%s, ",tolower($1)}' >"${tmp_file}"
 520                                 printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy memory; $(f_getelements "${tmp_file}") }"
 521                                 if [ -z "${feed_direction##*input*}" ]; then
 522                                         if [ "${ban_allowlistonly}" = "1" ]; then
 523                                                 printf "%s\n" "add rule inet banIP wan-input ip6 saddr != @${feed} ${log_input} counter drop"
 524                                         else
 525                                                 printf "%s\n" "add rule inet banIP wan-input ip6 saddr @${feed} counter accept"
 526                                         fi
 527                                 fi
 528                                 if [ -z "${feed_direction##*forwardwan*}" ]; then
 529                                         if [ "${ban_allowlistonly}" = "1" ]; then
 530                                                 printf "%s\n" "add rule inet banIP wan-forward ip6 saddr != @${feed} ${log_forwardwan} counter drop"
 531                                         else
 532                                                 printf "%s\n" "add rule inet banIP wan-forward ip6 saddr @${feed} counter accept"
 533                                         fi
 534                                 fi
 535                                 if [ -z "${feed_direction##*forwardlan*}" ]; then
 536                                         if [ "${ban_allowlistonly}" = "1" ]; then
 537                                                 printf "%s\n" "add rule inet banIP lan-forward ip6 daddr != @${feed} ${log_forwardlan} counter reject with icmpv6 type admin-prohibited"
 538                                         else
 539                                                 printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} counter accept"
 540                                         fi
 541                                 fi
 542                         fi
 543                 } >"${tmp_nft}"
 544                 feed_rc="${?}"
 545         elif [ "${feed%v*}" = "blocklist" ]; then
 546                 {
 547                         printf "%s\n\n" "#!/usr/sbin/nft -f"
 548                         [ -s "${tmp_flush}" ] && cat "${tmp_flush}"
 549                         if [ "${proto}" = "MAC" ]; then
 550                                 "${ban_awkcmd}" '/^([0-9A-f]{2}:){5}[0-9A-f]{2}([[:space:]]|$)/{printf "%s, ",tolower($1)}' "${ban_blocklist}" >"${tmp_file}"
 551                                 printf "%s\n" "add set inet banIP ${feed} { type ether_addr; policy memory; $(f_getelements "${tmp_file}") }"
 552                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ether saddr @${feed} ${log_forwardlan} counter reject"
 553                         elif [ "${proto}" = "4" ]; then
 554                                 if [ "${ban_deduplicate}" = "1" ]; then
 555                                         "${ban_awkcmd}" '/^(([0-9]{1,3}\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]]|$)/{printf "%s,\n",$1}' "${ban_blocklist}" >"${tmp_raw}"
 556                                         "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null >"${tmp_split}"
 557                                         "${ban_awkcmd}" 'BEGIN{FS="[ ,]"}NR==FNR{member[$1];next}!($1 in member)' "${ban_tmpfile}.deduplicate" "${ban_blocklist}" 2>/dev/null >"${tmp_raw}"
 558                                         cat "${tmp_raw}" 2>/dev/null >"${ban_blocklist}"
 559                                 else
 560                                         "${ban_awkcmd}" '/^(([0-9]{1,3}\.){3}(1?[0-9][0-9]?|2[0-4][0-9]|25[0-5])(\/(1?[0-9]|2?[0-9]|3?[0-2]))?)([[:space:]]|$)/{printf "%s,\n",$1}' "${ban_blocklist}" >"${tmp_split}"
 561                                 fi
 562                                 "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}"
 563                                 printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval, timeout; auto-merge; policy memory; $(f_getelements "${tmp_file}") }"
 564                                 [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip saddr @${feed} ${log_input} counter drop"
 565                                 [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip saddr @${feed} ${log_forwardwan} counter drop"
 566                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip daddr @${feed} ${log_forwardlan} counter reject with icmp type admin-prohibited"
 567                         elif [ "${proto}" = "6" ]; then
 568                                 if [ "${ban_deduplicate}" = "1" ]; then
 569                                         "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}([[:space:]]|$)/{printf "%s\n",$1}' "${ban_blocklist}" |
 570                                                 "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]]|$)/{printf "%s,\n",tolower($1)}' >"${tmp_raw}"
 571                                         "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null >"${tmp_split}"
 572                                         "${ban_awkcmd}" 'BEGIN{FS="[ ,]"}NR==FNR{member[$1];next}!($1 in member)' "${ban_tmpfile}.deduplicate" "${ban_blocklist}" 2>/dev/null >"${tmp_raw}"
 573                                         cat "${tmp_raw}" 2>/dev/null >"${ban_blocklist}"
 574                                 else
 575                                         "${ban_awkcmd}" '!/^([0-9A-f]{2}:){5}[0-9A-f]{2}([[:space:]]|$)/{printf "%s\n",$1}' "${ban_blocklist}" |
 576                                                 "${ban_awkcmd}" '/^(([0-9A-f]{0,4}:){1,7}[0-9A-f]{0,4}:?(\/(1?[0-2][0-8]|[0-9][0-9]))?)([[:space:]]|$)/{printf "%s,\n",tolower($1)}' >"${tmp_split}"
 577                                 fi
 578                                 "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}"
 579                                 printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval, timeout; auto-merge; policy memory; $(f_getelements "${tmp_file}") }"
 580                                 [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip6 saddr @${feed} ${log_input} counter drop"
 581                                 [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip6 saddr @${feed} ${log_forwardwan} counter drop"
 582                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} ${log_forwardlan} counter reject with icmpv6 type admin-prohibited"
 583                         fi
 584                 } >"${tmp_nft}"
 585                 feed_rc="${?}"
 586         # handle external downloads
 587         #
 588         elif [ "${restore_rc}" != "0" ] && [ "${feed_url}" != "local" ]; then
 589                 # handle country downloads
 590                 #
 591                 if [ "${feed%v*}" = "country" ]; then
 592                         for country in ${ban_country}; do
 593                                 feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}${country}-aggregated.zone" 2>&1)"
 594                                 feed_rc="${?}"
 595                                 [ "${feed_rc}" = "0" ] && cat "${tmp_raw}" 2>/dev/null >>"${tmp_load}"
 596                         done
 597                         rm -f "${tmp_raw}"
 598
 599                 # handle asn downloads
 600                 #
 601                 elif [ "${feed%v*}" = "asn" ]; then
 602                         for asn in ${ban_asn}; do
 603                                 feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}AS${asn}" 2>&1)"
 604                                 feed_rc="${?}"
 605                                 [ "${feed_rc}" = "0" ] && cat "${tmp_raw}" 2>/dev/null >>"${tmp_load}"
 606                         done
 607                         rm -f "${tmp_raw}"
 608
 609                 # handle compressed downloads
 610                 #
 611                 elif [ -n "${feed_flag}" ]; then
 612                         case "${feed_flag}" in
 613                                 "gz")
 614                                         feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_raw}" "${feed_url}" 2>&1)"
 615                                         feed_rc="${?}"
 616                                         if [ "${feed_rc}" = "0" ]; then
 617                                                 zcat "${tmp_raw}" 2>/dev/null >"${tmp_load}"
 618                                                 feed_rc="${?}"
 619                                         fi
 620                                         rm -f "${tmp_raw}"
 621                                         ;;
 622                         esac
 623
 624                 # handle normal downloads
 625                 #
 626                 else
 627                         feed_log="$("${ban_fetchcmd}" ${ban_fetchparm} "${tmp_load}" "${feed_url}" 2>&1)"
 628                         feed_rc="${?}"
 629                 fi
 630         fi
 631
 632         # backup/restore
 633         #
 634         if [ "${restore_rc}" != "0" ] && [ "${feed_rc}" = "0" ] && [ "${feed_url}" != "local" ] && [ ! -s "${tmp_nft}" ]; then
 635                 f_backup "${feed}" "${tmp_load}"
 636                 feed_rc="${?}"
 637         elif [ -z "${restore_rc}" ] && [ "${feed_rc}" != "0" ] && [ "${feed_url}" != "local" ] && [ ! -s "${tmp_nft}" ]; then
 638                 f_restore "${feed}" "${feed_url}" "${tmp_load}" "${feed_rc}"
 639                 feed_rc="${?}"
 640         fi
 641
 642         # build nft file with set and rules for regular downloads
 643         #
 644         if [ "${feed_rc}" = "0" ] && [ ! -s "${tmp_nft}" ]; then
 645                 # deduplicate sets
 646                 #
 647                 if [ "${ban_deduplicate}" = "1" ] && [ "${feed_url}" != "local" ]; then
 648                         "${ban_awkcmd}" "${feed_rule}" "${tmp_load}" 2>/dev/null >"${tmp_raw}"
 649                         "${ban_awkcmd}" 'NR==FNR{member[$0];next}!($0 in member)' "${ban_tmpfile}.deduplicate" "${tmp_raw}" 2>/dev/null | tee -a "${ban_tmpfile}.deduplicate" >"${tmp_split}"
 650                 else
 651                         "${ban_awkcmd}" "${feed_rule}" "${tmp_load}" 2>/dev/null >"${tmp_split}"
 652                 fi
 653                 feed_rc="${?}"
 654                 # split sets
 655                 #
 656                 if [ "${feed_rc}" = "0" ]; then
 657                         if [ -n "${ban_splitsize//[![:digit]]/}" ] && [ "${ban_splitsize//[![:digit]]/}" -gt "0" ]; then
 658                                 if ! "${ban_awkcmd}" "NR%${ban_splitsize//[![:digit]]/}==1{file=\"${tmp_file}.\"++i;}{ORS=\" \";print > file}" "${tmp_split}" 2>/dev/null; then
 659                                         rm -f "${tmp_file}".*
 660                                         f_log "info" "failed to split ${feed} set to size '${ban_splitsize//[![:digit]]/}'"
 661                                 fi
 662                         else
 663                                 "${ban_awkcmd}" '{ORS=" ";print}' "${tmp_split}" 2>/dev/null >"${tmp_file}.1"
 664                         fi
 665                         feed_rc="${?}"
 666                 fi
 667                 rm -f "${tmp_raw}" "${tmp_load}"
 668                 if [ "${feed_rc}" = "0" ] && [ "${proto}" = "4" ]; then
 669                         {
 670                                 # nft header (IPv4 set)
 671                                 #
 672                                 printf "%s\n\n" "#!/usr/sbin/nft -f"
 673                                 [ -s "${tmp_flush}" ] && cat "${tmp_flush}"
 674                                 printf "%s\n" "add set inet banIP ${feed} { type ipv4_addr; flags interval; auto-merge; policy memory; $(f_getelements "${tmp_file}.1") }"
 675
 676                                 # input and forward rules
 677                                 #
 678                                 [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip saddr @${feed} ${log_input} counter drop"
 679                                 [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip saddr @${feed} ${log_forwardwan} counter drop"
 680                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip daddr @${feed} ${log_forwardlan} counter reject with icmp type admin-prohibited"
 681                         } >"${tmp_nft}"
 682                 elif [ "${feed_rc}" = "0" ] && [ "${proto}" = "6" ]; then
 683                         {
 684                                 # nft header (IPv6 set)
 685                                 #
 686                                 printf "%s\n\n" "#!/usr/sbin/nft -f"
 687                                 [ -s "${tmp_flush}" ] && cat "${tmp_flush}"
 688                                 printf "%s\n" "add set inet banIP ${feed} { type ipv6_addr; flags interval; auto-merge; policy memory; $(f_getelements "${tmp_file}.1") }"
 689
 690                                 # input and forward rules
 691                                 #
 692                                 [ -z "${feed_direction##*input*}" ] && printf "%s\n" "add rule inet banIP wan-input ip6 saddr @${feed} ${log_input} counter drop"
 693                                 [ -z "${feed_direction##*forwardwan*}" ] && printf "%s\n" "add rule inet banIP wan-forward ip6 saddr @${feed} ${log_forwardwan} counter drop"
 694                                 [ -z "${feed_direction##*forwardlan*}" ] && printf "%s\n" "add rule inet banIP lan-forward ip6 daddr @${feed} ${log_forwardlan} counter reject with icmpv6 type admin-prohibited"
 695                         } >"${tmp_nft}"
 696                 fi
 697         fi
 698
 699         # load generated nft file in banIP table
 700         #
 701         if [ "${feed_rc}" = "0" ]; then
 702                 cnt_dl="$("${ban_awkcmd}" 'END{printf "%d",NR}' "${tmp_split}" 2>/dev/null)"
 703                 if [ "${cnt_dl:-"0"}" -gt "0" ] || [ "${feed_url}" = "local" ] || [ "${feed%v*}" = "allowlist" ] || [ "${feed%v*}" = "blocklist" ]; then
 704                         feed_log="$("${ban_nftcmd}" -f "${tmp_nft}" 2>&1)"
 705                         feed_rc="${?}"
 706                         # load additional split files
 707                         #
 708                         if [ "${feed_rc}" = "0" ]; then
 709                                 for split_file in "${tmp_file}".*; do
 710                                         [ ! -f "${split_file}" ] && break
 711                                         if [ "${split_file##*.}" = "1" ]; then
 712                                                 rm -f "${split_file}"
 713                                                 continue
 714                                         fi
 715                                         if ! "${ban_nftcmd}" add element inet banIP "${feed}" "{ $(cat "${split_file}") }" >/dev/null 2>&1; then
 716                                                 f_log "info" "failed to add split file '${split_file##*.}' to ${feed} set"
 717                                         fi
 718                                         rm -f "${split_file}"
 719                                 done
 720                                 cnt_set="$("${ban_nftcmd}" -j list set inet banIP "${feed}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)"
 721                         fi
 722                 else
 723                         f_log "info" "empty feed ${feed} will be skipped"
 724                 fi
 725         fi
 726         rm -f "${tmp_split}" "${tmp_nft}"
 727         end_ts="$(date +%s)"
 728
 729         f_log "debug" "f_down    ::: name: ${feed}, cnt_dl: ${cnt_dl:-"-"}, cnt_set: ${cnt_set:-"-"}, split_size: ${ban_splitsize:-"-"}, time: $((end_ts - start_ts)), rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}"
 730 }
 731
 732 # backup feeds
 733 #
 734 f_backup() {
 735         local backup_rc feed="${1}" feed_file="${2}"
 736
 737         gzip -cf "${feed_file}" >"${ban_backupdir}/banIP.${feed}.gz"
 738         backup_rc="${?}"
 739
 740         f_log "debug" "f_backup  ::: name: ${feed}, source: ${feed_file##*/}, target: banIP.${feed}.gz, rc: ${backup_rc}"
 741         return ${backup_rc}
 742 }
 743
 744 # restore feeds
 745 #
 746 f_restore() {
 747         local tmp_feed restore_rc="1" feed="${1}" feed_url="${2}" feed_file="${3}" feed_rc="${4:-"0"}"
 748
 749         [ "${feed_rc}" != "0" ] && restore_rc="${feed_rc}"
 750         [ "${feed_url}" = "local" ] && tmp_feed="${feed%v*}v4" || tmp_feed="${feed}"
 751         if [ -f "${ban_backupdir}/banIP.${tmp_feed}.gz" ]; then
 752                 zcat "${ban_backupdir}/banIP.${tmp_feed}.gz" 2>/dev/null >"${feed_file}"
 753                 restore_rc="${?}"
 754         fi
 755
 756         f_log "debug" "f_restore ::: name: ${feed}, source: banIP.${tmp_feed}.gz, target: ${feed_file##*/}, in_rc: ${feed_rc}, rc: ${restore_rc}"
 757         return ${restore_rc}
 758 }
 759
 760 # remove disabled feeds
 761 #
 762 f_rmset() {
 763         local tmp_del table_sets input_handles forwardwan_handles forwardlan_handles handle sets feed feed_log feed_rc
 764
 765         tmp_del="${ban_tmpfile}.final.delete"
 766         table_sets="$("${ban_nftcmd}" -t list table inet banIP 2>/dev/null | "${ban_awkcmd}" '/^[[:space:]]+set [[:alnum:]]+ /{printf "%s ",$2}' 2>/dev/null)"
 767         input_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP wan-input 2>/dev/null)"
 768         forwardwan_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP wan-forward 2>/dev/null)"
 769         forwardlan_handles="$("${ban_nftcmd}" -t --handle --numeric list chain inet banIP lan-forward 2>/dev/null)"
 770         {
 771                 printf "%s\n\n" "#!/usr/sbin/nft -f"
 772                 for feed in ${table_sets}; do
 773                         if ! printf "%s" "allowlist blocklist ${ban_feed}" | "${ban_grepcmd}" -q "${feed%v*}"; then
 774                                 sets="${sets}${feed}/"
 775                                 rm -f "${ban_backupdir}/banIP.${feed}.gz"
 776                                 printf "%s\n" "flush set inet banIP ${feed}"
 777                                 handle="$(printf "%s\n" "${input_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}" 2>/dev/null)"
 778                                 [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-input handle ${handle}"
 779                                 handle="$(printf "%s\n" "${forwardwan_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}" 2>/dev/null)"
 780                                 [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP wan-forward handle ${handle}"
 781                                 handle="$(printf "%s\n" "${forwardlan_handles}" | "${ban_awkcmd}" "/@${feed} /{print \$NF}" 2>/dev/null)"
 782                                 [ -n "${handle}" ] && printf "%s\n" "delete rule inet banIP lan-forward handle ${handle}"
 783                                 printf "%s\n\n" "delete set inet banIP ${feed}"
 784                         fi
 785                 done
 786         } >"${tmp_del}"
 787
 788         if [ -n "${sets}" ]; then
 789                 feed_log="$("${ban_nftcmd}" -f "${tmp_del}" 2>&1)"
 790                 feed_rc="${?}"
 791         fi
 792         rm -f "${tmp_del}"
 793
 794         f_log "debug" "f_rmset   ::: sets: ${sets:-"-"}, tmp: ${tmp_del}, rc: ${feed_rc:-"-"}, log: ${feed_log:-"-"}"
 795 }
 796
 797 # generate status information
 798 #
 799 f_genstatus() {
 800         local object duration nft_table nft_feeds cnt_elements="0" split="0" status="${1}"
 801
 802         [ -z "${ban_dev}" ] && f_conf
 803         if [ "${status}" = "active" ]; then
 804                 if [ -n "${ban_starttime}" ]; then
 805                         ban_endtime="$(date "+%s")"
 806                         duration="$(((ban_endtime - ban_starttime) / 60))m $(((ban_endtime - ban_starttime) % 60))s"
 807                 fi
 808                 nft_table="$("${ban_nftcmd}" -t list table inet banIP 2>/dev/null)"
 809                 nft_feeds="$(f_trim "$(printf "%s\n" "${nft_table}" | "${ban_awkcmd}" '/^[[:space:]]+set [[:alnum:]]+ /{printf "%s ",$2}')")"
 810                 for object in ${nft_feeds}; do
 811                         cnt_elements="$((cnt_elements + $("${ban_nftcmd}" -j list set inet banIP "${object}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)))"
 812                 done
 813                 runtime="action: ${ban_action:-"-"}, duration: ${duration:-"-"}, date: $(date "+%Y-%m-%d %H:%M:%S")"
 814         fi
 815         f_system
 816         [ ${ban_splitsize:-"0"} -gt "0" ] && split="1"
 817
 818         : >"${ban_basedir}/ban_runtime.json"
 819         json_init
 820         json_load_file "${ban_basedir}/ban_runtime.json" >/dev/null 2>&1
 821         json_add_string "status" "${status}"
 822         json_add_string "version" "${ban_ver}"
 823         json_add_string "element_count" "${cnt_elements}"
 824         json_add_array "active_feeds"
 825         if [ "${status}" != "active" ]; then
 826                 json_add_object
 827                 json_add_string "feed" "-"
 828                 json_close_object
 829         else
 830                 for object in ${nft_feeds}; do
 831                         json_add_object
 832                         json_add_string "feed" "${object}"
 833                         json_close_object
 834                 done
 835         fi
 836         json_close_array
 837         json_add_array "active_devices"
 838         if [ "${status}" != "active" ]; then
 839                 json_add_object
 840                 json_add_string "device" "-"
 841                 json_close_object
 842         else
 843                 for object in ${ban_dev}; do
 844                         json_add_object
 845                         json_add_string "device" "${object}"
 846                         json_close_object
 847                 done
 848         fi
 849         json_close_array
 850         json_add_array "active_interfaces"
 851         if [ "${status}" != "active" ]; then
 852                 json_add_object
 853                 json_add_string "interface" "-"
 854                 json_close_object
 855         else
 856                 for object in ${ban_ifv4} ${ban_ifv6}; do
 857                         json_add_object
 858                         json_add_string "interface" "${object}"
 859                         json_close_object
 860                 done
 861         fi
 862         json_close_array
 863         json_add_array "active_subnets"
 864         if [ "${status}" != "active" ]; then
 865                 json_add_object
 866                 json_add_string "subnet" "-"
 867                 json_close_object
 868         else
 869                 for object in ${ban_sub}; do
 870                         json_add_object
 871                         json_add_string "subnet" "${object}"
 872                         json_close_object
 873                 done
 874         fi
 875         json_close_array
 876         json_add_string "run_info" "base: ${ban_basedir}, backup: ${ban_backupdir}, report: ${ban_reportdir}, feed: ${ban_feedfile}"
 877         json_add_string "run_flags" "protocol (4/6): $(f_char ${ban_protov4})/$(f_char ${ban_protov6}), log (wan-inp/wan-fwd/lan-fwd): $(f_char ${ban_loginput})/$(f_char ${ban_logforwardwan})/$(f_char ${ban_logforwardlan}), deduplicate: $(f_char ${ban_deduplicate}), split: $(f_char ${split}), allowed only: $(f_char ${ban_allowlistonly})"
 878         json_add_string "last_run" "${runtime:-"-"}"
 879         json_add_string "system_info" "cores: ${ban_cores}, memory: ${ban_memory}, device: ${ban_sysver}"
 880         json_dump >"${ban_basedir}/ban_runtime.json"
 881 }
 882
 883 # get status information
 884 #
 885 f_getstatus() {
 886         local key keylist type value index_value
 887
 888         [ -z "${ban_dev}" ] && f_conf
 889         json_load_file "${ban_basedir}/ban_runtime.json" >/dev/null 2>&1
 890         if json_get_keys keylist; then
 891                 printf "%s\n" "::: banIP runtime information"
 892                 for key in ${keylist}; do
 893                         json_get_var value "${key}" >/dev/null 2>&1
 894                         if [ "${key%_*}" = "active" ]; then
 895                                 json_select "${key}" >/dev/null 2>&1
 896                                 index=1
 897                                 while json_get_type type "${index}" && [ "${type}" = "object" ]; do
 898                                         json_get_values index_value "${index}" >/dev/null 2>&1
 899                                         if [ "${index}" = "1" ]; then
 900                                                 value="${index_value}"
 901                                         else
 902                                                 value="${value}, ${index_value}"
 903                                         fi
 904                                         index=$((index + 1))
 905                                 done
 906                                 json_select ".."
 907                         fi
 908                         value="$(
 909                                 printf "%s" "${value}" |
 910                                         awk '{NR=1;max=118;if(length($0)>max+1)while($0){if(NR==1){print substr($0,1,max)}else{printf"%-24s%s\n","",substr($0,1,max)}{$0=substr($0,max+1);NR=NR+1}}else print}'
 911                         )"
 912                         printf "  + %-17s : %s\n" "${key}" "${value:-"-"}"
 913                 done
 914         else
 915                 printf "%s\n" "::: no banIP runtime information available"
 916         fi
 917 }
 918
 919 # domain lookup
 920 #
 921 f_lookup() {
 922         local cnt list domain lookup ip start_time end_time duration cnt_domain="0" cnt_ip="0" feed="${1}"
 923
 924         start_time="$(date "+%s")"
 925         if [ "${feed}" = "allowlist" ]; then
 926                 list="$("${ban_awkcmd}" '/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_allowlist}" 2>/dev/null)"
 927         elif [ "${feed}" = "blocklist" ]; then
 928                 list="$("${ban_awkcmd}" '/^([[:alnum:]_-]{1,63}\.)+[[:alpha:]]+([[:space:]]|$)/{printf "%s ",tolower($1)}' "${ban_blocklist}" 2>/dev/null)"
 929         fi
 930
 931         for domain in ${list}; do
 932                 lookup="$("${ban_lookupcmd}" "${domain}" ${ban_resolver} 2>/dev/null | "${ban_awkcmd}" '/^Address[ 0-9]*: /{if(!seen[$NF]++)printf "%s ",$NF}' 2>/dev/null)"
 933                 for ip in ${lookup}; do
 934                         if [ "${ip%%.*}" = "0" ] || [ -z "${ip%%::*}" ]; then
 935                                 continue
 936                         else
 937                                 if { [ "${feed}" = "allowlist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_allowlist}"; } ||
 938                                         { [ "${feed}" = "blocklist" ] && ! "${ban_grepcmd}" -q "^${ip}" "${ban_blocklist}"; }; then
 939                                         cnt_ip="$((cnt_ip + 1))"
 940                                         if [ "${ip##*:}" = "${ip}" ]; then
 941                                                 if ! "${ban_nftcmd}" add element inet banIP "${feed}v4" "{ ${ip} }" >/dev/null 2>&1; then
 942                                                         f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v4 set"
 943                                                         continue
 944                                                 fi
 945                                         else
 946                                                 if ! "${ban_nftcmd}" add element inet banIP "${feed}v6" "{ ${ip} }" >/dev/null 2>&1; then
 947                                                         f_log "info" "failed to add IP '${ip}' (${domain}) to ${feed}v6 set"
 948                                                         continue
 949                                                 fi
 950                                         fi
 951                                         if [ "${feed}" = "allowlist" ] && [ "${ban_autoallowlist}" = "1" ]; then
 952                                                 printf "%-42s%s\n" "${ip}" "# ip of '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_allowlist}"
 953                                         elif [ "${feed}" = "blocklist" ] && [ "${ban_autoblocklist}" = "1" ]; then
 954                                                 printf "%-42s%s\n" "${ip}" "# ip of '${domain}' added on $(date "+%Y-%m-%d %H:%M:%S")" >>"${ban_blocklist}"
 955                                         fi
 956                                 fi
 957                         fi
 958                 done
 959                 cnt_domain="$((cnt_domain + 1))"
 960         done
 961         end_time="$(date "+%s")"
 962         duration="$(((end_time - start_time) / 60))m $(((end_time - start_time) % 60))s"
 963
 964         f_log "debug" "f_lookup  ::: name: ${feed}, cnt_domain: ${cnt_domain}, cnt_ip: ${cnt_ip}, duration: ${duration}"
 965 }
 966
 967 # banIP table statistics
 968 #
 969 f_report() {
 970         local report_jsn report_txt set tmp_val nft_raw nft_sets set_cnt set_input set_forwardwan set_forwardlan set_cntinput set_cntforwardwan set_cntforwardlan output="${1}"
 971         local detail set_details jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinput sum_setforwardwan sum_setforwardlan sum_setelements sum_cntinput sum_cntforwardwan sum_cntforwardlan
 972
 973         [ -z "${ban_dev}" ] && f_conf
 974         f_mkdir "${ban_reportdir}"
 975         report_jsn="${ban_reportdir}/ban_report.jsn"
 976         report_txt="${ban_reportdir}/ban_report.txt"
 977
 978         # json output preparation
 979         #
 980         nft_raw="$("${ban_nftcmd}" -tj list table inet banIP 2>/dev/null)"
 981         nft_sets="$(printf "%s" "${nft_raw}" | jsonfilter -qe '@.nftables[*].set.name')"
 982         sum_sets="0"
 983         sum_setinput="0"
 984         sum_setforwardwan="0"
 985         sum_setforwardlan="0"
 986         sum_setelements="0"
 987         sum_cntinput="0"
 988         sum_cntforwardwan="0"
 989         sum_cntforwardlan="0"
 990         timestamp="$(date "+%Y-%m-%d %H:%M:%S")"
 991         : >"${report_jsn}"
 992         {
 993                 printf "%s\n" "{"
 994                 printf "\t%s\n" '"sets": {'
 995                 for set in ${nft_sets}; do
 996                         set_cnt="$("${ban_nftcmd}" -j list set inet banIP "${set}" 2>/dev/null | jsonfilter -qe '@.nftables[*].set.elem[*]' | wc -l 2>/dev/null)"
 997                         sum_setelements="$((sum_setelements + set_cnt))"
 998                         set_cntinput="$(printf "%s" "${nft_raw}" | jsonfilter -qe "@.nftables[@.rule.chain=\"wan-input\"][@.expr[*].match.right=\"@${set}\"].expr[*].counter.packets")"
 999                         set_cntforwardwan="$(printf "%s" "${nft_raw}" | jsonfilter -qe "@.nftables[@.rule.chain=\"wan-forward\"][@.expr[*].match.right=\"@${set}\"].expr[*].counter.packets")"
1000                         set_cntforwardlan="$(printf "%s" "${nft_raw}" | jsonfilter -qe "@.nftables[@.rule.chain=\"lan-forward\"][@.expr[*].match.right=\"@${set}\"].expr[*].counter.packets")"
1001                         if [ -n "${set_cntinput}" ]; then
1002                                 set_input="OK"
1003                                 sum_setinput="$((sum_setinput + 1))"
1004                                 sum_cntinput="$((sum_cntinput + set_cntinput))"
1005                         else
1006                                 set_input="-"
1007                                 set_cntinput=""
1008                         fi
1009                         if [ -n "${set_cntforwardwan}" ]; then
1010                                 set_forwardwan="OK"
1011                                 sum_setforwardwan="$((sum_setforwardwan + 1))"
1012                                 sum_cntforwardwan="$((sum_cntforwardwan + set_cntforwardwan))"
1013                         else
1014                                 set_forwardwan="-"
1015                                 set_cntforwardwan=""
1016                         fi
1017                         if [ -n "${set_cntforwardlan}" ]; then
1018                                 set_forwardlan="OK"
1019                                 sum_setforwardlan="$((sum_setforwardlan + 1))"
1020                                 sum_cntforwardlan="$((sum_cntforwardlan + set_cntforwardlan))"
1021                         else
1022                                 set_forwardlan="-"
1023                                 set_cntforwardlan=""
1024                         fi
1025                         [ "${sum_sets}" -gt "0" ] && printf "%s\n" ","
1026                         printf "\t\t%s\n" "\"${set}\": {"
1027                         printf "\t\t\t%s\n" "\"cnt_elements\": \"${set_cnt}\","
1028                         printf "\t\t\t%s\n" "\"cnt_input\": \"${set_cntinput}\","
1029                         printf "\t\t\t%s\n" "\"input\": \"${set_input}\","
1030                         printf "\t\t\t%s\n" "\"cnt_forwardwan\": \"${set_cntforwardwan}\","
1031                         printf "\t\t\t%s\n" "\"wan_forward\": \"${set_forwardwan}\","
1032                         printf "\t\t\t%s\n" "\"cnt_forwardlan\": \"${set_cntforwardlan}\","
1033                         printf "\t\t\t%s\n" "\"lan_forward\": \"${set_forwardlan}\""
1034                         printf "\t\t%s" "}"
1035                         sum_sets="$((sum_sets + 1))"
1036                 done
1037                 printf "\n\t%s\n" "},"
1038                 printf "\t%s\n" "\"timestamp\": \"${timestamp}\","
1039                 printf "\t%s\n" "\"autoadd_allow\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_allowlist}")\","
1040                 printf "\t%s\n" "\"autoadd_block\": \"$("${ban_grepcmd}" -c "added on ${timestamp% *}" "${ban_blocklist}")\","
1041                 printf "\t%s\n" "\"sum_sets\": \"${sum_sets}\","
1042                 printf "\t%s\n" "\"sum_setinput\": \"${sum_setinput}\","
1043                 printf "\t%s\n" "\"sum_setforwardwan\": \"${sum_setforwardwan}\","
1044                 printf "\t%s\n" "\"sum_setforwardlan\": \"${sum_setforwardlan}\","
1045                 printf "\t%s\n" "\"sum_setelements\": \"${sum_setelements}\","
1046                 printf "\t%s\n" "\"sum_cntinput\": \"${sum_cntinput}\","
1047                 printf "\t%s\n" "\"sum_cntforwardwan\": \"${sum_cntforwardwan}\","
1048                 printf "\t%s\n" "\"sum_cntforwardlan\": \"${sum_cntforwardlan}\""
1049                 printf "%s\n" "}"
1050         } >>"${report_jsn}"
1051
1052         # text output preparation
1053         #
1054         if [ "${output}" != "json" ] && [ -s "${report_jsn}" ]; then
1055                 : >"${report_txt}"
1056                 json_init
1057                 if json_load_file "${report_jsn}" >/dev/null 2>&1; then
1058                         json_get_var timestamp "timestamp" >/dev/null 2>&1
1059                         json_get_var autoadd_allow "autoadd_allow" >/dev/null 2>&1
1060                         json_get_var autoadd_block "autoadd_block" >/dev/null 2>&1
1061                         json_get_var sum_sets "sum_sets" >/dev/null 2>&1
1062                         json_get_var sum_setinput "sum_setinput" >/dev/null 2>&1
1063                         json_get_var sum_setforwardwan "sum_setforwardwan" >/dev/null 2>&1
1064                         json_get_var sum_setforwardlan "sum_setforwardlan" >/dev/null 2>&1
1065                         json_get_var sum_setelements "sum_setelements" >/dev/null 2>&1
1066                         json_get_var sum_cntinput "sum_cntinput" >/dev/null 2>&1
1067                         json_get_var sum_cntforwardwan "sum_cntforwardwan" >/dev/null 2>&1
1068                         json_get_var sum_cntforwardlan "sum_cntforwardlan" >/dev/null 2>&1
1069                         {
1070                                 printf "%s\n%s\n%s\n" ":::" "::: banIP Set Statistics" ":::"
1071                                 printf "%s\n" "    Timestamp: ${timestamp}"
1072                                 printf "%s\n" "    ------------------------------"
1073                                 printf "%s\n" "    auto-added to allowlist: ${autoadd_allow}"
1074                                 printf "%s\n\n" "    auto-added to blocklist: ${autoadd_block}"
1075                                 json_select "sets" >/dev/null 2>&1
1076                                 json_get_keys nft_sets >/dev/null 2>&1
1077                                 if [ -n "${nft_sets}" ]; then
1078                                         printf "%-25s%-15s%-24s%-24s%s\n" "    Set" "| Elements" "| WAN-Input (packets)" "| WAN-Forward (packets)" "| LAN-Forward (packets)"
1079                                         printf "%s\n" "    ---------------------+--------------+-----------------------+-----------------------+------------------------"
1080                                         for set in ${nft_sets}; do
1081                                                 printf "    %-21s" "${set}"
1082                                                 json_select "${set}"
1083                                                 json_get_keys set_details
1084                                                 for detail in ${set_details}; do
1085                                                         json_get_var jsnval "${detail}" >/dev/null 2>&1
1086                                                         case "${detail}" in
1087                                                                 "cnt_elements")
1088                                                                         printf "%-15s" "| ${jsnval}"
1089                                                                         ;;
1090                                                                 "cnt_input" | "cnt_forwardwan" | "cnt_forwardlan")
1091                                                                         [ -n "${jsnval}" ] && tmp_val=": ${jsnval}"
1092                                                                         ;;
1093                                                                 *)
1094                                                                         printf "%-24s" "| ${jsnval}${tmp_val}"
1095                                                                         tmp_val=""
1096                                                                         ;;
1097                                                         esac
1098                                                 done
1099                                                 printf "\n"
1100                                                 json_select ".."
1101                                         done
1102                                         printf "%s\n" "    ---------------------+--------------+-----------------------+-----------------------+------------------------"
1103                                         printf "%-25s%-15s%-24s%-24s%s\n" "    ${sum_sets}" "| ${sum_setelements}" "| ${sum_setinput} (${sum_cntinput})" "| ${sum_setforwardwan} (${sum_cntforwardwan})" "| ${sum_setforwardlan} (${sum_cntforwardlan})"
1104                                 fi
1105                         } >>"${report_txt}"
1106                 fi
1107         fi
1108
1109         # output channel (text|json|mail)
1110         #
1111         case "${output}" in
1112                 "text")
1113                         [ -s "${report_txt}" ] && cat "${report_txt}"
1114                         ;;
1115                 "json")
1116                         [ -s "${report_jsn}" ] && cat "${report_jsn}"
1117                         ;;
1118                 "mail")
1119                         [ -x "${ban_mailcmd}" ] && f_mail
1120                         ;;
1121         esac
1122 }
1123
1124 # banIP set search
1125 #
1126 f_search() {
1127         local nft_sets ip proto run_search search="${1}"
1128
1129         f_system
1130         run_search="/var/run/banIP.search"
1131
1132         if [ -n "${search}" ]; then
1133                 ip="$(printf "%s" "${search}" | "${ban_awkcmd}" 'BEGIN{RS="(([0-9]{1,3}\\.){3}[0-9]{1,3})+"}{printf "%s",RT}')"
1134                 [ -n "${ip}" ] && proto="v4"
1135                 if [ -z "${proto}" ]; then
1136                         ip="$(printf "%s" "${search}" | "${ban_awkcmd}" 'BEGIN{RS="([A-Fa-f0-9]{1,4}::?){3,7}[A-Fa-f0-9]{1,4}"}{printf "%s",RT}')"
1137                         [ -n "${ip}" ] && proto="v6"
1138                 fi
1139                 if [ -n "${proto}" ]; then
1140                         nft_sets="$("${ban_nftcmd}" -tj list table inet banIP 2>/dev/null | jsonfilter -qe "@.nftables[@.set.type=\"ip${proto}_addr\"].set.name")"
1141                 else
1142                         printf "%s\n%s\n%s\n" ":::" "::: no valid search input (single IPv4/IPv6 address)" ":::"
1143                         return
1144                 fi
1145         else
1146                 printf "%s\n%s\n%s\n" ":::" "::: no valid search input (single IPv4/IPv6 address)" ":::"
1147                 return
1148         fi
1149         printf "%s\n%s\n%s\n" ":::" "::: banIP Search" ":::"
1150         printf "%s\n" "    Looking for IP ${ip} on $(date "+%Y-%m-%d %H:%M:%S")"
1151         printf "%s\n" "    ---"
1152         cnt=1
1153         for set in ${nft_sets}; do
1154                 (
1155                         if "${ban_nftcmd}" get element inet banIP "${set}" "{ ${ip} }" >/dev/null 2>&1; then
1156                                 printf "%s\n" "    IP found in set ${set}"
1157                                 : >"${run_search}"
1158                         fi
1159                 ) &
1160                 hold="$((cnt % ban_cores))"
1161                 [ "${hold}" = "0" ] && wait
1162                 cnt="$((cnt + 1))"
1163         done
1164         wait
1165         [ ! -f "${run_search}" ] && printf "%s\n" "    IP not found"
1166         rm -f "${run_search}"
1167 }
1168
1169 # send status mails
1170 #
1171 f_mail() {
1172         local msmtp_debug
1173
1174         # load mail template
1175         #
1176         [ ! -r "${ban_mailtemplate}" ] && f_log "err" "the mail template is missing"
1177         . "${ban_mailtemplate}"
1178
1179         [ -z "${ban_mailreceiver}" ] && f_log "err" "the option 'ban_mailreceiver' is missing"
1180         [ -z "${mail_text}" ] && f_log "err" "the 'mail_text' is empty"
1181         [ "${ban_debug}" = "1" ] && msmtp_debug="--debug"
1182
1183         # send mail
1184         #
1185         ban_mailhead="From: ${ban_mailsender}\nTo: ${ban_mailreceiver}\nSubject: ${ban_mailtopic}\nReply-to: ${ban_mailsender}\nMime-Version: 1.0\nContent-Type: text/html;charset=utf-8\nContent-Disposition: inline\n\n"
1186         if printf "%b" "${ban_mailhead}${mail_text}" | "${ban_mailcmd}" --timeout=10 ${msmtp_debug} -a "${ban_mailprofile}" "${ban_mailreceiver}" >/dev/null 2>&1; then
1187                 f_log "info" "status mail was sent successfully"
1188         else
1189                 f_log "info" "failed to send status mail (${?})"
1190         fi
1191
1192         f_log "debug" "f_mail    ::: template: ${ban_mailtemplate}, profile: ${ban_mailprofile}, receiver: ${ban_mailreceiver}, rc: ${?}"
1193 }
1194
1195 # check banIP availability and initial sourcing
1196 #
1197 if [ "${ban_action}" != "stop" ]; then
1198         if [ -r "/lib/functions.sh" ] && [ -r "/lib/functions/network.sh" ] && [ -r "/usr/share/libubox/jshn.sh" ]; then
1199                 . "/lib/functions.sh"
1200                 . "/lib/functions/network.sh"
1201                 . "/usr/share/libubox/jshn.sh"
1202         else
1203                 f_log "err" "system libraries not found"
1204         fi
1205         [ ! -d "/etc/banip" ] && f_log "err" "banIP config directory not found, please re-install the package"
1206         [ ! -r "/etc/banip/banip.feeds" ] && f_log "err" "banIP feed file not found, please re-install the package"
1207         [ ! -r "/etc/config/banip" ] && f_log "err" "banIP config not found, please re-install the package"
1208         [ "$(uci_get banip global ban_enabled)" = "0" ] && f_log "err" "banIP is currently disabled, please set the config option 'ban_enabled' to '1' to use this service"
1209 fi