net/adblock/files/adblock-helper.sh

   1 #!/bin/sh
   2 ##############################################
   3 # function library used by adblock-update.sh #
   4 # written by Dirk Brenken (dirk@brenken.org) #
   5 ##############################################
   6
   7 #####################################
   8 # f_envload: load adblock environment
   9 #
  10 f_envload()
  11 {
  12     # source in openwrt function library
  13     #
  14     if [ -r "/lib/functions.sh" ]
  15     then
  16         . /lib/functions.sh
  17     else
  18         /usr/bin/logger -t "adblock[${pid}]" "error: openwrt function library not found"
  19         f_deltemp
  20         exit 10
  21     fi
  22
  23     # source in openwrt json helpers library
  24     #
  25     if [ -r "/usr/share/libubox/jshn.sh" ]
  26     then
  27         . "/usr/share/libubox/jshn.sh"
  28     else
  29         /usr/bin/logger -t "adblock[${pid}]" "error: openwrt json helpers library not found"
  30         f_deltemp
  31         exit 15
  32     fi
  33
  34     # get list with all installed openwrt packages
  35     #
  36     pkg_list="$(opkg list-installed 2>/dev/null)"
  37     if [ -z "${pkg_list}" ]
  38     then
  39         /usr/bin/logger -t "adblock[${pid}]" "error: empty openwrt package list"
  40         f_deltemp
  41         exit 20
  42     fi
  43 }
  44
  45 ######################################################
  46 # f_envparse: parse adblock config and set environment
  47 #
  48 f_envparse()
  49 {
  50     # function to read/set global options by callback,
  51     # prepare list items and build option list for all others
  52     #
  53     config_cb()
  54     {
  55         local type="${1}"
  56         local name="${2}"
  57         if [ "${type}" = "adblock" ]
  58         then
  59             option_cb()
  60             {
  61                 local option="${1}"
  62                 local value="${2}"
  63                 eval "${option}=\"${value}\""
  64             }
  65         else
  66             option_cb()
  67             {
  68                 local option="${1}"
  69                 local value="${2}"
  70                 local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p')"
  71                 if [ -z "${opt_out}" ]
  72                 then
  73                     all_options="${all_options} ${option}"
  74                 fi
  75             }
  76             list_cb()
  77             {
  78                 local list="${1}"
  79                 local value="${2}"
  80                 if [ "${list}" = "adb_wanlist" ]
  81                 then
  82                     adb_wandev="${adb_wandev} ${value}"
  83                 elif [ "${list}" = "adb_ntplist" ]
  84                 then
  85                     adb_ntpsrv="${adb_ntpsrv} ${value}"
  86                 elif [ "${list}" = "adb_catlist" ]
  87                 then
  88                     adb_cat_shalla="${adb_cat_shalla} ${value}"
  89                 fi
  90             }
  91         fi
  92     }
  93
  94     # function to iterate through option list, read/set all options in "enabled" sections
  95     #
  96     parse_config()
  97     {
  98         local config="${1}"
  99         config_get switch "${config}" "enabled"
 100         if [ "${switch}" = "1" ]
 101         then
 102             for option in ${all_options}
 103             do
 104                 config_get value "${config}" "${option}"
 105                 if [ -n "${value}" ]
 106                 then
 107                     local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p')"
 108                     if [ -n "${opt_src}" ]
 109                     then
 110                         adb_sources="${adb_sources} ${value}"
 111                     else
 112                         eval "${option}=\"${value}\""
 113                     fi
 114                 fi
 115             done
 116         elif [ "${config}" = "wancheck" ]
 117         then
 118            unset adb_wandev
 119         elif [ "${config}" = "ntpcheck" ]
 120         then
 121            unset adb_ntpsrv
 122         elif [ "${config}" = "shalla" ]
 123         then
 124            unset adb_cat_shalla
 125         fi
 126     }
 127
 128     # load adblock config and start parsing functions
 129     #
 130     config_load adblock
 131     config_foreach parse_config service
 132     config_foreach parse_config source
 133
 134     # set temp variables and counter
 135     #
 136     adb_tmpfile="$(mktemp -tu)"
 137     adb_tmpdir="$(mktemp -d)"
 138     cnt=0
 139     max_cnt=30
 140     max_time=60
 141
 142     # set adblock source ruleset definitions
 143     #
 144     rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost//g; s/[\^#/:;_\.\t ]*$//g'"
 145     rset_end="sed '/^[#/:;_\s]*$/d'"
 146     rset_default="${rset_start} | ${rset_end}"
 147     rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
 148     rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
 149     rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
 150     rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
 151
 152     # set adblock/dnsmasq destination file and format
 153     #
 154     adb_dnsfile="/tmp/dnsmasq.d/adlist.conf"
 155     adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
 156 }
 157
 158 #############################################
 159 # f_envcheck: check environment prerequisites
 160 #
 161 f_envcheck()
 162 {
 163     # check adblock network device configuration
 164     #
 165     if [ ! -d "/sys/class/net/${adb_dev}" ]
 166     then
 167         /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network device input (${adb_dev})"
 168         f_deltemp
 169         exit 25
 170     fi
 171
 172     # check adblock network interface configuration
 173     #
 174     check_if="$(printf "${adb_if}" | sed -n '/[^_0-9A-Za-z]/p')"
 175     banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p')"
 176     if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
 177     then
 178         /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock network interface input (${adb_if})"
 179         f_deltemp
 180         exit 30
 181     fi
 182
 183     # check adblock ip address configuration
 184     #
 185     check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p')"
 186     if [ -z "${check_ip}" ]
 187     then
 188         /usr/bin/logger -t "adblock[${pid}]" "error: invalid adblock ip address input (${adb_ip})"
 189         f_deltemp
 190         exit 35
 191     fi
 192
 193     # check adblock blacklist/whitelist configuration
 194     #
 195     if [ ! -r "${adb_blacklist}" ]
 196     then
 197         /usr/bin/logger -t "adblock[${pid}]" "error: adblock blacklist not found"
 198         f_deltemp
 199         exit 40
 200     elif [ ! -r "${adb_whitelist}" ]
 201     then
 202         /usr/bin/logger -t "adblock[${pid}]" "error: adblock whitelist not found"
 203         f_deltemp
 204         exit 45
 205     fi
 206
 207     # check wan update configuration
 208     #
 209     if [ -n "${adb_wandev}" ]
 210     then
 211         wan_ok="true"
 212     else
 213         wan_ok="false"
 214         /usr/bin/logger -t "adblock[${pid}]" "info: wan update check will be disabled"
 215     fi
 216
 217     # check ntp sync configuration
 218     #
 219     if [ -n "${adb_ntpsrv}" ]
 220     then
 221         ntp_ok="true"
 222     else
 223         ntp_ok="false"
 224         /usr/bin/logger -t "adblock[${pid}]" "info: ntp time sync will be disabled"
 225     fi
 226
 227     # check backup configuration
 228     #
 229     adb_backupdir="${adb_backupfile%/*}"
 230     if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
 231     then
 232         backup_ok="true"
 233         adb_mounts="${adb_backupdir} ${adb_tmpdir}"
 234     else
 235         backup_ok="false"
 236         /usr/bin/logger -t "adblock[${pid}]" "info: backup/restore will be disabled"
 237     fi
 238
 239     # check error log configuration
 240     #
 241     adb_logdir="${adb_logfile%/*}"
 242     if [ -n "${adb_logfile}" ] && [ "${adb_logfile}" = "/dev/stdout" ]
 243     then
 244         log_ok="true"
 245         adb_logfile="/proc/self/fd/1"
 246     elif [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ] && [ "${ntp_ok}" = "true" ]
 247     then
 248         log_ok="true"
 249         adb_mounts="${adb_mounts} ${adb_logdir}"
 250     else
 251         log_ok="false"
 252         adb_logfile="/dev/null"
 253         /usr/bin/logger -t "adblock[${pid}]" "info: error logging will be disabled"
 254     fi
 255
 256     # check dns query log configuration
 257     #
 258     adb_querydir="${adb_queryfile%/*}"
 259     if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
 260     then
 261         # check find capabilities
 262         #
 263         check="$(find --help 2>&1 | grep "mtime")"
 264         if [ -z "${check}" ]
 265         then
 266             query_ok="false"
 267             /usr/bin/logger -t "adblock[${pid}]" "info: busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
 268         else
 269             query_ok="true"
 270             query_name="${adb_queryfile##*/}"
 271             query_ip="${adb_ip//./\\.}"
 272             query_pid="/var/run/query.pid"
 273             adb_mounts="${adb_mounts} ${adb_querydir}"
 274         fi
 275     else
 276         query_ok="false"
 277         /usr/bin/logger -t "adblock[${pid}]" "info: dns query logging will be disabled"
 278     fi
 279
 280     # check mount points & space requirements
 281     #
 282     adb_mounts="${adb_mounts} ${adb_tmpdir}"
 283     for mp in ${adb_mounts}
 284     do
 285         df "${mp}" 2>/dev/null |\
 286         tail -n1 |\
 287         while read filesystem overall used available scrap
 288         do
 289             av_space="${available}"
 290             if [ $((av_space)) -eq 0 ]
 291             then
 292                 /usr/bin/logger -t "adblock[${pid}]" "error: no space left on device/not mounted (${mp})"
 293                 exit 50
 294             elif [ $((av_space)) -lt $((adb_minspace)) ]
 295             then
 296                 /usr/bin/logger -t "adblock[${pid}]" "error: not enough space on device (${mp})"
 297                 exit 55
 298             fi
 299         done
 300         # subshell return code handling
 301         #
 302         rc=$?
 303         if [ $((rc)) -ne 0 ]
 304         then
 305             f_deltemp
 306             exit ${rc}
 307         fi
 308     done
 309
 310     # check curl package dependency
 311     #
 312     check="$(printf "${pkg_list}" | grep "^curl")"
 313     if [ -z "${check}" ]
 314     then
 315         /usr/bin/logger -t "adblock[${pid}]" "error: curl package not found"
 316         f_deltemp
 317         exit 60
 318     fi
 319
 320     # check wget package dependency
 321     #
 322     check="$(printf "${pkg_list}" | grep "^wget")"
 323     if [ -z "${check}" ]
 324     then
 325         /usr/bin/logger -t "adblock[${pid}]" "error: wget package not found"
 326         f_deltemp
 327         exit 65
 328     fi
 329
 330     # check dynamic/volatile adblock network interface configuration
 331     #
 332     rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf $?)"
 333     if [ $((rc)) -ne 0 ]
 334     then
 335         json_init
 336         json_add_string name "${adb_if}"
 337         json_add_string ifname "${adb_dev}"
 338         json_add_string proto "static"
 339         json_add_array ipaddr
 340         json_add_string "" "${adb_ip}"
 341         json_close_array
 342         json_close_object
 343         ubus call network add_dynamic "$(json_dump)"
 344         /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
 345     fi
 346
 347     # check dynamic/volatile adblock uhttpd instance configuration
 348     #
 349     rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf $?)"
 350     if [ $((rc)) -ne 0 ]
 351     then
 352         uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80"
 353         /usr/bin/logger -t "adblock[${pid}]" "info: created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
 354     fi
 355 }
 356
 357 ###################################################
 358 # f_deltemp: delete temporary files and directories
 359 #
 360 f_deltemp()
 361 {
 362     if [ -f "${adb_tmpfile}" ]
 363     then
 364        rm -f "${adb_tmpfile}" 2>/dev/null
 365     fi
 366     if [ -d "${adb_tmpdir}" ]
 367     then
 368        rm -rf "${adb_tmpdir}" 2>/dev/null
 369     fi
 370 }
 371
 372 ################################################################
 373 # f_remove: remove temporary files, start and maintain query log
 374 #
 375 f_remove()
 376 {
 377     # delete temporary files and directories
 378     #
 379     f_deltemp
 380
 381     # kill existing domain query log background process,
 382     # housekeeping and start of a new process on daily basis
 383     #
 384     if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
 385     then
 386         query_date="$(date "+%Y%m%d")"
 387         if [ -s "${query_pid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
 388         then
 389             kill -9 $(< "${query_pid}") 2>/dev/null
 390             > "${query_pid}"
 391             find "${adb_backupdir}" -maxdepth 1 -type f -mtime +${adb_queryhistory} -name "${query_name}.*" -exec rm -f {} \; 2>/dev/null
 392             /usr/bin/logger -t "adblock[${pid}]" "info: kill old query log background process and do logfile housekeeping"
 393         fi
 394         if [ ! -s "${query_pid}" ]
 395         then
 396             ( logread -f 2>/dev/null & printf -n "$!" > "${query_pid}" ) | egrep -o "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" >> "${adb_queryfile}.${query_date}" &
 397             /usr/bin/logger -t "adblock[${pid}]" "info: start new domain query log background process"
 398         fi
 399     fi
 400
 401     # final log entry
 402     #
 403     /usr/bin/logger -t "adblock[${pid}]" "info: domain adblock processing finished (${adb_version})"
 404 }
 405
 406 #####################################################
 407 # f_restore: if available, restore last adlist backup
 408 #
 409 f_restore()
 410 {
 411     if [ -z "${restore_msg}" ]
 412     then
 413         restore_msg="unknown"
 414     fi
 415
 416     if [ "${backup_ok}" = "true" ] && [ -f "${adb_backupfile}" ]
 417     then
 418         cp -f "${adb_backupfile}" "${adb_dnsfile}" 2>/dev/null
 419         /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, adlist backup restored"
 420         printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, adlist backup restored" >> "${adb_logfile}"
 421     else
 422         > "${adb_dnsfile}"
 423         /usr/bin/logger -t "adblock[${pid}]" "error: ${restore_msg}, empty adlist generated"
 424         printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ${restore_msg}, empty adlist generated" >> "${adb_logfile}"
 425     fi
 426
 427     # restart dnsmasq
 428     #
 429     /etc/init.d/dnsmasq restart >/dev/null 2>&1
 430
 431     # remove files and exit
 432     #
 433     f_remove
 434     exit 100
 435 }
 436
 437 #######################################################
 438 # f_wancheck: check for usable adblock update interface
 439 #
 440 f_wancheck()
 441 {
 442     if [ "${wan_ok}" = "true" ]
 443     then
 444         # wait for wan update interface(s)
 445         #
 446         while [ $((cnt)) -le $((max_cnt)) ]
 447         do
 448             for dev in ${adb_wandev}
 449             do
 450                 if [ -d "/sys/class/net/${dev}" ]
 451                 then
 452                     dev_out=$(< /sys/class/net/${dev}/operstate 2>/dev/null)
 453                     if [ "${dev_out}" = "up" ]
 454                     then
 455                         /usr/bin/logger -t "adblock[${pid}]" "info: get wan/update interface: ${dev}, after ${cnt} loops"
 456                         break 2
 457                     fi
 458                 fi
 459                 if [ $((cnt)) -eq $((max_cnt)) ]
 460                 then
 461                     /usr/bin/logger -t "adblock[${pid}]" "error: no wan/update interface(s) found (${adb_wandev})"
 462                     printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: no wan/update interface(s) found (${adb_wandev})" >> "${adb_logfile}"
 463                     restore_msg="no wan/update interface(s)"
 464                     f_restore
 465                 fi
 466             done
 467             sleep 1
 468             cnt=$((cnt + 1))
 469         done
 470     fi
 471 }
 472
 473 #####################################
 474 # f_ntpcheck: check/get ntp time sync
 475 #
 476 f_ntpcheck()
 477 {
 478     if [ "${ntp_ok}" = "true" ]
 479     then
 480         # prepare ntp server pool
 481         #
 482         unset ntp_pool
 483         for srv in ${adb_ntpsrv}
 484         do
 485             ntp_pool="${ntp_pool} -p ${srv}"
 486         done
 487
 488         # wait for ntp time sync
 489         #
 490         while [ $((cnt)) -le $((max_cnt)) ]
 491         do
 492             /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
 493             rc=$?
 494             if [ $((rc)) -eq 0 ]
 495             then
 496                 /usr/bin/logger -t "adblock[${pid}]" "info: get ntp time sync (${adb_ntpsrv}), after ${cnt} loops"
 497                 break
 498             fi
 499             if [ $((cnt)) -eq $((max_cnt)) ]
 500             then
 501                 ntp_ok="false"
 502                 /usr/bin/logger -t "adblock[${pid}]" "error: ntp time sync failed (${adb_ntpsrv})"
 503                 printf "$(/bin/date "+%d.%m.%Y %H:%M:%S") - error: ntp time sync failed (${adb_ntpsrv})" >> "${adb_logfile}"
 504                 restore_msg="time sync failed"
 505                 f_restore
 506             fi
 507             sleep 1
 508             cnt=$((cnt + 1))
 509         done
 510     fi
 511 }
 512
 513 #################################################################
 514 # f_dnscheck: dnsmasq health check with newly generated blocklist
 515 #
 516 f_dnscheck()
 517 {
 518     # check 1: dnsmasq startup
 519     #
 520     dns_status="$(logread -l 20 -e "dnsmasq" -e "FAILED to start up")"
 521     if [ -z "${dns_status}" ]
 522     then
 523         # check 2: nslookup probe
 524         #
 525         dns_status="$(nslookup "${adb_domain}" 2>/dev/null | grep "${adb_ip}")"
 526         if [ -z "${dns_status}" ]
 527         then
 528             # create backup of new block list only, if both checks are OK and backup enabled
 529             #
 530             if [ "${backup_ok}" = "true" ]
 531             then
 532                 cp -f "${adb_dnsfile}" "${adb_backupfile}" 2>/dev/null
 533                 /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, backup generated"
 534             else
 535                 /usr/bin/logger -t "adblock[${pid}]" "info: new block list with ${adb_count} domains loaded, no backup"
 536             fi
 537         else
 538             restore_msg="nslookup probe failed"
 539             f_restore
 540         fi
 541     else
 542             restore_msg="dnsmasq probe failed"
 543             f_restore
 544     fi
 545 }
 546
 547 ##########################################################
 548 # f_footer: write footer with a few statistics to dns file
 549 #
 550 f_footer()
 551 {
 552     adb_count="$(wc -l < "${adb_dnsfile}")"
 553     printf "%s\n" "###################################################" >> "${adb_dnsfile}"
 554     printf "%s\n" "# last adblock file update: $(date +"%d.%m.%Y - %T")" >> "${adb_dnsfile}"
 555     printf "%s\n" "# ${0##*/} (${adb_version}) - ${adb_count} ad/abuse domains blocked" >> "${adb_dnsfile}"
 556     printf "%s\n" "# domain blacklist sources:" >> "${adb_dnsfile}"
 557     for src in ${adb_sources}
 558     do
 559         url="${src//\&ruleset=*/}"
 560         printf "%s\n" "# ${url}" >> "${adb_dnsfile}"
 561     done
 562     printf "%s\n" "###################################################" >> "${adb_dnsfile}"
 563     printf "%s\n" "# domain whitelist source:" >> "${adb_dnsfile}"
 564     printf "%s\n" "# ${adb_whitelist}" >> "${adb_dnsfile}"
 565     printf "%s\n" "###################################################" >> "${adb_dnsfile}"
 566 }