net/adblock/files/adblock.sh

   1 #!/bin/sh
   2 # dns based ad/abuse domain blocking
   3 # written by Dirk Brenken (dev@brenken.org)
   4
   5 # This is free software, licensed under the GNU General Public License v3.
   6 # You should have received a copy of the GNU General Public License
   7 # along with this program. If not, see <http://www.gnu.org/licenses/>.
   8
   9 # set initial defaults
  10 #
  11 LC_ALL=C
  12 PATH="/usr/sbin:/usr/bin:/sbin:/bin"
  13 adb_ver="2.0.3"
  14 adb_enabled=1
  15 adb_debug=0
  16 adb_whitelist="/etc/adblock/adblock.whitelist"
  17 adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\"^\"\$1\"\\\|[.]\"\$1)}"
  18 adb_dns="dnsmasq"
  19 adb_dnsdir="/tmp/dnsmasq.d"
  20 adb_dnshidedir="${adb_dnsdir}/.adb_hidden"
  21 adb_dnsprefix="adb_list"
  22 adb_dnsformat="awk '{print \"local=/\"\$0\"/\"}'"
  23 adb_fetch="/usr/bin/wget"
  24 adb_fetchparm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --timeout=5 --no-check-certificate -O"
  25
  26 # f_envload: load adblock environment
  27 #
  28 f_envload()
  29 {
  30     # source in system library
  31     #
  32     if [ -r "/lib/functions.sh" ]
  33     then
  34         . "/lib/functions.sh"
  35     else
  36         f_log "error" "status ::: system library not found"
  37     fi
  38
  39     # parse global section by callback
  40     #
  41     config_cb()
  42     {
  43         local type="${1}"
  44         if [ "${type}" = "adblock" ]
  45         then
  46             option_cb()
  47             {
  48                 local option="${1}"
  49                 local value="${2}"
  50                 eval "${option}=\"${value}\""
  51             }
  52         else
  53             reset_cb
  54         fi
  55     }
  56
  57     # parse 'service' and 'source' sections
  58     #
  59     parse_config()
  60     {
  61         local value opt section="${1}" options="enabled adb_dir adb_src adb_src_rset adb_src_cat"
  62         if [ "${section}" != "backup" ]
  63         then
  64             eval "adb_sources=\"${adb_sources} ${section}\""
  65         fi
  66         for opt in ${options}
  67         do
  68             config_get value "${section}" "${opt}"
  69             if [ -n "${value}" ]
  70             then
  71                 eval "${opt}_${section}=\"${value}\""
  72             fi
  73         done
  74     }
  75
  76     # load adblock config
  77     #
  78     config_load adblock
  79     config_foreach parse_config service
  80     config_foreach parse_config source
  81 }
  82
  83 # f_envcheck: check/set environment prerequisites
  84 #
  85 f_envcheck()
  86 {
  87     # check 'enabled' option
  88     #
  89     if [ "${adb_enabled}" != "1" ]
  90     then
  91         if [ "$(ls -dA "${adb_dnsdir}/${adb_dnsprefix}"* >/dev/null 2>&1)" ]
  92         then
  93             f_rmdns
  94             f_dnsrestart
  95         fi
  96         f_log "info " "status ::: adblock is currently disabled, please set adb_enabled to '1' to use this service"
  97         exit 0
  98     fi
  99
 100     # check fetch utility
 101     #
 102     if [ -z "${adb_fetch}" ] || [ ! -f "${adb_fetch}" ]
 103     then
 104         f_log "error" "status ::: no download utility with ssl support found/configured"
 105     fi
 106
 107     # create dns hideout directory
 108     #
 109     if [ ! -d "${adb_dnshidedir}" ]
 110     then
 111         mkdir -p -m 660 "${adb_dnshidedir}"
 112     else
 113         rm -f "${adb_dnshidedir}/${adb_dnsprefix}"*
 114     fi
 115
 116     # create adblock temp file/directory
 117     #
 118     adb_tmpload="$(mktemp -tu)"
 119     adb_tmpfile="$(mktemp -tu)"
 120     adb_tmpdir="$(mktemp -p /tmp -d)"
 121
 122     # prepare whitelist entries
 123     #
 124     if [ -s "${adb_whitelist}" ]
 125     then
 126         awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist"
 127     fi
 128
 129     # get system information
 130     #
 131     adb_sysver="$(ubus -S call system board | jsonfilter -e '@.release.description')"
 132 }
 133
 134 # f_rmtemp: remove temporary files & directories
 135 #
 136 f_rmtemp()
 137 {
 138     rm -f "${adb_tmpload}"
 139     rm -f "${adb_tmpfile}"
 140     rm -rf "${adb_tmpdir}"
 141 }
 142
 143 # f_rmdns: remove dns related files & directories
 144 #
 145 f_rmdns()
 146 {
 147     rm -f "${adb_dnsdir}/${adb_dnsprefix}"*
 148     rm -f "${adb_dir_backup}/${adb_dnsprefix}"*.gz
 149     rm -rf "${adb_dnshidedir}"
 150     ubus call service delete "{\"name\":\"adblock_stats\",\"instances\":\"stats\"}" 2>/dev/null
 151 }
 152
 153 # f_dnsrestart: restart the dns server
 154 #
 155 f_dnsrestart()
 156 {
 157     local cnt=0
 158     dns_running="false"
 159
 160     sync
 161     killall -q -TERM "${adb_dns}"
 162     while [ ${cnt} -le 10 ]
 163     do
 164         dns_running="$(ubus -S call service list '{"name":"dnsmasq"}' | jsonfilter -l 1 -e '@.dnsmasq.instances.*.running')"
 165         if [ "${dns_running}" = "true" ]
 166         then
 167             return 0
 168         fi
 169         cnt=$((cnt+1))
 170         sleep 1
 171     done
 172     /etc/init.d/"${adb_dns}" restart
 173     sleep 1
 174 }
 175
 176 # f_list: backup/restore/remove block lists
 177 #
 178 f_list()
 179 {
 180     local mode="${1}"
 181
 182     if [ "${enabled_backup}" = "1" ] && [ -d "${adb_dir_backup}" ]
 183     then
 184         case "${mode}" in
 185             backup)
 186                 gzip -cf "${adb_tmpfile}" > "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
 187                 ;;
 188             restore)
 189                 rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
 190                 if [ -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
 191                 then
 192                     gunzip -cf "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}"
 193                 fi
 194                 ;;
 195             remove)
 196                 rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
 197                 if [ -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
 198                 then
 199                     rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
 200                 fi
 201                 ;;
 202         esac
 203     fi
 204 }
 205
 206 # f_switch: suspend/resume adblock processing
 207 #
 208 f_switch()
 209 {
 210     if [ -d "${adb_dnshidedir}" ]
 211     then
 212         local source target status mode="${1}"
 213         local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)"
 214         local dns_passive="$(find "${adb_dnshidedir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)"
 215
 216         if [ -n "${dns_active}" ] && [ "${mode}" = "suspend" ]
 217         then
 218             source="${adb_dnsdir}/${adb_dnsprefix}"
 219             target="${adb_dnshidedir}"
 220             status="suspended"
 221         elif [ -n "${dns_passive}" ] && [ "${mode}" = "resume" ]
 222         then
 223             source="${adb_dnshidedir}/${adb_dnsprefix}"
 224             target="${adb_dnsdir}"
 225             status="resumed"
 226         fi
 227         if [ -n "${status}" ]
 228         then
 229             mv -f "${source}"* "${target}"
 230             f_dnsrestart
 231             f_log "info " "status ::: adblock processing ${status}"
 232         fi
 233     fi
 234 }
 235
 236 # f_query: query block lists for certain (sub-)domains
 237 #
 238 f_query()
 239 {
 240     local search result cnt
 241     local domain="${1}"
 242     local tld="${domain#*.}"
 243     local dns_active="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print)"
 244
 245     if [ -z "${dns_active}" ]
 246     then
 247          printf "%s\n" ":: no active block lists found, please start adblock first"
 248     elif [ -z "${domain}" ] || [ "${domain}" = "${tld}" ]
 249     then
 250         printf "%s\n" ":: invalid domain input, please submit a specific (sub-)domain, i.e. 'www.abc.xyz'"
 251     else
 252         while [ "${domain}" != "${tld}" ]
 253         do
 254             search="${domain//./\.}"
 255             result="$(grep -Hm 1 "[/\.]${search}/" "${adb_dnsdir}/${adb_dnsprefix}"* | awk -F ':|/' '{print "   "$4"\t: "$6}')"
 256             cnt="$(grep -hc "[/\.]${search}/" "${adb_dnsdir}/${adb_dnsprefix}"* | awk '{sum += $1} END {printf sum}')"
 257             printf "%s\n" ":: distinct results for domain '${domain}' (overall ${cnt})"
 258             if [ -z "${result}" ]
 259             then
 260                 printf "%s\n" "   no matches in active block lists"
 261             else
 262                 printf "%s\n" "${result}"
 263             fi
 264             domain="${tld}"
 265             tld="${domain#*.}"
 266         done
 267     fi
 268 }
 269
 270 # f_log: write to syslog, exit on error
 271 #
 272 f_log()
 273 {
 274     local class="${1}"
 275     local log_msg="${2}"
 276
 277     if [ -n "${log_msg}" ] && ([ "${class}" != "debug" ] || [ ${adb_debug} -eq 1 ])
 278     then
 279         logger -t "adblock-[${adb_ver}] ${class}" "${log_msg}"
 280         if [ "${class}" = "error" ]
 281         then
 282             f_debug
 283             f_rmtemp
 284             f_rmdns
 285             f_dnsrestart
 286             exit 255
 287         fi
 288     fi
 289 }
 290
 291 # f_debug: gather memory & space information
 292 f_debug()
 293 {
 294         local mem_total=0 mem_free=0 mem_swap=0 tmp_space=0 backup_space=0
 295
 296         if [ ${adb_debug} -eq 1 ]
 297         then
 298         mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo")"
 299         mem_free="$(awk '$1 ~ /^MemFree/ {printf $2}' "/proc/meminfo")"
 300         mem_swap="$(awk '$1 ~ /^SwapTotal/ {printf $2}' "/proc/meminfo")"
 301         f_log "debug" "memory ::: total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap}"
 302
 303         if [ -d "${adb_tmpdir}" ]
 304         then
 305             tmp_space="$(df "${adb_tmpdir}" 2>/dev/null | tail -n1 | awk '{printf $4}')"
 306         fi
 307         if [ -d "${adb_dir_backup}" ]
 308         then
 309             backup_space="$(df "${adb_dir_backup}" 2>/dev/null | tail -n1 | awk '{printf $4}')"
 310         fi
 311         f_log "debug" "space  ::: tmp_dir: ${adb_tmpdir}, tmp_kb: ${tmp_space}, backup: ${enabled_backup}, backup_dir: ${adb_dir_backup}, backup_kb: ${backup_space}"
 312     fi
 313 }
 314
 315 # main function for block list processing
 316 #
 317 f_main()
 318 {
 319     local enabled url rc cnt sum_cnt=0
 320     local src_name src_rset shalla_file shalla_archive
 321
 322     f_debug
 323     f_log "debug" "main   ::: tool: ${adb_fetch}, parm: ${adb_fetchparm}"
 324     for src_name in ${adb_sources}
 325     do
 326         eval "enabled=\"\${enabled_${src_name}}\""
 327         eval "url=\"\${adb_src_${src_name}}\""
 328         eval "src_rset=\"\${adb_src_rset_${src_name}}\""
 329         adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
 330         > "${adb_tmpload}"
 331         > "${adb_tmpfile}"
 332
 333         # basic pre-checks
 334         #
 335         if [ "${enabled}" = "0" ] || [ -z "${url}" ] || [ -z "${src_rset}" ]
 336         then
 337             f_list remove
 338             continue
 339         fi
 340
 341         # download block list
 342         #
 343         f_log "debug" "loop   ::: name: ${src_name}, enabled: ${enabled}, dnsfile: ${adb_dnsfile}"
 344         if [ "${src_name}" = "blacklist" ]
 345         then
 346             cat "${url}" > "${adb_tmpload}"
 347             rc=${?}
 348         elif [ "${src_name}" = "shalla" ]
 349         then
 350             shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
 351             shalla_file="${adb_tmpdir}/shallalist.txt"
 352             "${adb_fetch}" ${adb_fetchparm} "${shalla_archive}" "${url}"
 353             rc=${?}
 354             if [ ${rc} -eq 0 ]
 355             then
 356                 > "${shalla_file}"
 357                 for category in ${adb_src_cat_shalla}
 358                 do
 359                     tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
 360                     rc=${?}
 361                     if [ ${rc} -ne 0 ]
 362                     then
 363                         break
 364                     fi
 365                 done
 366                 cat "${shalla_file}" > "${adb_tmpload}"
 367                 rm -f "${shalla_file}"
 368             fi
 369             rm -f "${shalla_archive}"
 370             rm -rf "${adb_tmpdir}/BL"
 371         else
 372             "${adb_fetch}" ${adb_fetchparm} "${adb_tmpload}" "${url}"
 373             rc=${?}
 374         fi
 375
 376         # check download result and prepare domain output (incl. list backup/restore)
 377         #
 378         f_log "debug" "loop   ::: name: ${src_name}, load-rc: ${rc}"
 379         if [ ${rc} -eq 0 ] && [ -s "${adb_tmpload}" ]
 380         then
 381             awk "${src_rset}" "${adb_tmpload}" > "${adb_tmpfile}"
 382             if [ -s "${adb_tmpfile}" ]
 383             then
 384                 f_list backup
 385             else
 386                 f_list restore
 387             fi
 388         else
 389             f_list restore
 390         fi
 391
 392         # remove whitelist domains, sort and make them unique, final list preparation
 393         #
 394         if [ -s "${adb_tmpfile}" ]
 395         then
 396             if [ -s "${adb_tmpdir}/tmp.whitelist" ]
 397             then
 398                 grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
 399             else
 400                 sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
 401             fi
 402             rc=${?}
 403             if [ ${rc} -ne 0 ]
 404             then
 405                 f_list remove
 406             fi
 407         fi
 408         f_log "debug" "loop   ::: name: ${src_name}, list-rc: ${rc}"
 409     done
 410
 411     # make overall sort, restart & check dns server
 412     #
 413     for src_name in $(ls -dASr "${adb_dnsdir}/${adb_dnsprefix}"* 2>/dev/null)
 414     do
 415         if [ -s "${adb_tmpdir}/blocklist.overall" ]
 416         then
 417             sort "${adb_tmpdir}/blocklist.overall" "${adb_tmpdir}/blocklist.overall" "${src_name}" | uniq -u > "${adb_tmpdir}/tmp.blocklist"
 418             cat "${adb_tmpdir}/tmp.blocklist" > "${src_name}"
 419         fi
 420         cat "${src_name}" >> "${adb_tmpdir}/blocklist.overall"
 421         cnt="$(wc -l < "${src_name}")"
 422         sum_cnt=$((sum_cnt + cnt))
 423         list="${src_name/*./}"
 424         if [ -z "${sum_list}" ]
 425         then
 426             sum_list="\"${list}\":\"${cnt}\""
 427         else
 428             sum_list="${sum_list},\"${list}\":\"${cnt}\""
 429         fi
 430     done
 431     f_dnsrestart
 432     if [ "${dns_running}" = "true" ]
 433     then
 434         f_debug
 435         f_rmtemp
 436         f_log "info " "status ::: block lists with overall ${sum_cnt} domains loaded (${adb_sysver})"
 437         ubus call service add "{\"name\":\"adblock_stats\",
 438                                 \"instances\":{\"stats\":{\"command\":[\"\"],
 439                                 \"data\":{\"blocked_domains\":\"${sum_cnt}\",
 440                                 \"last_rundate\":\"$(/bin/date "+%d.%m.%Y %H:%M:%S")\",
 441                                 \"active_lists\":[{${sum_list}}]}}}}"
 442         exit 0
 443     fi
 444     f_log "error" "status ::: dns server restart with active block lists failed"
 445 }
 446
 447 # handle different adblock actions
 448 #
 449 if [ "${adb_procd}" = "true" ]
 450 then
 451     f_envload
 452     case "${1}" in
 453         stop)
 454             f_rmtemp
 455             f_rmdns
 456             f_dnsrestart
 457             ;;
 458         suspend)
 459             f_switch suspend
 460             ;;
 461         resume)
 462             f_switch resume
 463             ;;
 464         query)
 465             f_query "${2}"
 466             ;;
 467         *)
 468             f_envcheck
 469             f_main
 470             ;;
 471     esac
 472 fi
 473 exit 0