adblock: update 1.1.10
[feed/packages.git] / net / adblock / files / adblock-helper.sh
1 #!/bin/sh
2 # function library used by adblock-update.sh
3 # written by Dirk Brenken (dev@brenken.org)
4
5 # f_envload: load adblock environment
6 #
7 f_envload()
8 {
9 # source in system function library
10 #
11 if [ -r "/lib/functions.sh" ]
12 then
13 . "/lib/functions.sh"
14 else
15 rc=110
16 f_log "system function library not found" "${rc}"
17 f_exit
18 fi
19
20 # source in system network library
21 #
22 if [ -r "/lib/functions/network.sh" ]
23 then
24 . "/lib/functions/network.sh"
25 else
26 rc=115
27 f_log "system network library not found" "${rc}"
28 f_exit
29 fi
30
31 # set initial defaults,
32 # may be overwritten by setting appropriate adblock config options in global section of /etc/config/adblock
33 #
34 adb_lanif="lan"
35 adb_nullport="65535"
36 adb_nullipv4="192.0.2.1"
37 adb_nullipv6="::ffff:c000:0201"
38 adb_whitelist="/etc/adblock/adblock.whitelist"
39 adb_whitelist_rset="\$1 ~/^([A-Za-z0-9_-]+\.){1,}[A-Za-z]+/{print tolower(\$1)}"
40 adb_forcedns=1
41
42 # function to parse global section by callback
43 #
44 config_cb()
45 {
46 local type="${1}"
47 if [ "${type}" = "adblock" ]
48 then
49 option_cb()
50 {
51 local option="${1}"
52 local value="${2}"
53 eval "${option}=\"${value}\""
54 }
55 else
56 reset_cb
57 fi
58 }
59
60 # function to parse 'service' and 'source' sections
61 #
62 parse_config()
63 {
64 local value opt section="${1}" options="adb_dir adb_src adb_src_rset adb_src_cat"
65 config_get switch "${section}" "enabled"
66 if [ "${switch}" = "1" ]
67 then
68 if [ "${section}" != "backup" ]
69 then
70 eval "adb_sources=\"${adb_sources} ${section}\""
71 fi
72 for opt in ${options}
73 do
74 config_get value "${section}" "${opt}"
75 if [ -n "${value}" ]
76 then
77 eval "${opt}_${section}=\"${value}\""
78 fi
79 done
80 fi
81 }
82
83 # check opkg availability
84 #
85 if [ -r "/var/lock/opkg.lock" ]
86 then
87 rc=-1
88 f_log "adblock installation finished successfully, 'opkg' currently locked by package installer"
89 f_exit
90 fi
91
92 # get list with all installed packages
93 #
94 pkg_list="$(opkg list-installed)"
95 if [ -z "${pkg_list}" ]
96 then
97 rc=120
98 f_log "empty package list" "${rc}"
99 f_exit
100 fi
101
102 # load adblock config and start parsing functions
103 #
104 config_load adblock
105 config_foreach parse_config service
106 config_foreach parse_config source
107
108 # set more script defaults (can't be overwritten by adblock config options)
109 #
110 adb_minspace=12000
111 adb_tmpfile="$(mktemp -tu)"
112 adb_tmpdir="$(mktemp -p /tmp -d)"
113 adb_dnsdir="/tmp/dnsmasq.d"
114 adb_dnsprefix="adb_list"
115 adb_uci="$(which uci)"
116 adb_iptv4="$(which iptables)"
117 adb_iptv6="$(which ip6tables)"
118 adb_fetch="$(which wget-ssl)"
119 unset adb_srclist adb_revsrclist adb_errsrclist
120
121 # check 'enabled' & 'version' config options
122 #
123 if [ -z "${adb_enabled}" ] || [ -z "${adb_cfgversion}" ] || [ "${adb_cfgversion}" != "${adb_scriptver%.*}" ]
124 then
125 rc=-1
126 f_log "outdated adblock configuration found, please copy latest version from '/etc/adblock/adblock.conf.default' to '/etc/config/adblock'"
127 f_exit
128 fi
129 if [ $((adb_enabled)) -ne 1 ]
130 then
131 rc=-1
132 f_log "adblock is currently disabled, please set adblock.global.adb_enabled=1' to use this service"
133 f_exit
134 fi
135
136 # check running dnsmasq instance
137 #
138 rc="$(ps | grep -q "[d]nsmasq"; printf ${?})"
139 if [ $((rc)) -ne 0 ]
140 then
141 rc=-1
142 f_log "please enable the local dnsmasq instance to use adblock"
143 f_exit
144 fi
145
146 # check running firewall
147 #
148 check="$(${adb_iptv4} -vnL | grep -F "DROP")"
149 if [ -z "${check}" ]
150 then
151 rc=-1
152 f_log "please enable the local firewall to use adblock"
153 f_exit
154 fi
155
156 # get lan ip addresses
157 #
158 network_get_ipaddr adb_ipv4 "${adb_lanif}"
159 network_get_ipaddr6 adb_ipv6 "${adb_lanif}"
160 if [ -z "${adb_ipv4}" ] && [ -z "${adb_ipv6}" ]
161 then
162 rc=-1
163 f_log "no valid IPv4/IPv6 configuration found (${adb_lanif}), please set 'adb_lanif' manually"
164 f_exit
165 fi
166
167 # check logical update interfaces (with default route)
168 #
169 network_find_wan adb_wanif4
170 network_find_wan6 adb_wanif6
171 if [ -z "${adb_wanif4}" ] && [ -z "${adb_wanif6}" ]
172 then
173 adb_wanif4="${adb_lanif}"
174 fi
175
176 # check AP mode
177 #
178 if [ "${adb_wanif4}" = "${adb_lanif}" ] || [ "${adb_wanif6}" = "${adb_lanif}" ]
179 then
180 adb_nullipv4="${adb_ipv4}"
181 adb_nullipv6="${adb_ipv6}"
182 if [ "$(uci get uhttpd.main.listen_http | grep -Fo "80")" = "80" ] ||
183 [ "$(uci get uhttpd.main.listen_https | grep -Fo "443")" = "443" ]
184 then
185 rc=-1
186 f_log "AP mode detected, set local LuCI instance to ports <> 80/443"
187 f_exit
188 else
189 apmode_ok="true"
190 fi
191 fi
192
193 # get system release level
194 #
195 adb_sysver="$(printf "${pkg_list}" | grep "^base-files -")"
196 adb_sysver="${adb_sysver##*-}"
197 }
198
199 # f_envcheck: check/set environment prerequisites
200 #
201 f_envcheck()
202 {
203 local check
204
205 if [ "${apmode_ok}" = "true" ]
206 then
207 f_log "AP mode enabled"
208 fi
209
210 # check general package dependencies
211 #
212 f_depend "busybox"
213 f_depend "uci"
214 f_depend "uhttpd"
215 f_depend "wget"
216 f_depend "iptables"
217 f_depend "kmod-ipt-nat"
218
219 # check ipv6 related package dependencies
220 #
221 if [ -n "${adb_wanif6}" ]
222 then
223 check="$(printf "${pkg_list}" | grep "^ip6tables -")"
224 if [ -z "${check}" ]
225 then
226 f_log "package 'ip6tables' not found, IPv6 support will be disabled"
227 unset adb_wanif6
228 else
229 check="$(printf "${pkg_list}" | grep "^kmod-ipt-nat6 -")"
230 if [ -z "${check}" ]
231 then
232 f_log "package 'kmod-ipt-nat6' not found, IPv6 support will be disabled"
233 unset adb_wanif6
234 fi
235 fi
236 fi
237
238 # check ca-certificates package and set fetch parms accordingly
239 #
240 fetch_parm="--no-config --quiet --tries=1 --no-cache --no-cookies --max-redirect=0 --dns-timeout=5 --connect-timeout=5 --read-timeout=5"
241 check="$(printf "${pkg_list}" | grep "^ca-certificates -")"
242 if [ -z "${check}" ]
243 then
244 fetch_parm="${fetch_parm} --no-check-certificate"
245 fi
246
247 # check adblock temp directory
248 #
249 if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ]
250 then
251 f_space "${adb_tmpdir}"
252 if [ "${space_ok}" = "false" ]
253 then
254 if [ $((av_space)) -le 2000 ]
255 then
256 rc=125
257 f_log "not enough free space in '${adb_tmpdir}' (avail. ${av_space} kb)" "${rc}"
258 f_exit
259 else
260 f_log "not enough free space to handle all adblock list sources at once in '${adb_tmpdir}' (avail. ${av_space} kb)"
261 fi
262 fi
263 else
264 rc=130
265 f_log "temp directory not found" "${rc}"
266 f_exit
267 fi
268
269 # check memory
270 #
271 mem_total="$(awk '$1 ~ /^MemTotal/ {printf $2}' "/proc/meminfo")"
272 mem_free="$(awk '$1 ~ /^MemFree/ {printf $2}' "/proc/meminfo")"
273 mem_swap="$(awk '$1 ~ /^SwapTotal/ {printf $2}' "/proc/meminfo")"
274 if [ $((mem_total)) -le 64000 ] && [ $((mem_swap)) -eq 0 ]
275 then
276 mem_ok="false"
277 f_log "not enough free memory, overall sort processing will be disabled (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})"
278 else
279 mem_ok="true"
280 fi
281
282 # check backup configuration
283 #
284 if [ -n "${adb_dir_backup}" ] && [ -d "${adb_dir_backup}" ]
285 then
286 f_space "${adb_dir_backup}"
287 if [ "${space_ok}" = "false" ]
288 then
289 f_log "not enough free space in '${adb_dir_backup}'(avail. ${av_space} kb), backup/restore will be disabled"
290 backup_ok="false"
291 else
292 f_log "backup/restore will be enabled"
293 backup_ok="true"
294 fi
295 else
296 backup_ok="false"
297 f_log "backup/restore will be disabled"
298 fi
299
300 # set dnsmasq defaults
301 #
302 if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
303 then
304 adb_dnsformat="awk -v ipv4="${adb_nullipv4}" -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv4\"\n\"\"address=/\"\$0\"/\"ipv6}'"
305 elif [ -n "${adb_wanif4}" ]
306 then
307 adb_dnsformat="awk -v ipv4="${adb_nullipv4}" '{print \"address=/\"\$0\"/\"ipv4}'"
308 else
309 adb_dnsformat="awk -v ipv6="${adb_nullipv6}" '{print \"address=/\"\$0\"/\"ipv6}'"
310 fi
311
312 # check ipv4/iptables configuration
313 #
314 if [ -n "${adb_wanif4}" ]
315 then
316 f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-nat" "-p tcp -d ${adb_nullipv4} -m multiport --dports 80,443 -j DNAT --to-destination ${adb_ipv4}:${adb_nullport}"
317 f_firewall "IPv4" "filter" "A" "forwarding_rule" "adb-fwd" "-p tcp -d ${adb_nullipv4} -j REJECT --reject-with tcp-reset"
318 f_firewall "IPv4" "filter" "A" "forwarding_rule" "adb-fwd" "-d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable"
319 f_firewall "IPv4" "filter" "A" "output_rule" "adb-out" "-p tcp -d ${adb_nullipv4} -j REJECT --reject-with tcp-reset"
320 f_firewall "IPv4" "filter" "A" "output_rule" "adb-out" "-d ${adb_nullipv4} -j REJECT --reject-with icmp-host-unreachable"
321 if [ $((adb_forcedns)) -eq 1 ]
322 then
323 f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-dns" "-p udp --dport 53 -j DNAT --to-destination ${adb_ipv4}:53"
324 f_firewall "IPv4" "nat" "A" "prerouting_rule" "adb-dns" "-p tcp --dport 53 -j DNAT --to-destination ${adb_ipv4}:53"
325 fi
326 if [ "${fw_done}" = "true" ]
327 then
328 f_log "created volatile IPv4 firewall ruleset"
329 fw_done="false"
330 fi
331 fi
332
333 # check ipv6/ip6tables configuration
334 #
335 if [ -n "${adb_wanif6}" ]
336 then
337 f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-nat" "-p tcp -d ${adb_nullipv6} -m multiport --dports 80,443 -j DNAT --to-destination [${adb_ipv6}]:${adb_nullport}"
338 f_firewall "IPv6" "filter" "A" "forwarding_rule" "adb-fwd" "-p tcp -d ${adb_nullipv6} -j REJECT --reject-with tcp-reset"
339 f_firewall "IPv6" "filter" "A" "forwarding_rule" "adb-fwd" "-d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable"
340 f_firewall "IPv6" "filter" "A" "output_rule" "adb-out" "-p tcp -d ${adb_nullipv6} -j REJECT --reject-with tcp-reset"
341 f_firewall "IPv6" "filter" "A" "output_rule" "adb-out" "-d ${adb_nullipv6} -j REJECT --reject-with icmp6-addr-unreachable"
342 if [ $((adb_forcedns)) -eq 1 ]
343 then
344 f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns" "-p udp --dport 53 -j DNAT --to-destination [${adb_ipv6}]:53"
345 f_firewall "IPv6" "nat" "A" "PREROUTING" "adb-dns" "-p tcp --dport 53 -j DNAT --to-destination [${adb_ipv6}]:53"
346 fi
347 if [ "${fw_done}" = "true" ]
348 then
349 f_log "created volatile IPv6 firewall ruleset"
350 fw_done="false"
351 fi
352 fi
353
354 # check volatile adblock uhttpd instance configuration
355 #
356 rc="$(ps | grep -q "[u]httpd.*\-h /www/adblock"; printf ${?})"
357 if [ $((rc)) -ne 0 ]
358 then
359 if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
360 then
361 uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_nullport}" -p "[${adb_ipv6}]:${adb_nullport}"
362 rc=${?}
363 elif [ -n "${adb_wanif4}" ]
364 then
365 uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "${adb_ipv4}:${adb_nullport}"
366 rc=${?}
367 else
368 uhttpd -h "/www/adblock" -k 0 -N 100 -t 0 -T 1 -D -S -E "/index.html" -p "[${adb_ipv6}]:${adb_nullport}"
369 rc=${?}
370 fi
371 if [ $((rc)) -eq 0 ]
372 then
373 f_log "created volatile uhttpd instance"
374 else
375 f_log "failed to initialize volatile uhttpd instance" "${rc}"
376 f_restore
377 fi
378 fi
379
380 # check whitelist entries
381 #
382 if [ -s "${adb_whitelist}" ]
383 then
384 awk "${adb_whitelist_rset}" "${adb_whitelist}" > "${adb_tmpdir}/tmp.whitelist"
385 fi
386
387 # remove no longer used opkg package list
388 #
389 unset pkg_list
390 }
391
392 # f_depend: check package dependencies
393 #
394 f_depend()
395 {
396 local check
397 local package="${1}"
398
399 check="$(printf "${pkg_list}" | grep "^${package} -")"
400 if [ -z "${check}" ]
401 then
402 rc=135
403 f_log "package '${package}' not found" "${rc}"
404 f_exit
405 fi
406 }
407
408 # f_firewall: set iptables rules for ipv4/ipv6
409 #
410 f_firewall()
411 {
412 local ipt="${adb_iptv4}"
413 local proto="${1}"
414 local table="${2}"
415 local ctype="${3}"
416 local chain="${4}"
417 local notes="${5}"
418 local rules="${6}"
419
420 # select appropriate iptables executable for IPv6
421 #
422 if [ "${proto}" = "IPv6" ]
423 then
424 ipt="${adb_iptv6}"
425 fi
426
427 # check whether iptables rule already applied and proceed accordingly
428 #
429 rc="$("${ipt}" -w -t "${table}" -C "${chain}" -m comment --comment "${notes}" ${rules}; printf ${?})"
430 if [ $((rc)) -ne 0 ]
431 then
432 "${ipt}" -w -t "${table}" -"${ctype}" "${chain}" -m comment --comment "${notes}" ${rules}
433 rc=${?}
434 if [ $((rc)) -eq 0 ]
435 then
436 fw_done="true"
437 else
438 f_log "failed to initialize volatile ${proto} firewall rule '${notes}'" "${rc}"
439 f_exit
440 fi
441 fi
442 }
443
444 # f_log: log messages to stdout and syslog
445 #
446 f_log()
447 {
448 local log_parm
449 local log_msg="${1}"
450 local log_rc="${2}"
451 local class="info "
452
453 # check for terminal session
454 #
455 if [ -t 1 ]
456 then
457 log_parm="-s"
458 fi
459
460 # log to different output devices and set log class accordingly
461 #
462 if [ -n "${log_msg}" ]
463 then
464 if [ $((log_rc)) -gt 0 ]
465 then
466 class="error"
467 log_rc=", rc: ${log_rc}"
468 log_msg="${log_msg}${log_rc}"
469 fi
470 "${adb_log}" ${log_parm} -t "adblock[${adb_pid}] ${class}" "${log_msg}" 2>&1
471 fi
472 }
473
474 ################################################
475 # f_space: check mount points/space requirements
476 #
477 f_space()
478 {
479 local mp="${1}"
480
481 if [ -d "${mp}" ]
482 then
483 av_space="$(df "${mp}" | tail -n1 | awk '{printf $4}')"
484 if [ $((av_space)) -lt $((adb_minspace)) ]
485 then
486 space_ok="false"
487 fi
488 fi
489 }
490
491 # f_cntconfig: calculate counters in config
492 #
493 f_cntconfig()
494 {
495 local src_name
496 local count=0
497 local count_sum=0
498
499 for src_name in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}"*)
500 do
501 count="$(wc -l < "${src_name}")"
502 src_name="${src_name#*.}"
503 if [ -n "${adb_wanif4}" ] && [ -n "${adb_wanif6}" ]
504 then
505 count=$((count / 2))
506 fi
507 "${adb_uci}" -q set "adblock.${src_name}.adb_src_count=${count}"
508 count_sum=$((count_sum + count))
509 done
510 "${adb_uci}" -q set "adblock.global.adb_overall_count=${count_sum}"
511 }
512
513 # f_rmconfig: remove counters & timestamps in given config sections
514 #
515 f_rmconfig()
516 {
517 local src_name
518 local rm_done="${1}"
519 local restore_done="${2:-false}"
520
521 for src_name in ${rm_done}
522 do
523 src_name="${src_name#*.}"
524 if [ "${restore_done}" = "true" ]
525 then
526 src_name="${src_name%.*}"
527 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=list restored"
528 else
529 "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
530 "${adb_uci}" -q delete "adblock.${src_name}.adb_src_timestamp"
531 fi
532 done
533 }
534
535 # f_restore: restore last adblock list backups and restart dnsmasq
536 #
537 f_restore()
538 {
539 local rm_done
540 local restore_done
541
542 # remove bogus adblock lists
543 #
544 if [ -n "${adb_revsrclist}" ]
545 then
546 rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec rm -f "{}" \;)"
547 rc=${?}
548 if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
549 then
550 f_rmconfig "${rm_done}"
551 f_log "all bogus adblock lists removed"
552 elif [ $((rc)) -ne 0 ]
553 then
554 f_log "error during removal of bogus adblock lists" "${rc}"
555 f_exit
556 fi
557 fi
558
559 # restore backups
560 #
561 if [ "${backup_ok}" = "true" ]
562 then
563 restore_done="$(find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}*.gz" -print -exec cp -pf "{}" "${adb_dnsdir}" \;)"
564 rc=${?}
565 if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ]
566 then
567 find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*.gz" -exec gunzip -f "{}" \;
568 f_log "all available backups restored"
569 elif [ $((rc)) -ne 0 ] && [ -n "${restore_done}" ]
570 then
571 f_log "error during restore of adblock lists" "${rc}"
572 f_exit
573 fi
574 else
575 f_log "backup service disabled, nothing to restore"
576 fi
577
578 # (re-)try dnsmasq restart without bogus adblock lists / with backups
579 #
580 if [ -n "${restore_done}" ] || [ -n "${rm_done}" ]
581 then
582 /etc/init.d/dnsmasq restart
583 sleep 1
584 rc="$(ps | grep -q "[d]nsmasq"; printf ${?})"
585 if [ $((rc)) -eq 0 ]
586 then
587 rc=0
588 f_cntconfig
589 f_log "adblock lists with overall ${adb_count} domains loaded"
590 else
591 rc=140
592 f_log "dnsmasq restart failed, please check 'logread' output" "${rc}"
593 fi
594 fi
595 f_exit
596 }
597
598 # f_exit: delete (temporary) files, generate statistics and exit
599 #
600 f_exit()
601 {
602 local ipv4_adblock=0
603 local ipv6_adblock=0
604
605 # delete temporary files & directories
606 #
607 if [ -f "${adb_tmpfile}" ]
608 then
609 rm -f "${adb_tmpfile}"
610 fi
611 if [ -d "${adb_tmpdir}" ]
612 then
613 rm -rf "${adb_tmpdir}"
614 fi
615
616 # final log message and iptables statistics
617 #
618 if [ $((rc)) -eq 0 ]
619 then
620 if [ -n "${adb_wanif4}" ]
621 then
622 ipv4_adblock="$(${adb_iptv4} -t nat -vnL | awk '$11 ~ /^adb-nat$/ {sum += $1} END {printf sum}')"
623 ipv4_adblock="$((${ipv4_adblock} + $(${adb_iptv4} -vnL | awk '$11 ~ /^adb-(fwd|out)$/ {sum += $1} END {printf sum}')))"
624 fi
625 if [ -n "${adb_wanif6}" ]
626 then
627 ipv6_adblock="$(${adb_iptv6} -t nat -vnL | awk '$10 ~ /^adb-nat$/ {sum += $1} END {printf sum}')"
628 ipv6_adblock="$((${ipv6_adblock} + $(${adb_iptv6} -vnL | awk '$10 ~ /^adb-(fwd|out)$/ {sum += $1} END {printf sum}')))"
629 fi
630 if [ -n "$(${adb_uci} -q changes adblock)" ]
631 then
632 "${adb_uci}" -q commit "adblock"
633 fi
634 f_log "firewall statistics (IPv4/IPv6): ${ipv4_adblock}/${ipv6_adblock} ad related packets blocked"
635 f_log "domain adblock processing finished successfully (${adb_scriptver}, ${adb_sysver}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
636 elif [ $((rc)) -gt 0 ]
637 then
638 if [ -n "$(${adb_uci} -q changes adblock)" ]
639 then
640 "${adb_uci}" -q revert "adblock"
641 fi
642 f_log "domain adblock processing failed (${adb_scriptver}, ${adb_sysver}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
643 else
644 rc=0
645 fi
646 rm -f "${adb_pidfile}"
647 exit ${rc}
648 }