adblock: bugfix
[feed/packages.git] / net / adblock / files / adblock-helper.sh
1 #!/bin/sh
2 ##############################################
3 # function library used by adblock-update.sh #
4 # written by Dirk Brenken (dirk@brenken.org) #
5 ##############################################
6
7 #####################################
8 # f_envload: load adblock environment
9 #
10 f_envload()
11 {
12 # source in openwrt function library
13 #
14 if [ -r "/lib/functions.sh" ]
15 then
16 . "/lib/functions.sh" 2>/dev/null
17 else
18 rc=500
19 f_log "openwrt function library not found" "${rc}"
20 f_deltemp
21 fi
22
23 # source in openwrt json helpers library
24 #
25 if [ -r "/usr/share/libubox/jshn.sh" ]
26 then
27 . "/usr/share/libubox/jshn.sh" 2>/dev/null
28 else
29 rc=505
30 f_log "openwrt json helpers library not found" "${rc}"
31 f_deltemp
32 fi
33
34 # get list with all installed openwrt packages
35 #
36 pkg_list="$(opkg list-installed 2>/dev/null)"
37 if [ -z "${pkg_list}" ]
38 then
39 rc=510
40 f_log "empty openwrt package list" "${rc}"
41 f_deltemp
42 fi
43 }
44
45 ######################################################
46 # f_envparse: parse adblock config and set environment
47 #
48 f_envparse()
49 {
50 # set the C locale, characters are single bytes, the charset is ASCII
51 # speeds up sort, grep etc.
52 #
53 LC_ALL=C
54
55 # set initial defaults (may be overwritten by setting appropriate adblock config options)
56 #
57 adb_if="adblock"
58 adb_minspace="20000"
59 adb_maxtime="60"
60 adb_maxloop="5"
61 adb_unique="1"
62 adb_blacklist="/etc/adblock/adblock.blacklist"
63 adb_whitelist="/etc/adblock/adblock.whitelist"
64
65 # adblock device name auto detection
66 # derived from first entry in openwrt lan ifname config
67 #
68 adb_dev="$(uci get network.lan.ifname 2>/dev/null)"
69 adb_dev="${adb_dev/ *}"
70
71 # adblock ntp server name auto detection
72 # derived from ntp list found in openwrt ntp server config
73 #
74 adb_ntpsrv="$(uci get system.ntp.server 2>/dev/null)"
75
76 # function to read/set global options by callback,
77 # prepare list items and build option list for all others
78 #
79 config_cb()
80 {
81 local type="${1}"
82 local name="${2}"
83 if [ "${type}" = "adblock" ]
84 then
85 option_cb()
86 {
87 local option="${1}"
88 local value="${2}"
89 eval "${option}=\"${value}\""
90 }
91 else
92 option_cb()
93 {
94 local option="${1}"
95 local value="${2}"
96 local opt_out="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p' 2>/dev/null)"
97 if [ -z "${opt_out}" ]
98 then
99 all_options="${all_options} ${option}"
100 fi
101 }
102 list_cb()
103 {
104 local list="${1}"
105 local value="${2}"
106 if [ "${list}" = "adb_wanlist" ]
107 then
108 adb_wandev="${adb_wandev} ${value}"
109 elif [ "${list}" = "adb_ntplist" ]
110 then
111 adb_ntpsrv="${adb_ntpsrv} ${value}"
112 elif [ "${list}" = "adb_catlist" ]
113 then
114 adb_cat_shalla="${adb_cat_shalla} ${value}"
115 fi
116 }
117 fi
118 }
119
120 # function to iterate through option list, read/set all options in "enabled" sections
121 #
122 parse_config()
123 {
124 local config="${1}"
125 config_get switch "${config}" "enabled"
126 if [ "${switch}" = "1" ]
127 then
128 for option in ${all_options}
129 do
130 config_get value "${config}" "${option}"
131 if [ -n "${value}" ]
132 then
133 local opt_src="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p' 2>/dev/null)"
134 if [ -n "${opt_src}" ]
135 then
136 adb_sources="${adb_sources} ${value}"
137 else
138 eval "${option}=\"${value}\""
139 fi
140 fi
141 done
142 elif [ "${config}" = "wancheck" ]
143 then
144 unset adb_wandev
145 elif [ "${config}" = "ntpcheck" ]
146 then
147 unset adb_ntpsrv
148 elif [ "${config}" = "shalla" ]
149 then
150 unset adb_cat_shalla
151 fi
152 }
153
154 # load adblock config and start parsing functions
155 #
156 config_load adblock
157 config_foreach parse_config service
158 config_foreach parse_config source
159
160 # set temp variables and defaults
161 #
162 adb_tmpfile="$(mktemp -tu 2>/dev/null)"
163 adb_tmpdir="$(mktemp -p /tmp -d 2>/dev/null)"
164 unset adb_srcfind
165 unset adb_revsrcfind
166
167 # set adblock source ruleset definitions
168 #
169 rset_start="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost.*//g; s/[\^#/:;_\.\t ]*$//g'"
170 rset_end="sed '/^[#/:;_\s]*$/d'"
171 rset_adaway="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
172 rset_blacklist="${rset_start} | ${rset_end}"
173 rset_disconnect="${rset_start} | ${rset_end}"
174 rset_dshield="${rset_start} | ${rset_end}"
175 rset_feodo="${rset_start} | ${rset_end}"
176 rset_malware="${rset_start} | ${rset_end}"
177 rset_palevo="${rset_start} | ${rset_end}"
178 rset_shalla="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
179 rset_spam404="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
180 rset_whocares="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
181 rset_winhelp="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
182 rset_yoyo="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
183 rset_zeus="${rset_start} | ${rset_end}"
184
185 # set dnsmasq defaults
186 #
187 adb_dnsdir="/tmp/dnsmasq.d"
188 adb_dnsformat="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
189 adb_dnsprefix="adb_list"
190 }
191
192 #############################################
193 # f_envcheck: check environment prerequisites
194 #
195 f_envcheck()
196 {
197 # check adblock config file
198 #
199 check_config="$(grep -F "ruleset=rset_default" /etc/config/adblock 2>/dev/null)"
200 if [ -n "${check_config}" ]
201 then
202 rc=515
203 grep -Fv "#" "/etc/adblock/samples/adblock.conf.sample" > /etc/config/adblock
204 f_log "new default adblock config applied, please check your configuration settings in /etc/config/adblock" "${rc}"
205 f_deltemp
206 fi
207
208 # check required config options
209 #
210 adb_varlist="adb_ip adb_dev adb_domain"
211 for var in ${adb_varlist}
212 do
213 if [ -z "$(eval printf \"\$"${var}"\")" ]
214 then
215 rc=520
216 f_log "missing adblock config option (${var})" "${rc}"
217 f_deltemp
218 fi
219 done
220
221 # check main uhttpd configuration
222 #
223 check_uhttpd="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0" 2>/dev/null)"
224 if [ -n "${check_uhttpd}" ]
225 then
226 rc=525
227 lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
228 f_log "please bind main uhttpd instance to LAN only (lan ip: ${lan_ip})" "${rc}"
229 f_deltemp
230 fi
231
232 # check adblock network device configuration
233 #
234 if [ ! -d "/sys/class/net/${adb_dev}" ]
235 then
236 rc=530
237 f_log "invalid adblock network device input (${adb_dev})" "${rc}"
238 f_deltemp
239 fi
240
241 # check adblock network interface configuration
242 #
243 check_if="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p' 2>/dev/null)"
244 banned_if="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p' 2>/dev/null)"
245 if [ -n "${check_if}" ] || [ -n "${banned_if}" ]
246 then
247 rc=535
248 f_log "invalid adblock network interface input (${adb_if})" "${rc}"
249 f_deltemp
250 fi
251
252 # check adblock ip address configuration
253 #
254 check_ip="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' 2>/dev/null)"
255 lan_ip="$(uci get network.lan.ipaddr 2>/dev/null)"
256 if [ -z "${check_ip}" ]
257 then
258 rc=540
259 f_log "invalid adblock ip address input (${adb_ip})" "${rc}"
260 f_deltemp
261 elif [ "${adb_ip}" = "${lan_ip}" ]
262 then
263 rc=545
264 f_log "adblock ip needs to be a different subnet from the normal LAN (adblock ip: ${adb_ip})" "${rc}"
265 f_deltemp
266 fi
267
268 # check adblock blacklist/whitelist configuration
269 #
270 if [ ! -r "${adb_blacklist}" ]
271 then
272 rc=550
273 f_log "adblock blacklist not found" "${rc}"
274 f_deltemp
275 elif [ ! -r "${adb_whitelist}" ]
276 then
277 rc=555
278 f_log "adblock whitelist not found" "${rc}"
279 f_deltemp
280 fi
281
282 # check adblock temp directory
283 #
284 if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ]
285 then
286 f_space "${adb_tmpdir}"
287 tmp_ok="true"
288 else
289 rc=560
290 tmp_ok="false"
291 f_log "temp directory not found" "${rc}"
292 f_deltemp
293 fi
294
295 # check curl package dependency
296 #
297 check="$(printf "${pkg_list}" | grep "^curl -" 2>/dev/null)"
298 if [ -z "${check}" ]
299 then
300 rc=565
301 f_log "curl package not found" "${rc}"
302 f_deltemp
303 fi
304
305 # check wget package dependency
306 #
307 check="$(printf "${pkg_list}" | grep "^wget -" 2>/dev/null)"
308 if [ -z "${check}" ]
309 then
310 rc=570
311 f_log "wget package not found" "${rc}"
312 f_deltemp
313 fi
314
315 # check ca-certificates package and set wget/curl options accordingly
316 #
317 check="$(printf "${pkg_list}" | grep "^ca-certificates -" 2>/dev/null)"
318 if [ -z "${check}" ]
319 then
320 curl_parm="-q --insecure --silent"
321 wget_parm="--no-config --no-hsts --no-check-certificate --quiet"
322 else
323 curl_parm="-q --silent"
324 wget_parm="--no-config --no-hsts --quiet"
325 fi
326
327 # check total and swap memory
328 #
329 mem_total="$(grep -F "MemTotal" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
330 mem_free="$(grep -F "MemFree" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
331 swap_total="$(grep -F "SwapTotal" "/proc/meminfo" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
332 if [ $((mem_total)) -le 64000 ] && [ $((swap_total)) -eq 0 ]
333 then
334 adb_unique=0
335 f_log "overall sort/unique processing will be disabled,"
336 f_log "please consider adding an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})"
337 fi
338
339 # check backup configuration
340 #
341 if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
342 then
343 f_space "${adb_backupdir}"
344 backup_ok="true"
345 else
346 backup_ok="false"
347 f_log "backup/restore will be disabled"
348 fi
349
350 # check dns query log configuration
351 #
352 adb_querydir="${adb_queryfile%/*}"
353 adb_querypid="/var/run/adb_query.pid"
354 if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
355 then
356 # check find capabilities
357 #
358 check="$(find --help 2>&1 | grep -F "mtime" 2>/dev/null)"
359 if [ -z "${check}" ]
360 then
361 query_ok="false"
362 f_log "busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
363 else
364 f_space "${adb_querydir}"
365 query_ok="true"
366 query_name="${adb_queryfile##*/}"
367 query_ip="${adb_ip//./\\.}"
368 fi
369 else
370 query_ok="false"
371 f_log "dns query logging will be disabled"
372 if [ -s "${adb_querypid}" ]
373 then
374 kill -9 "$(cat "${adb_querypid}")" >/dev/null 2>&1
375 f_log "remove old dns query log background process (pid: $(cat "${adb_querypid}" 2>/dev/null))"
376 > "${adb_querypid}"
377 fi
378 fi
379
380 # check debug log configuration
381 #
382 adb_logdir="${adb_logfile%/*}"
383 if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ]
384 then
385 f_space "${adb_logdir}"
386 log_ok="true"
387 else
388 log_ok="false"
389 f_log "debug logging will be disabled"
390 fi
391
392 # check wan update configuration
393 #
394 if [ -n "${adb_wandev}" ]
395 then
396 f_wancheck "${adb_maxloop}"
397 else
398 wan_ok="false"
399 f_log "wan update check will be disabled"
400 fi
401
402 # check ntp sync configuration
403 #
404 if [ -n "${adb_ntpsrv}" ]
405 then
406 f_ntpcheck "${adb_maxloop}"
407 else
408 ntp_ok="false"
409 f_log "ntp time sync will be disabled"
410 fi
411
412 # check dynamic/volatile adblock network interface configuration
413 #
414 rc="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf ${?})"
415 if [ $((rc)) -ne 0 ]
416 then
417 json_init
418 json_add_string name "${adb_if}"
419 json_add_string ifname "${adb_dev}"
420 json_add_string proto "static"
421 json_add_array ipaddr
422 json_add_string "" "${adb_ip}"
423 json_close_array
424 json_close_object
425 ubus call network add_dynamic "$(json_dump)"
426 rc=${?}
427 if [ $((rc)) -eq 0 ]
428 then
429 f_log "created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
430 else
431 f_log "failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}"
432 f_remove
433 fi
434 fi
435
436 # check dynamic/volatile adblock uhttpd instance configuration
437 #
438 rc="$(ps | grep "[u]httpd.*\-r ${adb_if}" >/dev/null 2>&1; printf ${?})"
439 if [ $((rc)) -ne 0 ]
440 then
441 uhttpd -h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" >/dev/null 2>&1
442 rc=${?}
443 if [ $((rc)) -eq 0 ]
444 then
445 f_log "created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
446 else
447 f_log "failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}"
448 f_remove
449 fi
450 fi
451
452 # remove no longer used package list
453 #
454 unset pkg_list
455 }
456
457 ################################################
458 # f_log: log messages to stdout, syslog, logfile
459 #
460 f_log()
461 {
462 local log_msg="${1}"
463 local log_rc="${2}"
464 local class="info "
465 if [ -n "${log_msg}" ]
466 then
467 if [ $((log_rc)) -ne 0 ]
468 then
469 class="error"
470 log_rc=", rc: ${log_rc}"
471 log_msg="${log_msg}${log_rc}"
472 fi
473 /usr/bin/logger -s -t "adblock[${pid}] ${class}" "${log_msg}"
474 if [ "${log_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
475 then
476 printf "%s\n" "$(/bin/date "+%d.%m.%Y %H:%M:%S") adblock[${pid}] ${class}: ${log_msg}" >> "${adb_logfile}"
477 fi
478 fi
479 }
480
481 ################################################
482 # f_space: check mount points/space requirements
483 #
484 f_space()
485 {
486 local mp="${1}"
487 if [ -d "${mp}" ]
488 then
489 df "${mp}" 2>/dev/null |\
490 tail -n1 2>/dev/null |\
491 while read filesystem overall used available scrap
492 do
493 av_space="${available}"
494 if [ $((av_space)) -eq 0 ]
495 then
496 rc=575
497 f_log "no space left on device/not mounted (${mp})" "${rc}"
498 exit ${rc}
499 elif [ $((av_space)) -lt $((adb_minspace)) ]
500 then
501 rc=580
502 f_log "not enough space left on device (${mp})" "${rc}"
503 exit ${rc}
504 fi
505 done
506 rc=${?}
507 if [ $((rc)) -eq 0 ]
508 then
509 space_ok="true"
510 else
511 space_ok="false"
512 f_deltemp
513 fi
514 fi
515 }
516
517 ####################################################
518 # f_deltemp: delete temp files, directories and exit
519 #
520 f_deltemp()
521 {
522 if [ -f "${adb_tmpfile}" ]
523 then
524 rm -f "${adb_tmpfile}" >/dev/null 2>&1
525 fi
526 if [ -d "${adb_tmpdir}" ]
527 then
528 rm -rf "${adb_tmpdir}" >/dev/null 2>&1
529 fi
530 f_log "domain adblock processing finished (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.%m.%Y %H:%M:%S"))"
531 exit ${rc}
532 }
533
534 ####################################################
535 # f_remove: maintain and (re-)start domain query log
536 #
537 f_remove()
538 {
539 local query_pid
540 local query_date
541 local query_total
542 local query_blocked
543 if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
544 then
545 query_date="$(date "+%Y%m%d")"
546 if [ -s "${adb_querypid}" ] && [ -f "${adb_queryfile}.${query_date}" ]
547 then
548 query_total="$(grep -F "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
549 query_blocked="$(grep -Fv "query[A]" "${adb_queryfile}.${query_date}" 2>/dev/null | wc -l)"
550 f_log "adblock statistics for query date ${query_date} (total: ${query_total}, blocked: ${query_blocked})"
551 fi
552 if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
553 then
554 query_pid="$(cat "${adb_querypid}" 2>/dev/null)"
555 > "${adb_querypid}"
556 kill -9 "${query_pid}" >/dev/null 2>&1
557 rc=${?}
558 if [ $((rc)) -eq 0 ]
559 then
560 find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f "{}" \; 2>/dev/null
561 rc=${?}
562 if [ $((rc)) -eq 0 ]
563 then
564 f_log "remove old domain query background process (pid: ${query_pid}) and do logfile housekeeping"
565 else
566 f_log "error during domain query logfile housekeeping" "${rc}"
567 fi
568 else
569 f_log "error during domain query background process removal (pid: ${query_pid})" "${rc}"
570 fi
571 fi
572 if [ ! -s "${adb_querypid}" ]
573 then
574 (logread -f 2>/dev/null & printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" 2>/dev/null >> "${adb_queryfile}.${query_date}" &
575 rc=${?}
576 if [ $((rc)) -eq 0 ]
577 then
578 sleep 1
579 f_log "new domain query log background process started (pid: $(cat "${adb_querypid}" 2>/dev/null))"
580 else
581 f_log "error during domain query background process start" "${rc}"
582 fi
583 fi
584 fi
585 f_deltemp
586 }
587
588 ################################################################
589 # f_restore: restore last adblocklist backup and restart dnsmasq
590 #
591 f_restore()
592 {
593 # remove bogus adblocklists
594 #
595 if [ -n "${adb_revsrclist}" ]
596 then
597 find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec rm -f "{}" \; 2>/dev/null
598 if [ $((rc)) -eq 0 ]
599 then
600 f_log "bogus adblocklists removed"
601 else
602 f_log "error during removal of bogus adblocklists" "${rc}"
603 f_remove
604 fi
605 fi
606
607 # restore backups
608 #
609 if [ "${backup_ok}" = "true" ] && [ -d "${adb_backupdir}" ] && [ "$(printf "${adb_backupdir}/${adb_dnsprefix}."*)" != "${adb_backupdir}/${adb_dnsprefix}.*" ]
610 then
611 cp -f "${adb_backupdir}/${adb_dnsprefix}."* "${adb_dnsdir}" >/dev/null 2>&1
612 rc=${?}
613 if [ $((rc)) -eq 0 ]
614 then
615 f_log "all available backups restored"
616 else
617 f_log "error during restore" "${rc}"
618 f_remove
619 fi
620 fi
621 /etc/init.d/dnsmasq restart >/dev/null 2>&1
622 f_remove
623 }
624
625 #######################################################
626 # f_wancheck: check for usable adblock update interface
627 #
628 f_wancheck()
629 {
630 local cnt=0
631 local cnt_max="${1}"
632 local dev
633 local dev_out
634 while [ $((cnt)) -le $((cnt_max)) ]
635 do
636 for dev in ${adb_wandev}
637 do
638 if [ -d "/sys/class/net/${dev}" ]
639 then
640 dev_out="$(cat /sys/class/net/${dev}/operstate 2>/dev/null)"
641 rc=${?}
642 if [ "${dev_out}" = "up" ]
643 then
644 wan_ok="true"
645 f_log "get wan/update interface (${dev}), after ${cnt} loops"
646 break 2
647 fi
648 fi
649 done
650 sleep 1
651 cnt=$((cnt + 1))
652 done
653 if [ -z "${wan_ok}" ]
654 then
655 rc=585
656 wan_ok="false"
657 f_log "no wan/update interface(s) found (${adb_wandev# })" "${rc}"
658 f_restore
659 fi
660 }
661
662 #####################################
663 # f_ntpcheck: check/get ntp time sync
664 #
665 f_ntpcheck()
666 {
667 local cnt=0
668 local cnt_max="${1}"
669 local ntp_pool
670 for srv in ${adb_ntpsrv}
671 do
672 ntp_pool="${ntp_pool} -p ${srv}"
673 done
674 while [ $((cnt)) -le $((cnt_max)) ]
675 do
676 /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
677 rc=${?}
678 if [ $((rc)) -eq 0 ]
679 then
680 ntp_ok="true"
681 f_log "get ntp time sync (${adb_ntpsrv# }), after ${cnt} loops"
682 break
683 fi
684 sleep 1
685 cnt=$((cnt + 1))
686 done
687 if [ -z "${ntp_ok}" ]
688 then
689 rc=590
690 ntp_ok="false"
691 f_log "ntp time sync failed (${adb_ntpsrv# })" "${rc}"
692 f_restore
693 fi
694 }