2 ##############################################
3 # function library used by adblock-update.sh #
4 # written by Dirk Brenken (dirk@brenken.org) #
5 ##############################################
7 #####################################
8 # f_envload: load adblock environment
12 # source in openwrt function library
14 if [ -r "/lib/functions.sh" ]
16 .
"/lib/functions.sh" 2>/dev
/null
19 f_log
"openwrt function library not found" "${rc}"
23 # source in openwrt json helpers library
25 if [ -r "/usr/share/libubox/jshn.sh" ]
27 .
"/usr/share/libubox/jshn.sh" 2>/dev
/null
30 f_log
"openwrt json helpers library not found" "${rc}"
34 # get list with all installed openwrt packages
36 pkg_list
="$(opkg list-installed 2>/dev/null)"
37 if [ -z "${pkg_list}" ]
40 f_log
"empty openwrt package list" "${rc}"
45 ######################################################
46 # f_envparse: parse adblock config and set environment
50 # set the C locale, characters are single bytes, the charset is ASCII
51 # speeds up sort, grep etc.
55 # set initial defaults (may be overwritten by setting appropriate adblock config options)
62 adb_blacklist
="/etc/adblock/adblock.blacklist"
63 adb_whitelist
="/etc/adblock/adblock.whitelist"
65 # adblock device name auto detection
66 # derived from first entry in openwrt lan ifname config
68 adb_dev
="$(uci get network.lan.ifname 2>/dev/null)"
69 adb_dev
="${adb_dev/ *}"
71 # adblock ntp server name auto detection
72 # derived from ntp list found in openwrt ntp server config
74 adb_ntpsrv
="$(uci get system.ntp.server 2>/dev/null)"
76 # function to read/set global options by callback,
77 # prepare list items and build option list for all others
83 if [ "${type}" = "adblock" ]
89 eval "${option}=\"${value}\""
96 local opt_out
="$(printf "${option}" | sed -n '/.*_ITEM[0-9]$/p; /.*_LENGTH$/p; /enabled/p' 2>/dev/null)"
97 if [ -z "${opt_out}" ]
99 all_options
="${all_options} ${option}"
106 if [ "${list}" = "adb_wanlist" ]
108 adb_wandev
="${adb_wandev} ${value}"
109 elif [ "${list}" = "adb_ntplist" ]
111 adb_ntpsrv
="${adb_ntpsrv} ${value}"
112 elif [ "${list}" = "adb_catlist" ]
114 adb_cat_shalla
="${adb_cat_shalla} ${value}"
120 # function to iterate through option list, read/set all options in "enabled" sections
125 config_get switch
"${config}" "enabled"
126 if [ "${switch}" = "1" ]
128 for option
in ${all_options}
130 config_get value
"${config}" "${option}"
133 local opt_src
="$(printf "${option}" | sed -n '/^adb_src_[a-z0-9]*$/p' 2>/dev/null)"
134 if [ -n "${opt_src}" ]
136 adb_sources
="${adb_sources} ${value}"
138 eval "${option}=\"${value}\""
142 elif [ "${config}" = "wancheck" ]
145 elif [ "${config}" = "ntpcheck" ]
148 elif [ "${config}" = "shalla" ]
154 # load adblock config and start parsing functions
157 config_foreach parse_config service
158 config_foreach parse_config
source
160 # set temp variables and defaults
162 adb_tmpfile
="$(mktemp -tu 2>/dev/null)"
163 adb_tmpdir
="$(mktemp -p /tmp -d 2>/dev/null)"
167 # set adblock source ruleset definitions
169 rset_start
="sed -r 's/[[:space:]]|[\[!#/:;_].*|[0-9\.]*localhost.*//g; s/[\^#/:;_\.\t ]*$//g'"
170 rset_end
="sed '/^[#/:;_\s]*$/d'"
171 rset_adaway
="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
172 rset_blacklist
="${rset_start} | ${rset_end}"
173 rset_disconnect
="${rset_start} | ${rset_end}"
174 rset_dshield
="${rset_start} | ${rset_end}"
175 rset_feodo
="${rset_start} | ${rset_end}"
176 rset_malware
="${rset_start} | ${rset_end}"
177 rset_palevo
="${rset_start} | ${rset_end}"
178 rset_shalla
="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}$//g' | ${rset_end}"
179 rset_spam404
="${rset_start} | sed 's/^\|\|//g' | ${rset_end}"
180 rset_whocares
="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
181 rset_winhelp
="${rset_start} | sed 's/\([0-9]\{1,3\}\.\)\{3\}[0-1]\{1,1\}//g' | ${rset_end}"
182 rset_yoyo
="${rset_start} | sed 's/,/\n/g' | ${rset_end}"
183 rset_zeus
="${rset_start} | ${rset_end}"
185 # set dnsmasq defaults
187 adb_dnsdir
="/tmp/dnsmasq.d"
188 adb_dnsformat
="sed 's/^/address=\//;s/$/\/'${adb_ip}'/'"
189 adb_dnsprefix
="adb_list"
192 #############################################
193 # f_envcheck: check environment prerequisites
197 # check adblock config file
199 check_config
="$(grep -F "ruleset
=rset_default
" /etc/config/adblock 2>/dev/null)"
200 if [ -n "${check_config}" ]
203 grep -Fv "#" "/etc/adblock/samples/adblock.conf.sample" > /etc
/config
/adblock
204 f_log
"new default adblock config applied, please check your configuration settings in /etc/config/adblock" "${rc}"
208 # check required config options
210 adb_varlist
="adb_ip adb_dev adb_domain"
211 for var
in ${adb_varlist}
213 if [ -z "$(eval printf \"\$"${var}"\")" ]
216 f_log
"missing adblock config option (${var})" "${rc}"
221 # check main uhttpd configuration
223 check_uhttpd
="$(uci get uhttpd.main.listen_http 2>/dev/null | grep -Fo "0.0.0.0" 2>/dev/null)"
224 if [ -n "${check_uhttpd}" ]
227 lan_ip
="$(uci get network.lan.ipaddr 2>/dev/null)"
228 f_log
"please bind main uhttpd instance to LAN only (lan ip: ${lan_ip})" "${rc}"
232 # check adblock network device configuration
234 if [ ! -d "/sys/class/net/${adb_dev}" ]
237 f_log
"invalid adblock network device input (${adb_dev})" "${rc}"
241 # check adblock network interface configuration
243 check_if
="$(printf "${adb_if}" | sed -n '/[^._0-9A-Za-z]/p' 2>/dev/null)"
244 banned_if
="$(printf "${adb_if}" | sed -n '/.*lan.*\|.*wan.*\|.*switch.*\|main\|globals\|loopback\|px5g/p' 2>/dev/null)"
245 if [ -n "${check_if}" ] ||
[ -n "${banned_if}" ]
248 f_log
"invalid adblock network interface input (${adb_if})" "${rc}"
252 # check adblock ip address configuration
254 check_ip
="$(printf "${adb_ip}" | sed -n '/\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/p' 2>/dev/null)"
255 lan_ip
="$(uci get network.lan.ipaddr 2>/dev/null)"
256 if [ -z "${check_ip}" ]
259 f_log
"invalid adblock ip address input (${adb_ip})" "${rc}"
261 elif [ "${adb_ip}" = "${lan_ip}" ]
264 f_log
"adblock ip needs to be a different subnet from the normal LAN (adblock ip: ${adb_ip})" "${rc}"
268 # check adblock blacklist/whitelist configuration
270 if [ ! -r "${adb_blacklist}" ]
273 f_log
"adblock blacklist not found" "${rc}"
275 elif [ ! -r "${adb_whitelist}" ]
278 f_log
"adblock whitelist not found" "${rc}"
282 # check adblock temp directory
284 if [ -n "${adb_tmpdir}" ] && [ -d "${adb_tmpdir}" ]
286 f_space
"${adb_tmpdir}"
291 f_log
"temp directory not found" "${rc}"
295 # check curl package dependency
297 check
="$(printf "${pkg_list}" | grep "^curl
-" 2>/dev/null)"
301 f_log
"curl package not found" "${rc}"
305 # check wget package dependency
307 check
="$(printf "${pkg_list}" | grep "^wget
-" 2>/dev/null)"
311 f_log
"wget package not found" "${rc}"
315 # check ca-certificates package and set wget/curl options accordingly
317 check
="$(printf "${pkg_list}" | grep "^ca-certificates
-" 2>/dev/null)"
320 curl_parm
="-q --insecure --silent"
321 wget_parm
="--no-config --no-hsts --no-check-certificate --quiet"
323 curl_parm
="-q --silent"
324 wget_parm
="--no-config --no-hsts --quiet"
327 # check total and swap memory
329 mem_total
="$(grep -F "MemTotal
" "/proc
/meminfo
" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
330 mem_free
="$(grep -F "MemFree
" "/proc
/meminfo
" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
331 swap_total
="$(grep -F "SwapTotal
" "/proc
/meminfo
" 2>/dev/null | grep -o "[0-9]*" 2>/dev/null)"
332 if [ $
((mem_total
)) -le 64000 ] && [ $
((swap_total
)) -eq 0 ]
335 f_log
"overall sort/unique processing will be disabled,"
336 f_log
"please consider adding an external swap device to supersize your /tmp directory (total: ${mem_total}, free: ${mem_free}, swap: ${mem_swap})"
339 # check backup configuration
341 if [ -n "${adb_backupdir}" ] && [ -d "${adb_backupdir}" ]
343 f_space
"${adb_backupdir}"
347 f_log
"backup/restore will be disabled"
350 # check dns query log configuration
352 adb_querydir
="${adb_queryfile%/*}"
353 adb_querypid
="/var/run/adb_query.pid"
354 if [ -n "${adb_querydir}" ] && [ -d "${adb_querydir}" ]
356 # check find capabilities
358 check
="$(find --help 2>&1 | grep -F "mtime
" 2>/dev/null)"
362 f_log
"busybox without 'find/mtime' support (min. r47362), dns query logging will be disabled"
364 f_space
"${adb_querydir}"
366 query_name
="${adb_queryfile##*/}"
367 query_ip
="${adb_ip//./\\.}"
371 f_log
"dns query logging will be disabled"
372 if [ -s "${adb_querypid}" ]
374 kill -9 "$(cat "${adb_querypid}")" >/dev
/null
2>&1
375 f_log
"remove old dns query log background process (pid: $(cat "${adb_querypid}" 2>/dev/null))"
380 # check debug log configuration
382 adb_logdir
="${adb_logfile%/*}"
383 if [ -n "${adb_logdir}" ] && [ -d "${adb_logdir}" ]
385 f_space
"${adb_logdir}"
389 f_log
"debug logging will be disabled"
392 # check wan update configuration
394 if [ -n "${adb_wandev}" ]
396 f_wancheck
"${adb_maxloop}"
399 f_log
"wan update check will be disabled"
402 # check ntp sync configuration
404 if [ -n "${adb_ntpsrv}" ]
406 f_ntpcheck
"${adb_maxloop}"
409 f_log
"ntp time sync will be disabled"
412 # check dynamic/volatile adblock network interface configuration
414 rc
="$(ifstatus "${adb_if}" >/dev/null 2>&1; printf ${?})"
418 json_add_string name
"${adb_if}"
419 json_add_string ifname
"${adb_dev}"
420 json_add_string proto
"static"
421 json_add_array ipaddr
422 json_add_string
"" "${adb_ip}"
425 ubus call network add_dynamic
"$(json_dump)"
429 f_log
"created new dynamic/volatile network interface (${adb_if}, ${adb_ip})"
431 f_log
"failed to initialize new dynamic/volatile network interface (${adb_if}, ${adb_ip})" "${rc}"
436 # check dynamic/volatile adblock uhttpd instance configuration
438 rc
="$(ps | grep "[u
]httpd.
*\
-r ${adb_if}" >/dev/null 2>&1; printf ${?})"
441 uhttpd
-h "/www/adblock" -r "${adb_if}" -E "/adblock.html" -p "${adb_ip}:80" >/dev
/null
2>&1
445 f_log
"created new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})"
447 f_log
"failed to initialize new dynamic/volatile uhttpd instance (${adb_if}, ${adb_ip})" "${rc}"
452 # remove no longer used package list
457 ################################################
458 # f_log: log messages to stdout, syslog, logfile
465 if [ -n "${log_msg}" ]
467 if [ $
((log_rc
)) -ne 0 ]
470 log_rc
=", rc: ${log_rc}"
471 log_msg
="${log_msg}${log_rc}"
473 /usr
/bin
/logger
-s -t "adblock[${pid}] ${class}" "${log_msg}"
474 if [ "${log_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
476 printf "%s\n" "$(/bin/date "+%d.
%m.
%Y
%H
:%M
:%S
") adblock[${pid}] ${class}: ${log_msg}" >> "${adb_logfile}"
481 ################################################
482 # f_space: check mount points/space requirements
489 df
"${mp}" 2>/dev
/null |\
490 tail -n1 2>/dev
/null |\
491 while read filesystem overall used available scrap
493 av_space
="${available}"
494 if [ $
((av_space
)) -eq 0 ]
497 f_log
"no space left on device/not mounted (${mp})" "${rc}"
499 elif [ $
((av_space
)) -lt $
((adb_minspace
)) ]
502 f_log
"not enough space left on device (${mp})" "${rc}"
517 ####################################################
518 # f_deltemp: delete temp files, directories and exit
522 if [ -f "${adb_tmpfile}" ]
524 rm -f "${adb_tmpfile}" >/dev
/null
2>&1
526 if [ -d "${adb_tmpdir}" ]
528 rm -rf "${adb_tmpdir}" >/dev
/null
2>&1
530 f_log
"domain adblock processing finished (${adb_version}, ${openwrt_version}, $(/bin/date "+%d.
%m.
%Y
%H
:%M
:%S
"))"
534 ####################################################
535 # f_remove: maintain and (re-)start domain query log
543 if [ "${query_ok}" = "true" ] && [ "${ntp_ok}" = "true" ]
545 query_date
="$(date "+%Y
%m
%d
")"
546 if [ -s "${adb_querypid}" ] && [ -f "${adb_queryfile}.${query_date}" ]
548 query_total
="$(grep -F "query
[A
]" "${adb_queryfile}.
${query_date}" 2>/dev/null | wc -l)"
549 query_blocked
="$(grep -Fv "query
[A
]" "${adb_queryfile}.
${query_date}" 2>/dev/null | wc -l)"
550 f_log
"adblock statistics for query date ${query_date} (total: ${query_total}, blocked: ${query_blocked})"
552 if [ -s "${adb_querypid}" ] && [ ! -f "${adb_queryfile}.${query_date}" ]
554 query_pid
="$(cat "${adb_querypid}" 2>/dev/null)"
556 kill -9 "${query_pid}" >/dev
/null
2>&1
560 find "${adb_backupdir}" -maxdepth 1 -type f -mtime +"${adb_queryhistory}" -name "${query_name}.*" -exec rm -f "{}" \
; 2>/dev
/null
564 f_log
"remove old domain query background process (pid: ${query_pid}) and do logfile housekeeping"
566 f_log
"error during domain query logfile housekeeping" "${rc}"
569 f_log
"error during domain query background process removal (pid: ${query_pid})" "${rc}"
572 if [ ! -s "${adb_querypid}" ]
574 (logread
-f 2>/dev
/null
& printf ${!} > "${adb_querypid}") | grep -Eo "(query\[A\].*)|([a-z0-9\.\-]* is ${query_ip}$)" 2>/dev/null >> "${adb_queryfile}.${query_date}" &
579 f_log "new domain query log background process started
(pid
: $
(cat "${adb_querypid}" 2>/dev
/null
))"
581 f_log "error during domain query background process start
" "${rc}"
588 ################################################################
589 # f_restore: restore last adblocklist backup and restart dnsmasq
593 # remove bogus adblocklists
595 if [ -n "${adb_revsrclist}" ]
597 find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrcfind} \) -exec rm -f "{}" \; 2>/dev/null
600 f_log "bogus adblocklists removed
"
602 f_log "error during removal of bogus adblocklists
" "${rc}"
609 if [ "${backup_ok}" = "true" ] && [ -d "${adb_backupdir}" ] && [ "$(printf "${adb_backupdir}/${adb_dnsprefix}."*)" != "${adb_backupdir}/${adb_dnsprefix}.
*" ]
611 cp -f "${adb_backupdir}/${adb_dnsprefix}."* "${adb_dnsdir}" >/dev/null 2>&1
615 f_log "all available backups restored
"
617 f_log "error during restore
" "${rc}"
621 /etc/init.d/dnsmasq restart >/dev/null 2>&1
625 #######################################################
626 # f_wancheck: check for usable adblock update interface
634 while [ $((cnt)) -le $((cnt_max)) ]
636 for dev in ${adb_wandev}
638 if [ -d "/sys
/class
/net
/${dev}" ]
640 dev_out="$
(cat /sys
/class
/net
/${dev}/operstate
2>/dev
/null
)"
642 if [ "${dev_out}" = "up
" ]
645 f_log "get wan
/update interface
(${dev}), after
${cnt} loops
"
653 if [ -z "${wan_ok}" ]
657 f_log "no wan
/update interface
(s
) found
(${adb_wandev# })" "${rc}"
662 #####################################
663 # f_ntpcheck: check/get ntp time sync
670 for srv in ${adb_ntpsrv}
672 ntp_pool="${ntp_pool} -p ${srv}"
674 while [ $((cnt)) -le $((cnt_max)) ]
676 /usr/sbin/ntpd -nq ${ntp_pool} >/dev/null 2>&1
681 f_log "get ntp
time sync
(${adb_ntpsrv# }), after
${cnt} loops
"
687 if [ -z "${ntp_ok}" ]
691 f_log "ntp
time sync failed
(${adb_ntpsrv# })" "${rc}"