883a0f5803b221ed70c0f01b39cdbfd5635e7427
2 # dns based ad/abuse domain blocking script
3 # written by Dirk Brenken (dev@brenken.org)
5 # This is free software, licensed under the GNU General Public License v3.
6 # You should have received a copy of the GNU General Public License
7 # along with this program. If not, see <http://www.gnu.org/licenses/>.
13 # script debug switch (disabled by default)
14 # set 'DEBUG=1' to enable script debugging
17 if [ $
((DEBUG
)) -eq 0 ]
25 adb_pidfile
="/var/run/adblock.pid"
26 adb_log
="$(which logger)"
28 if [ -r "${adb_pidfile}" ]
31 "${adb_log}" -s -t "adblock[${adb_pid}] error" "adblock service already running ($(cat ${adb_pidfile}))"
34 printf "${adb_pid}" > "${adb_pidfile}"
37 # get current directory and set script/config version
39 adb_scriptdir
="${0%/*}"
40 adb_scriptver
="1.1.12"
43 # source in adblock function library
45 if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
47 .
"${adb_scriptdir}/adblock-helper.sh"
50 "${adb_log}" -s -t "adblock[${adb_pid}] error" "adblock function library not found"
51 rm -f "${adb_pidfile}"
55 # call trap function on error signals (HUP, INT, QUIT, BUS, SEGV, TERM)
57 trap "rc=250; f_log 'error signal received/trapped' '${rc}'; f_exit" 1 2 3 10 11 15
65 f_log
"domain adblock processing started (${adb_scriptver}, ${adb_sysver}, $(/bin/date "+%d.
%m.
%Y
%H
:%M
:%S
"))"
71 # loop through active adblock domain sources,
72 # download sources, prepare output and store all extracted domains in temp file
74 for src_name
in ${adb_sources}
76 eval "url=\"\${adb_src_${src_name}}\""
77 eval "src_rset=\"\${adb_src_rset_${src_name}}\""
78 adb_dnsfile
="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
79 list_time
="$(${adb_uci} -q get "adblock.
${src_name}.adb_src_timestamp
")"
80 f_log
"=> processing adblock source '${src_name}'"
82 # check 'url' and 'src_rset' values
84 if [ -z "${url}" ] ||
[ -z "${src_rset}" ]
86 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=broken config"
87 f_log
" broken source configuration, check 'adb_src' and 'adb_src_rset' in config"
91 # prepare find statement with active adblock list sources
93 if [ -z "${adb_srclist}" ]
95 adb_srclist
="! -name ${adb_dnsprefix}.${src_name}*"
97 adb_srclist
="${adb_srclist} -a ! -name ${adb_dnsprefix}.${src_name}*"
100 # only download adblock list with newer/updated timestamp
102 if [ "${src_name}" = "blacklist" ]
104 url_time
="$(date -r "${url}")"
106 url_time
="$(${adb_fetch} ${fetch_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
108 if [ -z "${url_time}" ]
111 f_log
" no online timestamp received, current date will be used"
113 if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] || [ ! -r "${adb_dnsfile}" ] ||\
114 ([ "${backup_ok}" = "true" ] && [ ! -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ])
116 if [ "${src_name}" = "blacklist" ]
118 tmp_domains
="$(cat "${url}")"
120 elif [ "${src_name}" = "shalla" ]
122 shalla_archive
="${adb_tmpdir}/shallalist.tar.gz"
123 shalla_file
="${adb_tmpdir}/shallalist.txt"
124 "${adb_fetch}" ${fetch_parm} --output-document="${shalla_archive}" "${url}"
129 for category
in ${adb_src_cat_shalla}
131 tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
135 f_log
" archive extraction failed (${category})"
139 rm -f "${shalla_archive}"
140 rm -rf "${adb_tmpdir}/BL"
141 tmp_domains
="$(cat "${shalla_file}")"
145 tmp_domains
="$(${adb_fetch} ${fetch_parm} --output-document=- "${url}")"
149 f_log " source doesn
't change, no update required"
153 # check download result and prepare domain output by regex patterns
155 if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
157 count="$(printf "%s\n" "${tmp_domains}" | awk "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
158 f_log " source download finished (${count} entries)"
159 if [ "${src_name}" = "shalla" ]
161 rm -f "${shalla_file}"
164 elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
166 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty download"
167 f_log " empty source download finished"
171 if [ -z "${adb_errsrclist}" ]
173 adb_errsrclist="-name ${adb_dnsprefix}.${src_name}.gz"
175 adb_errsrclist="${adb_errsrclist} -o -name ${adb_dnsprefix}.${src_name}.gz"
177 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=download failed"
178 f_log " source download failed"
182 # remove whitelist domains, sort domains and make them unique,
183 # finally rewrite ad/abuse domain information to separate dnsmasq files
185 if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ]
187 if [ -s "${adb_tmpdir}/tmp.whitelist" ]
189 grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
192 sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
196 # prepare find statement with revised adblock list sources
198 if [ -z "${adb_revsrclist}" ]
200 adb_revsrclist="-name ${adb_dnsprefix}.${src_name}"
202 adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}"
205 # store source timestamp in config
209 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=${url_time}"
210 f_log " domain merging finished"
212 f_log " domain merging failed" "${rc}"
216 "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty domain input"
217 f_log " empty domain input received"
222 # remove disabled adblock lists and their backups
224 if [ -n "${adb_srclist}" ]
226 rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_srclist} \) -print -exec rm -f "{}" \;)"
228 if [ "${backup_ok}" = "true" ] && [ -n "${rm_done}" ]
230 find "${adb_dir_backup}" -maxdepth 1 -type f \( ${adb_srclist} \) -exec rm -f "{}" \;
233 rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -print -exec rm -f "{}" \;)"
235 if [ "${backup_ok}" = "true" ]
237 find "${adb_dir_backup}" -maxdepth 1 -type f -name "${adb_dnsprefix}*" -exec rm -f "{}" \;
240 if [ $((rc)) -eq 0 ] && [ -n "${rm_done}" ]
242 f_rmconfig "${rm_done}"
243 f_log "remove disabled adblock lists"
244 elif [ $((rc)) -ne 0 ] && [ -n "${rm_done}" ]
246 f_log "error during removal of disabled adblock lists" "${rc}"
250 # partial restore of adblock lists in case of download errors
252 if [ "${backup_ok}" = "true" ] && [ -n "${adb_errsrclist}" ]
254 restore_done="$(find "${adb_dir_backup}" -maxdepth 1 -type f \( ${adb_errsrclist} \) -print -exec cp -pf "{}" "${adb_dnsdir}" \;)"
256 if [ $((rc)) -eq 0 ] && [ -n "${restore_done}" ]
258 find "${adb_dnsdir}" -maxdepth 1 -type f -name "${adb_dnsprefix}*.gz" -exec gunzip -f "{}" \;
259 f_rmconfig "${restore_done}" "true"
260 f_log "partial restore done"
261 elif [ $((rc)) -ne 0 ]
263 f_log "error during partial restore" "${rc}"
268 # make separate adblock lists entries unique
270 if [ "${mem_ok}" = "true" ] && [ -n "${adb_revsrclist}" ]
272 f_log "remove duplicates in separate adblock lists"
274 # generate a unique overall block list
276 sort -u "${adb_dnsdir}/${adb_dnsprefix}."* > "${adb_tmpdir}/blocklist.overall"
278 # loop through all separate lists, ordered by size (ascending)
280 for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}"*)
282 # check overall block list vs. separate block list,
283 # write all duplicate entries to separate list
286 sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_tmpdir}/tmp.${list}"
287 mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
289 # write all unique entries back to overall block list
291 sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall"
292 mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall"
294 rm -f "${adb_tmpdir}/blocklist.overall"
297 # restart & check dnsmasq with newly generated set of adblock lists
300 adb_count="$(${adb_uci} -q get "adblock.global.adb_overall_count")"
301 if [ -n "${adb_revsrclist}" ] || [ -n "${rm_done}" ] || [ -n "${restore_done}" ]
303 /etc/init.d/dnsmasq restart
305 rc="$(ps | grep -q "[d]nsmasq"; printf ${?})"
308 f_log "adblock lists with overall ${adb_count} domains loaded"
311 f_log "dnsmasq restart failed, please check 'logread
' output" "${rc}"
315 f_log "adblock lists with overall ${adb_count} domains are still valid, no update required"
318 # create adblock list backups
320 if [ "${backup_ok}" = "true" ] && [ -n "${adb_revsrclist}" ]
322 backup_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec cp -pf "{}" "${adb_dir_backup}" \;)"
324 if [ $((rc)) -eq 0 ] && [ -n "${backup_done}" ]
326 find "${adb_dir_backup}" -maxdepth 1 -type f \( -name "${adb_dnsprefix}*" -a ! -name "${adb_dnsprefix}*.gz" \) -exec gzip -f "{}" \;
327 f_log "new adblock list backups generated"
328 elif [ $((rc)) -ne 0 ] && [ -n "${backup_done}" ]
330 f_log "error during backup of adblock lists" "${rc}"
335 # remove temporary files and exit