net/adblock/files/adblock-update.sh

   1 #!/bin/sh
   2 # dns based ad/abuse domain blocking script
   3 # written by Dirk Brenken (dev@brenken.org)
   4
   5 # This is free software, licensed under the GNU General Public License v3.
   6 # You should have received a copy of the GNU General Public License
   7 # along with this program. If not, see <http://www.gnu.org/licenses/>.
   8
   9 # prepare environment
  10 #
  11 adb_pid="${$}"
  12 adb_pidfile="/var/run/adblock.pid"
  13 adb_scriptver="1.2.7"
  14 adb_mincfgver="2.2"
  15 adb_scriptdir="${0%/*}"
  16 if [ -r "${adb_pidfile}" ]
  17 then
  18     rc=255
  19     logger -s -t "adblock[${adb_pid}] error" "adblock service already running ($(cat ${adb_pidfile}))"
  20     exit ${rc}
  21 else
  22     printf "${adb_pid}" > "${adb_pidfile}"
  23     if [ -r "${adb_scriptdir}/adblock-helper.sh" ]
  24     then
  25         . "${adb_scriptdir}/adblock-helper.sh"
  26         f_envload
  27     else
  28         rc=254
  29         logger -s -t "adblock[${adb_pid}] error" "adblock function library not found"
  30         rm -f "${adb_pidfile}"
  31         exit ${rc}
  32     fi
  33 fi
  34
  35 # call trap function on error signals (HUP, INT, QUIT, BUS, SEGV, TERM)
  36 #
  37 trap "rc=250; f_log 'error signal received/trapped'; f_exit" 1 2 3 10 11 15
  38
  39 # check environment
  40 #
  41 f_envcheck
  42
  43 # main loop for all block list sources
  44 #
  45 for src_name in ${adb_sources}
  46 do
  47     # check disabled sources
  48     #
  49     eval "enabled=\"\${enabled_${src_name}}\""
  50     if [ "${enabled}" = "0" ]
  51     then
  52         if [ -r "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" ]
  53         then
  54             rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
  55             if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
  56             then
  57                 rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
  58             fi
  59             rm_done="true"
  60             f_log "=> disabled source '${src_name}' removed"
  61         fi
  62         "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
  63         "${adb_uci}" -q delete "adblock.${src_name}.adb_src_timestamp"
  64         continue
  65     fi
  66
  67     f_log "=> processing source '${src_name}'"
  68     eval "url=\"\${adb_src_${src_name}}\""
  69     eval "src_rset=\"\${adb_src_rset_${src_name}}\""
  70     eval "list_time=\"\${CONFIG_${src_name}_adb_src_timestamp}\""
  71     adb_dnsfile="${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
  72
  73     # check 'url' and 'src_rset' values
  74     #
  75     if [ -z "${url}" ] || [ -z "${src_rset}" ]
  76     then
  77         "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=broken config"
  78         f_log "   broken source configuration, skipped"
  79         continue
  80     fi
  81
  82     # download only block list with newer/updated timestamp
  83     #
  84     if [ "${src_name}" = "blacklist" ]
  85     then
  86         url_time="$(date -r "${url}")"
  87     else
  88         url_time="$(${adb_fetch} ${fetch_parm} --server-response --spider "${url}" 2>&1 | awk '$0 ~ /Last-Modified/ {printf substr($0,18)}')"
  89     fi
  90     if [ -z "${url_time}" ]
  91     then
  92         url_time="$(date)"
  93         f_log "   no online timestamp"
  94     fi
  95     if [ -z "${list_time}" ] || [ "${list_time}" != "${url_time}" ] || [ ! -r "${adb_dnsfile}" ] ||\
  96       ([ "${backup_ok}" = "true" ] && [ ! -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ])
  97     then
  98         if [ "${src_name}" = "blacklist" ]
  99         then
 100             tmp_domains="$(cat "${url}")"
 101         elif [ "${src_name}" = "shalla" ]
 102         then
 103             shalla_archive="${adb_tmpdir}/shallalist.tar.gz"
 104             shalla_file="${adb_tmpdir}/shallalist.txt"
 105             "${adb_fetch}" ${fetch_parm} --output-document="${shalla_archive}" "${url}"
 106             rc=${?}
 107             if [ $((rc)) -eq 0 ]
 108             then
 109                 > "${shalla_file}"
 110                 for category in ${adb_src_cat_shalla}
 111                 do
 112                     tar -xOzf "${shalla_archive}" BL/${category}/domains >> "${shalla_file}"
 113                     rc=${?}
 114                     if [ $((rc)) -ne 0 ]
 115                     then
 116                         f_log "   archive extraction failed (${category})"
 117                         break
 118                     fi
 119                 done
 120                 tmp_domains="$(cat "${shalla_file}")"
 121                 rm -rf "${adb_tmpdir}/BL"
 122                 rm -f "${shalla_archive}"
 123                 rm -f "${shalla_file}"
 124             fi
 125         else
 126             tmp_domains="$(${adb_fetch} ${fetch_parm} --output-document=- "${url}")"
 127         fi
 128         rc=${?}
 129     else
 130         f_log "   source doesn't change, skipped"
 131         continue
 132     fi
 133
 134     # check download result and prepare domain output, backup/restore if needed
 135     #
 136     if [ $((rc)) -eq 0 ] && [ -n "${tmp_domains}" ]
 137     then
 138         count="$(printf "%s\n" "${tmp_domains}" | awk "${src_rset}" | tee "${adb_tmpfile}" | wc -l)"
 139         "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=${url_time}"
 140         if [ "${backup_ok}" = "true" ]
 141         then
 142             gzip -cf "${adb_tmpfile}" > "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
 143         fi
 144         f_log "   source download finished (${count} entries)"
 145         unset tmp_domains
 146     elif [ $((rc)) -eq 0 ] && [ -z "${tmp_domains}" ]
 147     then
 148         "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
 149         "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty download"
 150         f_log "   empty source download, skipped"
 151         continue
 152     else
 153         rc=0
 154         if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
 155         then
 156             gunzip -cf "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" > "${adb_tmpfile}"
 157             count="$(wc -l < "${adb_tmpfile}")"
 158             "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=list restored"
 159             f_log "   source download failed, restored (${count} entries)"
 160         else
 161             if [ -r "${adb_dnsdir}/${adb_dnsprefix}.${src_name}" ]
 162             then
 163                 rm -f "${adb_dnsdir}/${adb_dnsprefix}.${src_name}"
 164                 rm_done="true"
 165             fi
 166             "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
 167             "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=download failed"
 168             f_log "   source download failed, skipped"
 169             continue
 170         fi
 171     fi
 172
 173     # remove whitelist domains, sort domains and make them unique,
 174     # rewrite ad/abuse domain information to separate dnsmasq files
 175     #
 176     if [ $((count)) -gt 0 ] && [ -n "${adb_tmpfile}" ]
 177     then
 178         if [ -s "${adb_tmpdir}/tmp.whitelist" ]
 179         then
 180             grep -vf "${adb_tmpdir}/tmp.whitelist" "${adb_tmpfile}" | sort -u | eval "${adb_dnsformat}" > "${adb_dnsfile}"
 181         else
 182             sort -u "${adb_tmpfile}" | eval "${adb_dnsformat}" > "${adb_dnsfile}"
 183         fi
 184         rc=${?}
 185
 186         # finish domain processing, prepare find statement with revised block list source
 187         #
 188         if [ $((rc)) -eq 0 ]
 189         then
 190             if [ -z "${adb_revsrclist}" ]
 191             then
 192                 adb_revsrclist="-name ${adb_dnsprefix}.${src_name}"
 193             else
 194                 adb_revsrclist="${adb_revsrclist} -o -name ${adb_dnsprefix}.${src_name}"
 195             fi
 196             f_log "   domain merging finished"
 197         else
 198             rc=0
 199             rm -f "${adb_dnsfile}"
 200             if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
 201             then
 202                 rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
 203             fi
 204             "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
 205             "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=domain merging failed"
 206             f_log "   domain merging failed, skipped"
 207             continue
 208         fi
 209     else
 210         rm -f "${adb_dnsfile}"
 211         if [ "${backup_ok}" = "true" ] && [ -r "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz" ]
 212         then
 213             rm -f "${adb_dir_backup}/${adb_dnsprefix}.${src_name}.gz"
 214         fi
 215         "${adb_uci}" -q delete "adblock.${src_name}.adb_src_count"
 216         "${adb_uci}" -q set "adblock.${src_name}.adb_src_timestamp=empty domain input"
 217         f_log "   empty domain input, skipped"
 218         continue
 219     fi
 220 done
 221
 222 # make separate block lists entries unique
 223 #
 224 if [ "${mem_ok}" = "true" ] && [ -n "${adb_revsrclist}" ]
 225 then
 226     f_log "remove duplicates in separate block lists"
 227
 228     # generate a unique overall block list
 229     #
 230     sort -u "${adb_dnsdir}/${adb_dnsprefix}."* > "${adb_tmpdir}/blocklist.overall"
 231
 232     # loop through all separate lists, ordered by size (ascending)
 233     #
 234     for list in $(ls -ASr "${adb_dnsdir}/${adb_dnsprefix}"*)
 235     do
 236         # check overall block list vs. separate block list,
 237         # write all duplicate entries to separate list
 238         #
 239         list="${list/*./}"
 240         sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -d > "${adb_tmpdir}/tmp.${list}"
 241         mv -f "${adb_tmpdir}/tmp.${list}" "${adb_dnsdir}/${adb_dnsprefix}.${list}"
 242
 243         # write all unique entries back to overall block list
 244         #
 245         sort "${adb_tmpdir}/blocklist.overall" "${adb_dnsdir}/${adb_dnsprefix}.${list}" | uniq -u > "${adb_tmpdir}/tmp.overall"
 246         mv -f "${adb_tmpdir}/tmp.overall" "${adb_tmpdir}/blocklist.overall"
 247     done
 248     rm -f "${adb_tmpdir}/blocklist.overall"
 249 fi
 250
 251 # restart & check dnsmasq with newly generated set of block lists
 252 #
 253 if [ -n "${adb_revsrclist}" ] || [ "${rm_done}" = "true" ]
 254 then
 255     "${adb_uci}" -q delete "adblock.global.adb_dnstoggle"
 256     /etc/init.d/dnsmasq restart
 257     sleep 1
 258     check="$(pgrep -f "dnsmasq")"
 259     if [ -n "${check}" ]
 260     then
 261         dns_ok="true"
 262     else
 263         f_log "dnsmasq restart failed, retry without newly generated block lists"
 264         rm_done="$(find "${adb_dnsdir}" -maxdepth 1 -type f \( ${adb_revsrclist} \) -print -exec rm -f "{}" \;)"
 265         if [ -n "${rm_done}" ]
 266         then
 267             /etc/init.d/dnsmasq restart
 268             sleep 1
 269             check="$(pgrep -f "dnsmasq")"
 270             if [ -n "${check}" ]
 271             then
 272                 dns_ok="true"
 273             fi
 274         fi
 275     fi
 276     if [ "${dns_ok}" = "true" ]
 277     then
 278         f_cntconfig
 279         f_log "block lists with overall ${adb_count} domains loaded"
 280     else
 281         rc=100
 282         f_log "dnsmasq restart finally failed, please check 'logread' output"
 283         f_exit
 284     fi
 285 else
 286     f_cntconfig
 287     f_log "block lists with overall ${adb_count} domains are still valid, no update required"
 288 fi
 289
 290 # remove temporary files and exit
 291 #
 292 f_exit