webproxy: T563: use new bash blacklist updater

2 files changed, 104 insertions, 93 deletions

diff --git a/src/op_mode/webproxy_update_blacklist.py b/src/op_mode/webproxy_update_blacklist.py
deleted file mode 100755
index c6572c663..000000000
--- a/src/op_mode/webproxy_update_blacklist.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-#
-# Copyright (C) 2020 VyOS maintainers and contributors
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 or later as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-#blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz'
-blacklist_url = 'http://lnx01.mybll.net/~cpo/blacklists.tar.gz'
-global_data_dir = '/config/url-filtering'
-sg_dir = f'{global_data_dir}/squidguard'
-blacklist_dir = f'{sg_dir}/db'
-archive_dir = f'{sg_dir}/archive'
-target_file = '/tmp/blacklists.tar.gz'
-
-#
-# XXX: this is a proof of concept for downloading a file via Python
-#
-
-
-import os
-import shutil
-import argparse
-import urllib.request
-import tarfile
-
-from tqdm import tqdm
-from vyos.util import chown
-from vyos.util import chmod
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--update", help="Update SquidGuard blacklist",
-        action="store_true")
-args = parser.parse_args()
-
-class DownloadProgressBar(tqdm):
-    def update_to(self, b=1, bsize=1, tsize=None):
-        if tsize is not None:
-            self.total = tsize
-        self.update(b * bsize - self.n)
-
-def download_url(url, output_path):
-    with DownloadProgressBar(unit='B', unit_scale=True,
-                             miniters=1, desc=url.split('/')[-1]) as t:
-        urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)
-
-def squidguard_is_blacklist_installed():
-    return os.path.exists(blacklist_dir)
-
-
-def install_blacklist():
-    download_url(blacklist_url, target_file)
-
-    print('Uncompressing blacklist...')
-    tar = tarfile.open(target_file, "r:gz")
-    tar.extractall(path='/tmp')
-    tar.close()
-
-    if not os.path.exists(sg_dir):
-        os.makedirs(sg_dir, exist_ok=True)
-
-    if os.path.exists(archive_dir):
-        print('Removing old archive...')
-        shutil.rmtree(archive_dir)
-
-    if os.path.exists(blacklist_dir):
-        print('Archiving old blacklist...')
-        shutil.move(blacklist_dir, archive_dir)
-
-    shutil.move('/tmp/blacklists', blacklist_dir)
-
-    chown(blacklist_dir, 'proxy', 'proxy')
-    chmod(blacklist_dir, 0o755)
-
-
-if args.update:
-    if not squidguard_is_blacklist_installed():
-        print('Warning: No url-filtering blacklist installed')
-        input('Would you like to download a default blacklist? [confirm]')
-
-    else:
-        input('Would you like to re-download the blacklist? [confirm]')
-
-    install_blacklist()
diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh
new file mode 100755
index 000000000..43a4b79fc
--- /dev/null
+++ b/src/op_mode/webproxy_update_blacklist.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz'
+data_dir="/opt/vyatta/etc/config/url-filtering"
+archive="${data_dir}/squidguard/archive"
+db_dir="${data_dir}/squidguard/db"
+
+while [ $# -gt 0 ]
+do
+    case $1 in
+    --update-blacklist)
+        update="yes"
+        ;;
+    --auto-update-blacklist)
+        auto="yes"
+        ;;
+    (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;;
+    (*) break;;
+    esac
+    shift
+done
+
+if [ ! -d ${db_dir} ]; then
+    mkdir -p ${db_dir}
+    getent passwd proxy 2> /dev/null
+    if [ $? -ne 0 ]; then
+        echo "proxy system user does not exist"
+        exit 1
+    fi
+    getent group proxy 2> /dev/null
+    if [ $? -ne 0 ]; then
+        echo "proxy system group does not exist"
+        exit 1
+    fi
+    chown proxy:proxy ${db_dir}
+fi
+
+free_space=$(expr $(df ${db_dir} | grep -v Filesystem | awk '{print $4}') \* 1024)
+mb_size="100"
+required_space=$(expr $mb_size \* 1024 \* 1024) # 100 MB
+if [ ${free_space} -le ${required_space} ]; then
+    echo "Error: not enough disk space, required  ${mb_size} MiB"
+    exit 1
+fi
+
+if [[ -n $update ]] && [[ $update -eq "yes" ]]; then
+    tmp_blacklists='/tmp/blacklists.gz'
+    curl -o $tmp_blacklists $blacklist_url
+    if [ $? -ne 0 ]; then
+        echo "Unable to download [$blacklist_url]!"
+        exit 1
+    fi
+    echo "Uncompressing blacklist..."
+    tar --directory /tmp -xf $tmp_blacklists
+    if [ $? -ne 0 ]; then
+        echo "Unable to uncompress [$blacklist_url]!"
+    fi
+
+    if [ ! -d ${archive} ]; then
+        mkdir -p ${archive}
+    fi
+
+    rm -rf ${archive}/*
+    count_before=$(find ${db_dir} -type f \( -name domains -o -name urls \) | xargs wc -l | tail -n 1 | awk '{print $1}')
+    mv ${db_dir}/* ${archive} 2> /dev/null
+    mv /tmp/blacklists/* ${db_dir}
+    if [ $? -ne 0 ]; then
+        echo "Unable to install [$blacklist_url]"
+        exit 1
+    fi
+    mv ${archive}/local-* ${db_dir} 2> /dev/null
+    rm -rf /tmp/blacklists $tmp_blacklists 2> /dev/null
+    count_after=$(find ${db_dir} -type f \( -name domains -o -name urls \) | xargs wc -l | tail -n 1 | awk '{print $1}')
+
+    # fix permissions
+    chown -R proxy:proxy ${db_dir}
+    chmod 2770 ${db_dir}
+
+    logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})"
+
+else
+    echo "SquidGuard blacklist updater"
+    echo ""
+    echo "Usage:"
+    echo "--update-blacklist            Download latest version of the SquidGuard blacklist"
+    echo "--auto-update-blacklist       Automatically update"
+    echo ""
+    exit 1
+fi
+