summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rwxr-xr-xsrc/completion/list_webproxy_category.sh5
-rwxr-xr-xsrc/conf_mode/service_webproxy.py209
-rwxr-xr-xsrc/legacy/vyatta-sg-blacklist.pl682
-rwxr-xr-xsrc/op_mode/webproxy_update_blacklist.py93
4 files changed, 989 insertions, 0 deletions
diff --git a/src/completion/list_webproxy_category.sh b/src/completion/list_webproxy_category.sh
new file mode 100755
index 000000000..a5ad2398a
--- /dev/null
+++ b/src/completion/list_webproxy_category.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+DB_DIR="/opt/vyatta/etc/config/url-filtering/squidguard/db/"
+if [ -d ${DB_DIR} ]; then
+ ls -ald ${DB_DIR}/* | grep -E '^(d|l)' | awk '{print $9}' | sed s#${DB_DIR}/##
+fi
diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py
new file mode 100755
index 000000000..8dfae348a
--- /dev/null
+++ b/src/conf_mode/service_webproxy.py
@@ -0,0 +1,209 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import os
+
+from shutil import rmtree
+from sys import exit
+
+from vyos.config import Config
+from vyos.configdict import dict_merge
+from vyos.template import render
+from vyos.util import call
+from vyos.util import dict_search
+from vyos.util import write_file
+from vyos.validate import is_addr_assigned
+from vyos.xml import defaults
+from vyos import ConfigError
+from vyos import airbag
+airbag.enable()
+
+squid_config_file = '/etc/squid/squid.conf'
+squidguard_config_file = '/etc/squidguard/squidGuard.conf'
+squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db'
+user_group = 'proxy'
+
+def generate_sg_localdb(category, list_type, role, proxy):
+ cat_ = category.replace('-', '_')
+ if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy),
+ list):
+
+ # local block databases must be generated "on-the-fly"
+ tmp = {
+ 'squidguard_db_dir' : squidguard_db_dir,
+ 'category' : f'{category}-default',
+ 'list_type' : list_type,
+ 'rule' : role
+ }
+ sg_tmp_file = '/tmp/sg.conf'
+ db_file = f'{category}-default/{list_type}'
+ domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy))
+
+ # local file
+ write_file(f'{squidguard_db_dir}/{category}-default/local', '',
+ user=user_group, group=user_group)
+ # database input file
+ write_file(f'{squidguard_db_dir}/{db_file}', domains,
+ user=user_group, group=user_group)
+
+ # temporary config file, deleted after generation
+ render(sg_tmp_file, 'squid/sg_acl.conf.tmpl', tmp,
+ user=user_group, group=user_group)
+
+ call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"')
+
+ if os.path.exists(sg_tmp_file):
+ os.unlink(sg_tmp_file)
+
+ else:
+ # if category is not part of our configuration, clean out the
+ # squidguard lists
+ tmp = f'{squidguard_db_dir}/{category}-default'
+ if os.path.exists(tmp):
+ rmtree(f'{squidguard_db_dir}/{category}-default')
+
+def get_config(config=None):
+ if config:
+ conf = config
+ else:
+ conf = Config()
+ base = ['service', 'webproxy']
+ if not conf.exists(base):
+ return None
+
+ proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+ # We have gathered the dict representation of the CLI, but there are default
+ # options which we need to update into the dictionary retrived.
+ default_values = defaults(base)
+
+ # if no authentication method is supplied, no need to add defaults
+ if not dict_search('authentication.method', proxy):
+ default_values.pop('authentication')
+ # if no url_filteringurl-filtering method is supplied, no need to add defaults
+ if 'url_filtering' not in proxy:
+ default_values.pop('url_filtering')
+ else:
+ # store path to squidGuard config, used when generating Squid config
+ proxy['squidguard_conf'] = squidguard_config_file
+ proxy['squidguard_db_dir'] = squidguard_db_dir
+
+ # XXX: T2665: blend in proper cache-peer default values later
+ default_values.pop('cache_peer')
+ proxy = dict_merge(default_values, proxy)
+
+ # XXX: T2665: blend in proper cache-peer default values
+ if 'cache_peer' in proxy:
+ default_values = defaults(base + ['cache-peer'])
+ for peer in proxy['cache_peer']:
+ proxy['cache_peer'][peer] = dict_merge(default_values,
+ proxy['cache_peer'][peer])
+
+ return proxy
+
+def verify(proxy):
+ if not proxy:
+ return None
+
+ if 'listen_address' not in proxy:
+ raise ConfigError('listen-address needs to be configured!')
+
+ ldap_auth = dict_search('authentication.method', proxy) == 'ldap'
+
+ for address, config in proxy['listen_address'].items():
+ if not is_addr_assigned(address):
+ raise ConfigError(
+ f'listen-address "{address}" not assigned on any interface!')
+ if ldap_auth and 'disable_transparent' not in config:
+ raise ConfigError('Authentication can not be configured when ' \
+ 'proxy is in transparent mode')
+
+ if 'outgoing_address' in proxy:
+ address = proxy['outgoing_address']
+ if not is_addr_assigned(address):
+ raise ConfigError(
+ f'outgoing-address "{address}" not assigned on any interface!')
+
+ if 'authentication' in proxy:
+ if 'method' not in proxy['authentication']:
+ raise ConfigError('proxy authentication method required!')
+
+ if ldap_auth:
+ ldap_config = proxy['authentication']['ldap']
+
+ if 'server' not in ldap_config:
+ raise ConfigError(
+ 'LDAP authentication enabled, but no server set')
+
+ if 'password' in ldap_config and 'bind_dn' not in ldap_config:
+ raise ConfigError(
+ 'LDAP password can not be set when base-dn is undefined!')
+
+ if 'bind_dn' in ldap_config and 'password' not in ldap_config:
+ raise ConfigError(
+ 'LDAP bind DN can not be set without password!')
+
+ if 'base_dn' not in ldap_config:
+ raise ConfigError('LDAP base-dn must be set!')
+
+ if 'cache_peer' in proxy:
+ for peer, config in proxy['cache_peer'].items():
+ if 'address' not in config:
+ raise ConfigError(f'Cache-peer "{peer}" address must be set!')
+
+
+def generate(proxy):
+ if not proxy:
+ return None
+
+ render(squid_config_file, 'squid/squid.conf.tmpl', proxy)
+ render(squidguard_config_file, 'squid/squidGuard.conf.tmpl', proxy)
+
+ cat_dict = {
+ 'local-block' : 'domains',
+ 'local-block-keyword' : 'expressions',
+ 'local-block-url' : 'urls',
+ 'local-ok' : 'domains',
+ 'local-ok-url' : 'urls'
+ }
+ for category, list_type in cat_dict.items():
+ generate_sg_localdb(category, list_type, 'default', proxy)
+
+ return None
+
+def apply(proxy):
+ if not proxy:
+ # proxy is removed in the commit
+ call('systemctl stop squid.service')
+
+ if os.path.exists(squid_config_file):
+ os.unlink(squid_config_file)
+ if os.path.exists(squidguard_config_file):
+ os.unlink(squidguard_config_file)
+
+ return None
+
+ call('systemctl restart squid.service')
+ return None
+
+if __name__ == '__main__':
+ try:
+ c = get_config()
+ verify(c)
+ generate(c)
+ apply(c)
+ except ConfigError as e:
+ print(e)
+ exit(1)
diff --git a/src/legacy/vyatta-sg-blacklist.pl b/src/legacy/vyatta-sg-blacklist.pl
new file mode 100755
index 000000000..4104ac266
--- /dev/null
+++ b/src/legacy/vyatta-sg-blacklist.pl
@@ -0,0 +1,682 @@
+#!/usr/bin/perl
+#
+# Module: vyatta-sg-blacklist.pl
+#
+# **** License ****
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+# General Public License for more details.
+#
+# This code was originally developed by Vyatta, Inc.
+# Portions created by Vyatta are Copyright (C) 2008-2009 Vyatta, Inc.
+# All Rights Reserved.
+#
+# Author: Stig Thormodsrud
+# Date: October 2008
+# Description: script to download/update free url blacklist.
+#
+# **** End License ****
+#
+
+use Getopt::Long;
+use POSIX;
+use IO::Prompt;
+use Sys::Syslog qw(:standard :macros);
+use File::Copy;
+use Fcntl qw(:flock);
+use base qw(Exporter);
+use File::Basename;
+use File::Compare;
+
+use lib "/opt/vyatta/share/perl5";
+use Vyatta::Config;
+use Vyatta::File;
+
+use warnings;
+use strict;
+
+#
+# Default blacklist
+#
+# Below are some free blacklists we've tried:
+#
+# http://squidguard.mesd.k12.or.us/blacklists.tgz
+# http://ftp.teledanmark.no/pub/www/proxy/squidguard/contrib/blacklists.tar.gz
+# ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz
+#
+# Note: the auto install/update assumes that the blacklist url is a tar gz
+# file with the blacklist categorys in a "blacklist" directory. Some
+# of the commercially available blacklists are a cgi script instead, so
+# those blacklists will need a different install/update script. Of
+# course they can be manually installed/updated.
+#
+my $blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz';
+
+#squid globals
+my $squid_init = '/etc/init.d/squid';
+my $squid_mime_type = '/usr/share/squid/mime.conf';
+
+#squidGuard globals
+my $urlfilter_data_dir = '/opt/vyatta/etc/config/url-filtering';
+my $squidguard_blacklist_db = "$urlfilter_data_dir/squidguard/db";
+my $squidguard_log_dir = '/var/log/squid';
+my $squidguard_blacklist_log = "$squidguard_log_dir/blacklist.log";
+my $squidguard_safesearch = "/opt/vyatta/etc/safesearch_rewrites";
+
+#vyattaguard globals
+my $vyattaguard = '/opt/vyatta/sbin/vg';
+
+sub webproxy_get_global_data_dir {
+ return $urlfilter_data_dir;
+}
+
+my $global_data_dir = webproxy_get_global_data_dir();
+
+
+sub squid_get_mime {
+ my @mime_types = ();
+ open(my $FILE, "<", $squid_mime_type) or die "Error: read $!";
+ my @lines = <$FILE>;
+ close($FILE);
+ foreach my $line (@lines) {
+ next if $line =~ /^#/; # skip comments
+ if ($line =~ /^([\S]+)[\s]+([\S]+)[\s]+([\S]+)[\s]+([\S]+).*$/) {
+ my $type = $2;
+ push @mime_types, $type if $type =~ /\//;
+ }
+ }
+ return @mime_types;
+}
+
+sub squidguard_is_configured {
+ my $config = new Vyatta::Config;
+ $config->setLevel('service webproxy url-filtering');
+ # This checks the running config, so it is assumed
+ # to be called from op mode.
+ return 1 if $config->existsOrig('squidguard');
+ return 0;
+}
+
+sub squidguard_get_blacklist_dir {
+ return $squidguard_blacklist_db;
+}
+
+sub squidguard_get_blacklist_log {
+ return $squidguard_blacklist_log;
+}
+
+sub squidguard_get_safesearch_rewrites {
+ my @rewrites = ();
+ open(my $FILE, "<", $squidguard_safesearch) or die "Error: read $!";
+ my @lines = <$FILE>;
+ close($FILE);
+ chomp @lines;
+ foreach my $line (@lines) {
+ next if $line =~ /^#/; # skip comments
+ if ($line =~ /^s\@/) {
+ push @rewrites, $line;
+ }
+ }
+ return @rewrites;
+}
+
+sub squidguard_ec_get_categorys {
+ my %cat_hash;
+
+ die "Must enable vyattaguard" if ! squidguard_use_ec();
+ die "Missing vyattaguard package\n" if ! -e $vyattaguard;
+ exit 1 if ! -e "$urlfilter_data_dir/sitefilter/categories.txt";
+
+ my @lines = `$vyattaguard list`;
+ foreach my $line (@lines) {
+ my ($id, $category) = split ':', $line;
+ next if ! defined $category;
+ chomp $category;
+ $category =~ s/\s/\_/g;
+ $category =~ s/\&/\_and\_/g;
+ $cat_hash{$id} = $category;
+ }
+ return %cat_hash;
+}
+
+sub squidguard_ec_cat2name {
+ my ($cat) = @_;
+
+ my %cat_hash = squidguard_ec_get_categorys();
+ return $cat_hash{$cat} if defined $cat_hash{$cat};
+ return;
+}
+
+sub squidguard_ec_name2cat {
+ my ($name) = @_;
+
+ my %cat_hash = squidguard_ec_get_categorys();
+ foreach my $key (keys (%cat_hash)) {
+ if ($cat_hash{$key} eq $name) {
+ return $key;
+ }
+ }
+ return;
+}
+
+sub squidguard_use_ec {
+ my $rc = system("cli-shell-api inSession");
+ my ($exist_func, $value_func);
+ if ($rc == 0) {
+ $exist_func = 'exists';
+ $value_func = 'returnValue';
+ } else {
+ $exist_func = 'existsOrig';
+ $value_func = 'returnOrigValue';
+ }
+ my $config = new Vyatta::Config;
+ $config->setLevel('service webproxy url-filtering squidguard');
+ if ($config->$exist_func('vyattaguard')) {
+ return if ! -e $vyattaguard;
+ my $mode = $config->$value_func('vyattaguard mode');
+ return $mode;
+ }
+ return;
+}
+
+sub squidguard_get_blacklists {
+
+ my @blacklists = ();
+ if (squidguard_use_ec()) {
+ die "Missing vyattaguard package\n" if ! -e $vyattaguard;
+ my %cat_hash = squidguard_ec_get_categorys();
+ foreach my $key (keys (%cat_hash)) {
+ next if ! defined $cat_hash{$key};
+ push @blacklists, $cat_hash{$key};
+ }
+ } else {
+ my $dir = $squidguard_blacklist_db;
+ opendir(DIR, $dir) || die "can't opendir $dir: $!";
+ my @dirs = readdir(DIR);
+ closedir DIR;
+
+ foreach my $file (@dirs) {
+ next if $file eq '.';
+ next if $file eq '..';
+ if (-d "$dir/$file") {
+ push @blacklists, $file;
+ }
+ }
+ }
+ @blacklists = sort(@blacklists);
+ return @blacklists;
+}
+
+sub squidguard_generate_db {
+ my ($interactive, $category, $group) = @_;
+
+ my $db_dir = squidguard_get_blacklist_dir();
+ my $tmp_conf = "/tmp/sg.conf.$$";
+ my $output = "dbhome $db_dir\n";
+ $output .= squidguard_build_dest($category, 0, $group);
+ $output .= "\nacl {\n";
+ $output .= "\tdefault {\n";
+ $output .= "\t\tpass all\n";
+ $output .= "\t}\n}\n\n";
+ webproxy_write_file($tmp_conf, $output);
+
+ my $dir = "$db_dir/$category";
+ if ( -l $dir) {
+ print "Skip link for [$category] -> [", readlink($dir), "]\n"
+ if $interactive;
+ return;
+ }
+ foreach my $type ('domains', 'urls', 'expressions') {
+ my $path = "$category/$type";
+ my $file = "$db_dir/$path";
+ if (-e $file and -s _) { # check exists and non-zero
+ my $file_db = "$file.db";
+ if (! -e $file_db) {
+ #
+ # it appears that there is a bug in squidGuard that if
+ # the db file doesn't exist then running with -C leaves
+ # huge tmp files in /var/tmp.
+ #
+ system("touch $file.db");
+ system("chown -R proxy.proxy $file.db > /dev/null 2>&1");
+ }
+ my $wc = `cat $file| wc -l`; chomp $wc;
+ print "Building DB for [$path] - $wc entries\n" if $interactive;
+ my $cmd = "\"squidGuard -d -c $tmp_conf -C $path\"";
+ system("su - proxy -c $cmd > /dev/null 2>&1");
+ }
+ }
+ system("rm $tmp_conf");
+}
+
+sub squidguard_is_category_local {
+ my ($category) = @_;
+
+ my $db_dir = squidguard_get_blacklist_dir();
+ my $local_file = "$db_dir/$category/local";
+ return 1 if -e $local_file;
+ return 0;
+}
+
+sub squidguard_is_blacklist_installed {
+ if (squidguard_use_ec()) {
+ if (-e "$urlfilter_data_dir/sitefilter/urldb") {
+ return 1;
+ }
+ } else {
+ my @blacklists = squidguard_get_blacklists();
+ foreach my $category (@blacklists) {
+ next if squidguard_is_category_local($category);
+ return 1;
+ }
+ }
+ return 0;
+}
+
+sub squidguard_get_blacklist_domains_urls_exps {
+ my ($list) = shift;
+
+ my $dir = $squidguard_blacklist_db;
+ my ($domains, $urls, $exps) = undef;
+ $domains = "$list/domains" if -f "$dir/$list/domains" && -s _;
+ $urls = "$list/urls" if -f "$dir/$list/urls" && -s _;
+ $exps = "$list/expressions" if -f "$dir/$list/expressions" && -s _;
+ return ($domains, $urls, $exps);
+}
+
+sub squidguard_get_blacklist_files {
+ my ($type, $category) = @_;
+
+ my @lists = squidguard_get_blacklists();
+ my @files = ();
+ foreach my $list (@lists) {
+ my ($domain, $url, $exp) = squidguard_get_blacklist_domains_urls_exps(
+ $list);
+ if ($type eq 'domains') {
+ next if !defined $domain;
+ if (defined $category) {
+ next if $domain ne "$category/domains";
+ }
+ $domain = "$squidguard_blacklist_db/$domain";
+ push @files, $domain;
+ }
+ if ($type eq 'urls') {
+ next if !defined $url;
+ if (defined $category) {
+ next if $url ne "$category/urls";
+ }
+ $url = "$squidguard_blacklist_db/$url";
+ push @files, $url;
+ }
+ if ($type eq 'expressions') {
+ next if !defined $exp;
+ if (defined $category) {
+ next if $url ne "$category/expressions";
+ }
+ $exp = "$squidguard_blacklist_db/$exp";
+ push @files, $exp;
+ }
+
+ }
+ @files = sort(@files);
+ return @files;
+}
+
+sub squidguard_get_log_files {
+ open(my $LS, "-|", "ls $squidguard_log_dir/bl*.log* 2> /dev/null | sort -nr ");
+ my @log_files = <$LS>;
+ close $LS;
+ chomp @log_files;
+ return @log_files;
+}
+
+sub squidguard_build_dest {
+ my ($category, $logging, $group, $ec) = @_;
+
+ my $output = '';
+ my ($domains, $urls, $exps);
+ if (squidguard_is_category_local("$category-$group")) {
+ ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps(
+ "$category-$group");
+ } else {
+ ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps(
+ $category);
+ }
+
+ my $ec_cat = undef;
+ if (defined $ec) {
+ $ec_cat = squidguard_ec_name2cat($category);
+ }
+
+ $output = "dest $category-$group {\n";
+ $output .= "\tdomainlist $domains\n" if defined $domains;
+ $output .= "\turllist $urls\n" if defined $urls;
+ $output .= "\texpressionlist $exps\n" if defined $exps;
+ $output .= "\teccategory $ec_cat\n" if defined $ec_cat;
+ if ($logging) {
+ my $log = basename($squidguard_blacklist_log);
+ $output .= "\tlog $log\n";
+ }
+ $output .= "}\n\n";
+ return $output;
+}
+
+sub webproxy_read_file {
+ my ($file) = @_;
+ my @lines;
+ if ( -e $file) {
+ open(my $FILE, '<', $file) or die "Error: read $!";
+ @lines = <$FILE>;
+ close($FILE);
+ chomp @lines;
+ }
+ return @lines;
+}
+
+sub is_same_as_file {
+ my ($file, $value) = @_;
+
+ return if ! -e $file;
+
+ my $mem_file = '';
+ open my $MF, '+<', \$mem_file or die "couldn't open memfile $!\n";
+ print $MF $value;
+ seek($MF, 0, 0);
+
+ my $rc = compare($file, $MF);
+ return 1 if $rc == 0;
+ return;
+}
+
+sub webproxy_write_file {
+ my ($file, $config) = @_;
+
+ # Avoid unnecessary writes. At boot the file will be the
+ # regenerated with the same content.
+ return if is_same_as_file($file, $config);
+
+ open(my $fh, '>', $file) || die "Couldn't open $file - $!";
+ print $fh $config;
+ close $fh;
+ return 1;
+}
+
+sub webproxy_append_file {
+ my ($dst, $src) = @_;
+
+ open(my $ih, '<', $src) || die "Couldn't open $src - $!";
+ open(my $oh, '>>', $dst) || die "Couldn't open $dst - $!";
+ for (<$ih>) {
+ print $oh $_;
+ }
+ close($oh);
+ close($ih);
+ return 1;
+}
+
+sub webproxy_delete_local_entry {
+ my ($file, $value) = @_;
+
+ my $db_dir = squidguard_get_blacklist_dir();
+ $file = "$db_dir/$file";
+ my @lines = webproxy_read_file($file);
+ my $config = '';
+ foreach my $line (@lines) {
+ $config .= "$line\n" if $line ne $value;
+ }
+ if ($config eq '') {
+ unlink($file);
+ } else {
+ webproxy_write_file($file, $config);
+ }
+ return;
+}
+
+sub webproxy_delete_all_local {
+ my $db_dir = squidguard_get_blacklist_dir();
+ my @categorys = squidguard_get_blacklists();
+ foreach my $category (@categorys) {
+ if (squidguard_is_category_local($category)) {
+ system("rm -rf $db_dir/$category");
+ }
+ }
+ return;
+}
+
+sub print_err {
+ my ($interactive, $msg) = @_;
+ if ($interactive) {
+ print "$msg\n";
+ } else {
+ syslog(LOG_ERR, $msg);
+ }
+}
+
+sub squidguard_count_blacklist_entries {
+ my $db_dir = squidguard_get_blacklist_dir();
+
+ my $total = 0;
+ my @categories = squidguard_get_blacklists();
+ foreach my $category (@categories) {
+ foreach my $type ('domains', 'urls') {
+ my $path = "$category/$type";
+ my $file = "$db_dir/$path";
+ if (-e $file) {
+ my $wc = `cat $file| wc -l`; chomp $wc;
+ $total += $wc;
+ }
+ }
+ }
+ return $total;
+}
+
+sub squidguard_clean_tmpfiles {
+ #
+ # workaround for squidguard
+ # bug http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=494281
+ #
+ my @tmpfiles = </var/tmp/*>;
+ foreach my $file (@tmpfiles) {
+ my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime,
+ $mtime, $ctime, $blksize, $blocks) = stat($file);
+ my $name = (getpwuid($uid))[0] if $uid;
+ unlink($file) if $name and $name eq 'proxy';
+ }
+}
+
+sub squidguard_auto_update {
+ my ($interactive, $file) = @_;
+
+ my $rc;
+ my $db_dir = squidguard_get_blacklist_dir();
+ my $tmp_blacklists = '/tmp/blacklists.gz';
+
+ if (!squidguard_is_blacklist_installed()) {
+ my ($disk_free, $disk_required);
+ $disk_required = (30 * 1024 * 1024); # 30MB
+ $disk_free = `df $db_dir | grep -v Filesystem | awk '{ print \$4 }'`;
+ chomp($disk_free);
+ $disk_free *= 1024;
+ if ($disk_free < $disk_required) {
+ die "Error: not enough disk space $disk_required\/$disk_free";
+ }
+ }
+
+ if (defined $file) {
+ # use existing file
+ $rc = copy($file, $tmp_blacklists);
+ if (!$rc) {
+ print_err($interactive, "Unable to copy [$file] $!");
+ return 1;
+ }
+ } else {
+ # get from net
+ my $opt = '';
+ $opt = "-q" if ! $interactive;
+ $rc = system("wget -O $tmp_blacklists $opt $blacklist_url");
+ if ($rc) {
+ print_err($interactive, "Unable to download [$blacklist_url] $!");
+ return 1;
+ }
+ }
+
+ print "Uncompressing blacklist...\n" if $interactive;
+ $rc = system("tar --directory /tmp -zxvf $tmp_blacklists > /dev/null");
+ if ($rc) {
+ print_err($interactive, "Unable to uncompress [$blacklist_url] $!");
+ return 1;
+ }
+ my $b4_entries = squidguard_count_blacklist_entries();
+ my $archive = "$global_data_dir/squidguard/archive";
+ mkdir_p($archive) if ! -d $archive;
+ system("rm -rf $archive/*");
+ system("mv $db_dir/* $archive 2> /dev/null");
+ $rc = system("mv /tmp/blacklists/* $db_dir");
+ if ($rc) {
+ print_err($interactive, "Unable to install [$blacklist_url] $!");
+ return 1;
+ }
+ system("mv $archive/local-* $db_dir 2> /dev/null");
+ rm_rf($tmp_blacklists);
+ rm_rf("/tmp/blacklists");
+
+ my $after_entries = squidguard_count_blacklist_entries();
+ my $mode = "auto-update";
+ $mode = "manual" if $interactive;
+ syslog(LOG_WARNING,
+ "blacklist entries updated($mode) ($b4_entries/$after_entries)");
+ return 0;
+}
+
+sub squidguard_install_blacklist_def {
+ squidguard_auto_update(1, undef);
+}
+
+sub squidguard_update_blacklist {
+ my ($interactive, $update_category) = @_;
+
+ my @blacklists = squidguard_get_blacklists();
+ print "Checking permissions...\n" if $interactive;
+ my $db_dir = squidguard_get_blacklist_dir();
+ system("chown -R proxy.proxy $db_dir > /dev/null 2>&1");
+ chmod(2770, $db_dir);
+
+ #
+ # generate temporary config for each category & generate DB
+ #
+ foreach my $category (@blacklists) {
+ next if defined $update_category and $update_category ne $category;
+ squidguard_generate_db($interactive, $category, 'default');
+ }
+}
+
+
+#
+# main
+#
+my ($update_bl, $update_bl_cat, $update_bl_file, $auto_update_bl);
+
+GetOptions("update-blacklist!" => \$update_bl,
+ "update-blacklist-category=s" => \$update_bl_cat,
+ "update-blacklist-file=s" => \$update_bl_file,
+ "auto-update-blacklist!" => \$auto_update_bl,
+);
+
+my $sg_updatestatus_file = "$global_data_dir/squidguard/updatestatus";
+if (! -e "$global_data_dir/squidguard") {
+ system("mkdir -p $global_data_dir/squidguard/db");
+ my ($login, $pass, $uid, $gid) = getpwnam('proxy')
+ or die "proxy not in passwd file";
+ chown $uid, $gid, "$global_data_dir/squidguard/db";
+}
+touch($sg_updatestatus_file);
+system("echo update failed at `date` > $sg_updatestatus_file");
+system("sudo rm -f /var/lib/sitefilter/updatestatus");
+
+my $lock_file = '/tmp/vyatta_bl_lock';
+open(my $lck, ">", $lock_file) || die "Lock failed\n";
+flock($lck, LOCK_EX);
+
+if (defined $update_bl_cat) {
+ squidguard_update_blacklist(1, $update_bl_cat);
+ if (squidguard_is_configured()) {
+ print "\nThe webproxy daemon must be restarted\n";
+ if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') ||
+ prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) {
+ squid_restart(1);
+ }
+ }
+ squidguard_clean_tmpfiles();
+}
+
+if (defined $update_bl) {
+ my $updated = 0;
+ if (!squidguard_is_blacklist_installed()) {
+ print "Warning: No url-filtering blacklist installed\n";
+ if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') ||
+ prompt("Would you like to download a default blacklist? [confirm]",
+ -y1d=>"y")) {
+ exit 1 if squidguard_install_blacklist_def();
+ $updated = 1;
+ } else {
+ exit 1;
+ }
+ } else {
+ if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') ||
+ prompt("Would you like to re-download the blacklist? [confirm]",
+ -y1d=>"y")) {
+ my $rc = squidguard_auto_update(1, undef);
+ $updated = 1 if ! $rc;
+ }
+ }
+ if (! $updated) {
+ print "No blacklist updated\n";
+ if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') ||
+ !prompt("Do you still want to generate binary DB? [confirm]",
+ -y1d=>"y")) {
+ exit 1;
+ }
+ }
+ # if there was an update we need to re-gen the binary DBs
+ # and restart the daemon
+ squidguard_update_blacklist(1);
+ if (squidguard_is_configured()) {
+ print "\nThe webproxy daemon must be restarted\n";
+ if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') ||
+ prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) {
+ squid_restart(1);
+ }
+ }
+ squidguard_clean_tmpfiles();
+}
+
+if (defined $update_bl_file) {
+ if (! -e $update_bl_file) {
+ die "Error: file [$update_bl_file] doesn't exist";
+ }
+ my $rc = squidguard_auto_update(0, $update_bl_file);
+ exit 1 if $rc;
+ squidguard_update_blacklist(1);
+ squidguard_clean_tmpfiles();
+}
+
+if (defined $auto_update_bl) {
+ my $rc = squidguard_auto_update(0);
+ exit 1 if $rc;
+ squidguard_update_blacklist(0);
+ if (squidguard_is_configured()) {
+ squid_restart(0);
+ }
+ squidguard_clean_tmpfiles();
+}
+
+system("echo update succeeded at `date` > $sg_updatestatus_file");
+close($lck);
+exit 0;
+
+#end of file
diff --git a/src/op_mode/webproxy_update_blacklist.py b/src/op_mode/webproxy_update_blacklist.py
new file mode 100755
index 000000000..c6572c663
--- /dev/null
+++ b/src/op_mode/webproxy_update_blacklist.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+#blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz'
+blacklist_url = 'http://lnx01.mybll.net/~cpo/blacklists.tar.gz'
+global_data_dir = '/config/url-filtering'
+sg_dir = f'{global_data_dir}/squidguard'
+blacklist_dir = f'{sg_dir}/db'
+archive_dir = f'{sg_dir}/archive'
+target_file = '/tmp/blacklists.tar.gz'
+
+#
+# XXX: this is a proof of concept for downloading a file via Python
+#
+
+
+import os
+import shutil
+import argparse
+import urllib.request
+import tarfile
+
+from tqdm import tqdm
+from vyos.util import chown
+from vyos.util import chmod
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--update", help="Update SquidGuard blacklist",
+ action="store_true")
+args = parser.parse_args()
+
+class DownloadProgressBar(tqdm):
+ def update_to(self, b=1, bsize=1, tsize=None):
+ if tsize is not None:
+ self.total = tsize
+ self.update(b * bsize - self.n)
+
+def download_url(url, output_path):
+ with DownloadProgressBar(unit='B', unit_scale=True,
+ miniters=1, desc=url.split('/')[-1]) as t:
+ urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)
+
+def squidguard_is_blacklist_installed():
+ return os.path.exists(blacklist_dir)
+
+
+def install_blacklist():
+ download_url(blacklist_url, target_file)
+
+ print('Uncompressing blacklist...')
+ tar = tarfile.open(target_file, "r:gz")
+ tar.extractall(path='/tmp')
+ tar.close()
+
+ if not os.path.exists(sg_dir):
+ os.makedirs(sg_dir, exist_ok=True)
+
+ if os.path.exists(archive_dir):
+ print('Removing old archive...')
+ shutil.rmtree(archive_dir)
+
+ if os.path.exists(blacklist_dir):
+ print('Archiving old blacklist...')
+ shutil.move(blacklist_dir, archive_dir)
+
+ shutil.move('/tmp/blacklists', blacklist_dir)
+
+ chown(blacklist_dir, 'proxy', 'proxy')
+ chmod(blacklist_dir, 0o755)
+
+
+if args.update:
+ if not squidguard_is_blacklist_installed():
+ print('Warning: No url-filtering blacklist installed')
+ input('Would you like to download a default blacklist? [confirm]')
+
+ else:
+ input('Would you like to re-download the blacklist? [confirm]')
+
+ install_blacklist()