diff options
| -rw-r--r-- | debian/control | 1 | ||||
| -rwxr-xr-x | debian/rules | 4 | ||||
| -rw-r--r-- | debian/vyos-1x.install | 1 | ||||
| -rw-r--r-- | op-mode-definitions/webproxy.xml | 13 | ||||
| -rwxr-xr-x | src/legacy/vyatta-sg-blacklist.pl | 682 | ||||
| -rwxr-xr-x | src/op_mode/webproxy_update_blacklist.py | 93 | ||||
| -rwxr-xr-x | src/op_mode/webproxy_update_blacklist.sh | 104 |
7 files changed, 105 insertions, 793 deletions
diff --git a/debian/control b/debian/control index 9750f08e2..ccdaa8492 100644 --- a/debian/control +++ b/debian/control @@ -99,7 +99,6 @@ Depends: python3-pyudev, python3-six, python3-tabulate, - python3-tqdm, python3-vici (>= 5.7.2), python3-voluptuous, python3-waitress, diff --git a/debian/rules b/debian/rules index 5e275b8a3..ab0df7201 100755 --- a/debian/rules +++ b/debian/rules @@ -51,10 +51,6 @@ override_dh_auto_install: mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/completion cp -r src/completion/* $(DIR)/$(VYOS_LIBEXEC_DIR)/completion - # Install legacy components - mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/legacy - cp -r src/legacy/* $(DIR)/$(VYOS_LIBEXEC_DIR)/legacy - # Install helper scripts cp -r src/helpers/* $(DIR)/$(VYOS_LIBEXEC_DIR)/ diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index 691fa4731..bfc30f7e6 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -21,6 +21,5 @@ usr/libexec/vyos/op_mode usr/libexec/vyos/services usr/libexec/vyos/system usr/libexec/vyos/validators -usr/libexec/vyos/legacy usr/libexec/vyos/*.py usr/share diff --git a/op-mode-definitions/webproxy.xml b/op-mode-definitions/webproxy.xml index bccffd0b3..f8ec8fb0a 100644 --- a/op-mode-definitions/webproxy.xml +++ b/op-mode-definitions/webproxy.xml @@ -101,18 +101,7 @@ <properties> <help>Update the webproxy blacklist database</help> </properties> - <children> - <tagNode name="category"> - <properties> - <help>Update a category of the webproxy blacklist database</help> - <completionHelp> - <script>${vyos_completion_dir}/list_webproxy_category.sh</script> - </completionHelp> - </properties> - <command>sudo /usr/libexec/vyos/legacy/vyatta-sg-blacklist.pl --update-blacklist-category="$5"</command> - </tagNode> - </children> - <command>sudo /usr/libexec/vyos/legacy/vyatta-sg-blacklist.pl --update-blacklist</command> + <command>sudo ${vyos_op_scripts_dir}/webproxy_update_blacklist.sh --update-blacklist</command> </node> </children> </node> diff --git a/src/legacy/vyatta-sg-blacklist.pl b/src/legacy/vyatta-sg-blacklist.pl deleted file mode 100755 index 4104ac266..000000000 --- a/src/legacy/vyatta-sg-blacklist.pl +++ /dev/null @@ -1,682 +0,0 @@ -#!/usr/bin/perl -# -# Module: vyatta-sg-blacklist.pl -# -# **** License **** -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# This code was originally developed by Vyatta, Inc. -# Portions created by Vyatta are Copyright (C) 2008-2009 Vyatta, Inc. -# All Rights Reserved. -# -# Author: Stig Thormodsrud -# Date: October 2008 -# Description: script to download/update free url blacklist. -# -# **** End License **** -# - -use Getopt::Long; -use POSIX; -use IO::Prompt; -use Sys::Syslog qw(:standard :macros); -use File::Copy; -use Fcntl qw(:flock); -use base qw(Exporter); -use File::Basename; -use File::Compare; - -use lib "/opt/vyatta/share/perl5"; -use Vyatta::Config; -use Vyatta::File; - -use warnings; -use strict; - -# -# Default blacklist -# -# Below are some free blacklists we've tried: -# -# http://squidguard.mesd.k12.or.us/blacklists.tgz -# http://ftp.teledanmark.no/pub/www/proxy/squidguard/contrib/blacklists.tar.gz -# ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz -# -# Note: the auto install/update assumes that the blacklist url is a tar gz -# file with the blacklist categorys in a "blacklist" directory. Some -# of the commercially available blacklists are a cgi script instead, so -# those blacklists will need a different install/update script. Of -# course they can be manually installed/updated. -# -my $blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz'; - -#squid globals -my $squid_init = '/etc/init.d/squid'; -my $squid_mime_type = '/usr/share/squid/mime.conf'; - -#squidGuard globals -my $urlfilter_data_dir = '/opt/vyatta/etc/config/url-filtering'; -my $squidguard_blacklist_db = "$urlfilter_data_dir/squidguard/db"; -my $squidguard_log_dir = '/var/log/squid'; -my $squidguard_blacklist_log = "$squidguard_log_dir/blacklist.log"; -my $squidguard_safesearch = "/opt/vyatta/etc/safesearch_rewrites"; - -#vyattaguard globals -my $vyattaguard = '/opt/vyatta/sbin/vg'; - -sub webproxy_get_global_data_dir { - return $urlfilter_data_dir; -} - -my $global_data_dir = webproxy_get_global_data_dir(); - - -sub squid_get_mime { - my @mime_types = (); - open(my $FILE, "<", $squid_mime_type) or die "Error: read $!"; - my @lines = <$FILE>; - close($FILE); - foreach my $line (@lines) { - next if $line =~ /^#/; # skip comments - if ($line =~ /^([\S]+)[\s]+([\S]+)[\s]+([\S]+)[\s]+([\S]+).*$/) { - my $type = $2; - push @mime_types, $type if $type =~ /\//; - } - } - return @mime_types; -} - -sub squidguard_is_configured { - my $config = new Vyatta::Config; - $config->setLevel('service webproxy url-filtering'); - # This checks the running config, so it is assumed - # to be called from op mode. - return 1 if $config->existsOrig('squidguard'); - return 0; -} - -sub squidguard_get_blacklist_dir { - return $squidguard_blacklist_db; -} - -sub squidguard_get_blacklist_log { - return $squidguard_blacklist_log; -} - -sub squidguard_get_safesearch_rewrites { - my @rewrites = (); - open(my $FILE, "<", $squidguard_safesearch) or die "Error: read $!"; - my @lines = <$FILE>; - close($FILE); - chomp @lines; - foreach my $line (@lines) { - next if $line =~ /^#/; # skip comments - if ($line =~ /^s\@/) { - push @rewrites, $line; - } - } - return @rewrites; -} - -sub squidguard_ec_get_categorys { - my %cat_hash; - - die "Must enable vyattaguard" if ! squidguard_use_ec(); - die "Missing vyattaguard package\n" if ! -e $vyattaguard; - exit 1 if ! -e "$urlfilter_data_dir/sitefilter/categories.txt"; - - my @lines = `$vyattaguard list`; - foreach my $line (@lines) { - my ($id, $category) = split ':', $line; - next if ! defined $category; - chomp $category; - $category =~ s/\s/\_/g; - $category =~ s/\&/\_and\_/g; - $cat_hash{$id} = $category; - } - return %cat_hash; -} - -sub squidguard_ec_cat2name { - my ($cat) = @_; - - my %cat_hash = squidguard_ec_get_categorys(); - return $cat_hash{$cat} if defined $cat_hash{$cat}; - return; -} - -sub squidguard_ec_name2cat { - my ($name) = @_; - - my %cat_hash = squidguard_ec_get_categorys(); - foreach my $key (keys (%cat_hash)) { - if ($cat_hash{$key} eq $name) { - return $key; - } - } - return; -} - -sub squidguard_use_ec { - my $rc = system("cli-shell-api inSession"); - my ($exist_func, $value_func); - if ($rc == 0) { - $exist_func = 'exists'; - $value_func = 'returnValue'; - } else { - $exist_func = 'existsOrig'; - $value_func = 'returnOrigValue'; - } - my $config = new Vyatta::Config; - $config->setLevel('service webproxy url-filtering squidguard'); - if ($config->$exist_func('vyattaguard')) { - return if ! -e $vyattaguard; - my $mode = $config->$value_func('vyattaguard mode'); - return $mode; - } - return; -} - -sub squidguard_get_blacklists { - - my @blacklists = (); - if (squidguard_use_ec()) { - die "Missing vyattaguard package\n" if ! -e $vyattaguard; - my %cat_hash = squidguard_ec_get_categorys(); - foreach my $key (keys (%cat_hash)) { - next if ! defined $cat_hash{$key}; - push @blacklists, $cat_hash{$key}; - } - } else { - my $dir = $squidguard_blacklist_db; - opendir(DIR, $dir) || die "can't opendir $dir: $!"; - my @dirs = readdir(DIR); - closedir DIR; - - foreach my $file (@dirs) { - next if $file eq '.'; - next if $file eq '..'; - if (-d "$dir/$file") { - push @blacklists, $file; - } - } - } - @blacklists = sort(@blacklists); - return @blacklists; -} - -sub squidguard_generate_db { - my ($interactive, $category, $group) = @_; - - my $db_dir = squidguard_get_blacklist_dir(); - my $tmp_conf = "/tmp/sg.conf.$$"; - my $output = "dbhome $db_dir\n"; - $output .= squidguard_build_dest($category, 0, $group); - $output .= "\nacl {\n"; - $output .= "\tdefault {\n"; - $output .= "\t\tpass all\n"; - $output .= "\t}\n}\n\n"; - webproxy_write_file($tmp_conf, $output); - - my $dir = "$db_dir/$category"; - if ( -l $dir) { - print "Skip link for [$category] -> [", readlink($dir), "]\n" - if $interactive; - return; - } - foreach my $type ('domains', 'urls', 'expressions') { - my $path = "$category/$type"; - my $file = "$db_dir/$path"; - if (-e $file and -s _) { # check exists and non-zero - my $file_db = "$file.db"; - if (! -e $file_db) { - # - # it appears that there is a bug in squidGuard that if - # the db file doesn't exist then running with -C leaves - # huge tmp files in /var/tmp. - # - system("touch $file.db"); - system("chown -R proxy.proxy $file.db > /dev/null 2>&1"); - } - my $wc = `cat $file| wc -l`; chomp $wc; - print "Building DB for [$path] - $wc entries\n" if $interactive; - my $cmd = "\"squidGuard -d -c $tmp_conf -C $path\""; - system("su - proxy -c $cmd > /dev/null 2>&1"); - } - } - system("rm $tmp_conf"); -} - -sub squidguard_is_category_local { - my ($category) = @_; - - my $db_dir = squidguard_get_blacklist_dir(); - my $local_file = "$db_dir/$category/local"; - return 1 if -e $local_file; - return 0; -} - -sub squidguard_is_blacklist_installed { - if (squidguard_use_ec()) { - if (-e "$urlfilter_data_dir/sitefilter/urldb") { - return 1; - } - } else { - my @blacklists = squidguard_get_blacklists(); - foreach my $category (@blacklists) { - next if squidguard_is_category_local($category); - return 1; - } - } - return 0; -} - -sub squidguard_get_blacklist_domains_urls_exps { - my ($list) = shift; - - my $dir = $squidguard_blacklist_db; - my ($domains, $urls, $exps) = undef; - $domains = "$list/domains" if -f "$dir/$list/domains" && -s _; - $urls = "$list/urls" if -f "$dir/$list/urls" && -s _; - $exps = "$list/expressions" if -f "$dir/$list/expressions" && -s _; - return ($domains, $urls, $exps); -} - -sub squidguard_get_blacklist_files { - my ($type, $category) = @_; - - my @lists = squidguard_get_blacklists(); - my @files = (); - foreach my $list (@lists) { - my ($domain, $url, $exp) = squidguard_get_blacklist_domains_urls_exps( - $list); - if ($type eq 'domains') { - next if !defined $domain; - if (defined $category) { - next if $domain ne "$category/domains"; - } - $domain = "$squidguard_blacklist_db/$domain"; - push @files, $domain; - } - if ($type eq 'urls') { - next if !defined $url; - if (defined $category) { - next if $url ne "$category/urls"; - } - $url = "$squidguard_blacklist_db/$url"; - push @files, $url; - } - if ($type eq 'expressions') { - next if !defined $exp; - if (defined $category) { - next if $url ne "$category/expressions"; - } - $exp = "$squidguard_blacklist_db/$exp"; - push @files, $exp; - } - - } - @files = sort(@files); - return @files; -} - -sub squidguard_get_log_files { - open(my $LS, "-|", "ls $squidguard_log_dir/bl*.log* 2> /dev/null | sort -nr "); - my @log_files = <$LS>; - close $LS; - chomp @log_files; - return @log_files; -} - -sub squidguard_build_dest { - my ($category, $logging, $group, $ec) = @_; - - my $output = ''; - my ($domains, $urls, $exps); - if (squidguard_is_category_local("$category-$group")) { - ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps( - "$category-$group"); - } else { - ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps( - $category); - } - - my $ec_cat = undef; - if (defined $ec) { - $ec_cat = squidguard_ec_name2cat($category); - } - - $output = "dest $category-$group {\n"; - $output .= "\tdomainlist $domains\n" if defined $domains; - $output .= "\turllist $urls\n" if defined $urls; - $output .= "\texpressionlist $exps\n" if defined $exps; - $output .= "\teccategory $ec_cat\n" if defined $ec_cat; - if ($logging) { - my $log = basename($squidguard_blacklist_log); - $output .= "\tlog $log\n"; - } - $output .= "}\n\n"; - return $output; -} - -sub webproxy_read_file { - my ($file) = @_; - my @lines; - if ( -e $file) { - open(my $FILE, '<', $file) or die "Error: read $!"; - @lines = <$FILE>; - close($FILE); - chomp @lines; - } - return @lines; -} - -sub is_same_as_file { - my ($file, $value) = @_; - - return if ! -e $file; - - my $mem_file = ''; - open my $MF, '+<', \$mem_file or die "couldn't open memfile $!\n"; - print $MF $value; - seek($MF, 0, 0); - - my $rc = compare($file, $MF); - return 1 if $rc == 0; - return; -} - -sub webproxy_write_file { - my ($file, $config) = @_; - - # Avoid unnecessary writes. At boot the file will be the - # regenerated with the same content. - return if is_same_as_file($file, $config); - - open(my $fh, '>', $file) || die "Couldn't open $file - $!"; - print $fh $config; - close $fh; - return 1; -} - -sub webproxy_append_file { - my ($dst, $src) = @_; - - open(my $ih, '<', $src) || die "Couldn't open $src - $!"; - open(my $oh, '>>', $dst) || die "Couldn't open $dst - $!"; - for (<$ih>) { - print $oh $_; - } - close($oh); - close($ih); - return 1; -} - -sub webproxy_delete_local_entry { - my ($file, $value) = @_; - - my $db_dir = squidguard_get_blacklist_dir(); - $file = "$db_dir/$file"; - my @lines = webproxy_read_file($file); - my $config = ''; - foreach my $line (@lines) { - $config .= "$line\n" if $line ne $value; - } - if ($config eq '') { - unlink($file); - } else { - webproxy_write_file($file, $config); - } - return; -} - -sub webproxy_delete_all_local { - my $db_dir = squidguard_get_blacklist_dir(); - my @categorys = squidguard_get_blacklists(); - foreach my $category (@categorys) { - if (squidguard_is_category_local($category)) { - system("rm -rf $db_dir/$category"); - } - } - return; -} - -sub print_err { - my ($interactive, $msg) = @_; - if ($interactive) { - print "$msg\n"; - } else { - syslog(LOG_ERR, $msg); - } -} - -sub squidguard_count_blacklist_entries { - my $db_dir = squidguard_get_blacklist_dir(); - - my $total = 0; - my @categories = squidguard_get_blacklists(); - foreach my $category (@categories) { - foreach my $type ('domains', 'urls') { - my $path = "$category/$type"; - my $file = "$db_dir/$path"; - if (-e $file) { - my $wc = `cat $file| wc -l`; chomp $wc; - $total += $wc; - } - } - } - return $total; -} - -sub squidguard_clean_tmpfiles { - # - # workaround for squidguard - # bug http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=494281 - # - my @tmpfiles = </var/tmp/*>; - foreach my $file (@tmpfiles) { - my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, - $mtime, $ctime, $blksize, $blocks) = stat($file); - my $name = (getpwuid($uid))[0] if $uid; - unlink($file) if $name and $name eq 'proxy'; - } -} - -sub squidguard_auto_update { - my ($interactive, $file) = @_; - - my $rc; - my $db_dir = squidguard_get_blacklist_dir(); - my $tmp_blacklists = '/tmp/blacklists.gz'; - - if (!squidguard_is_blacklist_installed()) { - my ($disk_free, $disk_required); - $disk_required = (30 * 1024 * 1024); # 30MB - $disk_free = `df $db_dir | grep -v Filesystem | awk '{ print \$4 }'`; - chomp($disk_free); - $disk_free *= 1024; - if ($disk_free < $disk_required) { - die "Error: not enough disk space $disk_required\/$disk_free"; - } - } - - if (defined $file) { - # use existing file - $rc = copy($file, $tmp_blacklists); - if (!$rc) { - print_err($interactive, "Unable to copy [$file] $!"); - return 1; - } - } else { - # get from net - my $opt = ''; - $opt = "-q" if ! $interactive; - $rc = system("wget -O $tmp_blacklists $opt $blacklist_url"); - if ($rc) { - print_err($interactive, "Unable to download [$blacklist_url] $!"); - return 1; - } - } - - print "Uncompressing blacklist...\n" if $interactive; - $rc = system("tar --directory /tmp -zxvf $tmp_blacklists > /dev/null"); - if ($rc) { - print_err($interactive, "Unable to uncompress [$blacklist_url] $!"); - return 1; - } - my $b4_entries = squidguard_count_blacklist_entries(); - my $archive = "$global_data_dir/squidguard/archive"; - mkdir_p($archive) if ! -d $archive; - system("rm -rf $archive/*"); - system("mv $db_dir/* $archive 2> /dev/null"); - $rc = system("mv /tmp/blacklists/* $db_dir"); - if ($rc) { - print_err($interactive, "Unable to install [$blacklist_url] $!"); - return 1; - } - system("mv $archive/local-* $db_dir 2> /dev/null"); - rm_rf($tmp_blacklists); - rm_rf("/tmp/blacklists"); - - my $after_entries = squidguard_count_blacklist_entries(); - my $mode = "auto-update"; - $mode = "manual" if $interactive; - syslog(LOG_WARNING, - "blacklist entries updated($mode) ($b4_entries/$after_entries)"); - return 0; -} - -sub squidguard_install_blacklist_def { - squidguard_auto_update(1, undef); -} - -sub squidguard_update_blacklist { - my ($interactive, $update_category) = @_; - - my @blacklists = squidguard_get_blacklists(); - print "Checking permissions...\n" if $interactive; - my $db_dir = squidguard_get_blacklist_dir(); - system("chown -R proxy.proxy $db_dir > /dev/null 2>&1"); - chmod(2770, $db_dir); - - # - # generate temporary config for each category & generate DB - # - foreach my $category (@blacklists) { - next if defined $update_category and $update_category ne $category; - squidguard_generate_db($interactive, $category, 'default'); - } -} - - -# -# main -# -my ($update_bl, $update_bl_cat, $update_bl_file, $auto_update_bl); - -GetOptions("update-blacklist!" => \$update_bl, - "update-blacklist-category=s" => \$update_bl_cat, - "update-blacklist-file=s" => \$update_bl_file, - "auto-update-blacklist!" => \$auto_update_bl, -); - -my $sg_updatestatus_file = "$global_data_dir/squidguard/updatestatus"; -if (! -e "$global_data_dir/squidguard") { - system("mkdir -p $global_data_dir/squidguard/db"); - my ($login, $pass, $uid, $gid) = getpwnam('proxy') - or die "proxy not in passwd file"; - chown $uid, $gid, "$global_data_dir/squidguard/db"; -} -touch($sg_updatestatus_file); -system("echo update failed at `date` > $sg_updatestatus_file"); -system("sudo rm -f /var/lib/sitefilter/updatestatus"); - -my $lock_file = '/tmp/vyatta_bl_lock'; -open(my $lck, ">", $lock_file) || die "Lock failed\n"; -flock($lck, LOCK_EX); - -if (defined $update_bl_cat) { - squidguard_update_blacklist(1, $update_bl_cat); - if (squidguard_is_configured()) { - print "\nThe webproxy daemon must be restarted\n"; - if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || - prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) { - squid_restart(1); - } - } - squidguard_clean_tmpfiles(); -} - -if (defined $update_bl) { - my $updated = 0; - if (!squidguard_is_blacklist_installed()) { - print "Warning: No url-filtering blacklist installed\n"; - if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || - prompt("Would you like to download a default blacklist? [confirm]", - -y1d=>"y")) { - exit 1 if squidguard_install_blacklist_def(); - $updated = 1; - } else { - exit 1; - } - } else { - if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || - prompt("Would you like to re-download the blacklist? [confirm]", - -y1d=>"y")) { - my $rc = squidguard_auto_update(1, undef); - $updated = 1 if ! $rc; - } - } - if (! $updated) { - print "No blacklist updated\n"; - if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || - !prompt("Do you still want to generate binary DB? [confirm]", - -y1d=>"y")) { - exit 1; - } - } - # if there was an update we need to re-gen the binary DBs - # and restart the daemon - squidguard_update_blacklist(1); - if (squidguard_is_configured()) { - print "\nThe webproxy daemon must be restarted\n"; - if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || - prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) { - squid_restart(1); - } - } - squidguard_clean_tmpfiles(); -} - -if (defined $update_bl_file) { - if (! -e $update_bl_file) { - die "Error: file [$update_bl_file] doesn't exist"; - } - my $rc = squidguard_auto_update(0, $update_bl_file); - exit 1 if $rc; - squidguard_update_blacklist(1); - squidguard_clean_tmpfiles(); -} - -if (defined $auto_update_bl) { - my $rc = squidguard_auto_update(0); - exit 1 if $rc; - squidguard_update_blacklist(0); - if (squidguard_is_configured()) { - squid_restart(0); - } - squidguard_clean_tmpfiles(); -} - -system("echo update succeeded at `date` > $sg_updatestatus_file"); -close($lck); -exit 0; - -#end of file diff --git a/src/op_mode/webproxy_update_blacklist.py b/src/op_mode/webproxy_update_blacklist.py deleted file mode 100755 index c6572c663..000000000 --- a/src/op_mode/webproxy_update_blacklist.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2020 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -#blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz' -blacklist_url = 'http://lnx01.mybll.net/~cpo/blacklists.tar.gz' -global_data_dir = '/config/url-filtering' -sg_dir = f'{global_data_dir}/squidguard' -blacklist_dir = f'{sg_dir}/db' -archive_dir = f'{sg_dir}/archive' -target_file = '/tmp/blacklists.tar.gz' - -# -# XXX: this is a proof of concept for downloading a file via Python -# - - -import os -import shutil -import argparse -import urllib.request -import tarfile - -from tqdm import tqdm -from vyos.util import chown -from vyos.util import chmod - -parser = argparse.ArgumentParser() -parser.add_argument("--update", help="Update SquidGuard blacklist", - action="store_true") -args = parser.parse_args() - -class DownloadProgressBar(tqdm): - def update_to(self, b=1, bsize=1, tsize=None): - if tsize is not None: - self.total = tsize - self.update(b * bsize - self.n) - -def download_url(url, output_path): - with DownloadProgressBar(unit='B', unit_scale=True, - miniters=1, desc=url.split('/')[-1]) as t: - urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to) - -def squidguard_is_blacklist_installed(): - return os.path.exists(blacklist_dir) - - -def install_blacklist(): - download_url(blacklist_url, target_file) - - print('Uncompressing blacklist...') - tar = tarfile.open(target_file, "r:gz") - tar.extractall(path='/tmp') - tar.close() - - if not os.path.exists(sg_dir): - os.makedirs(sg_dir, exist_ok=True) - - if os.path.exists(archive_dir): - print('Removing old archive...') - shutil.rmtree(archive_dir) - - if os.path.exists(blacklist_dir): - print('Archiving old blacklist...') - shutil.move(blacklist_dir, archive_dir) - - shutil.move('/tmp/blacklists', blacklist_dir) - - chown(blacklist_dir, 'proxy', 'proxy') - chmod(blacklist_dir, 0o755) - - -if args.update: - if not squidguard_is_blacklist_installed(): - print('Warning: No url-filtering blacklist installed') - input('Would you like to download a default blacklist? [confirm]') - - else: - input('Would you like to re-download the blacklist? [confirm]') - - install_blacklist() diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh new file mode 100755 index 000000000..43a4b79fc --- /dev/null +++ b/src/op_mode/webproxy_update_blacklist.sh @@ -0,0 +1,104 @@ +#!/bin/sh +# +# Copyright (C) 2020 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz' +data_dir="/opt/vyatta/etc/config/url-filtering" +archive="${data_dir}/squidguard/archive" +db_dir="${data_dir}/squidguard/db" + +while [ $# -gt 0 ] +do + case $1 in + --update-blacklist) + update="yes" + ;; + --auto-update-blacklist) + auto="yes" + ;; + (-*) echo "$0: error - unrecognized option $1" 1>&2; exit 1;; + (*) break;; + esac + shift +done + +if [ ! -d ${db_dir} ]; then + mkdir -p ${db_dir} + getent passwd proxy 2> /dev/null + if [ $? -ne 0 ]; then + echo "proxy system user does not exist" + exit 1 + fi + getent group proxy 2> /dev/null + if [ $? -ne 0 ]; then + echo "proxy system group does not exist" + exit 1 + fi + chown proxy:proxy ${db_dir} +fi + +free_space=$(expr $(df ${db_dir} | grep -v Filesystem | awk '{print $4}') \* 1024) +mb_size="100" +required_space=$(expr $mb_size \* 1024 \* 1024) # 100 MB +if [ ${free_space} -le ${required_space} ]; then + echo "Error: not enough disk space, required ${mb_size} MiB" + exit 1 +fi + +if [[ -n $update ]] && [[ $update -eq "yes" ]]; then + tmp_blacklists='/tmp/blacklists.gz' + curl -o $tmp_blacklists $blacklist_url + if [ $? -ne 0 ]; then + echo "Unable to download [$blacklist_url]!" + exit 1 + fi + echo "Uncompressing blacklist..." + tar --directory /tmp -xf $tmp_blacklists + if [ $? -ne 0 ]; then + echo "Unable to uncompress [$blacklist_url]!" + fi + + if [ ! -d ${archive} ]; then + mkdir -p ${archive} + fi + + rm -rf ${archive}/* + count_before=$(find ${db_dir} -type f \( -name domains -o -name urls \) | xargs wc -l | tail -n 1 | awk '{print $1}') + mv ${db_dir}/* ${archive} 2> /dev/null + mv /tmp/blacklists/* ${db_dir} + if [ $? -ne 0 ]; then + echo "Unable to install [$blacklist_url]" + exit 1 + fi + mv ${archive}/local-* ${db_dir} 2> /dev/null + rm -rf /tmp/blacklists $tmp_blacklists 2> /dev/null + count_after=$(find ${db_dir} -type f \( -name domains -o -name urls \) | xargs wc -l | tail -n 1 | awk '{print $1}') + + # fix permissions + chown -R proxy:proxy ${db_dir} + chmod 2770 ${db_dir} + + logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})" + +else + echo "SquidGuard blacklist updater" + echo "" + echo "Usage:" + echo "--update-blacklist Download latest version of the SquidGuard blacklist" + echo "--auto-update-blacklist Automatically update" + echo "" + exit 1 +fi + |