From a8a019c4f318ba6ad2f83b9b4f605de3830c7b28 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 12 Dec 2020 14:03:54 +0100 Subject: webproxy: T563: migrate from old Perl code to XML and get_config_dict() Basic proxy functionality is working but the squidguard smoketest still fails as this is yet not implemented. --- debian/control | 3 +++ 1 file changed, 3 insertions(+) (limited to 'debian') diff --git a/debian/control b/debian/control index dc5094e8e..ccdaa8492 100644 --- a/debian/control +++ b/debian/control @@ -108,6 +108,9 @@ Depends: salt-minion, snmp, snmpd, + squid, + squidclient, + squidguard, ssl-cert, systemd, tcpdump, -- cgit v1.2.3 From 1efd20ab21e75e421487d563fc794a7f97361a3e Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sat, 26 Dec 2020 17:12:29 +0100 Subject: webproxy: T563: op-mode: initial command support --- debian/control | 1 + debian/rules | 4 + debian/vyos-1x.install | 1 + .../include/webproxy-squidguard.xml.i | 7 + op-mode-definitions/webproxy.xml | 121 ++++ src/completion/list_webproxy_category.sh | 5 + src/legacy/vyatta-sg-blacklist.pl | 682 +++++++++++++++++++++ src/op_mode/webproxy_update_blacklist.py | 93 +++ 8 files changed, 914 insertions(+) create mode 100644 op-mode-definitions/webproxy.xml create mode 100755 src/completion/list_webproxy_category.sh create mode 100755 src/legacy/vyatta-sg-blacklist.pl create mode 100755 src/op_mode/webproxy_update_blacklist.py (limited to 'debian') diff --git a/debian/control b/debian/control index ccdaa8492..9750f08e2 100644 --- a/debian/control +++ b/debian/control @@ -99,6 +99,7 @@ Depends: python3-pyudev, python3-six, python3-tabulate, + python3-tqdm, python3-vici (>= 5.7.2), python3-voluptuous, python3-waitress, diff --git a/debian/rules b/debian/rules index ab0df7201..5e275b8a3 100755 --- a/debian/rules +++ b/debian/rules @@ -51,6 +51,10 @@ override_dh_auto_install: mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/completion cp -r src/completion/* $(DIR)/$(VYOS_LIBEXEC_DIR)/completion + # Install legacy components + mkdir -p $(DIR)/$(VYOS_LIBEXEC_DIR)/legacy + cp -r src/legacy/* $(DIR)/$(VYOS_LIBEXEC_DIR)/legacy + # Install helper scripts cp -r src/helpers/* $(DIR)/$(VYOS_LIBEXEC_DIR)/ diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index bfc30f7e6..691fa4731 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -21,5 +21,6 @@ usr/libexec/vyos/op_mode usr/libexec/vyos/services usr/libexec/vyos/system usr/libexec/vyos/validators +usr/libexec/vyos/legacy usr/libexec/vyos/*.py usr/share diff --git a/interface-definitions/include/webproxy-squidguard.xml.i b/interface-definitions/include/webproxy-squidguard.xml.i index 23a2fee7a..6958056d4 100644 --- a/interface-definitions/include/webproxy-squidguard.xml.i +++ b/interface-definitions/include/webproxy-squidguard.xml.i @@ -2,6 +2,9 @@ Category to allow + + + @@ -14,6 +17,9 @@ Category to block + + + @@ -107,6 +113,7 @@ Log block category + all diff --git a/op-mode-definitions/webproxy.xml b/op-mode-definitions/webproxy.xml new file mode 100644 index 000000000..09cefb929 --- /dev/null +++ b/op-mode-definitions/webproxy.xml @@ -0,0 +1,121 @@ + + + + + + + Monitor WebProxy service + + ${vyatta_bindir}/vyatta-monitor Webproxy squid + + + + Monitor the last lines of the squid access log + + if [ -f /var/log/squid3/access.log ]; then sudo tail --follow=name /var/log/squid3/access.log; else echo "WebProxy cache-log does not exist"; fi + + + + Monitor Webproxy in the background + + + + + Start background monitoring of Webproxy + + ${vyatta_bindir}/vyatta-monitor-background Webproxy squid + + + + Stop background monitoring of Webproxy + + ${vyatta_bindir}/vyatta-monitor-background-stop Webproxy + + + + + + Monitor the last lines of the squid cache log + + if [ -f /var/log/squid3/cache.log ]; then sudo tail --follow=name /var/log/squid3/cache.log; else echo "WebProxy cache-log does not exist"; fi + + + + + + + + + + Restart WebProxy service + + if cli-shell-api existsActive service webproxy; then sudo systemctl restart squid.service; else echo "Service WebProxy not configured"; fi + + + + + + + + Show WebProxy information + + + + + + Show webproxy blacklist information + + + + + Show webproxy blacklist categories + + ${vyos_completion_dir}/list_webproxy_category.sh + + + + + + Show contents of WebProxy access log + + if [ -e /var/log/squid/access.log ]; then sudo less $_vyatta_less_options --prompt="file %i of %m, page %dt of %D" -- `printf "%s\n" /var/log/squid/access.log* | sort -nr`; else echo "No WebProxy log"; fi + + + + Show update log for url-filter database + + if [ -e /config/url-filtering/squidguard/updatestatus ]; then cat /config/url-filtering/squidguard/updatestatus; else echo "Update log not found"; fi + + + + + + + + + + Update WebProxy + + + + + Update the webproxy blacklist database + + + + + Update a category of the webproxy blacklist database + + + + + sudo /usr/libexec/vyos/legacy/vyatta-sg-blacklist.pl --update-blacklist-category="$5" + + + sudo /usr/libexec/vyos/legacy/vyatta-sg-blacklist.pl --update-blacklist + + + + + + diff --git a/src/completion/list_webproxy_category.sh b/src/completion/list_webproxy_category.sh new file mode 100755 index 000000000..19f26bf85 --- /dev/null +++ b/src/completion/list_webproxy_category.sh @@ -0,0 +1,5 @@ +#!/bin/sh +DB_DIR="/config/url-filtering/squidguard/db/" +if [ -d ${DB_DIR} ]; then + ls -ald ${DB_DIR}/* | grep -E '^(d|l)' | awk '{print $9}' | sed s#${DB_DIR}/## +fi diff --git a/src/legacy/vyatta-sg-blacklist.pl b/src/legacy/vyatta-sg-blacklist.pl new file mode 100755 index 000000000..4104ac266 --- /dev/null +++ b/src/legacy/vyatta-sg-blacklist.pl @@ -0,0 +1,682 @@ +#!/usr/bin/perl +# +# Module: vyatta-sg-blacklist.pl +# +# **** License **** +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# This code was originally developed by Vyatta, Inc. +# Portions created by Vyatta are Copyright (C) 2008-2009 Vyatta, Inc. +# All Rights Reserved. +# +# Author: Stig Thormodsrud +# Date: October 2008 +# Description: script to download/update free url blacklist. +# +# **** End License **** +# + +use Getopt::Long; +use POSIX; +use IO::Prompt; +use Sys::Syslog qw(:standard :macros); +use File::Copy; +use Fcntl qw(:flock); +use base qw(Exporter); +use File::Basename; +use File::Compare; + +use lib "/opt/vyatta/share/perl5"; +use Vyatta::Config; +use Vyatta::File; + +use warnings; +use strict; + +# +# Default blacklist +# +# Below are some free blacklists we've tried: +# +# http://squidguard.mesd.k12.or.us/blacklists.tgz +# http://ftp.teledanmark.no/pub/www/proxy/squidguard/contrib/blacklists.tar.gz +# ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz +# +# Note: the auto install/update assumes that the blacklist url is a tar gz +# file with the blacklist categorys in a "blacklist" directory. Some +# of the commercially available blacklists are a cgi script instead, so +# those blacklists will need a different install/update script. Of +# course they can be manually installed/updated. +# +my $blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz'; + +#squid globals +my $squid_init = '/etc/init.d/squid'; +my $squid_mime_type = '/usr/share/squid/mime.conf'; + +#squidGuard globals +my $urlfilter_data_dir = '/opt/vyatta/etc/config/url-filtering'; +my $squidguard_blacklist_db = "$urlfilter_data_dir/squidguard/db"; +my $squidguard_log_dir = '/var/log/squid'; +my $squidguard_blacklist_log = "$squidguard_log_dir/blacklist.log"; +my $squidguard_safesearch = "/opt/vyatta/etc/safesearch_rewrites"; + +#vyattaguard globals +my $vyattaguard = '/opt/vyatta/sbin/vg'; + +sub webproxy_get_global_data_dir { + return $urlfilter_data_dir; +} + +my $global_data_dir = webproxy_get_global_data_dir(); + + +sub squid_get_mime { + my @mime_types = (); + open(my $FILE, "<", $squid_mime_type) or die "Error: read $!"; + my @lines = <$FILE>; + close($FILE); + foreach my $line (@lines) { + next if $line =~ /^#/; # skip comments + if ($line =~ /^([\S]+)[\s]+([\S]+)[\s]+([\S]+)[\s]+([\S]+).*$/) { + my $type = $2; + push @mime_types, $type if $type =~ /\//; + } + } + return @mime_types; +} + +sub squidguard_is_configured { + my $config = new Vyatta::Config; + $config->setLevel('service webproxy url-filtering'); + # This checks the running config, so it is assumed + # to be called from op mode. + return 1 if $config->existsOrig('squidguard'); + return 0; +} + +sub squidguard_get_blacklist_dir { + return $squidguard_blacklist_db; +} + +sub squidguard_get_blacklist_log { + return $squidguard_blacklist_log; +} + +sub squidguard_get_safesearch_rewrites { + my @rewrites = (); + open(my $FILE, "<", $squidguard_safesearch) or die "Error: read $!"; + my @lines = <$FILE>; + close($FILE); + chomp @lines; + foreach my $line (@lines) { + next if $line =~ /^#/; # skip comments + if ($line =~ /^s\@/) { + push @rewrites, $line; + } + } + return @rewrites; +} + +sub squidguard_ec_get_categorys { + my %cat_hash; + + die "Must enable vyattaguard" if ! squidguard_use_ec(); + die "Missing vyattaguard package\n" if ! -e $vyattaguard; + exit 1 if ! -e "$urlfilter_data_dir/sitefilter/categories.txt"; + + my @lines = `$vyattaguard list`; + foreach my $line (@lines) { + my ($id, $category) = split ':', $line; + next if ! defined $category; + chomp $category; + $category =~ s/\s/\_/g; + $category =~ s/\&/\_and\_/g; + $cat_hash{$id} = $category; + } + return %cat_hash; +} + +sub squidguard_ec_cat2name { + my ($cat) = @_; + + my %cat_hash = squidguard_ec_get_categorys(); + return $cat_hash{$cat} if defined $cat_hash{$cat}; + return; +} + +sub squidguard_ec_name2cat { + my ($name) = @_; + + my %cat_hash = squidguard_ec_get_categorys(); + foreach my $key (keys (%cat_hash)) { + if ($cat_hash{$key} eq $name) { + return $key; + } + } + return; +} + +sub squidguard_use_ec { + my $rc = system("cli-shell-api inSession"); + my ($exist_func, $value_func); + if ($rc == 0) { + $exist_func = 'exists'; + $value_func = 'returnValue'; + } else { + $exist_func = 'existsOrig'; + $value_func = 'returnOrigValue'; + } + my $config = new Vyatta::Config; + $config->setLevel('service webproxy url-filtering squidguard'); + if ($config->$exist_func('vyattaguard')) { + return if ! -e $vyattaguard; + my $mode = $config->$value_func('vyattaguard mode'); + return $mode; + } + return; +} + +sub squidguard_get_blacklists { + + my @blacklists = (); + if (squidguard_use_ec()) { + die "Missing vyattaguard package\n" if ! -e $vyattaguard; + my %cat_hash = squidguard_ec_get_categorys(); + foreach my $key (keys (%cat_hash)) { + next if ! defined $cat_hash{$key}; + push @blacklists, $cat_hash{$key}; + } + } else { + my $dir = $squidguard_blacklist_db; + opendir(DIR, $dir) || die "can't opendir $dir: $!"; + my @dirs = readdir(DIR); + closedir DIR; + + foreach my $file (@dirs) { + next if $file eq '.'; + next if $file eq '..'; + if (-d "$dir/$file") { + push @blacklists, $file; + } + } + } + @blacklists = sort(@blacklists); + return @blacklists; +} + +sub squidguard_generate_db { + my ($interactive, $category, $group) = @_; + + my $db_dir = squidguard_get_blacklist_dir(); + my $tmp_conf = "/tmp/sg.conf.$$"; + my $output = "dbhome $db_dir\n"; + $output .= squidguard_build_dest($category, 0, $group); + $output .= "\nacl {\n"; + $output .= "\tdefault {\n"; + $output .= "\t\tpass all\n"; + $output .= "\t}\n}\n\n"; + webproxy_write_file($tmp_conf, $output); + + my $dir = "$db_dir/$category"; + if ( -l $dir) { + print "Skip link for [$category] -> [", readlink($dir), "]\n" + if $interactive; + return; + } + foreach my $type ('domains', 'urls', 'expressions') { + my $path = "$category/$type"; + my $file = "$db_dir/$path"; + if (-e $file and -s _) { # check exists and non-zero + my $file_db = "$file.db"; + if (! -e $file_db) { + # + # it appears that there is a bug in squidGuard that if + # the db file doesn't exist then running with -C leaves + # huge tmp files in /var/tmp. + # + system("touch $file.db"); + system("chown -R proxy.proxy $file.db > /dev/null 2>&1"); + } + my $wc = `cat $file| wc -l`; chomp $wc; + print "Building DB for [$path] - $wc entries\n" if $interactive; + my $cmd = "\"squidGuard -d -c $tmp_conf -C $path\""; + system("su - proxy -c $cmd > /dev/null 2>&1"); + } + } + system("rm $tmp_conf"); +} + +sub squidguard_is_category_local { + my ($category) = @_; + + my $db_dir = squidguard_get_blacklist_dir(); + my $local_file = "$db_dir/$category/local"; + return 1 if -e $local_file; + return 0; +} + +sub squidguard_is_blacklist_installed { + if (squidguard_use_ec()) { + if (-e "$urlfilter_data_dir/sitefilter/urldb") { + return 1; + } + } else { + my @blacklists = squidguard_get_blacklists(); + foreach my $category (@blacklists) { + next if squidguard_is_category_local($category); + return 1; + } + } + return 0; +} + +sub squidguard_get_blacklist_domains_urls_exps { + my ($list) = shift; + + my $dir = $squidguard_blacklist_db; + my ($domains, $urls, $exps) = undef; + $domains = "$list/domains" if -f "$dir/$list/domains" && -s _; + $urls = "$list/urls" if -f "$dir/$list/urls" && -s _; + $exps = "$list/expressions" if -f "$dir/$list/expressions" && -s _; + return ($domains, $urls, $exps); +} + +sub squidguard_get_blacklist_files { + my ($type, $category) = @_; + + my @lists = squidguard_get_blacklists(); + my @files = (); + foreach my $list (@lists) { + my ($domain, $url, $exp) = squidguard_get_blacklist_domains_urls_exps( + $list); + if ($type eq 'domains') { + next if !defined $domain; + if (defined $category) { + next if $domain ne "$category/domains"; + } + $domain = "$squidguard_blacklist_db/$domain"; + push @files, $domain; + } + if ($type eq 'urls') { + next if !defined $url; + if (defined $category) { + next if $url ne "$category/urls"; + } + $url = "$squidguard_blacklist_db/$url"; + push @files, $url; + } + if ($type eq 'expressions') { + next if !defined $exp; + if (defined $category) { + next if $url ne "$category/expressions"; + } + $exp = "$squidguard_blacklist_db/$exp"; + push @files, $exp; + } + + } + @files = sort(@files); + return @files; +} + +sub squidguard_get_log_files { + open(my $LS, "-|", "ls $squidguard_log_dir/bl*.log* 2> /dev/null | sort -nr "); + my @log_files = <$LS>; + close $LS; + chomp @log_files; + return @log_files; +} + +sub squidguard_build_dest { + my ($category, $logging, $group, $ec) = @_; + + my $output = ''; + my ($domains, $urls, $exps); + if (squidguard_is_category_local("$category-$group")) { + ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps( + "$category-$group"); + } else { + ($domains, $urls, $exps) = squidguard_get_blacklist_domains_urls_exps( + $category); + } + + my $ec_cat = undef; + if (defined $ec) { + $ec_cat = squidguard_ec_name2cat($category); + } + + $output = "dest $category-$group {\n"; + $output .= "\tdomainlist $domains\n" if defined $domains; + $output .= "\turllist $urls\n" if defined $urls; + $output .= "\texpressionlist $exps\n" if defined $exps; + $output .= "\teccategory $ec_cat\n" if defined $ec_cat; + if ($logging) { + my $log = basename($squidguard_blacklist_log); + $output .= "\tlog $log\n"; + } + $output .= "}\n\n"; + return $output; +} + +sub webproxy_read_file { + my ($file) = @_; + my @lines; + if ( -e $file) { + open(my $FILE, '<', $file) or die "Error: read $!"; + @lines = <$FILE>; + close($FILE); + chomp @lines; + } + return @lines; +} + +sub is_same_as_file { + my ($file, $value) = @_; + + return if ! -e $file; + + my $mem_file = ''; + open my $MF, '+<', \$mem_file or die "couldn't open memfile $!\n"; + print $MF $value; + seek($MF, 0, 0); + + my $rc = compare($file, $MF); + return 1 if $rc == 0; + return; +} + +sub webproxy_write_file { + my ($file, $config) = @_; + + # Avoid unnecessary writes. At boot the file will be the + # regenerated with the same content. + return if is_same_as_file($file, $config); + + open(my $fh, '>', $file) || die "Couldn't open $file - $!"; + print $fh $config; + close $fh; + return 1; +} + +sub webproxy_append_file { + my ($dst, $src) = @_; + + open(my $ih, '<', $src) || die "Couldn't open $src - $!"; + open(my $oh, '>>', $dst) || die "Couldn't open $dst - $!"; + for (<$ih>) { + print $oh $_; + } + close($oh); + close($ih); + return 1; +} + +sub webproxy_delete_local_entry { + my ($file, $value) = @_; + + my $db_dir = squidguard_get_blacklist_dir(); + $file = "$db_dir/$file"; + my @lines = webproxy_read_file($file); + my $config = ''; + foreach my $line (@lines) { + $config .= "$line\n" if $line ne $value; + } + if ($config eq '') { + unlink($file); + } else { + webproxy_write_file($file, $config); + } + return; +} + +sub webproxy_delete_all_local { + my $db_dir = squidguard_get_blacklist_dir(); + my @categorys = squidguard_get_blacklists(); + foreach my $category (@categorys) { + if (squidguard_is_category_local($category)) { + system("rm -rf $db_dir/$category"); + } + } + return; +} + +sub print_err { + my ($interactive, $msg) = @_; + if ($interactive) { + print "$msg\n"; + } else { + syslog(LOG_ERR, $msg); + } +} + +sub squidguard_count_blacklist_entries { + my $db_dir = squidguard_get_blacklist_dir(); + + my $total = 0; + my @categories = squidguard_get_blacklists(); + foreach my $category (@categories) { + foreach my $type ('domains', 'urls') { + my $path = "$category/$type"; + my $file = "$db_dir/$path"; + if (-e $file) { + my $wc = `cat $file| wc -l`; chomp $wc; + $total += $wc; + } + } + } + return $total; +} + +sub squidguard_clean_tmpfiles { + # + # workaround for squidguard + # bug http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=494281 + # + my @tmpfiles = ; + foreach my $file (@tmpfiles) { + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, $atime, + $mtime, $ctime, $blksize, $blocks) = stat($file); + my $name = (getpwuid($uid))[0] if $uid; + unlink($file) if $name and $name eq 'proxy'; + } +} + +sub squidguard_auto_update { + my ($interactive, $file) = @_; + + my $rc; + my $db_dir = squidguard_get_blacklist_dir(); + my $tmp_blacklists = '/tmp/blacklists.gz'; + + if (!squidguard_is_blacklist_installed()) { + my ($disk_free, $disk_required); + $disk_required = (30 * 1024 * 1024); # 30MB + $disk_free = `df $db_dir | grep -v Filesystem | awk '{ print \$4 }'`; + chomp($disk_free); + $disk_free *= 1024; + if ($disk_free < $disk_required) { + die "Error: not enough disk space $disk_required\/$disk_free"; + } + } + + if (defined $file) { + # use existing file + $rc = copy($file, $tmp_blacklists); + if (!$rc) { + print_err($interactive, "Unable to copy [$file] $!"); + return 1; + } + } else { + # get from net + my $opt = ''; + $opt = "-q" if ! $interactive; + $rc = system("wget -O $tmp_blacklists $opt $blacklist_url"); + if ($rc) { + print_err($interactive, "Unable to download [$blacklist_url] $!"); + return 1; + } + } + + print "Uncompressing blacklist...\n" if $interactive; + $rc = system("tar --directory /tmp -zxvf $tmp_blacklists > /dev/null"); + if ($rc) { + print_err($interactive, "Unable to uncompress [$blacklist_url] $!"); + return 1; + } + my $b4_entries = squidguard_count_blacklist_entries(); + my $archive = "$global_data_dir/squidguard/archive"; + mkdir_p($archive) if ! -d $archive; + system("rm -rf $archive/*"); + system("mv $db_dir/* $archive 2> /dev/null"); + $rc = system("mv /tmp/blacklists/* $db_dir"); + if ($rc) { + print_err($interactive, "Unable to install [$blacklist_url] $!"); + return 1; + } + system("mv $archive/local-* $db_dir 2> /dev/null"); + rm_rf($tmp_blacklists); + rm_rf("/tmp/blacklists"); + + my $after_entries = squidguard_count_blacklist_entries(); + my $mode = "auto-update"; + $mode = "manual" if $interactive; + syslog(LOG_WARNING, + "blacklist entries updated($mode) ($b4_entries/$after_entries)"); + return 0; +} + +sub squidguard_install_blacklist_def { + squidguard_auto_update(1, undef); +} + +sub squidguard_update_blacklist { + my ($interactive, $update_category) = @_; + + my @blacklists = squidguard_get_blacklists(); + print "Checking permissions...\n" if $interactive; + my $db_dir = squidguard_get_blacklist_dir(); + system("chown -R proxy.proxy $db_dir > /dev/null 2>&1"); + chmod(2770, $db_dir); + + # + # generate temporary config for each category & generate DB + # + foreach my $category (@blacklists) { + next if defined $update_category and $update_category ne $category; + squidguard_generate_db($interactive, $category, 'default'); + } +} + + +# +# main +# +my ($update_bl, $update_bl_cat, $update_bl_file, $auto_update_bl); + +GetOptions("update-blacklist!" => \$update_bl, + "update-blacklist-category=s" => \$update_bl_cat, + "update-blacklist-file=s" => \$update_bl_file, + "auto-update-blacklist!" => \$auto_update_bl, +); + +my $sg_updatestatus_file = "$global_data_dir/squidguard/updatestatus"; +if (! -e "$global_data_dir/squidguard") { + system("mkdir -p $global_data_dir/squidguard/db"); + my ($login, $pass, $uid, $gid) = getpwnam('proxy') + or die "proxy not in passwd file"; + chown $uid, $gid, "$global_data_dir/squidguard/db"; +} +touch($sg_updatestatus_file); +system("echo update failed at `date` > $sg_updatestatus_file"); +system("sudo rm -f /var/lib/sitefilter/updatestatus"); + +my $lock_file = '/tmp/vyatta_bl_lock'; +open(my $lck, ">", $lock_file) || die "Lock failed\n"; +flock($lck, LOCK_EX); + +if (defined $update_bl_cat) { + squidguard_update_blacklist(1, $update_bl_cat); + if (squidguard_is_configured()) { + print "\nThe webproxy daemon must be restarted\n"; + if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || + prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) { + squid_restart(1); + } + } + squidguard_clean_tmpfiles(); +} + +if (defined $update_bl) { + my $updated = 0; + if (!squidguard_is_blacklist_installed()) { + print "Warning: No url-filtering blacklist installed\n"; + if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || + prompt("Would you like to download a default blacklist? [confirm]", + -y1d=>"y")) { + exit 1 if squidguard_install_blacklist_def(); + $updated = 1; + } else { + exit 1; + } + } else { + if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || + prompt("Would you like to re-download the blacklist? [confirm]", + -y1d=>"y")) { + my $rc = squidguard_auto_update(1, undef); + $updated = 1 if ! $rc; + } + } + if (! $updated) { + print "No blacklist updated\n"; + if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || + !prompt("Do you still want to generate binary DB? [confirm]", + -y1d=>"y")) { + exit 1; + } + } + # if there was an update we need to re-gen the binary DBs + # and restart the daemon + squidguard_update_blacklist(1); + if (squidguard_is_configured()) { + print "\nThe webproxy daemon must be restarted\n"; + if ((defined($ENV{VYATTA_PROCESS_CLIENT}) && $ENV{VYATTA_PROCESS_CLIENT} eq 'gui2_rest') || + prompt("Would you like to restart it now? [confirm]",-y1d=>"y")) { + squid_restart(1); + } + } + squidguard_clean_tmpfiles(); +} + +if (defined $update_bl_file) { + if (! -e $update_bl_file) { + die "Error: file [$update_bl_file] doesn't exist"; + } + my $rc = squidguard_auto_update(0, $update_bl_file); + exit 1 if $rc; + squidguard_update_blacklist(1); + squidguard_clean_tmpfiles(); +} + +if (defined $auto_update_bl) { + my $rc = squidguard_auto_update(0); + exit 1 if $rc; + squidguard_update_blacklist(0); + if (squidguard_is_configured()) { + squid_restart(0); + } + squidguard_clean_tmpfiles(); +} + +system("echo update succeeded at `date` > $sg_updatestatus_file"); +close($lck); +exit 0; + +#end of file diff --git a/src/op_mode/webproxy_update_blacklist.py b/src/op_mode/webproxy_update_blacklist.py new file mode 100755 index 000000000..c6572c663 --- /dev/null +++ b/src/op_mode/webproxy_update_blacklist.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2020 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +#blacklist_url = 'ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/blacklists.tar.gz' +blacklist_url = 'http://lnx01.mybll.net/~cpo/blacklists.tar.gz' +global_data_dir = '/config/url-filtering' +sg_dir = f'{global_data_dir}/squidguard' +blacklist_dir = f'{sg_dir}/db' +archive_dir = f'{sg_dir}/archive' +target_file = '/tmp/blacklists.tar.gz' + +# +# XXX: this is a proof of concept for downloading a file via Python +# + + +import os +import shutil +import argparse +import urllib.request +import tarfile + +from tqdm import tqdm +from vyos.util import chown +from vyos.util import chmod + +parser = argparse.ArgumentParser() +parser.add_argument("--update", help="Update SquidGuard blacklist", + action="store_true") +args = parser.parse_args() + +class DownloadProgressBar(tqdm): + def update_to(self, b=1, bsize=1, tsize=None): + if tsize is not None: + self.total = tsize + self.update(b * bsize - self.n) + +def download_url(url, output_path): + with DownloadProgressBar(unit='B', unit_scale=True, + miniters=1, desc=url.split('/')[-1]) as t: + urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to) + +def squidguard_is_blacklist_installed(): + return os.path.exists(blacklist_dir) + + +def install_blacklist(): + download_url(blacklist_url, target_file) + + print('Uncompressing blacklist...') + tar = tarfile.open(target_file, "r:gz") + tar.extractall(path='/tmp') + tar.close() + + if not os.path.exists(sg_dir): + os.makedirs(sg_dir, exist_ok=True) + + if os.path.exists(archive_dir): + print('Removing old archive...') + shutil.rmtree(archive_dir) + + if os.path.exists(blacklist_dir): + print('Archiving old blacklist...') + shutil.move(blacklist_dir, archive_dir) + + shutil.move('/tmp/blacklists', blacklist_dir) + + chown(blacklist_dir, 'proxy', 'proxy') + chmod(blacklist_dir, 0o755) + + +if args.update: + if not squidguard_is_blacklist_installed(): + print('Warning: No url-filtering blacklist installed') + input('Would you like to download a default blacklist? [confirm]') + + else: + input('Would you like to re-download the blacklist? [confirm]') + + install_blacklist() -- cgit v1.2.3 From eeb78e842423319169b036d16601e73227dbffdd Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Sun, 27 Dec 2020 11:43:27 +0100 Subject: webproxy: T563: squidguard: support default ruleset --- data/templates/squid/sg_acl.conf.tmpl | 18 +++ data/templates/squid/squidGuard.conf.tmpl | 75 ++++++++++++- debian/vyos-1x.postinst | 3 + .../include/webproxy-squidguard.xml.i | 122 --------------------- .../include/webproxy-url-filtering.xml.i | 119 ++++++++++++++++++++ interface-definitions/service_webproxy.xml.in | 13 ++- op-mode-definitions/webproxy.xml | 2 +- python/vyos/template.py | 7 +- python/vyos/util.py | 24 ++++ src/completion/list_webproxy_category.sh | 2 +- src/conf_mode/service_webproxy.py | 57 +++++++++- 11 files changed, 309 insertions(+), 133 deletions(-) create mode 100644 data/templates/squid/sg_acl.conf.tmpl delete mode 100644 interface-definitions/include/webproxy-squidguard.xml.i create mode 100644 interface-definitions/include/webproxy-url-filtering.xml.i (limited to 'debian') diff --git a/data/templates/squid/sg_acl.conf.tmpl b/data/templates/squid/sg_acl.conf.tmpl new file mode 100644 index 000000000..cb1c3ccb0 --- /dev/null +++ b/data/templates/squid/sg_acl.conf.tmpl @@ -0,0 +1,18 @@ +### generated by service_webproxy.py ### +dbhome {{ squidguard_db_dir }} + +dest {{ category }}-{{ rule }} { +{% if list_type == 'domains' %} + domainlist {{ category }}/domains +{% elif list_type == 'urls' %} + urllist {{ category }}/urls +{% elif list_type == 'expressions' %} + expressionlist {{ category }}/expressions +{% endif %} +} + +acl { + default { + pass all + } +} diff --git a/data/templates/squid/squidGuard.conf.tmpl b/data/templates/squid/squidGuard.conf.tmpl index 907043614..74de3a651 100644 --- a/data/templates/squid/squidGuard.conf.tmpl +++ b/data/templates/squid/squidGuard.conf.tmpl @@ -1,7 +1,25 @@ ### generated by service_webproxy.py ### + +{% macro sg_rule(category, log, db_dir) %} +{% set expressions = db_dir + '/' + category + '/expressions' %} +dest {{ category }}-default { + domainlist {{ category }}/domains + urllist {{ category }}/urls +{% if expressions | is_file %} + expressionlist {{ category }}/expressions +{% endif %} +{% if log is defined %} + log blacklist.log +{% endif %} +} +{% endmacro %} + {% if url_filtering is defined and url_filtering.disable is not defined %} {% if url_filtering.squidguard is defined and url_filtering.squidguard is not none %} -dbhome /opt/vyatta/etc/config/url-filtering/squidguard/db +{% set sg_config = url_filtering.squidguard %} +{% set acl = namespace(value='local-ok-default') %} +{% set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %} +dbhome {{ squidguard_db_dir }} logdir /var/log/squid rewrite safesearch { @@ -14,5 +32,60 @@ rewrite safesearch { log rewrite.log } +{% if sg_config.local_ok is defined and sg_config.local_ok is not none %} +{% set acl.value = acl.value + ' local-ok-default' %} +dest local-ok-default { + domainlist local-ok-default/domains +} +{% endif %} +{% if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %} +{% set acl.value = acl.value + ' local-ok-url-default' %} +dest local-ok-url-default { + urllist local-ok-url-default/urls +} +{% endif %} +{% if sg_config.local_block is defined and sg_config.local_block is not none %} +{% set acl.value = acl.value + ' !local-block-default' %} +dest local-block-default { + domainlist local-block-default/domains +} +{% endif %} +{% if sg_config.local_block_url is defined and sg_config.local_block_url is not none %} +{% set acl.value = acl.value + ' !local-block-url-default' %} +dest local-block-url-default { + urllist local-block-url-default/urls +} +{% endif %} +{% if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %} +{% set acl.value = acl.value + ' !local-block-keyword-default' %} +dest local-block-keyword-default { + expressionlist local-block-keyword-default/expressions +} +{% endif %} + +{% if sg_config.block_category is defined and sg_config.block_category is not none %} +{% for category in sg_config.block_category %} +{{ sg_rule(category, sg_config.log, squidguard_db_dir) }} +{% set acl.value = acl.value + ' !' + category + '-default' %} +{% endfor %} +{% endif %} +{% if sg_config.allow_category is defined and sg_config.allow_category is not none %} +{% for category in sg_config.allow_category %} +{{ sg_rule(category, False, squidguard_db_dir) }} +{% set acl.value = acl.value + ' ' + category + '-default' %} +{% endfor %} +{% endif %} +acl { + default { +{% if sg_config.enable_safe_search is defined %} + rewrite safesearch +{% endif %} + pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'allow' }} + redirect 302:http://{{ sg_config.redirect_url }} +{% if sg_config.log is defined and sg_config.log is not none %} + log blacklist.log +{% endif %} + } +} {% endif %} {% endif %} diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst index dc129cb54..92948de12 100644 --- a/debian/vyos-1x.postinst +++ b/debian/vyos-1x.postinst @@ -30,3 +30,6 @@ if ! grep -q '^dhcpd' /etc/passwd; then adduser --quiet --system --disabled-login --no-create-home --home /run/dhcp-server dhcpd adduser --quiet dhcpd hostsd fi + +# ensure hte proxy user has a proper shell +chsh -s /bin/sh proxy diff --git a/interface-definitions/include/webproxy-squidguard.xml.i b/interface-definitions/include/webproxy-squidguard.xml.i deleted file mode 100644 index 6958056d4..000000000 --- a/interface-definitions/include/webproxy-squidguard.xml.i +++ /dev/null @@ -1,122 +0,0 @@ - - - - Category to allow - - - - - - - - - Allow IP address URLs - - - - - - Category to block - - - - - - - - - Default action - - allow block - - - allow - Default filter action to allow (default) - - - block - Default filter action to allow (default) - - - ^(allow|block)$ - - - - - - Enable safe-mode search on popular search engines - - - - - Local keyword to block - - keyword - Keyword (or regex) to block - - - - - - - Local URL to block - - url - Local URL to block (without http:\/\/ - - - ^(https?:\/\/)$ - - - - - - - Local site to block - - ipv4 - IP address of site to block - - - - - - - - - - Local URL to allow - - url - Local URL to allow (without http:\/\/ - - - ^(https?:\/\/)$ - - - - - - - Local site to allow - - ipv4 - IP address of site to allow - - - - - - - - - - Log block category - - - all - - - - - diff --git a/interface-definitions/include/webproxy-url-filtering.xml.i b/interface-definitions/include/webproxy-url-filtering.xml.i new file mode 100644 index 000000000..de6ebffde --- /dev/null +++ b/interface-definitions/include/webproxy-url-filtering.xml.i @@ -0,0 +1,119 @@ + + + + Category to allow + + + + + + + + + Allow IP address URLs + + + + + + Category to block + + + + + + + + + Default action (default: allow) + + allow block + + + allow + Default filter action is allow) + + + block + Default filter action is block + + + ^(allow|block)$ + + + + + + Enable safe-mode search on popular search engines + + + + + + Local keyword to block + + keyword + Keyword (or regex) to block + + + + + + + Local URL to block + + url + Local URL to block (without "http://") + + + + + + + Local site to block + + ipv4 + IP address of site to block + + + + + + + + + + + Local URL to allow + + url + Local URL to allow (without "http://") + + + + + + + Local site to allow + + ipv4 + IP address of site to allow + + + + + + + + + + + Log block category + + + all + + + + + diff --git a/interface-definitions/service_webproxy.xml.in b/interface-definitions/service_webproxy.xml.in index ba33a30f4..4cd8138ec 100644 --- a/interface-definitions/service_webproxy.xml.in +++ b/interface-definitions/service_webproxy.xml.in @@ -171,11 +171,11 @@ Hostname or IP address of peer ipv4 - Remote syslog server IPv4 address + Squid cache-peer IPv4 address hostname - Remote syslog server FQDN + Squid cache-peer hostname @@ -293,7 +293,10 @@ - IPv4 address for webproxy to listen on [REQUIRED] + IPv4 listen-address for WebProxy [REQUIRED] + + + ipv4 IPv4 address listen on @@ -402,7 +405,7 @@ URL filtering via squidGuard redirector - #include + #include Auto update settings @@ -446,7 +449,7 @@ SquidGuard rule must between 1-1024 - #include + #include Redirect URL for filtered websites diff --git a/op-mode-definitions/webproxy.xml b/op-mode-definitions/webproxy.xml index 09cefb929..bccffd0b3 100644 --- a/op-mode-definitions/webproxy.xml +++ b/op-mode-definitions/webproxy.xml @@ -84,7 +84,7 @@ Show update log for url-filter database - if [ -e /config/url-filtering/squidguard/updatestatus ]; then cat /config/url-filtering/squidguard/updatestatus; else echo "Update log not found"; fi + if [ -e /opt/vyatta/etc/config/url-filtering/squidguard/updatestatus ]; then cat /opt/vyatta/etc/config/url-filtering/squidguard/updatestatus; else echo "Update log not found"; fi diff --git a/python/vyos/template.py b/python/vyos/template.py index 63d400642..bf087c223 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -248,7 +248,6 @@ def dec_ip(address, decrement): from ipaddress import ip_interface return str(ip_interface(address).ip - int(decrement)) - @register_filter('isc_static_route') def isc_static_route(subnet, router): # https://ercpe.de/blog/pushing-static-routes-with-isc-dhcp-server @@ -270,3 +269,9 @@ def isc_static_route(subnet, router): string += ','.join(router.split('.')) return string + +@register_filter('is_file') +def is_file(filename): + if os.path.exists(filename): + return os.path.isfile(filename) + return False diff --git a/python/vyos/util.py b/python/vyos/util.py index fc6915687..494c8155e 100644 --- a/python/vyos/util.py +++ b/python/vyos/util.py @@ -215,6 +215,30 @@ def read_file(fname, defaultonfailure=None): return defaultonfailure raise e +def write_file(fname, data, defaultonfailure=None, user=None, group=None): + """ + Write content of data to given fname, should defaultonfailure be not None, + it is returned on failure to read. + + If directory of file is not present, it is auto-created. + """ + dirname = os.path.dirname(fname) + if not os.path.isdir(dirname): + os.makedirs(dirname, mode=0o755, exist_ok=False) + chown(dirname, user, group) + + try: + """ Write a file to string """ + bytes = 0 + with open(fname, 'w') as f: + bytes = f.write(data) + chown(fname, user, group) + return bytes + except Exception as e: + if defaultonfailure is not None: + return defaultonfailure + raise e + def read_json(fname, defaultonfailure=None): """ diff --git a/src/completion/list_webproxy_category.sh b/src/completion/list_webproxy_category.sh index 19f26bf85..a5ad2398a 100755 --- a/src/completion/list_webproxy_category.sh +++ b/src/completion/list_webproxy_category.sh @@ -1,5 +1,5 @@ #!/bin/sh -DB_DIR="/config/url-filtering/squidguard/db/" +DB_DIR="/opt/vyatta/etc/config/url-filtering/squidguard/db/" if [ -d ${DB_DIR} ]; then ls -ald ${DB_DIR}/* | grep -E '^(d|l)' | awk '{print $9}' | sed s#${DB_DIR}/## fi diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index 76b72ad48..8dfae348a 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -16,6 +16,7 @@ import os +from shutil import rmtree from sys import exit from vyos.config import Config @@ -23,6 +24,7 @@ from vyos.configdict import dict_merge from vyos.template import render from vyos.util import call from vyos.util import dict_search +from vyos.util import write_file from vyos.validate import is_addr_assigned from vyos.xml import defaults from vyos import ConfigError @@ -31,6 +33,47 @@ airbag.enable() squid_config_file = '/etc/squid/squid.conf' squidguard_config_file = '/etc/squidguard/squidGuard.conf' +squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db' +user_group = 'proxy' + +def generate_sg_localdb(category, list_type, role, proxy): + cat_ = category.replace('-', '_') + if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy), + list): + + # local block databases must be generated "on-the-fly" + tmp = { + 'squidguard_db_dir' : squidguard_db_dir, + 'category' : f'{category}-default', + 'list_type' : list_type, + 'rule' : role + } + sg_tmp_file = '/tmp/sg.conf' + db_file = f'{category}-default/{list_type}' + domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy)) + + # local file + write_file(f'{squidguard_db_dir}/{category}-default/local', '', + user=user_group, group=user_group) + # database input file + write_file(f'{squidguard_db_dir}/{db_file}', domains, + user=user_group, group=user_group) + + # temporary config file, deleted after generation + render(sg_tmp_file, 'squid/sg_acl.conf.tmpl', tmp, + user=user_group, group=user_group) + + call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') + + if os.path.exists(sg_tmp_file): + os.unlink(sg_tmp_file) + + else: + # if category is not part of our configuration, clean out the + # squidguard lists + tmp = f'{squidguard_db_dir}/{category}-default' + if os.path.exists(tmp): + rmtree(f'{squidguard_db_dir}/{category}-default') def get_config(config=None): if config: @@ -55,6 +98,7 @@ def get_config(config=None): else: # store path to squidGuard config, used when generating Squid config proxy['squidguard_conf'] = squidguard_config_file + proxy['squidguard_db_dir'] = squidguard_db_dir # XXX: T2665: blend in proper cache-peer default values later default_values.pop('cache_peer') @@ -67,8 +111,6 @@ def get_config(config=None): proxy['cache_peer'][peer] = dict_merge(default_values, proxy['cache_peer'][peer]) - import pprint - pprint.pprint(proxy) return proxy def verify(proxy): @@ -121,6 +163,7 @@ def verify(proxy): if 'address' not in config: raise ConfigError(f'Cache-peer "{peer}" address must be set!') + def generate(proxy): if not proxy: return None @@ -128,6 +171,16 @@ def generate(proxy): render(squid_config_file, 'squid/squid.conf.tmpl', proxy) render(squidguard_config_file, 'squid/squidGuard.conf.tmpl', proxy) + cat_dict = { + 'local-block' : 'domains', + 'local-block-keyword' : 'expressions', + 'local-block-url' : 'urls', + 'local-ok' : 'domains', + 'local-ok-url' : 'urls' + } + for category, list_type in cat_dict.items(): + generate_sg_localdb(category, list_type, 'default', proxy) + return None def apply(proxy): -- cgit v1.2.3