webproxy: T563: squidguard: support default ruleset

10 files changed, 201 insertions, 25 deletions

diff --git a/data/templates/squid/sg_acl.conf.tmpl b/data/templates/squid/sg_acl.conf.tmpl
new file mode 100644
index 000000000..cb1c3ccb0
--- /dev/null
+++ b/data/templates/squid/sg_acl.conf.tmpl
@@ -0,0 +1,18 @@
+### generated by service_webproxy.py ###

+dbhome {{ squidguard_db_dir }}

+

+dest {{ category }}-{{ rule }} {

+{% if list_type == 'domains' %}

+    domainlist      {{ category }}/domains

+{% elif list_type == 'urls' %}

+    urllist         {{ category }}/urls

+{% elif list_type == 'expressions' %}

+    expressionlist  {{ category }}/expressions

+{% endif %}

+}

+

+acl {

+    default {

+        pass all

+    }

+}

diff --git a/data/templates/squid/squidGuard.conf.tmpl b/data/templates/squid/squidGuard.conf.tmpl
index 907043614..74de3a651 100644
--- a/data/templates/squid/squidGuard.conf.tmpl
+++ b/data/templates/squid/squidGuard.conf.tmpl
@@ -1,7 +1,25 @@
 ### generated by service_webproxy.py ###

+

+{% macro sg_rule(category, log, db_dir) %}

+{%   set expressions = db_dir + '/' + category + '/expressions' %}

+dest {{ category }}-default {

+        domainlist     {{ category }}/domains

+        urllist        {{ category }}/urls

+{%   if expressions | is_file %}

+        expressionlist {{ category }}/expressions

+{%   endif %}

+{%   if log is defined %}

+        log            blacklist.log

+{%   endif %}

+}

+{% endmacro %}

+

 {% if url_filtering is defined and url_filtering.disable is not defined %}

 {%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}

-dbhome /opt/vyatta/etc/config/url-filtering/squidguard/db

+{%     set sg_config = url_filtering.squidguard %}

+{%     set acl = namespace(value='local-ok-default') %}

+{%     set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}

+dbhome {{ squidguard_db_dir }}

 logdir /var/log/squid

 

 rewrite safesearch {

@@ -14,5 +32,60 @@ rewrite safesearch {
         log     rewrite.log

 }

 

+{%     if sg_config.local_ok is defined and sg_config.local_ok is not none %}

+{%       set acl.value = acl.value + ' local-ok-default' %}

+dest local-ok-default {

+        domainlist     local-ok-default/domains

+}

+{% endif %}

+{%     if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}

+{%       set acl.value = acl.value + ' local-ok-url-default' %}

+dest local-ok-url-default {

+        urllist        local-ok-url-default/urls

+}

+{% endif %}

+{%     if sg_config.local_block is defined and sg_config.local_block is not none %}

+{%       set acl.value = acl.value + ' !local-block-default' %}

+dest local-block-default {

+        domainlist     local-block-default/domains

+}

+{% endif %}

+{%     if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}

+{%       set acl.value = acl.value + ' !local-block-url-default' %}

+dest local-block-url-default {

+        urllist        local-block-url-default/urls

+}

+{% endif %}

+{%     if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}

+{%       set acl.value = acl.value + ' !local-block-keyword-default' %}

+dest local-block-keyword-default {

+        expressionlist local-block-keyword-default/expressions

+}

+{% endif %}

+

+{%     if sg_config.block_category is defined and sg_config.block_category is not none %}

+{%       for category in sg_config.block_category %}

+{{ sg_rule(category, sg_config.log, squidguard_db_dir) }}

+{%         set acl.value = acl.value + ' !' + category + '-default' %}

+{%       endfor %}

+{%     endif %}

+{%     if sg_config.allow_category is defined and sg_config.allow_category is not none %}

+{%       for category in sg_config.allow_category %}

+{{ sg_rule(category, False, squidguard_db_dir) }}

+{%         set acl.value = acl.value + ' ' + category + '-default' %}

+{%       endfor %}

+{%     endif %}

+acl {

+    default {

+{%     if sg_config.enable_safe_search is defined %}

+        rewrite safesearch

+{%     endif %}

+        pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'allow' }}

+        redirect 302:http://{{ sg_config.redirect_url }}

+{%     if sg_config.log is defined and sg_config.log is not none %}

+        log blacklist.log

+{%     endif %}

+    }

+}

 {%   endif %}

 {% endif %}

diff --git a/debian/vyos-1x.postinst b/debian/vyos-1x.postinst
index dc129cb54..92948de12 100644
--- a/debian/vyos-1x.postinst
+++ b/debian/vyos-1x.postinst
@@ -30,3 +30,6 @@ if ! grep -q '^dhcpd' /etc/passwd; then
     adduser --quiet --system --disabled-login --no-create-home --home /run/dhcp-server dhcpd
     adduser --quiet dhcpd hostsd
 fi
+
+# ensure hte proxy user has a proper shell
+chsh -s /bin/sh proxy
diff --git a/interface-definitions/include/webproxy-squidguard.xml.i b/interface-definitions/include/webproxy-url-filtering.xml.i
index 6958056d4..de6ebffde 100644
--- a/interface-definitions/include/webproxy-squidguard.xml.i
+++ b/interface-definitions/include/webproxy-url-filtering.xml.i
@@ -1,4 +1,4 @@
-<!-- included start from webproxy-squidguard.xml.i -->

+<!-- included start from webproxy-url-filtering.xml.i -->

 <leafNode name="allow-category">

   <properties>

     <help>Category to allow</help>

@@ -25,17 +25,17 @@
 </leafNode>

 <leafNode name="default-action">

   <properties>

-    <help>Default action</help>

+    <help>Default action (default: allow)</help>

     <completionHelp>

       <list>allow block</list>

     </completionHelp>

     <valueHelp>

       <format>allow</format>

-      <description>Default filter action to allow (default)</description>

+      <description>Default filter action is allow)</description>

     </valueHelp>

     <valueHelp>

       <format>block</format>

-      <description>Default filter action to allow (default)</description>

+      <description>Default filter action is block</description>

     </valueHelp>

     <constraint>

       <regex>^(allow|block)$</regex>

@@ -45,6 +45,7 @@
 <leafNode name="enable-safe-search">

   <properties>

     <help>Enable safe-mode search on popular search engines</help>

+    <valueless/>

   </properties>

 </leafNode>

 <leafNode name="local-block-keyword">

@@ -62,11 +63,8 @@
     <help>Local URL to block</help>

     <valueHelp>

       <format>url</format>

-      <description>Local URL to block (without http:\/\/</description>

+      <description>Local URL to block (without "http://")</description>

     </valueHelp>

-    <constraint>

-      <regex>^(https?:\/\/)$</regex>

-    </constraint>

     <multi/>

   </properties>

 </leafNode>

@@ -78,7 +76,8 @@
       <description>IP address of site to block</description>

     </valueHelp>

     <constraint>

-      <validator name="ipv4-prefix"/>

+      <validator name="ipv4-address"/>

+      <validator name="fqdn"/>

     </constraint>

     <multi/>

   </properties>

@@ -88,11 +87,8 @@
     <help>Local URL to allow</help>

     <valueHelp>

       <format>url</format>

-      <description>Local URL to allow (without http:\/\/</description>

+      <description>Local URL to allow (without "http://")</description>

     </valueHelp>

-    <constraint>

-      <regex>^(https?:\/\/)$</regex>

-    </constraint>

     <multi/>

   </properties>

 </leafNode>

@@ -104,7 +100,8 @@
       <description>IP address of site to allow</description>

     </valueHelp>

     <constraint>

-      <validator name="ipv4-prefix"/>

+      <validator name="ipv4-address"/>

+      <validator name="fqdn"/>

     </constraint>

     <multi/>

   </properties>

diff --git a/interface-definitions/service_webproxy.xml.in b/interface-definitions/service_webproxy.xml.in
index ba33a30f4..4cd8138ec 100644
--- a/interface-definitions/service_webproxy.xml.in
+++ b/interface-definitions/service_webproxy.xml.in
@@ -171,11 +171,11 @@
                   <help>Hostname or IP address of peer</help>
                   <valueHelp>
                     <format>ipv4</format>
-                    <description>Remote syslog server IPv4 address</description>
+                    <description>Squid cache-peer IPv4 address</description>
                   </valueHelp>
                   <valueHelp>
                     <format>hostname</format>
-                    <description>Remote syslog server FQDN</description>
+                    <description>Squid cache-peer hostname</description>
                   </valueHelp>
                   <constraint>
                     <validator name="ip-address"/>
@@ -293,7 +293,10 @@
           </leafNode>
           <tagNode name="listen-address">
             <properties>
-              <help>IPv4 address for webproxy to listen on [REQUIRED]</help>
+              <help>IPv4 listen-address for WebProxy [REQUIRED]</help>
+              <completionHelp>
+                <script>${vyos_completion_dir}/list_local_ips.sh --ipv4</script>
+              </completionHelp>
               <valueHelp>
                 <format>ipv4</format>
                 <description>IPv4 address listen on</description>
@@ -402,7 +405,7 @@
                   <help>URL filtering via squidGuard redirector</help>
                 </properties>
                 <children>
-                  #include <include/webproxy-squidguard.xml.i>
+                  #include <include/webproxy-url-filtering.xml.i>
                   <node name="auto-update">
                     <properties>
                       <help>Auto update settings</help>
@@ -446,7 +449,7 @@
                       <constraintErrorMessage>SquidGuard rule must between 1-1024</constraintErrorMessage>
                     </properties>
                     <children>
-                      #include <include/webproxy-squidguard.xml.i>
+                      #include <include/webproxy-url-filtering.xml.i>
                       <leafNode name="redirect-url">
                         <properties>
                           <help>Redirect URL for filtered websites</help>
diff --git a/op-mode-definitions/webproxy.xml b/op-mode-definitions/webproxy.xml
index 09cefb929..bccffd0b3 100644
--- a/op-mode-definitions/webproxy.xml
+++ b/op-mode-definitions/webproxy.xml
@@ -84,7 +84,7 @@
             <properties>
               <help>Show update log for url-filter database</help>
             </properties>
-            <command>if [ -e /config/url-filtering/squidguard/updatestatus ]; then cat /config/url-filtering/squidguard/updatestatus; else echo "Update log not found"; fi</command>
+            <command>if [ -e /opt/vyatta/etc/config/url-filtering/squidguard/updatestatus ]; then cat /opt/vyatta/etc/config/url-filtering/squidguard/updatestatus; else echo "Update log not found"; fi</command>
           </node>
         </children>
       </node>
diff --git a/python/vyos/template.py b/python/vyos/template.py
index 63d400642..bf087c223 100644
--- a/python/vyos/template.py
+++ b/python/vyos/template.py
@@ -248,7 +248,6 @@ def dec_ip(address, decrement):
     from ipaddress import ip_interface
     return str(ip_interface(address).ip - int(decrement))
 
-
 @register_filter('isc_static_route')
 def isc_static_route(subnet, router):
     # https://ercpe.de/blog/pushing-static-routes-with-isc-dhcp-server
@@ -270,3 +269,9 @@ def isc_static_route(subnet, router):
     string += ','.join(router.split('.'))
 
     return string
+
+@register_filter('is_file')
+def is_file(filename):
+    if os.path.exists(filename):
+        return os.path.isfile(filename)
+    return False
diff --git a/python/vyos/util.py b/python/vyos/util.py
index fc6915687..494c8155e 100644
--- a/python/vyos/util.py
+++ b/python/vyos/util.py
@@ -215,6 +215,30 @@ def read_file(fname, defaultonfailure=None):
             return defaultonfailure
         raise e
 
+def write_file(fname, data, defaultonfailure=None, user=None, group=None):
+    """
+    Write content of data to given fname, should defaultonfailure be not None,
+    it is returned on failure to read.
+
+    If directory of file is not present, it is auto-created.
+    """
+    dirname = os.path.dirname(fname)
+    if not os.path.isdir(dirname):
+        os.makedirs(dirname, mode=0o755, exist_ok=False)
+        chown(dirname, user, group)
+
+    try:
+        """ Write a file to string """
+        bytes = 0
+        with open(fname, 'w') as f:
+            bytes = f.write(data)
+        chown(fname, user, group)
+        return bytes
+    except Exception as e:
+        if defaultonfailure is not None:
+            return defaultonfailure
+        raise e
+
 
 def read_json(fname, defaultonfailure=None):
     """
diff --git a/src/completion/list_webproxy_category.sh b/src/completion/list_webproxy_category.sh
index 19f26bf85..a5ad2398a 100755
--- a/src/completion/list_webproxy_category.sh
+++ b/src/completion/list_webproxy_category.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
-DB_DIR="/config/url-filtering/squidguard/db/"
+DB_DIR="/opt/vyatta/etc/config/url-filtering/squidguard/db/"
 if [ -d ${DB_DIR} ]; then
     ls -ald ${DB_DIR}/* | grep -E '^(d|l)' | awk '{print $9}' | sed s#${DB_DIR}/##
 fi
diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py
index 76b72ad48..8dfae348a 100755
--- a/src/conf_mode/service_webproxy.py
+++ b/src/conf_mode/service_webproxy.py
@@ -16,6 +16,7 @@
 
 import os
 
+from shutil import rmtree
 from sys import exit
 
 from vyos.config import Config
@@ -23,6 +24,7 @@ from vyos.configdict import dict_merge
 from vyos.template import render
 from vyos.util import call
 from vyos.util import dict_search
+from vyos.util import write_file
 from vyos.validate import is_addr_assigned
 from vyos.xml import defaults
 from vyos import ConfigError
@@ -31,6 +33,47 @@ airbag.enable()
 
 squid_config_file = '/etc/squid/squid.conf'
 squidguard_config_file = '/etc/squidguard/squidGuard.conf'
+squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db'
+user_group = 'proxy'
+
+def generate_sg_localdb(category, list_type, role, proxy):
+    cat_ = category.replace('-', '_')
+    if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy),
+                  list):
+
+        # local block databases must be generated "on-the-fly"
+        tmp = {
+            'squidguard_db_dir' : squidguard_db_dir,
+            'category' : f'{category}-default',
+            'list_type' : list_type,
+            'rule' : role
+        }
+        sg_tmp_file = '/tmp/sg.conf'
+        db_file = f'{category}-default/{list_type}'
+        domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy))
+
+        # local file
+        write_file(f'{squidguard_db_dir}/{category}-default/local', '',
+                   user=user_group, group=user_group)
+        # database input file
+        write_file(f'{squidguard_db_dir}/{db_file}', domains,
+                   user=user_group, group=user_group)
+
+        # temporary config file, deleted after generation
+        render(sg_tmp_file, 'squid/sg_acl.conf.tmpl', tmp,
+               user=user_group, group=user_group)
+
+        call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"')
+
+        if os.path.exists(sg_tmp_file):
+            os.unlink(sg_tmp_file)
+
+    else:
+        # if category is not part of our configuration, clean out the
+        # squidguard lists
+        tmp = f'{squidguard_db_dir}/{category}-default'
+        if os.path.exists(tmp):
+            rmtree(f'{squidguard_db_dir}/{category}-default')
 
 def get_config(config=None):
     if config:
@@ -55,6 +98,7 @@ def get_config(config=None):
     else:
         # store path to squidGuard config, used when generating Squid config
         proxy['squidguard_conf'] = squidguard_config_file
+        proxy['squidguard_db_dir'] = squidguard_db_dir
 
     # XXX: T2665: blend in proper cache-peer default values later
     default_values.pop('cache_peer')
@@ -67,8 +111,6 @@ def get_config(config=None):
             proxy['cache_peer'][peer] = dict_merge(default_values,
                 proxy['cache_peer'][peer])
 
-    import pprint
-    pprint.pprint(proxy)
     return proxy
 
 def verify(proxy):
@@ -121,6 +163,7 @@ def verify(proxy):
             if 'address' not in config:
                 raise ConfigError(f'Cache-peer "{peer}" address must be set!')
 
+
 def generate(proxy):
     if not proxy:
         return None
@@ -128,6 +171,16 @@ def generate(proxy):
     render(squid_config_file, 'squid/squid.conf.tmpl', proxy)
     render(squidguard_config_file, 'squid/squidGuard.conf.tmpl', proxy)
 
+    cat_dict = {
+        'local-block' : 'domains',
+        'local-block-keyword' : 'expressions',
+        'local-block-url' : 'urls',
+        'local-ok' : 'domains',
+        'local-ok-url' : 'urls'
+    }
+    for category, list_type in cat_dict.items():
+        generate_sg_localdb(category, list_type, 'default', proxy)
+
     return None
 
 def apply(proxy):