Merge pull request #1722 from aapostoliuk/webproxybackport

T3810: Fixed all issues in T3810

7 files changed, 303 insertions, 102 deletions

diff --git a/data/templates/squid/sg_acl.conf.tmpl b/data/templates/squid/sg_acl.conf.tmpl
index ce72b173a..78297a2b8 100644
--- a/data/templates/squid/sg_acl.conf.tmpl
+++ b/data/templates/squid/sg_acl.conf.tmpl
@@ -1,6 +1,5 @@
 ### generated by service_webproxy.py ###
 dbhome {{ squidguard_db_dir }}
-
 dest {{ category }}-{{ rule }} {
 {% if list_type == 'domains' %}
     domainlist      {{ category }}/domains
diff --git a/data/templates/squid/squid.conf.tmpl b/data/templates/squid/squid.conf.tmpl
index 8754e762d..88870d5a9 100644
--- a/data/templates/squid/squid.conf.tmpl
+++ b/data/templates/squid/squid.conf.tmpl
@@ -16,25 +16,30 @@ acl Safe_ports port 488         # gss-http
 acl Safe_ports port 591         # filemaker
 acl Safe_ports port 777         # multiling http
 acl CONNECT method CONNECT
-
+{% if domain_block is defined and domain_block is not none %}
+{%     for domain in domain_block %}
+acl BLOCKDOMAIN dstdomain {{ domain }}
+{%     endfor %}
+http_access deny BLOCKDOMAIN
+{% endif %}
 {% if authentication is defined and authentication is not none %}
-{%   if authentication.children is defined and authentication.children is not none %}
+{%     if authentication.children is defined and authentication.children is not none %}
 auth_param basic children {{ authentication.children }}
-{%   endif %}
-{%   if authentication.credentials_ttl is defined and authentication.credentials_ttl is not none %}
+{%     endif %}
+{%     if authentication.credentials_ttl is defined and authentication.credentials_ttl is not none %}
 auth_param basic credentialsttl {{ authentication.credentials_ttl }} minute
-{%   endif %}
-{%   if authentication.realm is defined and authentication.realm is not none %}
+{%     endif %}
+{%     if authentication.realm is defined and authentication.realm is not none %}
 auth_param basic realm "{{ authentication.realm }}"
-{%   endif %}
+{%     endif %}
 {# LDAP based Authentication #}
-{%   if authentication.method is defined and authentication.method is not none  %}
-{%     if authentication.ldap is defined and authentication.ldap is not none and authentication.method == 'ldap' %}
+{%     if authentication.method is defined and authentication.method is not none %}
+{%         if authentication.ldap is defined and authentication.ldap is not none and authentication.method == 'ldap' %}
 auth_param basic program /usr/lib/squid/basic_ldap_auth -v {{ authentication.ldap.version }} -b "{{ authentication.ldap.base_dn }}" {{ '-D "' + authentication.ldap.bind_dn + '"' if authentication.ldap.bind_dn is defined }} {{ '-w "' + authentication.ldap.password + '"' if authentication.ldap.password is defined }} {{ '-f "' + authentication.ldap.filter_expression + '"' if authentication.ldap.filter_expression is defined }} {{ '-u "' + authentication.ldap.username_attribute + '"' if authentication.ldap.username_attribute is defined }} -p {{ authentication.ldap.port }} {{ '-ZZ' if authentication.ldap.use_ssl is defined }} -R -h "{{ authentication.ldap.server }}"
-{%     endif %}
+{%         endif %}
 acl auth proxy_auth REQUIRED
 http_access allow auth
-{%   endif %}
+{%     endif %}
 {% endif %}
 
 http_access allow manager localhost
@@ -46,18 +51,18 @@ http_access allow net
 http_access deny all
 
 {% if reply_block_mime is defined and reply_block_mime is not none %}
-{%   for mime_type in reply_block_mime %}
+{%     for mime_type in reply_block_mime %}
 acl BLOCK_MIME rep_mime_type {{ mime_type }}
-{%   endfor %}
+{%     endfor %}
 http_reply_access deny BLOCK_MIME
 {% endif %}
 
 {% if cache_size is defined and cache_size is not none %}
-{%   if cache_size | int > 0 %}
+{%     if cache_size | int > 0 %}
 cache_dir ufs /var/spool/squid {{ cache_size }} 16 256
-{%   else %}
+{%     else %}
 # disabling disk cache
-{%   endif %}
+{%     endif %}
 {% endif %}
 {% if mem_cache_size is defined and mem_cache_size is not none %}
 cache_mem {{ mem_cache_size }} MB
@@ -89,9 +94,9 @@ tcp_outgoing_address {{ outgoing_address }}
 
 
 {% if listen_address is defined and listen_address is not none %}
-{%   for address, config in listen_address.items() %}
+{%     for address, config in listen_address.items() %}
 http_port {{ address }}:{{ config.port if config.port is defined else default_port }} {{ 'intercept' if config.disable_transparent is not defined }}
-{%   endfor %}
+{%     endfor %}
 {% endif %}
 http_port 127.0.0.1:{{ default_port }}
 
@@ -100,16 +105,16 @@ forwarded_for off
 
 {# SquidGuard #}
 {% if url_filtering is defined and url_filtering.disable is not defined %}
-{%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
-redirect_program /usr/bin/squidGuard -c {{ squidguard_conf }}
-redirect_children 8
-redirector_bypass on
-{%   endif %}
+{%     if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
+url_rewrite_program /usr/bin/squidGuard -c {{ squidguard_conf }}
+url_rewrite_children 8
+url_rewrite_bypass on
+{%     endif %}
 {% endif %}
 
 {% if cache_peer is defined and cache_peer is not none %}
-{%   for peer, config in cache_peer.items() %}
+{%     for peer, config in cache_peer.items() %}
 cache_peer {{ config.address }} {{ config.type }} {{ config.http_port }} {{ config.icp_port }} {{ config.options }}
-{%   endfor %}
+{%     endfor %}
 never_direct allow all
 {% endif %}
diff --git a/data/templates/squid/squidGuard.conf.tmpl b/data/templates/squid/squidGuard.conf.tmpl
index f530d1072..47bc8ee75 100644
--- a/data/templates/squid/squidGuard.conf.tmpl
+++ b/data/templates/squid/squidGuard.conf.tmpl
@@ -1,24 +1,31 @@
 ### generated by service_webproxy.py ###
 
-{% macro sg_rule(category, log, db_dir) %}
-{%   set expressions = db_dir + '/' + category + '/expressions' %}
-dest {{ category }}-default {
+{% macro sg_rule(category, rule, log, db_dir) %}
+{% set domains = db_dir + '/' + category + '/domains' %}
+{% set urls = db_dir + '/' + category + '/urls' %}
+{% set expressions = db_dir + '/' + category + '/expressions' %}
+dest {{ category }}-{{ rule }}{
+{% if domains | is_file %}
         domainlist     {{ category }}/domains
+{% endif %}
+{% if urls | is_file %}
         urllist        {{ category }}/urls
-{%   if expressions | is_file %}
+{% endif %}
+{% if expressions | is_file %}
         expressionlist {{ category }}/expressions
-{%   endif %}
-{%   if log is defined %}
+{% endif %}
+{% if log is defined %}
         log            blacklist.log
-{%   endif %}
+{% endif %}
 }
 {% endmacro %}
 
 {% if url_filtering is defined and url_filtering.disable is not defined %}
-{%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
-{%     set sg_config = url_filtering.squidguard %}
-{%     set acl = namespace(value='local-ok-default') %}
-{%     set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%     if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
+{%         set sg_config = url_filtering.squidguard %}
+{%         set acl = namespace(value='') %}
+{%         set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%         set ruleacls = {} %}
 dbhome {{ squidguard_db_dir }}
 logdir /var/log/squid
 
@@ -32,60 +39,168 @@ rewrite safesearch {
         log     rewrite.log
 }
 
-{%     if sg_config.local_ok is defined and sg_config.local_ok is not none %}
-{%       set acl.value = acl.value + ' local-ok-default' %}
+{%         if sg_config.local_ok is defined and sg_config.local_ok is not none %}
+{%             set acl.value = acl.value + ' local-ok-default' %}
 dest local-ok-default {
         domainlist     local-ok-default/domains
 }
-{% endif %}
-{%     if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
-{%       set acl.value = acl.value + ' local-ok-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
+{%             set acl.value = acl.value + ' local-ok-url-default' %}
 dest local-ok-url-default {
         urllist        local-ok-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block is defined and sg_config.local_block is not none %}
-{%       set acl.value = acl.value + ' !local-block-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block is defined and sg_config.local_block is not none %}
+{%             set acl.value = acl.value + ' !local-block-default' %}
 dest local-block-default {
         domainlist     local-block-default/domains
 }
-{% endif %}
-{%     if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
-{%       set acl.value = acl.value + ' !local-block-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
+{%             set acl.value = acl.value + ' !local-block-url-default' %}
 dest local-block-url-default {
         urllist        local-block-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
-{%       set acl.value = acl.value + ' !local-block-keyword-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
+{%             set acl.value = acl.value + ' !local-block-keyword-default' %}
 dest local-block-keyword-default {
         expressionlist local-block-keyword-default/expressions
 }
-{% endif %}
+{%         endif %}
+
+{%         if sg_config.block_category is defined and sg_config.block_category is not none %}
+{%             for category in sg_config.block_category %}
+{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' !' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+{%         if sg_config.allow_category is defined and sg_config.allow_category is not none %}
+{%             for category in sg_config.allow_category %}
+{{ sg_rule(category, 'default', False, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' ' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+{%                 if rule_config.local_ok is defined and rule_config.local_ok is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %}
+{%                     else %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %}
+{%                     endif %}
+dest local-ok-{{ rule }} {
+    domainlist     local-ok-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_ok_url is defined and rule_config.local_ok_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %}
+{%                     endif   %}
+dest local-ok-url-{{ rule }} {
+    urllist     local-ok-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block is defined and rule_config.local_block is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %}
+{%                     endif   %}
+dest local-block-{{ rule }} {
+    domainlist     local-block-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_url is defined and rule_config.local_block_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %}
+{%                     endif   %}
+dest local-block-url-{{ rule }} {
+    urllist     local-block-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_keyword is defined and rule_config.local_block_keyword is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %}
+{%                     endif   %}
+dest local-block-keyword-{{ rule }} {
+    expressionlist     local-block-keyword-{{ rule }}/expressions
+}
+{%                 endif %}
+
+{%                 if rule_config.block_category is defined and rule_config.block_category is not none %}
+{%                     for b_category in rule_config.block_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+
+{%                 if rule_config.allow_category is defined and rule_config.allow_category is not none %}
+{%                     for a_category in rule_config.allow_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.source_group is defined and sg_config.source_group is not none %}
+{%             for sgroup, sg_config in sg_config.source_group.items() %}
+{%                 if sg_config.address is defined and sg_config.address is not none %}
+src {{ sgroup }} {
+{%                     for address in sg_config.address %}
+        ip {{ address }}
+{%                     endfor %}
+}
+{%                 endif %}
+{%             endfor %}
+{%         endif %}
 
-{%     if sg_config.block_category is defined and sg_config.block_category is not none %}
-{%       for category in sg_config.block_category %}
-{{ sg_rule(category, sg_config.log, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' !' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
-{%     if sg_config.allow_category is defined and sg_config.allow_category is not none %}
-{%       for category in sg_config.allow_category %}
-{{ sg_rule(category, False, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' ' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
 acl {
-    default {
-{%     if sg_config.enable_safe_search is defined %}
-        rewrite safesearch
-{%     endif %}
-        pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'allow' }}
-        redirect 302:http://{{ sg_config.redirect_url }}
-{%     if sg_config.log is defined and sg_config.log is not none %}
-        log blacklist.log
-{%     endif %}
-    }
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+        {{ rule_config.source_group }} {
+            pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is defined and rule_config.default_action == 'block' else 'any' }}
+        }
+{%             endfor %}
+{%         endif %}
+
+        default {
+{%         if sg_config.enable_safe_search is defined and sg_config.enable_safe_search is not none %}
+            rewrite safesearch
+{%         endif %}
+            pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'any' }}
+            redirect 302:http://{{ sg_config.redirect_url }}
+{%         if sg_config.log is defined and sg_config.log is not none %}
+            log blacklist.log
+{%         endif %}
+        }
 }
-{%   endif %}
+{%     endif %}
 {% endif %}
diff --git a/interface-definitions/service_webproxy.xml.in b/interface-definitions/service_webproxy.xml.in
index bd5396291..9136e2fe7 100644
--- a/interface-definitions/service_webproxy.xml.in
+++ b/interface-definitions/service_webproxy.xml.in
@@ -513,6 +513,7 @@
                             <validator name="ipv4-prefix"/>
                             <validator name="ipv4-range"/>
                           </constraint>
+                          <multi/>
                         </properties>
                       </leafNode>
                       <leafNode name="description">
diff --git a/smoketest/scripts/cli/test_service_webproxy.py b/smoketest/scripts/cli/test_service_webproxy.py
index dfccced0a..75486337a 100755
--- a/smoketest/scripts/cli/test_service_webproxy.py
+++ b/smoketest/scripts/cli/test_service_webproxy.py
@@ -241,8 +241,8 @@ class TestServiceWebProxy(VyOSUnitTestSHIM.TestCase):
         config = read_file(PROXY_CONF)
         self.assertIn(f'http_port {listen_ip}:3128 intercept', config)
 
-        self.assertIn(f'redirect_program /usr/bin/squidGuard -c /etc/squidguard/squidGuard.conf', config)
-        self.assertIn(f'redirect_children 8', config)
+        self.assertIn(f'url_rewrite_program /usr/bin/squidGuard -c /etc/squidguard/squidGuard.conf', config)
+        self.assertIn(f'url_rewrite_children 8', config)
 
         # Check SquidGuard config
         sg_config = read_file('/etc/squidguard/squidGuard.conf')
diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py
index cbbd2e0bc..9e0d73a5b 100755
--- a/src/conf_mode/service_webproxy.py
+++ b/src/conf_mode/service_webproxy.py
@@ -23,12 +23,15 @@ from vyos.config import Config
 from vyos.configdict import dict_merge
 from vyos.template import render
 from vyos.util import call
+from vyos.util import chmod_755
 from vyos.util import dict_search
 from vyos.util import write_file
 from vyos.validate import is_addr_assigned
 from vyos.xml import defaults
+from vyos.base import Warning
 from vyos import ConfigError
 from vyos import airbag
+
 airbag.enable()
 
 squid_config_file = '/etc/squid/squid.conf'
@@ -36,24 +39,56 @@ squidguard_config_file = '/etc/squidguard/squidGuard.conf'
 squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db'
 user_group = 'proxy'
 
-def generate_sg_localdb(category, list_type, role, proxy):
+
+def check_blacklist_categorydb(config_section):
+    if 'block_category' in config_section:
+        for category in config_section['block_category']:
+            check_categorydb(category)
+    if 'allow_category' in config_section:
+        for category in config_section['allow_category']:
+            check_categorydb(category)
+
+
+def check_categorydb(category: str):
+    """
+    Check if category's db exist
+    :param category:
+    :type str:
+    """
+    path_to_cat: str = f'{squidguard_db_dir}/{category}'
+    if not os.path.exists(f'{path_to_cat}/domains.db') \
+            and not os.path.exists(f'{path_to_cat}/urls.db') \
+            and not os.path.exists(f'{path_to_cat}/expressions.db'):
+        Warning(f'DB of category {category} does not exist.\n '
+                f'Use [update webproxy blacklists] '
+                f'or delete undefined category!')
+
+
+def generate_sg_rule_localdb(category, list_type, role, proxy):
+    if not category or not list_type or not role:
+        return None
     cat_ = category.replace('-', '_')
-    if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy),
-                  list):
 
+    if role == 'default':
+        path_to_cat = f'{cat_}'
+    else:
+        path_to_cat = f'rule.{role}.{cat_}'
+    if isinstance(
+            dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy),
+            list):
         # local block databases must be generated "on-the-fly"
         tmp = {
-            'squidguard_db_dir' : squidguard_db_dir,
-            'category' : f'{category}-default',
-            'list_type' : list_type,
-            'rule' : role
+            'squidguard_db_dir': squidguard_db_dir,
+            'category': f'{category}-{role}',
+            'list_type': list_type,
+            'rule': role
         }
         sg_tmp_file = '/tmp/sg.conf'
-        db_file = f'{category}-default/{list_type}'
-        domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy))
-
+        db_file = f'{category}-{role}/{list_type}'
+        domains = '\n'.join(
+            dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy))
         # local file
-        write_file(f'{squidguard_db_dir}/{category}-default/local', '',
+        write_file(f'{squidguard_db_dir}/{category}-{role}/local', '',
                    user=user_group, group=user_group)
         # database input file
         write_file(f'{squidguard_db_dir}/{db_file}', domains,
@@ -63,17 +98,18 @@ def generate_sg_localdb(category, list_type, role, proxy):
         render(sg_tmp_file, 'squid/sg_acl.conf.tmpl', tmp,
                user=user_group, group=user_group)
 
-        call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"')
+        call(
+            f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"')
 
         if os.path.exists(sg_tmp_file):
             os.unlink(sg_tmp_file)
-
     else:
         # if category is not part of our configuration, clean out the
         # squidguard lists
-        tmp = f'{squidguard_db_dir}/{category}-default'
+        tmp = f'{squidguard_db_dir}/{category}-{role}'
         if os.path.exists(tmp):
-            rmtree(f'{squidguard_db_dir}/{category}-default')
+            rmtree(f'{squidguard_db_dir}/{category}-{role}')
+
 
 def get_config(config=None):
     if config:
@@ -84,7 +120,8 @@ def get_config(config=None):
     if not conf.exists(base):
         return None
 
-    proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True)
+    proxy = conf.get_config_dict(base, key_mangling=('-', '_'),
+                                 get_first_key=True)
     # We have gathered the dict representation of the CLI, but there are default
     # options which we need to update into the dictionary retrived.
     default_values = defaults(base)
@@ -109,10 +146,11 @@ def get_config(config=None):
         default_values = defaults(base + ['cache-peer'])
         for peer in proxy['cache_peer']:
             proxy['cache_peer'][peer] = dict_merge(default_values,
-                proxy['cache_peer'][peer])
+                                                   proxy['cache_peer'][peer])
 
     return proxy
 
+
 def verify(proxy):
     if not proxy:
         return None
@@ -169,17 +207,30 @@ def generate(proxy):
     render(squidguard_config_file, 'squid/squidGuard.conf.tmpl', proxy)
 
     cat_dict = {
-        'local-block' : 'domains',
-        'local-block-keyword' : 'expressions',
-        'local-block-url' : 'urls',
-        'local-ok' : 'domains',
-        'local-ok-url' : 'urls'
+        'local-block': 'domains',
+        'local-block-keyword': 'expressions',
+        'local-block-url': 'urls',
+        'local-ok': 'domains',
+        'local-ok-url': 'urls'
     }
-    for category, list_type in cat_dict.items():
-        generate_sg_localdb(category, list_type, 'default', proxy)
+    if dict_search(f'url_filtering.squidguard', proxy) is not None:
+        squidgard_config_section = proxy['url_filtering']['squidguard']
+
+        for category, list_type in cat_dict.items():
+            generate_sg_rule_localdb(category, list_type, 'default', proxy)
+        check_blacklist_categorydb(squidgard_config_section)
+
+        if 'rule' in squidgard_config_section:
+            for rule in squidgard_config_section['rule']:
+                rule_config_section = squidgard_config_section['rule'][
+                    rule]
+                for category, list_type in cat_dict.items():
+                    generate_sg_rule_localdb(category, list_type, rule, proxy)
+                check_blacklist_categorydb(rule_config_section)
 
     return None
 
+
 def apply(proxy):
     if not proxy:
         # proxy is removed in the commit
@@ -192,9 +243,12 @@ def apply(proxy):
 
         return None
 
+    if os.path.exists(squidguard_db_dir):
+        chmod_755(squidguard_db_dir)
     call('systemctl restart squid.service')
     return None
 
+
 if __name__ == '__main__':
     try:
         c = get_config()
diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh
index 43a4b79fc..4fb9a54c6 100755
--- a/src/op_mode/webproxy_update_blacklist.sh
+++ b/src/op_mode/webproxy_update_blacklist.sh
@@ -18,6 +18,23 @@ blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/black
 data_dir="/opt/vyatta/etc/config/url-filtering"
 archive="${data_dir}/squidguard/archive"
 db_dir="${data_dir}/squidguard/db"
+conf_file="/etc/squidguard/squidGuard.conf"
+tmp_conf_file="/tmp/sg_update_db.conf"
+
+#$1-category
+#$2-type
+#$3-list
+create_sg_db ()
+{
+       FILE=$db_dir/$1/$2
+       if test -f "$FILE"; then
+                rm -f ${tmp_conf_file}
+                printf "dbhome $db_dir\ndest $1 {\n     $3      $1/$2\n}\nacl {\n       default {\n             pass    any\n   }\n}" >> ${tmp_conf_file}
+                /usr/bin/squidGuard -b -c ${tmp_conf_file} -C $FILE
+                rm -f ${tmp_conf_file}
+       fi
+
+}
 
 while [ $# -gt 0 ]
 do
@@ -88,7 +105,17 @@ if [[ -n $update ]] && [[ $update -eq "yes" ]]; then
 
     # fix permissions
     chown -R proxy:proxy ${db_dir}
-    chmod 2770 ${db_dir}
+
+    #create db
+    category_list=(`find $db_dir -type d -exec basename {} \; `)
+    for category in ${category_list[@]}
+    do
+        create_sg_db $category "domains" "domainlist"
+        create_sg_db $category "urls" "urllist"
+        create_sg_db $category "expressions" "expressionlist"
+    done
+    chown -R proxy:proxy ${db_dir}
+    chmod 755 ${db_dir}
 
     logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})"