T3810: Fixed all issues in T3810

1. Added in script update webproxy blacklists generation of all DBs
2. Fixed: if the blacklist category does not have generated db,
   the template generates an empty dest category
   in squidGuard.conf and a Warning message.
3. Added template generation for local's categories
   in the rule section.
4. Changed syntax in the generation dest section for blacklist's
   categories
5. Fixed generation dest local sections in squidGuard.conf
6. Fixed bug in syntax. The word 'allow' changed to the word 'any'
   in acl squidGuard.conf
7. Backported all changes from 1.4 to 1.3 which were made in T3810
8. Fixed webproxy smoketest

1 files changed, 164 insertions, 49 deletions

diff --git a/data/templates/squid/squidGuard.conf.tmpl b/data/templates/squid/squidGuard.conf.tmpl
index f530d1072..47bc8ee75 100644
--- a/data/templates/squid/squidGuard.conf.tmpl
+++ b/data/templates/squid/squidGuard.conf.tmpl
@@ -1,24 +1,31 @@
 ### generated by service_webproxy.py ###
 
-{% macro sg_rule(category, log, db_dir) %}
-{%   set expressions = db_dir + '/' + category + '/expressions' %}
-dest {{ category }}-default {
+{% macro sg_rule(category, rule, log, db_dir) %}
+{% set domains = db_dir + '/' + category + '/domains' %}
+{% set urls = db_dir + '/' + category + '/urls' %}
+{% set expressions = db_dir + '/' + category + '/expressions' %}
+dest {{ category }}-{{ rule }}{
+{% if domains | is_file %}
         domainlist     {{ category }}/domains
+{% endif %}
+{% if urls | is_file %}
         urllist        {{ category }}/urls
-{%   if expressions | is_file %}
+{% endif %}
+{% if expressions | is_file %}
         expressionlist {{ category }}/expressions
-{%   endif %}
-{%   if log is defined %}
+{% endif %}
+{% if log is defined %}
         log            blacklist.log
-{%   endif %}
+{% endif %}
 }
 {% endmacro %}
 
 {% if url_filtering is defined and url_filtering.disable is not defined %}
-{%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
-{%     set sg_config = url_filtering.squidguard %}
-{%     set acl = namespace(value='local-ok-default') %}
-{%     set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%     if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
+{%         set sg_config = url_filtering.squidguard %}
+{%         set acl = namespace(value='') %}
+{%         set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%         set ruleacls = {} %}
 dbhome {{ squidguard_db_dir }}
 logdir /var/log/squid
 
@@ -32,60 +39,168 @@ rewrite safesearch {
         log     rewrite.log
 }
 
-{%     if sg_config.local_ok is defined and sg_config.local_ok is not none %}
-{%       set acl.value = acl.value + ' local-ok-default' %}
+{%         if sg_config.local_ok is defined and sg_config.local_ok is not none %}
+{%             set acl.value = acl.value + ' local-ok-default' %}
 dest local-ok-default {
         domainlist     local-ok-default/domains
 }
-{% endif %}
-{%     if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
-{%       set acl.value = acl.value + ' local-ok-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
+{%             set acl.value = acl.value + ' local-ok-url-default' %}
 dest local-ok-url-default {
         urllist        local-ok-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block is defined and sg_config.local_block is not none %}
-{%       set acl.value = acl.value + ' !local-block-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block is defined and sg_config.local_block is not none %}
+{%             set acl.value = acl.value + ' !local-block-default' %}
 dest local-block-default {
         domainlist     local-block-default/domains
 }
-{% endif %}
-{%     if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
-{%       set acl.value = acl.value + ' !local-block-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
+{%             set acl.value = acl.value + ' !local-block-url-default' %}
 dest local-block-url-default {
         urllist        local-block-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
-{%       set acl.value = acl.value + ' !local-block-keyword-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
+{%             set acl.value = acl.value + ' !local-block-keyword-default' %}
 dest local-block-keyword-default {
         expressionlist local-block-keyword-default/expressions
 }
-{% endif %}
+{%         endif %}
+
+{%         if sg_config.block_category is defined and sg_config.block_category is not none %}
+{%             for category in sg_config.block_category %}
+{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' !' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+{%         if sg_config.allow_category is defined and sg_config.allow_category is not none %}
+{%             for category in sg_config.allow_category %}
+{{ sg_rule(category, 'default', False, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' ' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+{%                 if rule_config.local_ok is defined and rule_config.local_ok is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %}
+{%                     else %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %}
+{%                     endif %}
+dest local-ok-{{ rule }} {
+    domainlist     local-ok-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_ok_url is defined and rule_config.local_ok_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %}
+{%                     endif   %}
+dest local-ok-url-{{ rule }} {
+    urllist     local-ok-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block is defined and rule_config.local_block is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %}
+{%                     endif   %}
+dest local-block-{{ rule }} {
+    domainlist     local-block-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_url is defined and rule_config.local_block_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %}
+{%                     endif   %}
+dest local-block-url-{{ rule }} {
+    urllist     local-block-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_keyword is defined and rule_config.local_block_keyword is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %}
+{%                     endif   %}
+dest local-block-keyword-{{ rule }} {
+    expressionlist     local-block-keyword-{{ rule }}/expressions
+}
+{%                 endif %}
+
+{%                 if rule_config.block_category is defined and rule_config.block_category is not none %}
+{%                     for b_category in rule_config.block_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+
+{%                 if rule_config.allow_category is defined and rule_config.allow_category is not none %}
+{%                     for a_category in rule_config.allow_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.source_group is defined and sg_config.source_group is not none %}
+{%             for sgroup, sg_config in sg_config.source_group.items() %}
+{%                 if sg_config.address is defined and sg_config.address is not none %}
+src {{ sgroup }} {
+{%                     for address in sg_config.address %}
+        ip {{ address }}
+{%                     endfor %}
+}
+{%                 endif %}
+{%             endfor %}
+{%         endif %}
 
-{%     if sg_config.block_category is defined and sg_config.block_category is not none %}
-{%       for category in sg_config.block_category %}
-{{ sg_rule(category, sg_config.log, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' !' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
-{%     if sg_config.allow_category is defined and sg_config.allow_category is not none %}
-{%       for category in sg_config.allow_category %}
-{{ sg_rule(category, False, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' ' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
 acl {
-    default {
-{%     if sg_config.enable_safe_search is defined %}
-        rewrite safesearch
-{%     endif %}
-        pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'allow' }}
-        redirect 302:http://{{ sg_config.redirect_url }}
-{%     if sg_config.log is defined and sg_config.log is not none %}
-        log blacklist.log
-{%     endif %}
-    }
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+        {{ rule_config.source_group }} {
+            pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is defined and rule_config.default_action == 'block' else 'any' }}
+        }
+{%             endfor %}
+{%         endif %}
+
+        default {
+{%         if sg_config.enable_safe_search is defined and sg_config.enable_safe_search is not none %}
+            rewrite safesearch
+{%         endif %}
+            pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'any' }}
+            redirect 302:http://{{ sg_config.redirect_url }}
+{%         if sg_config.log is defined and sg_config.log is not none %}
+            log blacklist.log
+{%         endif %}
+        }
 }
-{%   endif %}
+{%     endif %}
 {% endif %}