T3810: Fixed all issues in T3810

1. Added in script update webproxy blacklists generation of all DBs
2. Fixed: if the blacklist category does not have generated db,
   the template generates an empty dest category
   in squidGuard.conf and a Warning message.
3. Added template generation for local's categories
   in the rule section.
4. Changed syntax in the generation dest section for blacklist's
   categories
5. Fixed generation dest local sections in squidGuard.conf
6. Fixed bug in syntax. The word 'allow' changed to the word 'any'
   in acl squidGuard.conf
7. Backported all changes from 1.4 to 1.3 which were made in T3810
8. Fixed webproxy smoketest

3 files changed, 194 insertions, 75 deletions

diff --git a/data/templates/squid/sg_acl.conf.tmpl b/data/templates/squid/sg_acl.conf.tmpl
index ce72b173a..78297a2b8 100644
--- a/data/templates/squid/sg_acl.conf.tmpl
+++ b/data/templates/squid/sg_acl.conf.tmpl
@@ -1,6 +1,5 @@
 ### generated by service_webproxy.py ###
 dbhome {{ squidguard_db_dir }}
-
 dest {{ category }}-{{ rule }} {
 {% if list_type == 'domains' %}
     domainlist      {{ category }}/domains
diff --git a/data/templates/squid/squid.conf.tmpl b/data/templates/squid/squid.conf.tmpl
index 8754e762d..88870d5a9 100644
--- a/data/templates/squid/squid.conf.tmpl
+++ b/data/templates/squid/squid.conf.tmpl
@@ -16,25 +16,30 @@ acl Safe_ports port 488         # gss-http
 acl Safe_ports port 591         # filemaker
 acl Safe_ports port 777         # multiling http
 acl CONNECT method CONNECT
-
+{% if domain_block is defined and domain_block is not none %}
+{%     for domain in domain_block %}
+acl BLOCKDOMAIN dstdomain {{ domain }}
+{%     endfor %}
+http_access deny BLOCKDOMAIN
+{% endif %}
 {% if authentication is defined and authentication is not none %}
-{%   if authentication.children is defined and authentication.children is not none %}
+{%     if authentication.children is defined and authentication.children is not none %}
 auth_param basic children {{ authentication.children }}
-{%   endif %}
-{%   if authentication.credentials_ttl is defined and authentication.credentials_ttl is not none %}
+{%     endif %}
+{%     if authentication.credentials_ttl is defined and authentication.credentials_ttl is not none %}
 auth_param basic credentialsttl {{ authentication.credentials_ttl }} minute
-{%   endif %}
-{%   if authentication.realm is defined and authentication.realm is not none %}
+{%     endif %}
+{%     if authentication.realm is defined and authentication.realm is not none %}
 auth_param basic realm "{{ authentication.realm }}"
-{%   endif %}
+{%     endif %}
 {# LDAP based Authentication #}
-{%   if authentication.method is defined and authentication.method is not none  %}
-{%     if authentication.ldap is defined and authentication.ldap is not none and authentication.method == 'ldap' %}
+{%     if authentication.method is defined and authentication.method is not none %}
+{%         if authentication.ldap is defined and authentication.ldap is not none and authentication.method == 'ldap' %}
 auth_param basic program /usr/lib/squid/basic_ldap_auth -v {{ authentication.ldap.version }} -b "{{ authentication.ldap.base_dn }}" {{ '-D "' + authentication.ldap.bind_dn + '"' if authentication.ldap.bind_dn is defined }} {{ '-w "' + authentication.ldap.password + '"' if authentication.ldap.password is defined }} {{ '-f "' + authentication.ldap.filter_expression + '"' if authentication.ldap.filter_expression is defined }} {{ '-u "' + authentication.ldap.username_attribute + '"' if authentication.ldap.username_attribute is defined }} -p {{ authentication.ldap.port }} {{ '-ZZ' if authentication.ldap.use_ssl is defined }} -R -h "{{ authentication.ldap.server }}"
-{%     endif %}
+{%         endif %}
 acl auth proxy_auth REQUIRED
 http_access allow auth
-{%   endif %}
+{%     endif %}
 {% endif %}
 
 http_access allow manager localhost
@@ -46,18 +51,18 @@ http_access allow net
 http_access deny all
 
 {% if reply_block_mime is defined and reply_block_mime is not none %}
-{%   for mime_type in reply_block_mime %}
+{%     for mime_type in reply_block_mime %}
 acl BLOCK_MIME rep_mime_type {{ mime_type }}
-{%   endfor %}
+{%     endfor %}
 http_reply_access deny BLOCK_MIME
 {% endif %}
 
 {% if cache_size is defined and cache_size is not none %}
-{%   if cache_size | int > 0 %}
+{%     if cache_size | int > 0 %}
 cache_dir ufs /var/spool/squid {{ cache_size }} 16 256
-{%   else %}
+{%     else %}
 # disabling disk cache
-{%   endif %}
+{%     endif %}
 {% endif %}
 {% if mem_cache_size is defined and mem_cache_size is not none %}
 cache_mem {{ mem_cache_size }} MB
@@ -89,9 +94,9 @@ tcp_outgoing_address {{ outgoing_address }}
 
 
 {% if listen_address is defined and listen_address is not none %}
-{%   for address, config in listen_address.items() %}
+{%     for address, config in listen_address.items() %}
 http_port {{ address }}:{{ config.port if config.port is defined else default_port }} {{ 'intercept' if config.disable_transparent is not defined }}
-{%   endfor %}
+{%     endfor %}
 {% endif %}
 http_port 127.0.0.1:{{ default_port }}
 
@@ -100,16 +105,16 @@ forwarded_for off
 
 {# SquidGuard #}
 {% if url_filtering is defined and url_filtering.disable is not defined %}
-{%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
-redirect_program /usr/bin/squidGuard -c {{ squidguard_conf }}
-redirect_children 8
-redirector_bypass on
-{%   endif %}
+{%     if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
+url_rewrite_program /usr/bin/squidGuard -c {{ squidguard_conf }}
+url_rewrite_children 8
+url_rewrite_bypass on
+{%     endif %}
 {% endif %}
 
 {% if cache_peer is defined and cache_peer is not none %}
-{%   for peer, config in cache_peer.items() %}
+{%     for peer, config in cache_peer.items() %}
 cache_peer {{ config.address }} {{ config.type }} {{ config.http_port }} {{ config.icp_port }} {{ config.options }}
-{%   endfor %}
+{%     endfor %}
 never_direct allow all
 {% endif %}
diff --git a/data/templates/squid/squidGuard.conf.tmpl b/data/templates/squid/squidGuard.conf.tmpl
index f530d1072..47bc8ee75 100644
--- a/data/templates/squid/squidGuard.conf.tmpl
+++ b/data/templates/squid/squidGuard.conf.tmpl
@@ -1,24 +1,31 @@
 ### generated by service_webproxy.py ###
 
-{% macro sg_rule(category, log, db_dir) %}
-{%   set expressions = db_dir + '/' + category + '/expressions' %}
-dest {{ category }}-default {
+{% macro sg_rule(category, rule, log, db_dir) %}
+{% set domains = db_dir + '/' + category + '/domains' %}
+{% set urls = db_dir + '/' + category + '/urls' %}
+{% set expressions = db_dir + '/' + category + '/expressions' %}
+dest {{ category }}-{{ rule }}{
+{% if domains | is_file %}
         domainlist     {{ category }}/domains
+{% endif %}
+{% if urls | is_file %}
         urllist        {{ category }}/urls
-{%   if expressions | is_file %}
+{% endif %}
+{% if expressions | is_file %}
         expressionlist {{ category }}/expressions
-{%   endif %}
-{%   if log is defined %}
+{% endif %}
+{% if log is defined %}
         log            blacklist.log
-{%   endif %}
+{% endif %}
 }
 {% endmacro %}
 
 {% if url_filtering is defined and url_filtering.disable is not defined %}
-{%   if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
-{%     set sg_config = url_filtering.squidguard %}
-{%     set acl = namespace(value='local-ok-default') %}
-{%     set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%     if url_filtering.squidguard is defined and url_filtering.squidguard is not none %}
+{%         set sg_config = url_filtering.squidguard %}
+{%         set acl = namespace(value='') %}
+{%         set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %}
+{%         set ruleacls = {} %}
 dbhome {{ squidguard_db_dir }}
 logdir /var/log/squid
 
@@ -32,60 +39,168 @@ rewrite safesearch {
         log     rewrite.log
 }
 
-{%     if sg_config.local_ok is defined and sg_config.local_ok is not none %}
-{%       set acl.value = acl.value + ' local-ok-default' %}
+{%         if sg_config.local_ok is defined and sg_config.local_ok is not none %}
+{%             set acl.value = acl.value + ' local-ok-default' %}
 dest local-ok-default {
         domainlist     local-ok-default/domains
 }
-{% endif %}
-{%     if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
-{%       set acl.value = acl.value + ' local-ok-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_ok_url is defined and sg_config.local_ok_url is not none %}
+{%             set acl.value = acl.value + ' local-ok-url-default' %}
 dest local-ok-url-default {
         urllist        local-ok-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block is defined and sg_config.local_block is not none %}
-{%       set acl.value = acl.value + ' !local-block-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block is defined and sg_config.local_block is not none %}
+{%             set acl.value = acl.value + ' !local-block-default' %}
 dest local-block-default {
         domainlist     local-block-default/domains
 }
-{% endif %}
-{%     if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
-{%       set acl.value = acl.value + ' !local-block-url-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_url is defined and sg_config.local_block_url is not none %}
+{%             set acl.value = acl.value + ' !local-block-url-default' %}
 dest local-block-url-default {
         urllist        local-block-url-default/urls
 }
-{% endif %}
-{%     if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
-{%       set acl.value = acl.value + ' !local-block-keyword-default' %}
+{%         endif %}
+
+{%         if sg_config.local_block_keyword is defined and sg_config.local_block_keyword is not none %}
+{%             set acl.value = acl.value + ' !local-block-keyword-default' %}
 dest local-block-keyword-default {
         expressionlist local-block-keyword-default/expressions
 }
-{% endif %}
+{%         endif %}
+
+{%         if sg_config.block_category is defined and sg_config.block_category is not none %}
+{%             for category in sg_config.block_category %}
+{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' !' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+{%         if sg_config.allow_category is defined and sg_config.allow_category is not none %}
+{%             for category in sg_config.allow_category %}
+{{ sg_rule(category, 'default', False, squidguard_db_dir) }}
+{%                 set acl.value = acl.value + ' ' + category + '-default' %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+{%                 if rule_config.local_ok is defined and rule_config.local_ok is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %}
+{%                     else %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %}
+{%                     endif %}
+dest local-ok-{{ rule }} {
+    domainlist     local-ok-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_ok_url is defined and rule_config.local_ok_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %}
+{%                     endif   %}
+dest local-ok-url-{{ rule }} {
+    urllist     local-ok-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block is defined and rule_config.local_block is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %}
+{%                     endif   %}
+dest local-block-{{ rule }} {
+    domainlist     local-block-{{ rule }}/domains
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_url is defined and rule_config.local_block_url is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %}
+{%                     endif   %}
+dest local-block-url-{{ rule }} {
+    urllist     local-block-url-{{ rule }}/urls
+}
+{%                 endif %}
+
+{%                 if rule_config.local_block_keyword is defined and rule_config.local_block_keyword is not none %}
+{%                     if rule in ruleacls %}
+{%                         set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %}
+{%                     else    %}
+{%                         set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %}
+{%                     endif   %}
+dest local-block-keyword-{{ rule }} {
+    expressionlist     local-block-keyword-{{ rule }}/expressions
+}
+{%                 endif %}
+
+{%                 if rule_config.block_category is defined and rule_config.block_category is not none %}
+{%                     for b_category in rule_config.block_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+
+{%                 if rule_config.allow_category is defined and rule_config.allow_category is not none %}
+{%                     for a_category in rule_config.allow_category %}
+{%                         if rule in ruleacls %}
+{%                             set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %}
+{%                         else    %}
+{%                             set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %}
+{%                         endif   %}
+{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }}
+{%                     endfor %}
+{%                 endif   %}
+{%             endfor %}
+{%         endif %}
+
+
+{%         if sg_config.source_group is defined and sg_config.source_group is not none %}
+{%             for sgroup, sg_config in sg_config.source_group.items() %}
+{%                 if sg_config.address is defined and sg_config.address is not none %}
+src {{ sgroup }} {
+{%                     for address in sg_config.address %}
+        ip {{ address }}
+{%                     endfor %}
+}
+{%                 endif %}
+{%             endfor %}
+{%         endif %}
 
-{%     if sg_config.block_category is defined and sg_config.block_category is not none %}
-{%       for category in sg_config.block_category %}
-{{ sg_rule(category, sg_config.log, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' !' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
-{%     if sg_config.allow_category is defined and sg_config.allow_category is not none %}
-{%       for category in sg_config.allow_category %}
-{{ sg_rule(category, False, squidguard_db_dir) }}
-{%         set acl.value = acl.value + ' ' + category + '-default' %}
-{%       endfor %}
-{%     endif %}
 acl {
-    default {
-{%     if sg_config.enable_safe_search is defined %}
-        rewrite safesearch
-{%     endif %}
-        pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'allow' }}
-        redirect 302:http://{{ sg_config.redirect_url }}
-{%     if sg_config.log is defined and sg_config.log is not none %}
-        log blacklist.log
-{%     endif %}
-    }
+{%         if sg_config.rule is defined and sg_config.rule is not none %}
+{%             for rule, rule_config in sg_config.rule.items() %}
+        {{ rule_config.source_group }} {
+            pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is defined and rule_config.default_action == 'block' else 'any' }}
+        }
+{%             endfor %}
+{%         endif %}
+
+        default {
+{%         if sg_config.enable_safe_search is defined and sg_config.enable_safe_search is not none %}
+            rewrite safesearch
+{%         endif %}
+            pass {{ acl.value }} {{ 'none' if sg_config.default_action is defined and sg_config.default_action == 'block' else 'any' }}
+            redirect 302:http://{{ sg_config.redirect_url }}
+{%         if sg_config.log is defined and sg_config.log is not none %}
+            log blacklist.log
+{%         endif %}
+        }
 }
-{%   endif %}
+{%     endif %}
 {% endif %}