From e78235213c7409ae0ddb50edc1ba83095d1c9080 Mon Sep 17 00:00:00 2001 From: aapostoliuk <108394744+aapostoliuk@users.noreply.github.com> Date: Sat, 17 Dec 2022 09:20:56 +0200 Subject: webproxy: T3810: multiple squidGuard fixes 1. Added in script update webproxy blacklists generation of all DBs 2. Fixed: if the blacklist category does not have generated db, the template generates an empty dest category in squidGuard.conf and a Warning message. 3. Added template generation for local's categories in the rule section. 4. Changed syntax in the generation dest section for blacklist's categories 4. Fixed generation dest local sections in squidGuard.conf 5. Fixed bug in syntax. The word 'allow' changed to the word 'any' in acl squidGuard.conf --- data/templates/squid/sg_acl.conf.j2 | 1 - data/templates/squid/squidGuard.conf.j2 | 122 ++++++++++++++++++++++++++----- src/conf_mode/service_webproxy.py | 100 +++++++++++++++++++------ src/op_mode/webproxy_update_blacklist.sh | 27 +++++++ 4 files changed, 205 insertions(+), 45 deletions(-) diff --git a/data/templates/squid/sg_acl.conf.j2 b/data/templates/squid/sg_acl.conf.j2 index ce72b173a..78297a2b8 100644 --- a/data/templates/squid/sg_acl.conf.j2 +++ b/data/templates/squid/sg_acl.conf.j2 @@ -1,6 +1,5 @@ ### generated by service_webproxy.py ### dbhome {{ squidguard_db_dir }} - dest {{ category }}-{{ rule }} { {% if list_type == 'domains' %} domainlist {{ category }}/domains diff --git a/data/templates/squid/squidGuard.conf.j2 b/data/templates/squid/squidGuard.conf.j2 index 1bc4c984f..a93f878df 100644 --- a/data/templates/squid/squidGuard.conf.j2 +++ b/data/templates/squid/squidGuard.conf.j2 @@ -1,10 +1,16 @@ ### generated by service_webproxy.py ### -{% macro sg_rule(category, log, db_dir) %} +{% macro sg_rule(category, rule, log, db_dir) %} +{% set domains = db_dir + '/' + category + '/domains' %} +{% set urls = db_dir + '/' + category + '/urls' %} {% set expressions = db_dir + '/' + category + '/expressions' %} -dest {{ category }}-default { +dest {{ category }}-{{ rule }}{ +{% if domains | is_file %} domainlist {{ category }}/domains +{% endif %} +{% if urls | is_file %} urllist {{ category }}/urls +{% endif %} {% if expressions | is_file %} expressionlist {{ category }}/expressions {% endif %} @@ -17,8 +23,9 @@ dest {{ category }}-default { {% if url_filtering is vyos_defined and url_filtering.disable is not vyos_defined %} {% if url_filtering.squidguard is vyos_defined %} {% set sg_config = url_filtering.squidguard %} -{% set acl = namespace(value='local-ok-default') %} +{% set acl = namespace(value='') %} {% set acl.value = acl.value + ' !in-addr' if sg_config.allow_ipaddr_url is not defined else acl.value %} +{% set ruleacls = {} %} dbhome {{ squidguard_db_dir }} logdir /var/log/squid @@ -38,24 +45,28 @@ dest local-ok-default { domainlist local-ok-default/domains } {% endif %} + {% if sg_config.local_ok_url is vyos_defined %} {% set acl.value = acl.value + ' local-ok-url-default' %} dest local-ok-url-default { urllist local-ok-url-default/urls } {% endif %} + {% if sg_config.local_block is vyos_defined %} {% set acl.value = acl.value + ' !local-block-default' %} dest local-block-default { domainlist local-block-default/domains } {% endif %} + {% if sg_config.local_block_url is vyos_defined %} {% set acl.value = acl.value + ' !local-block-url-default' %} dest local-block-url-default { urllist local-block-url-default/urls } {% endif %} + {% if sg_config.local_block_keyword is vyos_defined %} {% set acl.value = acl.value + ' !local-block-keyword-default' %} dest local-block-keyword-default { @@ -65,16 +76,100 @@ dest local-block-keyword-default { {% if sg_config.block_category is vyos_defined %} {% for category in sg_config.block_category %} -{{ sg_rule(category, sg_config.log, squidguard_db_dir) }} +{{ sg_rule(category, 'default', sg_config.log, squidguard_db_dir) }} {% set acl.value = acl.value + ' !' + category + '-default' %} {% endfor %} {% endif %} {% if sg_config.allow_category is vyos_defined %} {% for category in sg_config.allow_category %} -{{ sg_rule(category, False, squidguard_db_dir) }} +{{ sg_rule(category, 'default', False, squidguard_db_dir) }} {% set acl.value = acl.value + ' ' + category + '-default' %} {% endfor %} {% endif %} + + +{% if sg_config.rule is vyos_defined %} +{% for rule, rule_config in sg_config.rule.items() %} +{% if rule_config.local_ok is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-' + rule}) %} +{% endif %} +dest local-ok-{{ rule }} { + domainlist local-ok-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_ok_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' local-ok-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'local-ok-url-' + rule}) %} +{% endif %} +dest local-ok-url-{{ rule }} { + urllist local-ok-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-' + rule}) %} +{% endif %} +dest local-block-{{ rule }} { + domainlist local-block-{{ rule }}/domains +} +{% endif %} + +{% if rule_config.local_block_url is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-url-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!ocal-block-url-' + rule}) %} +{% endif %} +dest local-block-url-{{ rule }} { + urllist local-block-url-{{ rule }}/urls +} +{% endif %} + +{% if rule_config.local_block_keyword is vyos_defined %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !local-block-keyword-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!local-block-keyword-' + rule}) %} +{% endif %} +dest local-block-keyword-{{ rule }} { + expressionlist local-block-keyword-{{ rule }}/expressions +} +{% endif %} + +{% if rule_config.block_category is vyos_defined %} +{% for b_category in rule_config.block_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' !' + b_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:'!' + b_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(b_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} + +{% if rule_config.allow_category is vyos_defined %} +{% for a_category in rule_config.allow_category %} +{% if rule in ruleacls %} +{% set _dummy = ruleacls.update({rule: ruleacls[rule] + ' ' + a_category + '-' + rule}) %} +{% else %} +{% set _dummy = ruleacls.update({rule:a_category + '-' + rule}) %} +{% endif %} +{{ sg_rule(a_category, rule, sg_config.log, squidguard_db_dir) }} +{% endfor %} +{% endif %} +{% endfor %} +{% endif %} + + {% if sg_config.source_group is vyos_defined %} {% for sgroup, sg_config in sg_config.source_group.items() %} {% if sg_config.address is vyos_defined %} @@ -83,28 +178,15 @@ src {{ sgroup }} { ip {{ address }} {% endfor %} } - {% endif %} {% endfor %} {% endif %} -{% if sg_config.rule is vyos_defined %} -{% for rule, rule_config in sg_config.rule.items() %} -{% for b_category in rule_config.block_category %} -dest {{ b_category }} { - domainlist {{ b_category }}/domains - urllist {{ b_category }}/urls -} -{% endfor %} -{% endfor %} -{% endif %} acl { {% if sg_config.rule is vyos_defined %} {% for rule, rule_config in sg_config.rule.items() %} {{ rule_config.source_group }} { -{% for b_category in rule_config.block_category %} - pass local-ok-1 !in-addr !{{ b_category }} all -{% endfor %} + pass {{ ruleacls[rule] }} {{ 'none' if rule_config.default_action is vyos_defined('block') else 'any' }} } {% endfor %} {% endif %} @@ -113,7 +195,7 @@ acl { {% if sg_config.enable_safe_search is vyos_defined %} rewrite safesearch {% endif %} - pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'allow' }} + pass {{ acl.value }} {{ 'none' if sg_config.default_action is vyos_defined('block') else 'any' }} redirect 302:http://{{ sg_config.redirect_url }} {% if sg_config.log is vyos_defined %} log blacklist.log diff --git a/src/conf_mode/service_webproxy.py b/src/conf_mode/service_webproxy.py index 32af31bde..41a1deaa3 100755 --- a/src/conf_mode/service_webproxy.py +++ b/src/conf_mode/service_webproxy.py @@ -28,8 +28,10 @@ from vyos.util import dict_search from vyos.util import write_file from vyos.validate import is_addr_assigned from vyos.xml import defaults +from vyos.base import Warning from vyos import ConfigError from vyos import airbag + airbag.enable() squid_config_file = '/etc/squid/squid.conf' @@ -37,24 +39,57 @@ squidguard_config_file = '/etc/squidguard/squidGuard.conf' squidguard_db_dir = '/opt/vyatta/etc/config/url-filtering/squidguard/db' user_group = 'proxy' -def generate_sg_localdb(category, list_type, role, proxy): + +def check_blacklist_categorydb(config_section): + if 'block_category' in config_section: + for category in config_section['block_category']: + check_categorydb(category) + if 'allow_category' in config_section: + for category in config_section['allow_category']: + check_categorydb(category) + + +def check_categorydb(category: str): + """ + Check if category's db exist + :param category: + :type str: + """ + path_to_cat: str = f'{squidguard_db_dir}/{category}' + if not os.path.exists(f'{path_to_cat}/domains.db') \ + and not os.path.exists(f'{path_to_cat}/urls.db') \ + and not os.path.exists(f'{path_to_cat}/expressions.db'): + Warning(f'DB of category {category} does not exist.\n ' + f'Use [update webproxy blacklists] ' + f'or delete undefined category!') + + +def generate_sg_rule_localdb(category, list_type, role, proxy): + if not category or not list_type or not role: + return None + cat_ = category.replace('-', '_') - if isinstance(dict_search(f'url_filtering.squidguard.{cat_}', proxy), - list): + if role == 'default': + path_to_cat = f'{cat_}' + else: + path_to_cat = f'rule.{role}.{cat_}' + if isinstance( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy), + list): # local block databases must be generated "on-the-fly" tmp = { - 'squidguard_db_dir' : squidguard_db_dir, - 'category' : f'{category}-default', - 'list_type' : list_type, - 'rule' : role + 'squidguard_db_dir': squidguard_db_dir, + 'category': f'{category}-{role}', + 'list_type': list_type, + 'rule': role } sg_tmp_file = '/tmp/sg.conf' - db_file = f'{category}-default/{list_type}' - domains = '\n'.join(dict_search(f'url_filtering.squidguard.{cat_}', proxy)) - + db_file = f'{category}-{role}/{list_type}' + domains = '\n'.join( + dict_search(f'url_filtering.squidguard.{path_to_cat}', proxy)) # local file - write_file(f'{squidguard_db_dir}/{category}-default/local', '', + write_file(f'{squidguard_db_dir}/{category}-{role}/local', '', user=user_group, group=user_group) # database input file write_file(f'{squidguard_db_dir}/{db_file}', domains, @@ -64,17 +99,18 @@ def generate_sg_localdb(category, list_type, role, proxy): render(sg_tmp_file, 'squid/sg_acl.conf.j2', tmp, user=user_group, group=user_group) - call(f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') + call( + f'su - {user_group} -c "squidGuard -d -c {sg_tmp_file} -C {db_file}"') if os.path.exists(sg_tmp_file): os.unlink(sg_tmp_file) - else: # if category is not part of our configuration, clean out the # squidguard lists - tmp = f'{squidguard_db_dir}/{category}-default' + tmp = f'{squidguard_db_dir}/{category}-{role}' if os.path.exists(tmp): - rmtree(f'{squidguard_db_dir}/{category}-default') + rmtree(f'{squidguard_db_dir}/{category}-{role}') + def get_config(config=None): if config: @@ -85,7 +121,8 @@ def get_config(config=None): if not conf.exists(base): return None - proxy = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + proxy = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True) # We have gathered the dict representation of the CLI, but there are default # options which we need to update into the dictionary retrived. default_values = defaults(base) @@ -110,10 +147,11 @@ def get_config(config=None): default_values = defaults(base + ['cache-peer']) for peer in proxy['cache_peer']: proxy['cache_peer'][peer] = dict_merge(default_values, - proxy['cache_peer'][peer]) + proxy['cache_peer'][peer]) return proxy + def verify(proxy): if not proxy: return None @@ -170,17 +208,30 @@ def generate(proxy): render(squidguard_config_file, 'squid/squidGuard.conf.j2', proxy) cat_dict = { - 'local-block' : 'domains', - 'local-block-keyword' : 'expressions', - 'local-block-url' : 'urls', - 'local-ok' : 'domains', - 'local-ok-url' : 'urls' + 'local-block': 'domains', + 'local-block-keyword': 'expressions', + 'local-block-url': 'urls', + 'local-ok': 'domains', + 'local-ok-url': 'urls' } - for category, list_type in cat_dict.items(): - generate_sg_localdb(category, list_type, 'default', proxy) + if dict_search(f'url_filtering.squidguard', proxy) is not None: + squidgard_config_section = proxy['url_filtering']['squidguard'] + + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, 'default', proxy) + check_blacklist_categorydb(squidgard_config_section) + + if 'rule' in squidgard_config_section: + for rule in squidgard_config_section['rule']: + rule_config_section = squidgard_config_section['rule'][ + rule] + for category, list_type in cat_dict.items(): + generate_sg_rule_localdb(category, list_type, rule, proxy) + check_blacklist_categorydb(rule_config_section) return None + def apply(proxy): if not proxy: # proxy is removed in the commit @@ -198,6 +249,7 @@ def apply(proxy): call('systemctl restart squid.service') return None + if __name__ == '__main__': try: c = get_config() diff --git a/src/op_mode/webproxy_update_blacklist.sh b/src/op_mode/webproxy_update_blacklist.sh index d5f301b75..4fb9a54c6 100755 --- a/src/op_mode/webproxy_update_blacklist.sh +++ b/src/op_mode/webproxy_update_blacklist.sh @@ -18,6 +18,23 @@ blacklist_url='ftp://ftp.univ-tlse1.fr/pub/reseau/cache/squidguard_contrib/black data_dir="/opt/vyatta/etc/config/url-filtering" archive="${data_dir}/squidguard/archive" db_dir="${data_dir}/squidguard/db" +conf_file="/etc/squidguard/squidGuard.conf" +tmp_conf_file="/tmp/sg_update_db.conf" + +#$1-category +#$2-type +#$3-list +create_sg_db () +{ + FILE=$db_dir/$1/$2 + if test -f "$FILE"; then + rm -f ${tmp_conf_file} + printf "dbhome $db_dir\ndest $1 {\n $3 $1/$2\n}\nacl {\n default {\n pass any\n }\n}" >> ${tmp_conf_file} + /usr/bin/squidGuard -b -c ${tmp_conf_file} -C $FILE + rm -f ${tmp_conf_file} + fi + +} while [ $# -gt 0 ] do @@ -88,6 +105,16 @@ if [[ -n $update ]] && [[ $update -eq "yes" ]]; then # fix permissions chown -R proxy:proxy ${db_dir} + + #create db + category_list=(`find $db_dir -type d -exec basename {} \; `) + for category in ${category_list[@]} + do + create_sg_db $category "domains" "domainlist" + create_sg_db $category "urls" "urllist" + create_sg_db $category "expressions" "expressionlist" + done + chown -R proxy:proxy ${db_dir} chmod 755 ${db_dir} logger --priority WARNING "webproxy blacklist entries updated (${count_before}/${count_after})" -- cgit v1.2.3