From 5563bcd9eaef5d5d8a020b92d43a86028e3c26c1 Mon Sep 17 00:00:00 2001 From: Roman Khramshin Date: Fri, 31 May 2024 16:04:42 +0700 Subject: T5307: QoS - traffic-class-map services (#3492) added new syntax to work with class match filters in QoS policy --- .../include/qos/class-match-group.xml.i | 15 ++++ .../include/qos/class-match-ipv4.xml.i | 31 +++++++ .../include/qos/class-match-ipv6.xml.i | 31 +++++++ .../include/qos/class-match-mark.xml.i | 14 ++++ .../include/qos/class-match-vif.xml.i | 15 ++++ .../include/qos/class-match.xml.i | 89 ++------------------ interface-definitions/qos.xml.in | 39 +++++++++ smoketest/scripts/cli/test_qos.py | 95 ++++++++++++++++++++++ src/completion/qos/list_traffic_match_group.py | 35 ++++++++ src/conf_mode/qos.py | 77 ++++++++++++++++-- 10 files changed, 352 insertions(+), 89 deletions(-) create mode 100644 interface-definitions/include/qos/class-match-group.xml.i create mode 100644 interface-definitions/include/qos/class-match-ipv4.xml.i create mode 100644 interface-definitions/include/qos/class-match-ipv6.xml.i create mode 100644 interface-definitions/include/qos/class-match-mark.xml.i create mode 100644 interface-definitions/include/qos/class-match-vif.xml.i create mode 100644 src/completion/qos/list_traffic_match_group.py diff --git a/interface-definitions/include/qos/class-match-group.xml.i b/interface-definitions/include/qos/class-match-group.xml.i new file mode 100644 index 000000000..40e3b7259 --- /dev/null +++ b/interface-definitions/include/qos/class-match-group.xml.i @@ -0,0 +1,15 @@ + + + + Filter group for QoS policy + + txt + Match group name + + + + + + + + diff --git a/interface-definitions/include/qos/class-match-ipv4.xml.i b/interface-definitions/include/qos/class-match-ipv4.xml.i new file mode 100644 index 000000000..dc44d32d5 --- /dev/null +++ b/interface-definitions/include/qos/class-match-ipv4.xml.i @@ -0,0 +1,31 @@ + + + + Match IP protocol header + + + + + Match on destination port or address + + + #include + #include + + + #include + #include + #include + + + Match on source port or address + + + #include + #include + + + #include + + + diff --git a/interface-definitions/include/qos/class-match-ipv6.xml.i b/interface-definitions/include/qos/class-match-ipv6.xml.i new file mode 100644 index 000000000..ed7aceff9 --- /dev/null +++ b/interface-definitions/include/qos/class-match-ipv6.xml.i @@ -0,0 +1,31 @@ + + + + Match IPv6 protocol header + + + + + Match on destination port or address + + + #include + #include + + + #include + #include + #include + + + Match on source port or address + + + #include + #include + + + #include + + + diff --git a/interface-definitions/include/qos/class-match-mark.xml.i b/interface-definitions/include/qos/class-match-mark.xml.i new file mode 100644 index 000000000..a7481c6aa --- /dev/null +++ b/interface-definitions/include/qos/class-match-mark.xml.i @@ -0,0 +1,14 @@ + + + + Match on mark applied by firewall + + u32 + FW mark to match + + + + + + + diff --git a/interface-definitions/include/qos/class-match-vif.xml.i b/interface-definitions/include/qos/class-match-vif.xml.i new file mode 100644 index 000000000..ec58db606 --- /dev/null +++ b/interface-definitions/include/qos/class-match-vif.xml.i @@ -0,0 +1,15 @@ + + + + Virtual Local Area Network (VLAN) ID for this match + + u32:0-4095 + Virtual Local Area Network (VLAN) tag + + + + + VLAN ID must be between 0 and 4095 + + + diff --git a/interface-definitions/include/qos/class-match.xml.i b/interface-definitions/include/qos/class-match.xml.i index 4ba12f8f7..77d1933a3 100644 --- a/interface-definitions/include/qos/class-match.xml.i +++ b/interface-definitions/include/qos/class-match.xml.i @@ -5,7 +5,7 @@ [^-].* - Match queue name cannot start with hyphen (-) + Match queue name cannot start with hyphen #include @@ -89,89 +89,10 @@ #include - - - Match IP protocol header - - - - - Match on destination port or address - - - #include - #include - - - #include - #include - #include - - - Match on source port or address - - - #include - #include - - - #include - - - - - Match IPv6 protocol header - - - - - Match on destination port or address - - - #include - #include - - - #include - #include - #include - - - Match on source port or address - - - #include - #include - - - #include - - - - - Match on mark applied by firewall - - u32 - FW mark to match - - - - - - - - - Virtual Local Area Network (VLAN) ID for this match - - u32:0-4095 - Virtual Local Area Network (VLAN) tag - - - - - VLAN ID must be between 0 and 4095 - - + #include + #include + #include + #include diff --git a/interface-definitions/qos.xml.in b/interface-definitions/qos.xml.in index 8f9ae3fa6..927594c11 100644 --- a/interface-definitions/qos.xml.in +++ b/interface-definitions/qos.xml.in @@ -281,6 +281,7 @@ #include #include #include + #include #include 20 @@ -415,6 +416,7 @@ #include #include #include + #include #include #include @@ -542,6 +544,8 @@ #include #include #include + #include + Packet scheduling quantum @@ -645,6 +649,7 @@ #include #include #include + #include #include #include #include @@ -767,6 +772,7 @@ #include + #include Realtime class settings @@ -830,6 +836,39 @@ + + + Filter group for QoS policy + + txt + Match group name + + + [^-].* + + Match group name cannot start with hyphen + + + #include + + + Class matching rule name + + [^-].* + + Match queue name cannot start with hyphen + + + #include + #include + #include + #include + #include + + + #include + + diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index 5977b2f41..b98c0e9b7 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -759,6 +759,101 @@ class TestQoS(VyOSUnitTestSHIM.TestCase): self.assertIn('filter parent ffff: protocol all pref 255 basic chain 0', tc_filters) self.assertIn('action order 1: police 0x2 rate 1Gbit burst 125000000b mtu 2Kb action drop overhead 0b', tc_filters) + def test_15_traffic_match_group(self): + interface = self._interfaces[0] + self.cli_set(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + base_policy_path = ['qos', 'policy', 'shaper', 'VyOS-HTB'] + + #old syntax + self.cli_set(base_policy_path + ['bandwidth', '100mbit']) + self.cli_set(base_policy_path + ['class', '10', 'bandwidth', '40%']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF11', 'ip', 'dscp', 'AF11']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF41', 'ip', 'dscp', 'AF41']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF43', 'ip', 'dscp', 'AF43']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'CS4', 'ip', 'dscp', 'CS4']) + self.cli_set(base_policy_path + ['class', '10', 'priority', '1']) + self.cli_set(base_policy_path + ['class', '10', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['class', '20', 'bandwidth', '30%']) + self.cli_set(base_policy_path + ['class', '20', 'match', 'EF', 'ip', 'dscp', 'EF']) + self.cli_set(base_policy_path + ['class', '20', 'match', 'CS5', 'ip', 'dscp', 'CS5']) + self.cli_set(base_policy_path + ['class', '20', 'priority', '2']) + self.cli_set(base_policy_path + ['class', '20', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '20%']) + self.cli_set(base_policy_path + ['default', 'queue-type', 'fair-queue']) + self.cli_commit() + + tc_filters_old = cmd(f'tc -details filter show dev {interface}') + self.assertIn('match 00280000/00ff0000', tc_filters_old) + self.assertIn('match 00880000/00ff0000', tc_filters_old) + self.assertIn('match 00980000/00ff0000', tc_filters_old) + self.assertIn('match 00800000/00ff0000', tc_filters_old) + self.assertIn('match 00a00000/00ff0000', tc_filters_old) + self.assertIn('match 00b80000/00ff0000', tc_filters_old) + # delete config by old syntax + self.cli_delete(base_policy_path) + self.cli_delete(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + self.cli_commit() + self.assertEqual('', cmd(f'tc -s filter show dev {interface}')) + + self.cli_set(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + # prepare traffic match group + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'description', 'voice shaper']) + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'match', 'EF', 'ip', 'dscp', 'EF']) + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'match', 'CS5', 'ip', 'dscp', 'CS5']) + + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'description', 'real time common filters']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'match', 'AF43', 'ip', 'dscp', 'AF43']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'match', 'CS4', 'ip', 'dscp', 'CS4']) + + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'description', 'real time shaper']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'match', 'AF41', 'ip', 'dscp', 'AF41']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'match-group', 'REAL_TIME_COMMON']) + + # new syntax + self.cli_set(base_policy_path + ['bandwidth', '100mbit']) + self.cli_set(base_policy_path + ['class', '10', 'bandwidth', '40%']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF11', 'ip', 'dscp', 'AF11']) + self.cli_set(base_policy_path + ['class', '10', 'match-group', 'REAL_TIME']) + self.cli_set(base_policy_path + ['class', '10', 'priority', '1']) + self.cli_set(base_policy_path + ['class', '10', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['class', '20', 'bandwidth', '30%']) + self.cli_set(base_policy_path + ['class', '20', 'match-group', 'VOICE']) + self.cli_set(base_policy_path + ['class', '20', 'priority', '2']) + self.cli_set(base_policy_path + ['class', '20', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '20%']) + self.cli_set(base_policy_path + ['default', 'queue-type', 'fair-queue']) + self.cli_commit() + + self.assertEqual(tc_filters_old, cmd(f'tc -details filter show dev {interface}')) + + def test_16_wrong_traffic_match_group(self): + interface = self._interfaces[0] + self.cli_set(['qos', 'interface', interface]) + + # Can not use both IPv6 and IPv4 in one match + self.cli_set(['qos', 'traffic-match-group', '1', 'match', 'one', 'ip', 'dscp', 'EF']) + self.cli_set(['qos', 'traffic-match-group', '1', 'match', 'one', 'ipv6', 'dscp', 'EF']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + + # check contain itself, should commit success + self.cli_delete(['qos', 'traffic-match-group', '1', 'match', 'one', 'ipv6']) + self.cli_set(['qos', 'traffic-match-group', '1', 'match-group', '1']) + self.cli_commit() + + # check cycle dependency, should commit success + self.cli_set(['qos', 'traffic-match-group', '1', 'match-group', '3']) + self.cli_set(['qos', 'traffic-match-group', '2', 'match', 'one', 'ip', 'dscp', 'CS4']) + self.cli_set(['qos', 'traffic-match-group', '2', 'match-group', '1']) + + self.cli_set(['qos', 'traffic-match-group', '3', 'match', 'one', 'ipv6', 'dscp', 'CS4']) + self.cli_set(['qos', 'traffic-match-group', '3', 'match-group', '2']) + self.cli_commit() + + # inherit from non exist group, should commit success with warning + self.cli_set(['qos', 'traffic-match-group', '3', 'match-group', 'unexpected']) + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/completion/qos/list_traffic_match_group.py b/src/completion/qos/list_traffic_match_group.py new file mode 100644 index 000000000..015d7ada9 --- /dev/null +++ b/src/completion/qos/list_traffic_match_group.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from vyos.config import Config + + +def get_qos_traffic_match_group(): + config = Config() + base = ['qos', 'traffic-match-group'] + conf = config.get_config_dict(base, key_mangling=('-', '_')) + groups = [] + + for group in conf.get('traffic_match_group', []): + groups.append(group) + + return groups + + +if __name__ == "__main__": + groups = get_qos_traffic_match_group() + print(" ".join(groups)) + diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 8a590cbc6..45248fb4a 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -17,6 +17,7 @@ from sys import exit from netifaces import interfaces +from vyos.base import Warning from vyos.config import Config from vyos.configdep import set_dependents from vyos.configdep import call_dependents @@ -89,6 +90,36 @@ def _clean_conf_dict(conf): return conf +def _get_group_filters(config: dict, group_name: str, visited=None) -> dict: + filters = dict() + if not visited: + visited = [group_name, ] + else: + if group_name in visited: + return filters + visited.append(group_name) + + for filter, filter_config in config.get(group_name, {}).items(): + if filter == 'match': + for match, match_config in filter_config.items(): + filters[f'{group_name}-{match}'] = match_config + elif filter == 'match_group': + for group in filter_config: + filters.update(_get_group_filters(config, group, visited)) + + return filters + + +def _get_group_match(config:dict, group_name:str) -> dict: + match = dict() + for key, val in _get_group_filters(config, group_name).items(): + # delete duplicate matches + if val not in match.values(): + match[key] = val + + return match + + def get_config(config=None): if config: conf = config @@ -135,11 +166,27 @@ def get_config(config=None): qos = conf.merge_defaults(qos, recursive=True) + if 'traffic_match_group' in qos: + for group, group_config in qos['traffic_match_group'].items(): + if 'match_group' in group_config: + qos['traffic_match_group'][group]['match'] = _get_group_match(qos['traffic_match_group'], group) + for policy in qos.get('policy', []): for p_name, p_config in qos['policy'][policy].items(): # cleanup empty match config if 'class' in p_config: for cls, cls_config in p_config['class'].items(): + if 'match_group' in cls_config: + # merge group match to match + for group in cls_config['match_group']: + for match, match_conf in qos['traffic_match_group'].get(group, {'match': {}})['match'].items(): + if 'match' not in cls_config: + cls_config['match'] = dict() + if match in cls_config['match']: + cls_config['match'][f'{group}-{match}'] = match_conf + else: + cls_config['match'][match] = match_conf + if 'match' in cls_config: cls_config['match'] = _clean_conf_dict(cls_config['match']) if cls_config['match'] == {}: @@ -147,6 +194,22 @@ def get_config(config=None): return qos + +def _verify_match(cls_config: dict) -> None: + if 'match' in cls_config: + for match, match_config in cls_config['match'].items(): + if {'ip', 'ipv6'} <= set(match_config): + raise ConfigError( + f'Can not use both IPv6 and IPv4 in one match ({match})!') + + +def _verify_match_group_exist(cls_config, qos): + if 'match_group' in cls_config: + for group in cls_config['match_group']: + if 'traffic_match_group' not in qos or group not in qos['traffic_match_group']: + Warning(f'Match group "{group}" does not exist!') + + def verify(qos): if not qos or 'interface' not in qos: return None @@ -174,11 +237,8 @@ def verify(qos): # bandwidth is not mandatory for priority-queue - that is why this is on the exception list if 'bandwidth' not in cls_config and policy_type not in ['priority_queue', 'round_robin', 'shaper_hfsc']: raise ConfigError(f'Bandwidth must be defined for policy "{policy}" class "{cls}"!') - if 'match' in cls_config: - for match, match_config in cls_config['match'].items(): - if {'ip', 'ipv6'} <= set(match_config): - raise ConfigError(f'Can not use both IPv6 and IPv4 in one match ({match})!') - + _verify_match(cls_config) + _verify_match_group_exist(cls_config, qos) if policy_type in ['random_detect']: if 'precedence' in policy_config: for precedence, precedence_config in policy_config['precedence'].items(): @@ -216,8 +276,14 @@ def verify(qos): if direction not in tmp: raise ConfigError(f'Selected QoS policy on interface "{interface}" only supports "{tmp}"!') + if 'traffic_match_group' in qos: + for group, group_config in qos['traffic_match_group'].items(): + _verify_match(group_config) + _verify_match_group_exist(group_config, qos) + return None + def generate(qos): if not qos or 'interface' not in qos: return None @@ -254,6 +320,7 @@ def apply(qos): return None + if __name__ == '__main__': try: c = get_config() -- cgit v1.2.3