diff options
| -rw-r--r-- | .github/labeler.yml | 12 | ||||
| -rw-r--r-- | .github/workflows/add-pr-labels.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/auto-author-assign.yml | 2 | ||||
| -rw-r--r-- | .github/workflows/chceck-pr-message.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/check-pr-conflicts.yml | 1 | ||||
| -rw-r--r-- | .github/workflows/check-stale.yml | 4 | ||||
| -rw-r--r-- | .github/workflows/check-unused-imports.yml | 8 | ||||
| -rw-r--r-- | .github/workflows/label-backport.yml | 6 | ||||
| -rw-r--r-- | .github/workflows/linit-j2.yml | 4 | ||||
| -rw-r--r-- | CODEOWNERS | 1 | ||||
| -rw-r--r-- | interface-definitions/nat.xml.in | 1 | ||||
| -rw-r--r-- | interface-definitions/nat_cgnat.xml.in | 1 | ||||
| -rw-r--r-- | op-mode-definitions/force-commit-archive.xml.in | 2 | ||||
| -rw-r--r-- | op-mode-definitions/nat.xml.in | 20 | ||||
| -rw-r--r-- | op-mode-definitions/show-log.xml.in | 50 | ||||
| -rw-r--r-- | python/vyos/config_mgmt.py | 2 | ||||
| -rw-r--r-- | python/vyos/nat.py | 6 | ||||
| -rw-r--r-- | python/vyos/qos/base.py | 11 | ||||
| -rwxr-xr-x | smoketest/scripts/cli/test_qos.py | 21 | ||||
| -rwxr-xr-x | src/conf_mode/nat_cgnat.py | 110 | ||||
| -rwxr-xr-x | src/op_mode/cgnat.py | 43 |
21 files changed, 233 insertions, 80 deletions
diff --git a/.github/labeler.yml b/.github/labeler.yml deleted file mode 100644 index e0b9ee430..000000000 --- a/.github/labeler.yml +++ /dev/null @@ -1,12 +0,0 @@ -equuleus: - - any: - - base-branch: 'equuleus' -current: - - any: - - base-branch: 'current' -crux: - - any: - - base-branch: 'crux' -sagitta: - - any: - - base-branch: 'sagitta' diff --git a/.github/workflows/add-pr-labels.yml b/.github/workflows/add-pr-labels.yml index 78d619f4a..1723cceb0 100644 --- a/.github/workflows/add-pr-labels.yml +++ b/.github/workflows/add-pr-labels.yml @@ -9,6 +9,10 @@ on: - equuleus - sagitta +permissions: + pull-requests: write + contents: read + jobs: add-pr-label: uses: vyos/.github/.github/workflows/add-pr-labels.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/auto-author-assign.yml b/.github/workflows/auto-author-assign.yml index 1f69f4807..c3696ea47 100644 --- a/.github/workflows/auto-author-assign.yml +++ b/.github/workflows/auto-author-assign.yml @@ -3,8 +3,10 @@ on: pull_request_target: types: [opened, reopened, ready_for_review, locked] + permissions: pull-requests: write + contents: read jobs: assign-author: diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/chceck-pr-message.yml index 95c5b69ce..e7e456961 100644 --- a/.github/workflows/chceck-pr-message.yml +++ b/.github/workflows/chceck-pr-message.yml @@ -8,6 +8,10 @@ on: - crux - equuleus +permissions: + pull-requests: write + contents: read + jobs: check-pr-title: uses: vyos/.github/.github/workflows/check-pr-message.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/check-pr-conflicts.yml b/.github/workflows/check-pr-conflicts.yml index 62a37a7fa..0c659e6ed 100644 --- a/.github/workflows/check-pr-conflicts.yml +++ b/.github/workflows/check-pr-conflicts.yml @@ -6,6 +6,7 @@ on: permissions: pull-requests: write + contents: read jobs: check-pr-conflict-call: diff --git a/.github/workflows/check-stale.yml b/.github/workflows/check-stale.yml index 0b88acdb7..b5ec533f1 100644 --- a/.github/workflows/check-stale.yml +++ b/.github/workflows/check-stale.yml @@ -3,6 +3,10 @@ on: schedule: - cron: "0 0 * * *" +permissions: + pull-requests: write + contents: read + jobs: stale: uses: vyos/.github/.github/workflows/check-stale.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml index 468543d6e..aada264f7 100644 --- a/.github/workflows/check-unused-imports.yml +++ b/.github/workflows/check-unused-imports.yml @@ -1,11 +1,15 @@ name: Check for unused imports using Pylint on: - pull_request_target: + pull_request: branches: - current - sagitta + workflow_dispatch: + +permissions: + contents: read jobs: - Check-Unused-Imports: + check-unused-imports: uses: vyos/.github/.github/workflows/check-unused-imports.yml@feature/T6349-reusable-workflows secrets: inherit diff --git a/.github/workflows/label-backport.yml b/.github/workflows/label-backport.yml index 581363eb1..9192b8184 100644 --- a/.github/workflows/label-backport.yml +++ b/.github/workflows/label-backport.yml @@ -2,7 +2,11 @@ name: Mergifyio backport on: [issue_comment] +permissions: + pull-requests: write + contents: read + jobs: - mergifyio_backport: + mergifyio-backport: uses: vyos/.github/.github/workflows/label-backport.yml@feature/T6349-reusable-workflows secrets: inherit diff --git a/.github/workflows/linit-j2.yml b/.github/workflows/linit-j2.yml index 093fe7ffe..364a65a14 100644 --- a/.github/workflows/linit-j2.yml +++ b/.github/workflows/linit-j2.yml @@ -8,6 +8,10 @@ on: - crux - equuleus +permissions: + pull-requests: write + contents: read + jobs: j2lint: uses: vyos/.github/.github/workflows/lint-j2.yml@feature/T6349-reusable-workflows diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 000000000..191394298 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @vyos/reviewers
\ No newline at end of file diff --git a/interface-definitions/nat.xml.in b/interface-definitions/nat.xml.in index 0a639bd80..73a748137 100644 --- a/interface-definitions/nat.xml.in +++ b/interface-definitions/nat.xml.in @@ -141,6 +141,7 @@ </children> </node> #include <include/inbound-interface.xml.i> + #include <include/firewall/log.xml.i> <node name="translation"> <properties> <help>Translation address or prefix</help> diff --git a/interface-definitions/nat_cgnat.xml.in b/interface-definitions/nat_cgnat.xml.in index caa26b4d9..fce5e655d 100644 --- a/interface-definitions/nat_cgnat.xml.in +++ b/interface-definitions/nat_cgnat.xml.in @@ -123,6 +123,7 @@ <validator name="ipv4-host"/> <validator name="ipv4-range"/> </constraint> + <multi/> </properties> </leafNode> </children> diff --git a/op-mode-definitions/force-commit-archive.xml.in b/op-mode-definitions/force-commit-archive.xml.in index 162323c20..46836f967 100644 --- a/op-mode-definitions/force-commit-archive.xml.in +++ b/op-mode-definitions/force-commit-archive.xml.in @@ -6,7 +6,7 @@ <properties> <help>Manually archive configuration</help> </properties> - <command>/usr/bin/config-mgmt</command> + <command>/etc/commit/post-hooks.d/02vyos-commit-archive; printf "\n"</command> </leafNode> </children> </node> diff --git a/op-mode-definitions/nat.xml.in b/op-mode-definitions/nat.xml.in index 6398c0e07..13e7fd81d 100644 --- a/op-mode-definitions/nat.xml.in +++ b/op-mode-definitions/nat.xml.in @@ -16,6 +16,26 @@ <properties> <help>Show allocated CGNAT parameters</help> </properties> + <children> + <tagNode name="external-address"> + <properties> + <help>Show CGNAT allocations for an external IP address</help> + <completionHelp> + <list><x.x.x.x></list> + </completionHelp> + </properties> + <command>sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation --external-address "$6"</command> + </tagNode> + <tagNode name="internal-address"> + <properties> + <help>Show CGNAT allocations for an internal IP address</help> + <completionHelp> + <list><x.x.x.x></list> + </completionHelp> + </properties> + <command>sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation --internal-address "$6"</command> + </tagNode> + </children> <command>sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation</command> </node> </children> diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index e13270364..c3aa324ba 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -464,12 +464,56 @@ </properties> <command>journalctl --no-hostname --boot --unit lldpd.service</command> </leafNode> - <leafNode name="nat"> + <node name="nat"> <properties> <help>Show log for Network Address Translation (NAT)</help> </properties> - <command>egrep -i "kernel:.*\[NAT-[A-Z]{3,}-[0-9]+(-MASQ)?\]" $(find /var/log -maxdepth 1 -type f -name messages\* | sort -t. -k2nr)</command> - </leafNode> + <children> + <node name="destination"> + <properties> + <help>Show NAT destination log</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[DST-NAT-[0-9]+\]"</command> + <children> + <tagNode name="rule"> + <properties> + <help>Show NAT destination log for specified rule</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[DST-NAT-$6\]"</command> + </tagNode> + </children> + </node> + <node name="source"> + <properties> + <help>Show NAT source log</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[SRC-NAT-[0-9]+(-MASQ)?\]"""</command> + <children> + <tagNode name="rule"> + <properties> + <help>Show NAT source log for specified rule</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[SRC-NAT-$6(-MASQ)?\]"</command> + </tagNode> + </children> + </node> + <node name="static"> + <properties> + <help>Show NAT static log</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[STATIC-(SRC|DST)-NAT-[0-9]+\]"</command> + <children> + <tagNode name="rule"> + <properties> + <help>Show NAT static log for specified rule</help> + </properties> + <command>journalctl --no-hostname --boot -k | egrep "\[STATIC-(SRC|DST)-NAT-$6\]"</command> + </tagNode> + </children> + </node> + </children> + <command>journalctl --no-hostname --boot -k | egrep "\[(STATIC-)?(DST|SRC)-NAT-[0-9]+(-MASQ)?\]"</command> + </node> <leafNode name="ndp-proxy"> <properties> <help>Show log for Neighbor Discovery Protocol (NDP) Proxy</help> diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index fc51d781c..70b6ea203 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -283,6 +283,8 @@ Proceed ?''' rollback_ct = self._get_config_tree_revision(rev) try: load(rollback_ct, switch='explicit') + print('Rollback diff has been applied.') + print('Use "compare" to review the changes or "commit" to apply them.') except LoadConfigError as e: raise ConfigMgmtError(e) from e diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 2ada29add..e54548788 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -300,12 +300,12 @@ def parse_nat_static_rule(rule_conf, rule_id, nat_type): output.append('counter') - if translation_str: - output.append(translation_str) - if 'log' in rule_conf: output.append(f'log prefix "[{log_prefix}{log_suffix}]"') + if translation_str: + output.append(translation_str) + output.append(f'comment "{log_prefix}"') return " ".join(output) diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index 87927ba9d..98e486e42 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -247,9 +247,15 @@ class QoSBase: filter_cmd_base += ' protocol all' if 'match' in cls_config: - is_filtered = False + has_filter = False for index, (match, match_config) in enumerate(cls_config['match'].items(), start=1): filter_cmd = filter_cmd_base + if not has_filter: + for key in ['mark', 'vif', 'ip', 'ipv6']: + if key in match_config: + has_filter = True + break + if self.qostype == 'shaper' and 'prio ' not in filter_cmd: filter_cmd += f' prio {index}' if 'mark' in match_config: @@ -332,13 +338,12 @@ class QoSBase: cls = int(cls) filter_cmd += f' flowid {self._parent:x}:{cls:x}' self._cmd(filter_cmd) - is_filtered = True vlan_expression = "match.*.vif" match_vlan = jmespath.search(vlan_expression, cls_config) if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in cls_config) \ - and is_filtered: + and has_filter: # For "vif" "basic match" is used instead of "action police" T5961 if not match_vlan: filter_cmd += f' action police' diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index bcf5139c7..5977b2f41 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -738,6 +738,27 @@ class TestQoS(VyOSUnitTestSHIM.TestCase): self.cli_commit() self.assertEqual('', cmd(f'tc filter show dev {interface}')) + def test_14_policy_limiter_marked_traffic(self): + policy_name = 'smoke_test' + base_policy_path = ['qos', 'policy', 'limiter', policy_name] + + self.cli_set(['qos', 'interface', self._interfaces[0], 'ingress', policy_name]) + self.cli_set(base_policy_path + ['class', '100', 'bandwidth', '20gbit']) + self.cli_set(base_policy_path + ['class', '100', 'burst', '3760k']) + self.cli_set(base_policy_path + ['class', '100', 'match', 'INTERNAL', 'mark', '100']) + self.cli_set(base_policy_path + ['class', '100', 'priority', '20']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '1gbit']) + self.cli_set(base_policy_path + ['default', 'burst', '125000000b']) + self.cli_commit() + + tc_filters = cmd(f'tc filter show dev {self._interfaces[0]} ingress') + # class 100 + self.assertIn('filter parent ffff: protocol all pref 20 fw chain 0', tc_filters) + self.assertIn('action order 1: police 0x1 rate 20Gbit burst 3847500b mtu 2Kb action drop overhead 0b', tc_filters) + # default + self.assertIn('filter parent ffff: protocol all pref 255 basic chain 0', tc_filters) + self.assertIn('action order 1: police 0x2 rate 1Gbit burst 125000000b mtu 2Kb action drop overhead 0b', tc_filters) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index 9a20a3c54..5ad65de80 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -189,11 +189,6 @@ def verify(config): if 'rule' not in config: raise ConfigError(f'Rule must be defined!') - # As PoC allow only one rule for CGNAT translations - # one internal pool and one external pool - if len(config['rule']) > 1: - raise ConfigError(f'Only one rule is allowed for translations!') - for pool in ('external', 'internal'): if pool not in config['pool']: raise ConfigError(f'{pool} pool must be defined!') @@ -208,6 +203,8 @@ def verify(config): internal_pools_query = "keys(pool.internal)" internal_pools: list = jmespath.search(internal_pools_query, config) + used_external_pools = {} + used_internal_pools = {} for rule, rule_config in config['rule'].items(): if 'source' not in rule_config: raise ConfigError(f'Rule "{rule}" source pool must be defined!') @@ -217,57 +214,82 @@ def verify(config): if 'translation' not in rule_config: raise ConfigError(f'Rule "{rule}" translation pool must be defined!') + # Check if pool exists internal_pool = rule_config['source']['pool'] if internal_pool not in internal_pools: raise ConfigError(f'Internal pool "{internal_pool}" does not exist!') - external_pool = rule_config['translation']['pool'] if external_pool not in external_pools: raise ConfigError(f'External pool "{external_pool}" does not exist!') + # Check pool duplication in different rules + if external_pool in used_external_pools: + raise ConfigError( + f'External pool "{external_pool}" is already used in rule ' + f'{used_external_pools[external_pool]} and cannot be used in ' + f'rule {rule}!' + ) + + if internal_pool in used_internal_pools: + raise ConfigError( + f'Internal pool "{internal_pool}" is already used in rule ' + f'{used_internal_pools[internal_pool]} and cannot be used in ' + f'rule {rule}!' + ) + + used_external_pools[external_pool] = rule + used_internal_pools[internal_pool] = rule + def generate(config): if not config: return None - # first external pool as we allow only one as PoC - ext_pool_name = jmespath.search("rule.*.translation | [0]", config).get('pool') - int_pool_name = jmespath.search("rule.*.source | [0]", config).get('pool') - ext_query = f'pool.external."{ext_pool_name}".range | keys(@)' - int_query = f'pool.internal."{int_pool_name}".range' - external_ranges = jmespath.search(ext_query, config) - internal_ranges = [jmespath.search(int_query, config)] - - external_list_count = [] - external_list_hosts = [] - internal_list_count = [] - internal_list_hosts = [] - for ext_range in external_ranges: - # External hosts count - e_count = IPOperations(ext_range).get_ips_count() - external_list_count.append(e_count) - # External hosts list - e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() - external_list_hosts.extend(e_hosts) - for int_range in internal_ranges: - # Internal hosts count - i_count = IPOperations(int_range).get_ips_count() - internal_list_count.append(i_count) - # Internal hosts list - i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() - internal_list_hosts.extend(i_hosts) - - external_host_count = sum(external_list_count) - internal_host_count = sum(internal_list_count) - ports_per_user = int( - jmespath.search(f'pool.external."{ext_pool_name}".per_user_limit.port', config) - ) - external_port_range: str = jmespath.search( - f'pool.external."{ext_pool_name}".external_port_range', config - ) - proto_maps, other_maps = generate_port_rules( - external_list_hosts, internal_list_hosts, ports_per_user, external_port_range - ) + proto_maps = [] + other_maps = [] + + for rule, rule_config in config['rule'].items(): + ext_pool_name: str = rule_config['translation']['pool'] + int_pool_name: str = rule_config['source']['pool'] + + external_ranges: list = [range for range in config['pool']['external'][ext_pool_name]['range']] + internal_ranges: list = [range for range in config['pool']['internal'][int_pool_name]['range']] + external_list_hosts_count = [] + external_list_hosts = [] + internal_list_hosts_count = [] + internal_list_hosts = [] + + for ext_range in external_ranges: + # External hosts count + e_count = IPOperations(ext_range).get_ips_count() + external_list_hosts_count.append(e_count) + # External hosts list + e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() + external_list_hosts.extend(e_hosts) + + for int_range in internal_ranges: + # Internal hosts count + i_count = IPOperations(int_range).get_ips_count() + internal_list_hosts_count.append(i_count) + # Internal hosts list + i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() + internal_list_hosts.extend(i_hosts) + + external_host_count = sum(external_list_hosts_count) + internal_host_count = sum(internal_list_hosts_count) + ports_per_user = int( + jmespath.search(f'pool.external."{ext_pool_name}".per_user_limit.port', config) + ) + external_port_range: str = jmespath.search( + f'pool.external."{ext_pool_name}".external_port_range', config + ) + + rule_proto_maps, rule_other_maps = generate_port_rules( + external_list_hosts, internal_list_hosts, ports_per_user, external_port_range + ) + + proto_maps.extend(rule_proto_maps) + other_maps.extend(rule_other_maps) config['proto_map_elements'] = ', '.join(proto_maps) config['other_map_elements'] = ', '.join(other_maps) diff --git a/src/op_mode/cgnat.py b/src/op_mode/cgnat.py index a98269a15..9ad8f92f9 100755 --- a/src/op_mode/cgnat.py +++ b/src/op_mode/cgnat.py @@ -28,15 +28,11 @@ from vyos.utils.process import cmd CGNAT_TABLE = 'cgnat' -def _get_raw_data(): - """ Get CGNAT dictionary - """ +def _get_raw_data(external_address: str = '', internal_address: str = '') -> list[dict]: + """Get CGNAT dictionary and filter by external or internal address if provided.""" cmd_output = cmd(f'nft --json list table ip {CGNAT_TABLE}') data = json.loads(cmd_output) - return data - -def _get_formatted_output(data): elements = data['nftables'][2]['map']['elem'] allocations = [] for elem in elements: @@ -45,23 +41,48 @@ def _get_formatted_output(data): start_port = elem[1]['concat'][1]['range'][0] end_port = elem[1]['concat'][1]['range'][1] port_range = f'{start_port}-{end_port}' - allocations.append((internal, external, port_range)) + if (internal_address and internal != internal_address) or ( + external_address and external != external_address + ): + continue + + allocations.append( + { + 'internal_address': internal, + 'external_address': external, + 'port_range': port_range, + } + ) + + return allocations + + +def _get_formatted_output(allocations: list[dict]) -> str: + # Convert the list of dictionaries to a list of tuples for tabulate headers = ['Internal IP', 'External IP', 'Port range'] - output = tabulate(allocations, headers, numalign="left") + data = [ + (alloc['internal_address'], alloc['external_address'], alloc['port_range']) + for alloc in allocations + ] + output = tabulate(data, headers, numalign="left") return output -def show_allocation(raw: bool): +def show_allocation( + raw: bool, + external_address: typing.Optional[str], + internal_address: typing.Optional[str], +) -> str: config = ConfigTreeQuery() if not config.exists('nat cgnat'): raise vyos.opmode.UnconfiguredSubsystem('CGNAT is not configured') if raw: - return _get_raw_data() + return _get_raw_data(external_address, internal_address) else: - raw_data = _get_raw_data() + raw_data = _get_raw_data(external_address, internal_address) return _get_formatted_output(raw_data) |