diff options
| -rw-r--r-- | data/templates/frr/nhrpd.frr.j2 | 2 | ||||
| -rw-r--r-- | data/templates/frr/staticd.frr.j2 | 4 | ||||
| -rw-r--r-- | data/templates/load-balancing/haproxy.cfg.j2 | 5 | ||||
| -rw-r--r-- | interface-definitions/load-balancing_haproxy.xml.in | 32 | ||||
| -rw-r--r-- | python/vyos/kea.py | 35 | ||||
| -rwxr-xr-x | python/vyos/xml_ref/generate_op_cache.py | 95 | ||||
| -rwxr-xr-x | smoketest/scripts/cli/test_load-balancing_haproxy.py | 23 | ||||
| -rw-r--r-- | src/conf_mode/load-balancing_haproxy.py | 7 | ||||
| -rwxr-xr-x | src/op_mode/dhcp.py | 29 | ||||
| -rwxr-xr-x | src/op_mode/image_installer.py | 19 |
10 files changed, 160 insertions, 91 deletions
diff --git a/data/templates/frr/nhrpd.frr.j2 b/data/templates/frr/nhrpd.frr.j2 index 2b2aba256..813a9384b 100644 --- a/data/templates/frr/nhrpd.frr.j2 +++ b/data/templates/frr/nhrpd.frr.j2 @@ -58,5 +58,3 @@ exit {% endfor %} {% endif %} ! -exit -! diff --git a/data/templates/frr/staticd.frr.j2 b/data/templates/frr/staticd.frr.j2 index 90d17ec14..18d300dae 100644 --- a/data/templates/frr/staticd.frr.j2 +++ b/data/templates/frr/staticd.frr.j2 @@ -94,14 +94,14 @@ vrf {{ vrf }} {% if pppoe is vyos_defined %} {% for interface, interface_config in pppoe.items() if interface_config.no_default_route is not vyos_defined %} {{ ip_prefix }} route 0.0.0.0/0 {{ interface }} tag 210 {{ interface_config.default_route_distance if interface_config.default_route_distance is vyos_defined }} -{%- endfor %} +{% endfor %} {% endif %} {# IPv6 routing #} {% if route6 is vyos_defined %} {% for prefix, prefix_config in route6.items() %} {{ static_routes(ipv6_prefix, prefix, prefix_config) }} {# j2lint: disable=jinja-statements-delimeter #} -{%- endfor %} +{% endfor %} {% endif %} {% if vrf is vyos_defined %} exit-vrf diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 786ebfb21..c98b739e2 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -93,6 +93,11 @@ frontend {{ front }} http-response set-header {{ header }} '{{ header_config['value'] }}' {% endfor %} {% endif %} +{% if front_config.http_compression is vyos_defined %} + filter compression + compression algo {{ front_config.http_compression.algorithm }} + compression type {{ front_config.http_compression.mime_type | join(' ') }} +{% endif %} {% if front_config.rule is vyos_defined %} {% for rule, rule_config in front_config.rule.items() %} # rule {{ rule }} diff --git a/interface-definitions/load-balancing_haproxy.xml.in b/interface-definitions/load-balancing_haproxy.xml.in index 742272436..ca089d3f0 100644 --- a/interface-definitions/load-balancing_haproxy.xml.in +++ b/interface-definitions/load-balancing_haproxy.xml.in @@ -48,6 +48,38 @@ <valueless/> </properties> </leafNode> + <node name="http-compression"> + <properties> + <help>Compress HTTP responses</help> + </properties> + <children> + <leafNode name="algorithm"> + <properties> + <help>Compression algorithm</help> + <completionHelp> + <list>gzip deflate identity raw-deflate</list> + </completionHelp> + <constraint> + <regex>(gzip|deflate|identity|raw-deflate)</regex> + </constraint> + </properties> + </leafNode> + <leafNode name="mime-type"> + <properties> + <help>MIME types to compress</help> + <valueHelp> + <format>txt</format> + <description>MIME type to compress</description> + </valueHelp> + <multi/> + <constraint> + <regex>\w+\/[-+.\w]+</regex> + </constraint> + <constraintErrorMessage>Invalid MIME type specified</constraintErrorMessage> + </properties> + </leafNode> + </children> + </node> <node name="ssl"> <properties> <help>SSL Certificate, SSL Key and CA</help> diff --git a/python/vyos/kea.py b/python/vyos/kea.py index 951c83693..65e2d99b4 100644 --- a/python/vyos/kea.py +++ b/python/vyos/kea.py @@ -474,12 +474,11 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list data = [] for lease in leases: lifetime = lease['valid-lft'] - expiry = lease['cltt'] + lifetime + start = lease['cltt'] + expiry = start + lifetime - lease['start_timestamp'] = datetime.fromtimestamp( - expiry - lifetime, timezone.utc - ) - lease['expire_timestamp'] = ( + lease['start_time'] = datetime.fromtimestamp(start, timezone.utc) + lease['expire_time'] = ( datetime.fromtimestamp(expiry, timezone.utc) if expiry else None ) @@ -493,18 +492,18 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list else '-' ) data_lease['end'] = ( - lease['expire_timestamp'].timestamp() if lease['expire_timestamp'] else None + lease['expire_time'].timestamp() if lease['expire_time'] else None ) data_lease['origin'] = 'local' # TODO: Determine remote in HA # remove trailing dot in 'hostname' to ensure consistency for `vyos-hostsd-client` - data_lease['hostname'] = lease.get('hostname', '-').rstrip('.') + data_lease['hostname'] = lease.get('hostname', '').rstrip('.') or '-' if inet == '4': data_lease['mac'] = lease['hw-address'] - data_lease['start'] = lease['start_timestamp'].timestamp() + data_lease['start'] = lease['start_time'].timestamp() if inet == '6': - data_lease['last_communication'] = lease['start_timestamp'].timestamp() + data_lease['last_communication'] = lease['start_time'].timestamp() data_lease['duid'] = _format_hex_string(lease['duid']) data_lease['type'] = lease['type'] @@ -512,21 +511,17 @@ def kea_get_server_leases(config, inet, pools=[], state=[], origin=None) -> list prefix_len = lease['prefix-len'] data_lease['ip'] += f'/{prefix_len}' - data_lease['remaining'] = '-' - - if lease['valid-lft'] > 0: - data_lease['remaining'] = lease['expire_timestamp'] - datetime.now( - timezone.utc - ) + data_lease['remaining'] = '' - if data_lease['remaining'].days >= 0: - # substraction gives us a timedelta object which can't be formatted with strftime - # so we use str(), split gets rid of the microseconds - data_lease['remaining'] = str(data_lease['remaining']).split('.')[0] + now = datetime.now(timezone.utc) + if lease['valid-lft'] > 0 and lease['expire_time'] > now: + # substraction gives us a timedelta object which can't be formatted + # with strftime so we use str(), split gets rid of the microseconds + data_lease['remaining'] = str(lease['expire_time'] - now).split('.')[0] # Do not add old leases if ( - data_lease['remaining'] + data_lease['remaining'] != '' and data_lease['pool'] in pools and data_lease['state'] != 'free' and (not state or state == 'all' or data_lease['state'] in state) diff --git a/python/vyos/xml_ref/generate_op_cache.py b/python/vyos/xml_ref/generate_op_cache.py index cd2ac890e..95779d066 100755 --- a/python/vyos/xml_ref/generate_op_cache.py +++ b/python/vyos/xml_ref/generate_op_cache.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2024 VyOS maintainers and contributors +# Copyright (C) 2024-2025 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -33,9 +33,9 @@ _here = dirname(__file__) sys.path.append(join(_here, '..')) from defaults import directories -from op_definition import NodeData from op_definition import PathData + xml_op_cache_json = 'xml_op_cache.json' xml_op_tmp = join('/tmp', xml_op_cache_json) op_ref_cache = abspath(join(_here, 'op_cache.py')) @@ -74,7 +74,7 @@ def translate_op_script(s: str) -> str: return s -def insert_node(n: Element, l: list[PathData], path = None) -> None: +def insert_node(n: Element, l: list[PathData], path=None) -> None: # pylint: disable=too-many-locals,too-many-branches prop: OptElement = n.find('properties') children: OptElement = n.find('children') @@ -95,65 +95,67 @@ def insert_node(n: Element, l: list[PathData], path = None) -> None: if command_text is not None: command_text = translate_command(command_text, path) - comp_help = None + comp_help = {} if prop is not None: - che = prop.findall("completionHelp") + che = prop.findall('completionHelp') + for c in che: - lists = c.findall("list") - paths = c.findall("path") - scripts = c.findall("script") - - comp_help = {} - list_l = [] - for i in lists: - list_l.append(i.text) - path_l = [] - for i in paths: - path_str = re.sub(r'\s+', '/', i.text) - path_l.append(path_str) - script_l = [] - for i in scripts: - script_str = translate_op_script(i.text) - script_l.append(script_str) - - comp_help['list'] = list_l - comp_help['fs_path'] = path_l - comp_help['script'] = script_l - - for d in l: - if name in list(d): - break - else: - d = {} - l.append(d) - - inner_l = d.setdefault(name, []) - - inner_d: PathData = {'node_data': NodeData(node_type=node_type, - help_text=help_text, - comp_help=comp_help, - command=command_text, - path=path)} - inner_l.append(inner_d) + comp_list_els = c.findall('list') + comp_path_els = c.findall('path') + comp_script_els = c.findall('script') + + comp_lists = [] + for i in comp_list_els: + comp_lists.append(i.text) + + comp_paths = [] + for i in comp_path_els: + comp_paths.append(i.text) + + comp_scripts = [] + for i in comp_script_els: + comp_script_str = translate_op_script(i.text) + comp_scripts.append(comp_script_str) + + if comp_lists: + comp_help['list'] = comp_lists + if comp_paths: + comp_help['path'] = comp_paths + if comp_scripts: + comp_help['script'] = comp_scripts + + cur_node_dict = {} + cur_node_dict['name'] = name + cur_node_dict['type'] = node_type + cur_node_dict['comp_help'] = comp_help + cur_node_dict['help'] = help_text + cur_node_dict['command'] = command_text + cur_node_dict['path'] = path + cur_node_dict['children'] = [] + l.append(cur_node_dict) if children is not None: - inner_nodes = children.iterfind("*") + inner_nodes = children.iterfind('*') for inner_n in inner_nodes: inner_path = path[:] - insert_node(inner_n, inner_l, inner_path) + insert_node(inner_n, cur_node_dict['children'], inner_path) def parse_file(file_path, l): tree = ET.parse(file_path) root = tree.getroot() - for n in root.iterfind("*"): + for n in root.iterfind('*'): insert_node(n, l) def main(): parser = ArgumentParser(description='generate dict from xml defintions') - parser.add_argument('--xml-dir', type=str, required=True, - help='transcluded xml op-mode-definition file') + parser.add_argument( + '--xml-dir', + type=str, + required=True, + help='transcluded xml op-mode-definition file', + ) args = vars(parser.parse_args()) @@ -170,5 +172,6 @@ def main(): with open(op_ref_cache, 'w') as f: f.write(f'op_reference = {str(l)}') + if __name__ == '__main__': main() diff --git a/smoketest/scripts/cli/test_load-balancing_haproxy.py b/smoketest/scripts/cli/test_load-balancing_haproxy.py index 967eb3869..9f412aa95 100755 --- a/smoketest/scripts/cli/test_load-balancing_haproxy.py +++ b/smoketest/scripts/cli/test_load-balancing_haproxy.py @@ -498,5 +498,28 @@ class TestLoadBalancingReverseProxy(VyOSUnitTestSHIM.TestCase): self.assertIn('log /dev/log local5 notice', config) self.assertIn('log /dev/log local6 crit', config) + def test_10_lb_reverse_proxy_http_compression(self): + # Setup base + self.configure_pki() + self.base_config() + + # Configure compression in frontend + self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'algorithm', 'gzip']) + self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/html']) + self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/javascript']) + self.cli_set(base_path + ['service', 'https_front', 'http-compression', 'mime-type', 'text/plain']) + self.cli_commit() + + # Test compression is present in generated configuration file + config = read_file(HAPROXY_CONF) + self.assertIn('filter compression', config) + self.assertIn('compression algo gzip', config) + self.assertIn('compression type text/html text/javascript text/plain', config) + + # Test setting compression without specifying any mime-types fails verification + self.cli_delete(base_path + ['service', 'https_front', 'http-compression', 'mime-type']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/load-balancing_haproxy.py b/src/conf_mode/load-balancing_haproxy.py index 45042dd52..5fd1beec9 100644 --- a/src/conf_mode/load-balancing_haproxy.py +++ b/src/conf_mode/load-balancing_haproxy.py @@ -78,6 +78,13 @@ def verify(lb): not is_listen_port_bind_service(int(tmp_port), 'haproxy'): raise ConfigError(f'"TCP" port "{tmp_port}" is used by another service') + if 'http_compression' in front_config: + if front_config['mode'] != 'http': + raise ConfigError(f'service {front} must be set to http mode to use http-compression!') + if len(front_config['http_compression']['mime_type']) == 0: + raise ConfigError(f'service {front} must have at least one mime-type configured to use' + f'http_compression!') + for back, back_config in lb['backend'].items(): if 'http_check' in back_config: http_check = back_config['http_check'] diff --git a/src/op_mode/dhcp.py b/src/op_mode/dhcp.py index b3d7d4dd3..8eed2c6cd 100755 --- a/src/op_mode/dhcp.py +++ b/src/op_mode/dhcp.py @@ -19,6 +19,7 @@ import sys import typing from datetime import datetime +from datetime import timezone from glob import glob from ipaddress import ip_address from tabulate import tabulate @@ -81,12 +82,6 @@ ArgState = typing.Literal[ ArgOrigin = typing.Literal['local', 'remote'] -def _utc_to_local(utc_dt): - return datetime.fromtimestamp( - (datetime.fromtimestamp(utc_dt) - datetime(1970, 1, 1)).total_seconds() - ) - - def _get_raw_server_leases( config, family='inet', pool=None, sorted=None, state=[], origin=None ) -> list: @@ -110,10 +105,12 @@ def _get_formatted_server_leases(raw_data, family='inet'): ipaddr = lease.get('ip') hw_addr = lease.get('mac') state = lease.get('state') - start = lease.get('start') - start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') - end = lease.get('end') - end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') if end else '-' + start = datetime.fromtimestamp(lease.get('start'), timezone.utc) + end = ( + datetime.fromtimestamp(lease.get('end'), timezone.utc) + if lease.get('end') + else '-' + ) remain = lease.get('remaining') pool = lease.get('pool') hostname = lease.get('hostname') @@ -138,10 +135,14 @@ def _get_formatted_server_leases(raw_data, family='inet'): for lease in raw_data: ipaddr = lease.get('ip') state = lease.get('state') - start = lease.get('last_communication') - start = _utc_to_local(start).strftime('%Y/%m/%d %H:%M:%S') - end = lease.get('end') - end = _utc_to_local(end).strftime('%Y/%m/%d %H:%M:%S') + start = datetime.fromtimestamp( + lease.get('last_communication'), timezone.utc + ) + end = ( + datetime.fromtimestamp(lease.get('end'), timezone.utc) + if lease.get('end') + else '-' + ) remain = lease.get('remaining') lease_type = lease.get('type') pool = lease.get('pool') diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 1da112673..91d69c463 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright 2023-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright 2023-2025 VyOS maintainers and contributors <maintainers@vyos.io> # # This file is part of VyOS. # @@ -47,6 +47,7 @@ MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' MSG_ERR_IMPROPER_IMAGE: str = 'Missing sha256sum.txt.\nEither this image is corrupted, or of era 1.2.x (md5sum) and would downgrade image tools;\ndisallowed in either case.' MSG_ERR_ARCHITECTURE_MISMATCH: str = 'Upgrading to a different image architecture will break your system.' +MSG_ERR_FLAVOR_MISMATCH: str = 'The current image flavor is "{0}", the new image is "{1}". Upgrading to a non-matching flavor can have unpredictable consequences.' MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install VyOS to your permanent storage.' MSG_INFO_INSTALL_EXIT: str = 'Exiting from VyOS installation' MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' @@ -79,7 +80,6 @@ MSG_WARN_ROOT_SIZE_TOOSMALL: str = 'The size is too small. Try again' MSG_WARN_IMAGE_NAME_WRONG: str = 'The suggested name is unsupported!\n'\ 'It must be between 1 and 64 characters long and contains only the next characters: .+-_ a-z A-Z 0-9' MSG_WARN_PASSWORD_CONFIRM: str = 'The entered values did not match. Try again' -MSG_WARN_FLAVOR_MISMATCH: str = 'The running image flavor is "{0}". The new image flavor is "{1}".\n' \ 'Installing a different image flavor may cause functionality degradation or break your system.\n' \ 'Do you want to continue with installation?' CONST_MIN_DISK_SIZE: int = 2147483648 # 2 GB @@ -701,7 +701,7 @@ def is_raid_install(install_object: Union[disk.DiskDetails, raid.RaidDetails]) - return False -def validate_compatibility(iso_path: str) -> None: +def validate_compatibility(iso_path: str, force: bool = False) -> None: """Check architecture and flavor compatibility with the running image Args: @@ -721,7 +721,8 @@ def validate_compatibility(iso_path: str) -> None: exit(MSG_INFO_INSTALL_EXIT) if not old_flavor == new_flavor: - if not ask_yes_no(MSG_WARN_FLAVOR_MISMATCH.format(old_flavor, new_flavor), default=False): + print(MSG_ERR_FLAVOR_MISMATCH.format(old_flavor, new_flavor)) + if not force: cleanup() exit(MSG_INFO_INSTALL_EXIT) @@ -893,7 +894,7 @@ def install_image() -> None: @compat.grub_cfg_update def add_image(image_path: str, vrf: str = None, username: str = '', - password: str = '', no_prompt: bool = False) -> None: + password: str = '', no_prompt: bool = False, force: bool = False) -> None: """Add a new image Args: @@ -910,7 +911,7 @@ def add_image(image_path: str, vrf: str = None, username: str = '', disk.partition_mount(iso_path, DIR_ISO_MOUNT, 'iso9660') print('Validating image compatibility') - validate_compatibility(DIR_ISO_MOUNT) + validate_compatibility(DIR_ISO_MOUNT, force=force) # check sums print('Validating image checksums') @@ -1031,6 +1032,9 @@ def parse_arguments() -> Namespace: parser.add_argument('--image-path', help='a path (HTTP or local file) to an image that needs to be installed' ) + parser.add_argument('--force', action='store_true', + help='Ignore flavor compatibility requirements.' + ) # parser.add_argument('--image_new_name', help='a new name for image') args: Namespace = parser.parse_args() # Validate arguments @@ -1047,7 +1051,8 @@ if __name__ == '__main__': install_image() if args.action == 'add': add_image(args.image_path, args.vrf, - args.username, args.password, args.no_prompt) + args.username, args.password, + args.no_prompt, args.force) exit() |