diff options
| -rw-r--r-- | data/templates/system/curlrc.tmpl | 8 | ||||
| -rw-r--r-- | interface-definitions/include/source-address-ipv4-ipv6.xml.i | 17 | ||||
| -rw-r--r-- | interface-definitions/include/source-interface.xml.i | 12 | ||||
| -rw-r--r-- | interface-definitions/interfaces-vxlan.xml.in | 13 | ||||
| -rw-r--r-- | interface-definitions/system-options.xml.in | 11 | ||||
| -rw-r--r-- | python/vyos/config.py | 32 | ||||
| -rw-r--r-- | python/vyos/xml/definition.py | 2 | ||||
| -rwxr-xr-x | src/conf_mode/interfaces-pseudo-ethernet.py | 11 | ||||
| -rwxr-xr-x | src/conf_mode/system-options.py | 75 | ||||
| -rwxr-xr-x | src/op_mode/flow_accounting_op.py | 54 |
10 files changed, 155 insertions, 80 deletions
diff --git a/data/templates/system/curlrc.tmpl b/data/templates/system/curlrc.tmpl new file mode 100644 index 000000000..675e35a0c --- /dev/null +++ b/data/templates/system/curlrc.tmpl @@ -0,0 +1,8 @@ +{% if http_client is defined %}
+{% if http_client.source_interface is defined %}
+--interface "{{ http_client.source_interface }}"
+{% endif %}
+{% if http_client.source_address is defined %}
+--interface "{{ http_client.source_address }}"
+{% endif %}
+{% endif %}
diff --git a/interface-definitions/include/source-address-ipv4-ipv6.xml.i b/interface-definitions/include/source-address-ipv4-ipv6.xml.i new file mode 100644 index 000000000..6d2d77c95 --- /dev/null +++ b/interface-definitions/include/source-address-ipv4-ipv6.xml.i @@ -0,0 +1,17 @@ +<leafNode name="source-address"> + <properties> + <help>IPv4/IPv6 source address</help> + <valueHelp> + <format>ipv4</format> + <description>IPv4 source-address</description> + </valueHelp> + <valueHelp> + <format>ipv6</format> + <description>IPv6 source-address</description> + </valueHelp> + <constraint> + <validator name="ipv4-address"/> + <validator name="ipv6-address"/> + </constraint> + </properties> +</leafNode> diff --git a/interface-definitions/include/source-interface.xml.i b/interface-definitions/include/source-interface.xml.i new file mode 100644 index 000000000..ae579c2a6 --- /dev/null +++ b/interface-definitions/include/source-interface.xml.i @@ -0,0 +1,12 @@ +<leafNode name="source-interface"> + <properties> + <help>Physical interface used for connection</help> + <valueHelp> + <format>interface</format> + <description>Physical interface used for connection</description> + </valueHelp> + <completionHelp> + <script>${vyos_completion_dir}/list_interfaces.py</script> + </completionHelp> + </properties> +</leafNode> diff --git a/interface-definitions/interfaces-vxlan.xml.in b/interface-definitions/interfaces-vxlan.xml.in index fdde57525..bd3ab4022 100644 --- a/interface-definitions/interfaces-vxlan.xml.in +++ b/interface-definitions/interfaces-vxlan.xml.in @@ -64,18 +64,7 @@ </constraint> </properties> </leafNode> - <leafNode name="source-interface"> - <properties> - <help>Physical Interface used for this connection</help> - <valueHelp> - <format>interface</format> - <description>Interface used for VXLAN underlay</description> - </valueHelp> - <completionHelp> - <script>${vyos_completion_dir}/list_interfaces.py</script> - </completionHelp> - </properties> - </leafNode> + #include <include/source-interface.xml.i> #include <include/interface-mtu-1200-9000.xml.i> <leafNode name="remote"> <properties> diff --git a/interface-definitions/system-options.xml.in b/interface-definitions/system-options.xml.in index 48bc353ab..194773329 100644 --- a/interface-definitions/system-options.xml.in +++ b/interface-definitions/system-options.xml.in @@ -33,7 +33,7 @@ <description>Poweroff VyOS</description> </valueHelp> <constraint> - <regex>(ignore|reboot|poweroff)</regex> + <regex>^(ignore|reboot|poweroff)$</regex> </constraint> <constraintErrorMessage>Must be ignore, reboot, or poweroff</constraintErrorMessage> </properties> @@ -44,6 +44,15 @@ <valueless/> </properties> </leafNode> + <node name="http-client"> + <properties> + <help>Global options used for HTTP based commands</help> + </properties> + <children> + #include <include/source-interface.xml.i> + #include <include/source-address-ipv4-ipv6.xml.i> + </children> + </node> </children> </node> </children> diff --git a/python/vyos/config.py b/python/vyos/config.py index 780b48a7b..5d58316e7 100644 --- a/python/vyos/config.py +++ b/python/vyos/config.py @@ -67,6 +67,7 @@ import os import re import json import subprocess +from copy import deepcopy import vyos.util import vyos.configtree @@ -91,6 +92,8 @@ class Config(object): def __init__(self, session_env=None): self._cli_shell_api = "/bin/cli-shell-api" self._level = [] + self._dict_cache = {} + if session_env: self.__session_env = session_env else: @@ -287,6 +290,24 @@ class Config(object): self.__session_env = save_env return(default) + def get_cached_dict(self, effective=False): + cached = self._dict_cache.get(effective, {}) + if cached: + config_dict = cached + else: + config_dict = {} + + if effective: + if self._running_config: + config_dict = json.loads((self._running_config).to_json()) + else: + if self._session_config: + config_dict = json.loads((self._session_config).to_json()) + + self._dict_cache[effective] = config_dict + + return config_dict + def get_config_dict(self, path=[], effective=False, key_mangling=None, get_first_key=False): """ Args: @@ -297,14 +318,7 @@ class Config(object): Returns: a dict representation of the config under path """ - config_dict = {} - - if effective: - if self._running_config: - config_dict = json.loads((self._running_config).to_json()) - else: - if self._session_config: - config_dict = json.loads((self._session_config).to_json()) + config_dict = self.get_cached_dict(effective) config_dict = vyos.util.get_sub_dict(config_dict, self._make_path(path), get_first_key) @@ -316,6 +330,8 @@ class Config(object): raise ValueError("key_mangling must be a tuple of two strings") else: config_dict = vyos.util.mangle_dict_keys(config_dict, key_mangling[0], key_mangling[1]) + else: + config_dict = deepcopy(config_dict) return config_dict diff --git a/python/vyos/xml/definition.py b/python/vyos/xml/definition.py index 5421007e0..b0339b228 100644 --- a/python/vyos/xml/definition.py +++ b/python/vyos/xml/definition.py @@ -248,7 +248,7 @@ class XML(dict): def defaults(self, lpath, flat): d = self[kw.default] for k in lpath: - d = d[k] + d = d.get(k, {}) if not flat: r = {} diff --git a/src/conf_mode/interfaces-pseudo-ethernet.py b/src/conf_mode/interfaces-pseudo-ethernet.py index 70710e97c..fb8237bee 100755 --- a/src/conf_mode/interfaces-pseudo-ethernet.py +++ b/src/conf_mode/interfaces-pseudo-ethernet.py @@ -36,7 +36,7 @@ default_config_data = { 'ip_arp_cache_tmo': 30, 'ip_proxy_arp_pvlan': 0, 'source_interface': '', - 'source_interface_changed': False, + 'recreating_required': False, 'mode': 'private', 'vif_s': {}, 'vif_s_remove': [], @@ -79,11 +79,14 @@ def get_config(): peth['source_interface'] = conf.return_value(['source-interface']) tmp = conf.return_effective_value(['source-interface']) if tmp != peth['source_interface']: - peth['source_interface_changed'] = True + peth['recreating_required'] = True # MACvlan mode if conf.exists(['mode']): peth['mode'] = conf.return_value(['mode']) + tmp = conf.return_effective_value(['mode']) + if tmp != peth['mode']: + peth['recreating_required'] = True add_to_dict(conf, disabled, peth, 'vif', 'vif') add_to_dict(conf, disabled, peth, 'vif-s', 'vif_s') @@ -139,10 +142,10 @@ def apply(peth): return None # Check if MACVLAN interface already exists. Parameters like the underlaying - # source-interface device can not be changed on the fly and the interface + # source-interface device or mode can not be changed on the fly and the interface # needs to be recreated from the bottom. if peth['intf'] in interfaces(): - if peth['source_interface_changed']: + if peth['recreating_required']: MACVLANIf(peth['intf']).remove() # MACVLAN interface needs to be created on-block instead of passing a ton diff --git a/src/conf_mode/system-options.py b/src/conf_mode/system-options.py index 8de3b6fa2..d7c5c0443 100755 --- a/src/conf_mode/system-options.py +++ b/src/conf_mode/system-options.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019 VyOS maintainers and contributors +# Copyright (C) 2019-2020 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -16,67 +16,62 @@ import os +from netifaces import interfaces from sys import exit -from copy import deepcopy + from vyos.config import Config +from vyos.template import render +from vyos.util import call from vyos import ConfigError -from vyos.util import run - from vyos import airbag airbag.enable() -systemd_ctrl_alt_del = '/lib/systemd/system/ctrl-alt-del.target' - -default_config_data = { - 'beep_if_fully_booted': False, - 'ctrl_alt_del': 'ignore', - 'reboot_on_panic': True -} +config_file = r'/etc/curlrc' +systemd_action_file = '/lib/systemd/system/ctrl-alt-del.target' def get_config(): - opt = deepcopy(default_config_data) conf = Config() - conf.set_level('system options') - if conf.exists(''): - if conf.exists('ctrl-alt-del-action'): - opt['ctrl_alt_del'] = conf.return_value('ctrl-alt-del-action') + base = ['system', 'options'] + options = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) + return options - opt['beep_if_fully_booted'] = conf.exists('beep-if-fully-booted') - opt['reboot_on_panic'] = conf.exists('reboot-on-panic') +def verify(options): + if 'http_client' in options.keys(): + config = options['http_client'] + if 'source_interface' in config.keys(): + if not config['source_interface'] in interfaces(): + raise ConfigError(f'Source interface {source_interface} does not ' + f'exist'.format(**config)) - return opt + if {'source_address', 'source_interface'} <= set(config): + raise ConfigError('Can not define both HTTP source-interface and source-address') -def verify(opt): - pass + return None -def generate(opt): - pass +def generate(options): + render(config_file, 'system/curlrc.tmpl', options, trim_blocks=True) + return None -def apply(opt): +def apply(options): # Beep action - if opt['beep_if_fully_booted']: - run('systemctl enable vyos-beep.service') + if 'beep_if_fully_booted' in options.keys(): + call('systemctl enable vyos-beep.service') else: - run('systemctl disable vyos-beep.service') + call('systemctl disable vyos-beep.service') # Ctrl-Alt-Delete action - if opt['ctrl_alt_del'] == 'ignore': - if os.path.exists(systemd_ctrl_alt_del): - os.unlink('/lib/systemd/system/ctrl-alt-del.target') - - elif opt['ctrl_alt_del'] == 'reboot': - if os.path.exists(systemd_ctrl_alt_del): - os.unlink(systemd_ctrl_alt_del) - os.symlink('/lib/systemd/system/reboot.target', systemd_ctrl_alt_del) + if os.path.exists(systemd_action_file): + os.unlink(systemd_action_file) - elif opt['ctrl_alt_del'] == 'poweroff': - if os.path.exists(systemd_ctrl_alt_del): - os.unlink(systemd_ctrl_alt_del) - os.symlink('/lib/systemd/system/poweroff.target', systemd_ctrl_alt_del) + if 'ctrl_alt_del_action' in options.keys(): + if options['ctrl_alt_del_action'] == 'reboot': + os.symlink('/lib/systemd/system/reboot.target', systemd_action_file) + elif options['ctrl_alt_del_action'] == 'poweroff': + os.symlink('/lib/systemd/system/poweroff.target', systemd_action_file) # Reboot system on kernel panic with open('/proc/sys/kernel/panic', 'w') as f: - if opt['reboot_on_panic']: + if 'reboot_on_panic' in options.keys(): f.write('60') else: f.write('0') diff --git a/src/op_mode/flow_accounting_op.py b/src/op_mode/flow_accounting_op.py index bf8c39fd6..9d0417cd4 100755 --- a/src/op_mode/flow_accounting_op.py +++ b/src/op_mode/flow_accounting_op.py @@ -21,8 +21,9 @@ import re import ipaddress import os.path from tabulate import tabulate - +from json import loads from vyos.util import cmd, run +from vyos.logger import syslog # some default values uacctd_pidfile = '/var/run/uacctd.pid' @@ -32,28 +33,28 @@ uacctd_pipefile = '/tmp/uacctd.pipe' # check if ports argument have correct format def _is_ports(ports): # define regex for checking - regex_filter = re.compile('^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$|^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])-(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$|^((\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]),)+(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') + regex_filter = re.compile(r'^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$|^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])-(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$|^((\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]),)+(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') if not regex_filter.search(ports): raise argparse.ArgumentTypeError("Invalid ports: {}".format(ports)) # check which type nitation is used: single port, ports list, ports range # single port - regex_filter = re.compile('^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') + regex_filter = re.compile(r'^(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') if regex_filter.search(ports): - filter_ports = { 'type': 'single', 'value': int(ports) } + filter_ports = {'type': 'single', 'value': int(ports)} # ports list - regex_filter = re.compile('^((\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]),)+(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])') + regex_filter = re.compile(r'^((\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5]),)+(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])') if regex_filter.search(ports): - filter_ports = { 'type': 'list', 'value': list(map(int, ports.split(','))) } + filter_ports = {'type': 'list', 'value': list(map(int, ports.split(',')))} # ports range - regex_filter = re.compile('^(?P<first>\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])-(?P<second>\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') + regex_filter = re.compile(r'^(?P<first>\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])-(?P<second>\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])$') if regex_filter.search(ports): # check if second number is greater than the first if int(regex_filter.search(ports).group('first')) >= int(regex_filter.search(ports).group('second')): raise argparse.ArgumentTypeError("Invalid ports: {}".format(ports)) - filter_ports = { 'type': 'range', 'value': range(int(regex_filter.search(ports).group('first')), int(regex_filter.search(ports).group('second'))) } + filter_ports = {'type': 'range', 'value': range(int(regex_filter.search(ports).group('first')), int(regex_filter.search(ports).group('second')))} # if all above failed if not filter_ports: @@ -61,6 +62,7 @@ def _is_ports(ports): else: return filter_ports + # check if host argument have correct format def _is_host(host): # define regex for checking @@ -68,11 +70,13 @@ def _is_host(host): raise argparse.ArgumentTypeError("Invalid host: {}".format(host)) return host + # check if flow-accounting running def _uacctd_running(): command = 'systemctl status uacctd.service > /dev/null' return run(command) == 0 + # get list of interfaces def _get_ifaces_dict(): # run command to get ifaces list @@ -83,7 +87,7 @@ def _get_ifaces_dict(): # make a dictionary with interfaces and indexes ifaces_dict = {} - regex_filter = re.compile('^(?P<iface_index>\d+):\ (?P<iface_name>[\w\d\.]+)[:@].*$') + regex_filter = re.compile(r'^(?P<iface_index>\d+):\ (?P<iface_name>[\w\d\.]+)[:@].*$') for iface_line in ifaces_out: if regex_filter.search(iface_line): ifaces_dict[int(regex_filter.search(iface_line).group('iface_index'))] = regex_filter.search(iface_line).group('iface_name') @@ -91,11 +95,12 @@ def _get_ifaces_dict(): # return dictioanry return ifaces_dict + # get list of flows def _get_flows_list(): # run command to get flows list out = cmd(f'/usr/bin/pmacct -s -O json -T flows -p {uacctd_pipefile}', - message='Failed to get flows list') + message='Failed to get flows list') # read output flows_out = out.splitlines() @@ -103,11 +108,15 @@ def _get_flows_list(): # make a list with flows flows_list = [] for flow_line in flows_out: - flows_list.append(eval(flow_line)) + try: + flows_list.append(loads(flow_line)) + except Exception as err: + syslog.error('Unable to read flow info: {}'.format(err)) # return list of flows return flows_list + # filter and format flows def _flows_filter(flows, ifaces): # predefine filtered flows list @@ -149,14 +158,29 @@ def _flows_filter(flows, ifaces): # return filtered flows return flows_filtered + # print flow table def _flows_table_print(flows): - #define headers and body - table_headers = [ 'IN_IFACE', 'SRC_MAC', 'DST_MAC', 'SRC_IP', 'DST_IP', 'SRC_PORT', 'DST_PORT', 'PROTOCOL', 'TOS', 'PACKETS', 'FLOWS', 'BYTES' ] + # define headers and body + table_headers = ['IN_IFACE', 'SRC_MAC', 'DST_MAC', 'SRC_IP', 'DST_IP', 'SRC_PORT', 'DST_PORT', 'PROTOCOL', 'TOS', 'PACKETS', 'FLOWS', 'BYTES'] table_body = [] # convert flows to list for flow in flows: - table_body.append([flow['iface_in_name'], flow['mac_src'], flow['mac_dst'], flow['ip_src'], flow['ip_dst'], flow['port_src'], flow['port_dst'], flow['ip_proto'], flow['tos'], flow['packets'], flow['flows'], flow['bytes'] ]) + table_line = [ + flow.get('iface_in_name'), + flow.get('mac_src'), + flow.get('mac_dst'), + flow.get('ip_src'), + flow.get('ip_dst'), + flow.get('port_src'), + flow.get('port_dst'), + flow.get('ip_proto'), + flow.get('tos'), + flow.get('packets'), + flow.get('flows'), + flow.get('bytes') + ] + table_body.append(table_line) # configure and fill table table = tabulate(table_body, table_headers, tablefmt="simple") @@ -168,12 +192,14 @@ def _flows_table_print(flows): except KeyboardInterrupt: sys.exit(0) + # check if in-memory table is active def _check_imt(): if not os.path.exists(uacctd_pipefile): print("In-memory table is not available") sys.exit(1) + # define program arguments cmd_args_parser = argparse.ArgumentParser(description='show flow-accounting') cmd_args_parser.add_argument('--action', choices=['show', 'clear', 'restart'], required=True, help='command to flow-accounting daemon') |