From dd2b753fb7c60cba0c79d9a3a19611312282dbce Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 6 May 2022 16:35:39 +0000 Subject: monitoring: T4418: Add output plugin azure-data-explorer Add output telegraf Plugin Azure Data Explorer set service monitoring telegraf azure-data-explorer authentication client-id 'x' set service monitoring telegraf azure-data-explorer authentication client-secret 'x' set service monitoring telegraf azure-data-explorer authentication tenant-id 'x' set service monitoring telegraf azure-data-explorer database 'x' set service monitoring telegraf azure-data-explorer group-metrics 'single-table' set service monitoring telegraf azure-data-explorer url 'http://localhost.loc' --- data/templates/monitoring/telegraf.j2 | 17 ++++ interface-definitions/include/monitoring/url.xml.i | 15 +++ .../service_monitoring_telegraf.xml.in | 102 ++++++++++++++++++--- src/conf_mode/service_monitoring_telegraf.py | 35 +++++++ 4 files changed, 156 insertions(+), 13 deletions(-) create mode 100644 interface-definitions/include/monitoring/url.xml.i diff --git a/data/templates/monitoring/telegraf.j2 b/data/templates/monitoring/telegraf.j2 index d1a94366b..a732fb5de 100644 --- a/data/templates/monitoring/telegraf.j2 +++ b/data/templates/monitoring/telegraf.j2 @@ -14,6 +14,23 @@ logfile = "" hostname = "" omit_hostname = false +{% if azure_data_explorer is vyos_defined %} +### Azure Data Explorer ### +[[outputs.azure_data_explorer]] + ## The URI property of the Azure Data Explorer resource on Azure + endpoint_url = "{{ azure_data_explorer.url }}" + + ## The Azure Data Explorer database that the metrics will be ingested into. + ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion. + database = "{{ azure_data_explorer.database }}" + metrics_grouping_type = "{{ azure_data_explorer.group_metrics }}" + + ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable"). +{% if azure_data_explorer.table is vyos_defined and azure_data_explorer.group_metrics == 'SingleTable' %} + table_name = "{{ azure_data_explorer.table }}" +{% endif %} +### End Azure Data Explorer ### +{% endif %} {% if influxdb_configured is vyos_defined %} ### InfluxDB2 ### [[outputs.influxdb_v2]] diff --git a/interface-definitions/include/monitoring/url.xml.i b/interface-definitions/include/monitoring/url.xml.i new file mode 100644 index 000000000..32c81122d --- /dev/null +++ b/interface-definitions/include/monitoring/url.xml.i @@ -0,0 +1,15 @@ + + + + Remote URL [REQUIRED] + + url + Remote URL + + + (http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}?(\/.*)? + + Incorrect URL format + + + diff --git a/interface-definitions/service_monitoring_telegraf.xml.in b/interface-definitions/service_monitoring_telegraf.xml.in index ff4c8c55f..bd528ea33 100644 --- a/interface-definitions/service_monitoring_telegraf.xml.in +++ b/interface-definitions/service_monitoring_telegraf.xml.in @@ -42,6 +42,94 @@ + + + Output plugin Azure Data Explorer + + + + + Authentication parameters + + + + + Application client id + + [-_a-zA-Z0-9]+ + + Client-id is limited to alphanumerical characters and can contain hyphen and underscores + + + + + Application client secret + + [-_a-zA-Z0-9]+ + + Client-secret is limited to alphanumerical characters and can contain hyphen and underscores + + + + + Set tenant id + + [-_a-zA-Z0-9]+ + + Tenant-id is limited to alphanumerical characters and can contain hyphen and underscores + + + + + + + Remote database name [REQUIRED] + + txt + Remote database name + + + [-_a-zA-Z0-9]+ + + Database is limited to alphanumerical characters and can contain hyphen and underscores + + + + + Type of metrics grouping when push to Azure Data Explorer + + single-table table-per-metric + + + single-table + Metrics stores in one table + + + table-per-metric + One table per gorups of metric by the metric name + + + (single-table|table-per-metric) + + + table-per-metric + + + + Name of the single table [Only if set group-metrics single-table] + + txt + Table name + + + [-_a-zA-Z0-9]+ + + Table is limited to alphanumerical characters and can contain hyphen and underscores + + + #include + + Remote bucket @@ -206,19 +294,7 @@ - - - Remote URL [REQUIRED] - - url - Remote URL to InfluxDB v2 - - - (http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}?(\/.*)? - - Incorrect URL format. - - + #include #include 8086 diff --git a/src/conf_mode/service_monitoring_telegraf.py b/src/conf_mode/service_monitoring_telegraf.py index 102a87318..daf75d740 100755 --- a/src/conf_mode/service_monitoring_telegraf.py +++ b/src/conf_mode/service_monitoring_telegraf.py @@ -103,11 +103,28 @@ def get_config(config=None): 'url' in monitoring: monitoring['influxdb_configured'] = True + # Redefine azure group-metrics 'single-table' and 'table-per-metric' + if 'azure_data_explorer' in monitoring: + if 'single-table' in monitoring['azure_data_explorer']['group_metrics']: + monitoring['azure_data_explorer']['group_metrics'] = 'SingleTable' + else: + monitoring['azure_data_explorer']['group_metrics'] = 'TablePerMetric' + # Set azure env + if 'authentication' in monitoring['azure_data_explorer']: + auth_config = monitoring['azure_data_explorer']['authentication'] + if {'client_id', 'client_secret', 'tenant_id'} <= set(auth_config): + os.environ['AZURE_CLIENT_ID'] = auth_config['client_id'] + os.environ['AZURE_CLIENT_SECRET'] = auth_config['client_secret'] + os.environ['AZURE_TENANT_ID'] = auth_config['tenant_id'] + # Ignore default XML values if config doesn't exists # Delete key from dict if not conf.exists(base + ['prometheus-client']): del monitoring['prometheus_client'] + if not conf.exists(base + ['azure-data-explorer']): + del monitoring['azure_data_explorer'] + return monitoring def verify(monitoring): @@ -124,6 +141,24 @@ def verify(monitoring): if 'url' not in monitoring: raise ConfigError(f'Monitoring "url" is mandatory!') + # Verify azure-data-explorer + if 'azure_data_explorer' in monitoring: + if 'authentication' not in monitoring['azure_data_explorer'] or \ + 'client_id' not in monitoring['azure_data_explorer']['authentication'] or \ + 'client_secret' not in monitoring['azure_data_explorer']['authentication'] or \ + 'tenant_id' not in monitoring['azure_data_explorer']['authentication']: + raise ConfigError(f'Authentication "client-id, client-secret and tenant-id" are mandatory!') + + if 'database' not in monitoring['azure_data_explorer']: + raise ConfigError(f'Monitoring "database" is mandatory!') + + if 'url' not in monitoring['azure_data_explorer']: + raise ConfigError(f'Monitoring "url" is mandatory!') + + if monitoring['azure_data_explorer']['group_metrics'] == 'SingleTable' and \ + 'table' not in monitoring['azure_data_explorer']: + raise ConfigError(f'Monitoring "table" name for single-table mode is mandatory!') + # Verify Splunk if 'splunk' in monitoring: if 'authentication' not in monitoring['splunk'] or \ -- cgit v1.2.3