Merge pull request #1317 from sever-sever/T4418

monitoring: T4418: Add output plugin azure-data-explorer

4 files changed, 156 insertions, 13 deletions

diff --git a/data/templates/monitoring/telegraf.j2 b/data/templates/monitoring/telegraf.j2
index d1a94366b..a732fb5de 100644
--- a/data/templates/monitoring/telegraf.j2
+++ b/data/templates/monitoring/telegraf.j2
@@ -14,6 +14,23 @@
   logfile = ""
   hostname = ""
   omit_hostname = false
+{% if azure_data_explorer is vyos_defined %}
+### Azure Data Explorer ###
+[[outputs.azure_data_explorer]]
+  ## The URI property of the Azure Data Explorer resource on Azure
+  endpoint_url = "{{ azure_data_explorer.url }}"
+
+  ## The Azure Data Explorer database that the metrics will be ingested into.
+  ## The plugin will NOT generate this database automatically, it's expected that this database already exists before ingestion.
+  database = "{{ azure_data_explorer.database }}"
+  metrics_grouping_type = "{{ azure_data_explorer.group_metrics }}"
+
+  ## Name of the single table to store all the metrics (Only needed if metrics_grouping_type is "SingleTable").
+{%     if azure_data_explorer.table is vyos_defined and azure_data_explorer.group_metrics == 'SingleTable' %}
+  table_name = "{{ azure_data_explorer.table }}"
+{%     endif %}
+### End Azure Data Explorer ###
+{% endif %}
 {% if influxdb_configured is vyos_defined %}
 ### InfluxDB2 ###
 [[outputs.influxdb_v2]]
diff --git a/interface-definitions/include/monitoring/url.xml.i b/interface-definitions/include/monitoring/url.xml.i
new file mode 100644
index 000000000..32c81122d
--- /dev/null
+++ b/interface-definitions/include/monitoring/url.xml.i
@@ -0,0 +1,15 @@
+<!-- include start from monitoring/url.xml.i -->
+<leafNode name="url">
+  <properties>
+    <help>Remote URL [REQUIRED]</help>
+    <valueHelp>
+      <format>url</format>
+      <description>Remote URL</description>
+    </valueHelp>
+    <constraint>
+      <regex>(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}?(\/.*)?</regex>
+    </constraint>
+    <constraintErrorMessage>Incorrect URL format</constraintErrorMessage>
+  </properties>
+</leafNode>
+<!-- include end -->
diff --git a/interface-definitions/service_monitoring_telegraf.xml.in b/interface-definitions/service_monitoring_telegraf.xml.in
index ff4c8c55f..bd528ea33 100644
--- a/interface-definitions/service_monitoring_telegraf.xml.in
+++ b/interface-definitions/service_monitoring_telegraf.xml.in
@@ -42,6 +42,94 @@
                   </leafNode>
                 </children>
               </node>
+              <node name="azure-data-explorer">
+                <properties>
+                  <help>Output plugin Azure Data Explorer</help>
+                </properties>
+                <children>
+                  <node name="authentication">
+                    <properties>
+                      <help>Authentication parameters</help>
+                    </properties>
+                    <children>
+                      <leafNode name="client-id">
+                        <properties>
+                          <help>Application client id</help>
+                          <constraint>
+                            <regex>[-_a-zA-Z0-9]+</regex>
+                          </constraint>
+                          <constraintErrorMessage>Client-id is limited to alphanumerical characters and can contain hyphen and underscores</constraintErrorMessage>
+                        </properties>
+                      </leafNode>
+                      <leafNode name="client-secret">
+                        <properties>
+                          <help>Application client secret</help>
+                          <constraint>
+                            <regex>[-_a-zA-Z0-9]+</regex>
+                          </constraint>
+                          <constraintErrorMessage>Client-secret is limited to alphanumerical characters and can contain hyphen and underscores</constraintErrorMessage>
+                        </properties>
+                      </leafNode>
+                      <leafNode name="tenant-id">
+                        <properties>
+                          <help>Set tenant id</help>
+                          <constraint>
+                            <regex>[-_a-zA-Z0-9]+</regex>
+                          </constraint>
+                          <constraintErrorMessage>Tenant-id is limited to alphanumerical characters and can contain hyphen and underscores</constraintErrorMessage>
+                        </properties>
+                      </leafNode>
+                    </children>
+                  </node>
+                  <leafNode name="database">
+                    <properties>
+                      <help>Remote database name [REQUIRED]</help>
+                      <valueHelp>
+                        <format>txt</format>
+                        <description>Remote database name</description>
+                      </valueHelp>
+                      <constraint>
+                        <regex>[-_a-zA-Z0-9]+</regex>
+                      </constraint>
+                      <constraintErrorMessage>Database is limited to alphanumerical characters and can contain hyphen and underscores</constraintErrorMessage>
+                    </properties>
+                  </leafNode>
+                  <leafNode name="group-metrics">
+                    <properties>
+                      <help>Type of metrics grouping when push to Azure Data Explorer</help>
+                      <completionHelp>
+                        <list>single-table table-per-metric</list>
+                      </completionHelp>
+                      <valueHelp>
+                        <format>single-table</format>
+                        <description>Metrics stores in one table</description>
+                      </valueHelp>
+                      <valueHelp>
+                        <format>table-per-metric</format>
+                        <description>One table per gorups of metric by the metric name</description>
+                      </valueHelp>
+                      <constraint>
+                        <regex>(single-table|table-per-metric)</regex>
+                      </constraint>
+                    </properties>
+                    <defaultValue>table-per-metric</defaultValue>
+                  </leafNode>
+                  <leafNode name="table">
+                    <properties>
+                      <help>Name of the single table [Only if set group-metrics single-table]</help>
+                      <valueHelp>
+                        <format>txt</format>
+                        <description>Table name</description>
+                      </valueHelp>
+                      <constraint>
+                        <regex>[-_a-zA-Z0-9]+</regex>
+                      </constraint>
+                      <constraintErrorMessage>Table is limited to alphanumerical characters and can contain hyphen and underscores</constraintErrorMessage>
+                    </properties>
+                  </leafNode>
+                  #include <include/monitoring/url.xml.i>
+                </children>
+              </node>
               <leafNode name="bucket">
                 <properties>
                   <help>Remote bucket</help>
@@ -206,19 +294,7 @@
                   </leafNode>
                 </children>
               </node>
-              <leafNode name="url">
-                <properties>
-                  <help>Remote URL [REQUIRED]</help>
-                  <valueHelp>
-                    <format>url</format>
-                    <description>Remote URL to InfluxDB v2</description>
-                  </valueHelp>
-                  <constraint>
-                    <regex>(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}?(\/.*)?</regex>
-                  </constraint>
-                  <constraintErrorMessage>Incorrect URL format.</constraintErrorMessage>
-                </properties>
-              </leafNode>
+              #include <include/monitoring/url.xml.i>
               #include <include/port-number.xml.i>
               <leafNode name="port">
                 <defaultValue>8086</defaultValue>
diff --git a/src/conf_mode/service_monitoring_telegraf.py b/src/conf_mode/service_monitoring_telegraf.py
index 102a87318..daf75d740 100755
--- a/src/conf_mode/service_monitoring_telegraf.py
+++ b/src/conf_mode/service_monitoring_telegraf.py
@@ -103,11 +103,28 @@ def get_config(config=None):
        'url' in monitoring:
         monitoring['influxdb_configured'] = True
 
+    # Redefine azure group-metrics 'single-table' and 'table-per-metric'
+    if 'azure_data_explorer' in monitoring:
+        if 'single-table' in monitoring['azure_data_explorer']['group_metrics']:
+            monitoring['azure_data_explorer']['group_metrics'] = 'SingleTable'
+        else:
+            monitoring['azure_data_explorer']['group_metrics'] = 'TablePerMetric'
+        # Set azure env
+        if 'authentication' in monitoring['azure_data_explorer']:
+            auth_config = monitoring['azure_data_explorer']['authentication']
+            if {'client_id', 'client_secret', 'tenant_id'} <= set(auth_config):
+                os.environ['AZURE_CLIENT_ID'] = auth_config['client_id']
+                os.environ['AZURE_CLIENT_SECRET'] = auth_config['client_secret']
+                os.environ['AZURE_TENANT_ID'] = auth_config['tenant_id']
+
     # Ignore default XML values if config doesn't exists
     # Delete key from dict
     if not conf.exists(base + ['prometheus-client']):
         del monitoring['prometheus_client']
 
+    if not conf.exists(base + ['azure-data-explorer']):
+        del monitoring['azure_data_explorer']
+
     return monitoring
 
 def verify(monitoring):
@@ -124,6 +141,24 @@ def verify(monitoring):
         if 'url' not in monitoring:
             raise ConfigError(f'Monitoring "url" is mandatory!')
 
+    # Verify azure-data-explorer
+    if 'azure_data_explorer' in monitoring:
+        if 'authentication' not in monitoring['azure_data_explorer'] or \
+           'client_id' not in monitoring['azure_data_explorer']['authentication'] or \
+           'client_secret' not in monitoring['azure_data_explorer']['authentication'] or \
+           'tenant_id' not in monitoring['azure_data_explorer']['authentication']:
+            raise ConfigError(f'Authentication "client-id, client-secret and tenant-id" are mandatory!')
+
+        if 'database' not in monitoring['azure_data_explorer']:
+            raise ConfigError(f'Monitoring "database" is mandatory!')
+
+        if 'url' not in monitoring['azure_data_explorer']:
+            raise ConfigError(f'Monitoring "url" is mandatory!')
+
+        if monitoring['azure_data_explorer']['group_metrics'] == 'SingleTable' and \
+            'table' not in monitoring['azure_data_explorer']:
+             raise ConfigError(f'Monitoring "table" name for single-table mode is mandatory!')
+
     # Verify Splunk
     if 'splunk' in monitoring:
         if 'authentication' not in monitoring['splunk'] or \