Merge branch 'nat-integration' of github.com:c-po/vyos-1x into current

* 'nat-integration' of github.com:c-po/vyos-1x:
  nat: T2460: fix KeyError: 'sport'
  nat: T2460: migrate to new Python implementation
  nat: T2460: add src/op_mode/show_nat_translations.py

4 files changed, 299 insertions, 366 deletions

diff --git a/debian/control b/debian/control
index a30b80b7a..1312d9462 100644
--- a/debian/control
+++ b/debian/control
@@ -33,6 +33,7 @@ Depends: python3,
   python3-netaddr,
   python3-zmq,
   python3-jmespath,
+  python3-xmltodict,
   bsdmainutils,
   cron,
   systemd,
@@ -94,7 +95,6 @@ Depends: python3,
   pmacct (>= 1.6.0),
   python3-certbot-nginx,
   pppoe,
-  libxml-simple-perl,
   salt-minion,
   vyos-utils,
   nftables (>= 0.9.3),
diff --git a/op-mode-definitions/nat.xml b/op-mode-definitions/nat.xml
index ffaa2cba3..f6c0fa748 100644
--- a/op-mode-definitions/nat.xml
+++ b/op-mode-definitions/nat.xml
@@ -1,98 +1,98 @@
-<?xml version="1.0" encoding="UTF-8"?>

-<interfaceDefinition>

-  <node name="show">

-    <children>

-      <node name="nat">

-        <properties>

-          <help>Show Network Address Translation (NAT) information</help>

-        </properties>

-        <children>

-          <node name="source">

-            <properties>

-              <help>Show source Network Address Translation (NAT) information</help>

-            </properties>

-            <children>

-              <node name="rules">

-                <properties>

-                  <help>Show configured source NAT rules</help>

-                </properties>

-                <command>echo To be migrated to Python - https://phabricator.vyos.net/T2459</command>

-              </node>

-              <node name="statistics">

-                <properties>

-                  <help>Show statistics for configured source NAT rules</help>

-                </properties>

-                <command>${vyos_op_scripts_dir}/show_nat_statistics.py --source</command>

-              </node>

-              <node name="translations">

-                <properties>

-                  <help>Show active source NAT translations</help>

-                </properties>

-                <children>

-                  <tagNode name="address">

-                    <properties>

-                      <help>Show active source NAT translations for an IP address</help>

-                      <completionHelp>

-                        <list>&lt;x.x.x.x&gt;</list>

-                      </completionHelp>

-                    </properties>

-                    <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=source --verbose --ipaddr="$6"</command>

-                  </tagNode>

-                  <node name="detail">

-                    <properties>

-                      <help>Show active source NAT translations detail</help>

-                    </properties>

-                    <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=source --verbose</command>

-                  </node>

-                </children>

-                <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=source</command>

-              </node>

-            </children>

-          </node>

-          <node name="destination">

-            <properties>

-              <help>Show destination Network Address Translation (NAT) information</help>

-            </properties>

-            <children>

-              <node name="rules">

-                <properties>

-                  <help>Show configured destination NAT rules</help>

-                </properties>

-                <command>echo To be migrated to Python - https://phabricator.vyos.net/T2459</command>

-              </node>

-              <node name="statistics">

-                <properties>

-                  <help>Show statistics for configured destination NAT rules</help>

-                </properties>

-                <command>${vyos_op_scripts_dir}/show_nat_statistics.py --destination</command>

-              </node>

-              <node name="translations">

-                <properties>

-                  <help>Show active destination NAT translations</help>

-                </properties>

-                <children>

-                  <tagNode name="address">

-                    <properties>

-                      <help>Show active NAT destination translations for an IP address</help>

-                      <completionHelp>

-                        <list>&lt;x.x.x.x&gt;</list>

-                      </completionHelp>

-                    </properties>

-                    <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=destination --verbose --ipaddr="$6"</command>

-                  </tagNode>

-                  <node name="detail">

-                    <properties>

-                      <help>Show active destination NAT translations detail</help>

-                    </properties>

-                    <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=destination --verbose</command>

-                  </node>

-                </children>

-                <command>${vyos_op_scripts_dir}/to_be_migrated/vyatta-nat-translations.pl --type=destination</command>

-              </node>

-            </children>

-          </node>

-        </children>

-      </node>

-    </children>

-  </node>

-</interfaceDefinition>

+<?xml version="1.0" encoding="UTF-8"?>
+<interfaceDefinition>
+  <node name="show">
+    <children>
+      <node name="nat">
+        <properties>
+          <help>Show Network Address Translation (NAT) information</help>
+        </properties>
+        <children>
+          <node name="source">
+            <properties>
+              <help>Show source Network Address Translation (NAT) information</help>
+            </properties>
+            <children>
+              <node name="rules">
+                <properties>
+                  <help>Show configured source NAT rules</help>
+                </properties>
+                <command>echo To be migrated to Python - https://phabricator.vyos.net/T2459</command>
+              </node>
+              <node name="statistics">
+                <properties>
+                  <help>Show statistics for configured source NAT rules</help>
+                </properties>
+                <command>${vyos_op_scripts_dir}/show_nat_statistics.py --source</command>
+              </node>
+              <node name="translations">
+                <properties>
+                  <help>Show active source NAT translations</help>
+                </properties>
+                <children>
+                  <tagNode name="address">
+                    <properties>
+                      <help>Show active source NAT translations for an IP address</help>
+                      <completionHelp>
+                        <list>&lt;x.x.x.x&gt;</list>
+                      </completionHelp>
+                    </properties>
+                    <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=source --verbose --ipaddr="$6"</command>
+                  </tagNode>
+                  <node name="detail">
+                    <properties>
+                      <help>Show active source NAT translations detail</help>
+                    </properties>
+                    <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=source --verbose</command>
+                  </node>
+                </children>
+                <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=source</command>
+              </node>
+            </children>
+          </node>
+          <node name="destination">
+            <properties>
+              <help>Show destination Network Address Translation (NAT) information</help>
+            </properties>
+            <children>
+              <node name="rules">
+                <properties>
+                  <help>Show configured destination NAT rules</help>
+                </properties>
+                <command>echo To be migrated to Python - https://phabricator.vyos.net/T2459</command>
+              </node>
+              <node name="statistics">
+                <properties>
+                  <help>Show statistics for configured destination NAT rules</help>
+                </properties>
+                <command>${vyos_op_scripts_dir}/show_nat_statistics.py --destination</command>
+              </node>
+              <node name="translations">
+                <properties>
+                  <help>Show active destination NAT translations</help>
+                </properties>
+                <children>
+                  <tagNode name="address">
+                    <properties>
+                      <help>Show active NAT destination translations for an IP address</help>
+                      <completionHelp>
+                        <list>&lt;x.x.x.x&gt;</list>
+                      </completionHelp>
+                    </properties>
+                    <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=destination --verbose --ipaddr="$6"</command>
+                  </tagNode>
+                  <node name="detail">
+                    <properties>
+                      <help>Show active destination NAT translations detail</help>
+                    </properties>
+                    <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=destination --verbose</command>
+                  </node>
+                </children>
+                <command>${vyos_op_scripts_dir}/show_nat_translations.py --type=destination</command>
+              </node>
+            </children>
+          </node>
+        </children>
+      </node>
+    </children>
+  </node>
+</interfaceDefinition>
diff --git a/src/op_mode/show_nat_translations.py b/src/op_mode/show_nat_translations.py
new file mode 100755
index 000000000..3af33b78e
--- /dev/null
+++ b/src/op_mode/show_nat_translations.py
@@ -0,0 +1,200 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020 VyOS maintainers and contributors
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 2 or later as
+# published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''
+show nat translations
+'''
+
+import os
+import sys
+import ipaddress
+import argparse
+import xmltodict
+
+from vyos.util import popen
+from vyos.util import DEVNULL
+
+conntrack = '/usr/sbin/conntrack'
+
+verbose_format = "%-20s %-18s %-20s %-18s"
+normal_format = "%-20s %-20s %-4s  %-8s %s"
+
+
+def headers(verbose, pipe):
+    if verbose:
+        return verbose_format % ('Pre-NAT src', 'Pre-NAT dst', 'Post-NAT src', 'Post-NAT dst')
+    return normal_format % ('Pre-NAT', 'Post-NAT', 'Prot', 'Timeout', 'Type' if pipe else '')
+
+
+def command(srcdest, proto, ipaddr):
+    command = f'{conntrack} -o xml -L'
+
+    if proto:
+        command += f' -p {proto}'
+
+    if srcdest == 'source':
+        command += ' -n'
+        if ipaddr:
+            command += f' --orig-src {ipaddr}'
+    if srcdest == 'destination':
+        command += ' -g'
+
+    return command
+
+
+def run(command):
+    xml, code = popen(command,stderr=DEVNULL)
+    if code:
+        sys.exit('conntrack failed')
+    return xml
+
+
+def content(xmlfile):
+    xml = ''
+    with open(xmlfile,'r') as r:
+        xml += r.read()
+    return xml
+
+
+def pipe():
+    xml = ''
+    while True:
+        line = sys.stdin.readline()
+        xml += line
+        if '</conntrack>' in line:
+            break
+
+    sys.stdin = open('/dev/tty')
+    return xml
+
+
+def process(data, stats, protocol, pipe, verbose, flowtype=''):
+    if not data:
+        return
+
+    parsed = xmltodict.parse(data)
+
+    print(headers(verbose, pipe))
+
+    # to help the linter to detect typos
+    ORIGINAL = 'original'
+    REPLY = 'reply'
+    INDEPENDANT = 'independent'
+    SPORT = 'sport'
+    DPORT = 'dport'
+    SRC = 'src'
+    DST = 'dst'
+
+    for rule in parsed['conntrack']['flow']:
+        src, dst, sport, dport, proto = {}, {}, {}, {}, {}
+        packet_count, byte_count = {}, {}
+        timeout, use = 0, 0
+
+        rule_type = rule.get('type', '')
+
+        for meta in rule['meta']:
+            # print(meta)
+            direction = meta['@direction']
+
+            if direction in (ORIGINAL, REPLY):
+                if 'layer3' in meta:
+                    l3 = meta['layer3']
+                    src[direction] = l3[SRC]
+                    dst[direction] = l3[DST]
+
+                if 'layer4' in meta:
+                    l4 = meta['layer4']
+                    sp = l4.get(SPORT, '')
+                    dp = l4.get(DPORT, '')
+                    if sp:
+                        sport[direction] = sp
+                    if dp:
+                        dport[direction] = dp
+                    proto[direction] = l4.get('@protoname','')
+
+                if stats and 'counters' in meta:
+                    packet_count[direction] = meta['packets']
+                    byte_count[direction] = meta['bytes']
+                continue
+
+            if direction == INDEPENDANT:
+                timeout = meta['timeout']
+                use = meta['use']
+                continue
+
+        in_src = '%s:%s' % (src[ORIGINAL], sport[ORIGINAL]) if ORIGINAL in sport else src[ORIGINAL]
+        in_dst = '%s:%s' % (dst[ORIGINAL], dport[ORIGINAL]) if ORIGINAL in dport else dst[ORIGINAL]
+
+        # inverted the the perl code !!?
+        out_dst = '%s:%s' % (dst[REPLY], dport[REPLY]) if REPLY in dport else dst[REPLY]
+        out_src = '%s:%s' % (src[REPLY], sport[REPLY]) if REPLY in sport else src[REPLY]
+
+        if flowtype == 'source':
+            v = ORIGINAL in sport and REPLY in dport
+            f = '%s:%s' % (src[ORIGINAL], sport[ORIGINAL]) if v else src[ORIGINAL]
+            t = '%s:%s' % (dst[REPLY], dport[REPLY]) if v else dst[REPLY]
+        else:
+            v = ORIGINAL in dport and REPLY in sport
+            f = '%s:%s' % (dst[ORIGINAL], dport[ORIGINAL]) if v else dst[ORIGINAL]
+            t = '%s:%s' % (src[REPLY], sport[REPLY]) if v else src[REPLY]
+
+        # Thomas: I do not believe proto should be an option
+        p = proto.get('original', '')
+        if protocol and p != protocol:
+            continue
+
+        if verbose:
+            msg = verbose_format % (in_src, in_dst, out_dst, out_src)
+            p = f'{p}: ' if p else ''
+            msg += f'\n  {p}{f} ==> {t}'
+            msg += f' timeout: {timeout}' if timeout else ''
+            msg += f' use: {use} ' if use else ''
+            msg += f' type: {rule_type}' if rule_type else ''
+            print(msg)
+        else:
+            print(normal_format % (f, t, p, timeout, rule_type if rule_type else ''))
+
+        if stats:
+            for direction in ('original', 'reply'):
+                if direction in packet_count:
+                    print('  %-8s: packets %s, bytes %s' % direction, packet_count[direction], byte_count[direction])
+
+
+def main():
+    parser = argparse.ArgumentParser(description=sys.modules[__name__].__doc__)
+    parser.add_argument('--verbose', help='provide more details about the flows', action='store_true')
+    parser.add_argument('--proto', help='filter by protocol', default='', type=str)
+    parser.add_argument('--file', help='read the conntrack xml from a file', type=str)
+    parser.add_argument('--stats', help='add usage statistics', action='store_true')
+    parser.add_argument('--type', help='NAT type (source, destination)', required=True, type=str)
+    parser.add_argument('--ipaddr', help='source ip address to filter on', type=ipaddress.ip_address)
+    parser.add_argument('--pipe', help='read conntrack xml data from stdin', action='store_true')
+
+    arg = parser.parse_args()
+
+    if arg.type not in ('source', 'destination'):
+        sys.exit('Unknown NAT type!')
+
+    if arg.pipe:
+        process(pipe(), arg.stats, arg.proto, arg.pipe, arg.verbose, arg.type)
+    elif arg.file:
+        process(content(arg.file), arg.stats, arg.proto, arg.pipe, arg.verbose, arg.type)
+    else:
+        process(run(command(arg.type, arg.proto, arg.ipaddr)), arg.stats, arg.proto, arg.pipe, arg.verbose, arg.type)
+
+
+if __name__ == '__main__':
+    main()
diff --git a/src/op_mode/to_be_migrated/vyatta-nat-translations.pl b/src/op_mode/to_be_migrated/vyatta-nat-translations.pl
deleted file mode 100755
index 94ed74bad..000000000
--- a/src/op_mode/to_be_migrated/vyatta-nat-translations.pl
+++ /dev/null
@@ -1,267 +0,0 @@
-#!/usr/bin/perl
-#
-# Module: vyatta-nat-translate.pl
-# 
-# **** License ****
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 2 as
-# published by the Free Software Foundation.
-# 
-# This program is distributed in the hope that it will be useful, but
-# WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# General Public License for more details.
-# 
-# This code was originally developed by Vyatta, Inc.
-# Portions created by Vyatta are Copyright (C) 2007 Vyatta, Inc.
-# All Rights Reserved.
-# 
-# Author: Stig Thormodsrud
-# Date: July 2008
-# Description: Script to display nat translations
-# 
-# **** End License ****
-#
-
-use Getopt::Long;
-use XML::Simple;
-use Data::Dumper;
-use POSIX;
-
-use warnings;
-use strict;
-
-my $dump = 0;
-my ($xml_file, $verbose, $proto, $stats, $ipaddr, $pipe);
-my $type;
-my $verbose_format = "%-20s %-18s %-20s %-18s\n";
-my $format         = "%-20s %-20s %-4s  %-8s";
-
-sub add_xml_root {
-    my $xml = shift;
-
-    $xml = '<data>' . $xml . '</data>';
-    return $xml;
-}
-
-
-sub read_xml_file {
-    my $file = shift;
-
-    local($/, *FD);  # slurp mode
-    open FD, "<", $file or die "Couldn't open $file\n";
-    my $xml = <FD>;
-    close FD;
-    return $xml;
-}
-
-sub print_xml {
-    my $data = shift;
-    print Dumper($data); 
-}
-
-sub guess_snat_dnat {
-    my ($src, $dst) = @_;
-
-    if ($src->{original} eq $dst->{reply}) {
-	return "dnat";
-    }
-    if ($dst->{original} eq $src->{reply}) {
-	return "snat";
-    }
-    return "unkn";
-}
-
-sub nat_print_xml {
-    my ($data, $type) = @_;
-
-    my $flow = 0;
-
-    my %flowh;
-    while (1) {
-	my $meta = 0;
-	last if ! defined $data->{flow}[$flow];
-	my $flow_ref = $data->{flow}[$flow];
-	my $flow_type = $flow_ref->{type};
-	my (%src, %dst, %sport, %dport, %proto);
-	my (%packets, %bytes);
-	my $timeout = undef;
-	my $uses = undef;
-	while (1) {
-	    my $meta_ref = $flow_ref->{meta}[$meta];
-	    last if ! defined $meta_ref;
-	    my $dir = $meta_ref->{direction};
-	    if ($dir eq 'original' or $dir eq 'reply') {
-		my $l3_ref    = $meta_ref->{layer3}[0];
-		my $l4_ref    = $meta_ref->{layer4}[0];
-		my $count_ref = $meta_ref->{counters}[0];
-		if (defined $l3_ref) {
-		    $src{$dir} = $l3_ref->{src}[0];
-		    $dst{$dir} = $l3_ref->{dst}[0];
-		    if (defined $l4_ref) {
-			$sport{$dir} = $l4_ref->{sport}[0];
-			$dport{$dir} = $l4_ref->{dport}[0];
-			$proto{$dir} = $l4_ref->{protoname};
-		    }
-		}
-		if (defined $stats and defined $count_ref) {
-		    $packets{$dir} = $count_ref->{packets}[0];
-		    $bytes{$dir}   = $count_ref->{bytes}[0];
-		}
-	    } elsif ($dir eq 'independent') {
-		$timeout = $meta_ref->{timeout}[0];
-		$uses    = $meta_ref->{'use'}[0];
-	    }
-	    $meta++;
-	}
-	my ($proto, $in_src, $in_dst, $out_src, $out_dst);
-	$proto    = $proto{original};
-	$in_src   = "$src{original}";
-	$in_src  .= ":$sport{original}" if defined $sport{original};
-	$in_dst   = "$dst{original}";
-	$in_dst  .= ":$dport{original}" if defined $dport{original};
-	$out_src  = "$dst{reply}";
-	$out_src .= ":$dport{reply}" if defined $dport{reply};
-	$out_dst  = "$src{reply}";
-	$out_dst .= ":$sport{reply}" if defined $sport{reply};
-	if (defined $verbose) {
-	    printf($verbose_format, $in_src, $in_dst, $out_src, $out_dst);
-	}
-#	if (! defined $type) {
-#	    $type = guess_snat_dnat(\%src, \%dst);
-#	}
-	if (defined $type) {
-	    my ($from, $to);
-	    if ($type eq 'source') {
-		$from = "$src{original}";
-		$to   = "$dst{reply}";
-		if (defined $sport{original} and defined $dport{reply}) {
-		    if ($sport{original} ne $dport{reply}) {
-			$from .= ":$sport{original}";
-			$to   .= ":$dport{reply}";
-		    }
-		}
-	    } else {
-		$from = "$dst{original}";
-		$to   = "$src{reply}";
-		if (defined $dport{original} and defined $sport{reply}) {
-		    if ($dport{original} ne $sport{reply}) {
-			$from .= ":$dport{original}";
-			$to   .= ":$sport{reply}";
-		    }
-		}
-	    }
-	    if (defined $verbose) {
-		print "  $proto: $from ==> $to";
-	    } else {
-		my $timeout2 = "";
-		if (defined $timeout) {
-		    $timeout2 = $timeout;
-		}
-		printf($format, $from, $to, $proto, $timeout2);
-		print " $flow_type" if defined $flow_type;
-		print "\n";
-	    }
-	}
-	if (defined $verbose) {
-	    print "  timeout: $timeout" if defined $timeout;
-	    print " use: $uses " if defined $uses;
-	    print " type: $flow_type" if defined $flow_type;
-	    print "\n";
-	}
-	if (defined $stats) {
-	    foreach my $dir ('original', 'reply') {
-		if (defined $packets{$dir}) {
-                    printf("  %-8s: packets %s, bytes %s\n",
-		           $dir, $packets{$dir}, $bytes{$dir});
-		}
-	    }
-	}
-	$flow++;
-    }
-    return $flow;
-}
-
-
-#
-# main
-#
-GetOptions("verbose"  => \$verbose,
-	   "proto=s"  => \$proto,
-	   "file=s"   => \$xml_file,
-	   "stats"    => \$stats,
-	   "type=s"   => \$type,
-	   "ipaddr=s" => \$ipaddr,
-	   "pipe"     => \$pipe,
-);
-
-my $conntrack = '/usr/sbin/conntrack';
-if  (! -f $conntrack) {
-    die "Package [conntrack] not installed";
-}
-
-die "Must specify NAT type!" if !defined($type);
-die "Unknown NAT type!" if (($type ne 'source') && ($type ne 'destination'));
-
-my $xs = XML::Simple->new(ForceArray => 1, KeepRoot => 0);
-my ($xml, $data);
-
-# flush stdout after every write for pipe mode
-$| = 1 if defined $pipe; 
-
-if (defined $verbose) {
-    printf($verbose_format, 'Pre-NAT src', 'Pre-NAT dst', 
-	   'Post-NAT src', 'Post-NAT dst');
-} else {
-    printf($format, 'Pre-NAT', 'Post-NAT', 'Prot', 'Timeout');
-    print " Type" if defined $pipe;
-    print "\n";
-}
-
-if (defined $xml_file) {
-    $xml = read_xml_file($xml_file);
-    $data = $xs->XMLin($xml);
-    if ($dump) {
-	print_xml($data);
-	exit;
-    }
-    nat_print_xml($data, 'snat');
-
-} elsif (defined $pipe) {
-    while ($xml = <STDIN>) {
-	$xml =~ s/\<\?xml version=\"1\.0\" encoding=\"utf-8\"\?\>//;
-	$xml =~ s/\<conntrack\>//;
-	$xml = add_xml_root($xml);
-	$data = $xs->XMLin($xml);
-	nat_print_xml($data, $type);
-    }
-} else {
-    if (defined $proto) {
-	$proto = "-p $proto" 
-    } else {
-	$proto = "";
-    }
-    if ($type eq 'source') {
-	my $ipopt = "";
-	if (defined $ipaddr) {
-	    $ipopt = "--orig-src $ipaddr";
-	} 
-	$xml = `sudo $conntrack -L -n $ipopt -o xml $proto 2>/dev/null`;
-	chomp $xml;
-	$data = undef;
-	$data = $xs->XMLin($xml) if ! $xml eq '';
-    }
-    if ($type eq 'destination') {
-	my $ipopt = "";
-	if (defined $ipaddr) {
-	    $ipopt = "--orig-dst $ipaddr";
-	} 
-	$xml = `sudo $conntrack -L -g $ipopt -o xml $proto 2>/dev/null`;
-	chomp $xml;
-	$data = undef;
-	$data = $xs->XMLin($xml) if ! $xml eq '';
-    }
-    nat_print_xml($data, $type) if defined $data;
-}
-
-# end of file