geoip: T7926: Refactor geoip handling

* Move core logic to separate vyos.geoip module
* Use a sqlite database for storing and querying address ranges by country
* Remove downloaded geoip ranges once loaded into sqlite db
* No longer rebuild geoip sets on each commit unless necessary
* Allows for extensibility using other geoip data vendors

3 files changed, 88 insertions, 75 deletions

diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index a609e305c..18c250d08 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -22,13 +22,13 @@ from glob import glob
 from sys import exit
 from vyos.base import Warning
 from vyos.config import Config
-from vyos.configdict import is_node_changed
-from vyos.configdiff import get_config_diff, Diff
+from vyos.configdict import is_node_changed, node_changed
+from vyos.configdiff import Diff
 from vyos.configdep import set_dependents, call_dependents
 from vyos.configverify import verify_interface_exists
 from vyos.ethtool import Ethtool
 from vyos.firewall import fqdn_config_parse
-from vyos.firewall import geoip_update
+from vyos.geoip import geoip_refresh, geoip_update
 from vyos.template import render
 from vyos.utils.dict import dict_search
 from vyos.utils.dict import dict_search_args
@@ -80,42 +80,29 @@ snmp_event_source = 1
 snmp_trap_mib = 'VYATTA-TRAP-MIB'
 snmp_trap_name = 'mgmtEventTrap'
 
-def geoip_updated(conf, firewall):
-    diff = get_config_diff(conf)
-    node_diff = diff.get_child_nodes_diff(['firewall'], expand_nodes=Diff.DELETE, recursive=True)
+def geoip_sets(firewall):
+    out = {'name': [], 'ipv6_name': []}
 
-    out = {
-        'name': [],
-        'ipv6_name': [],
-        'deleted_name': [],
-        'deleted_ipv6_name': []
-    }
-    updated = False
-
-    for key, path in dict_search_recursive(firewall, 'geoip'):
-        set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
+    for _, path in dict_search_recursive(firewall, 'geoip'):
         if (path[0] == 'ipv4'):
-            out['name'].append(set_name)
+            out['name'].append(f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}')
         elif (path[0] == 'ipv6'):
-            set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}'
-            out['ipv6_name'].append(set_name)
+            out['ipv6_name'].append(f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}')
 
-        updated = True
+    return out
 
-    if 'delete' in node_diff:
-        for key, path in dict_search_recursive(node_diff['delete'], 'geoip'):
-            set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
-            if (path[0] == 'ipv4'):
-                out['deleted_name'].append(set_name)
-            elif (path[0] == 'ipv6'):
-                set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
-                out['deleted_ipv6_name'].append(set_name)
-            updated = True
+def geoip_updated(conf):
+    changes = node_changed(conf, ['firewall'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
+    updated = False
 
-    if updated:
-        return out
+    for _, path in dict_search_recursive(changes, 'geoip'):
+        updated = True
+        break
 
-    return False
+    return updated
 
 def get_config(config=None):
     if config:
@@ -135,7 +122,8 @@ def get_config(config=None):
         # Update nat and policy-route as firewall groups were updated
         set_dependents('group_resync', conf)
 
-    firewall['geoip_updated'] = geoip_updated(conf, firewall)
+    firewall['geoip_sets'] = geoip_sets(firewall)
+    firewall['geoip_updated'] = geoip_updated(conf)
 
     fqdn_config_parse(firewall, 'firewall')
 
@@ -725,11 +713,12 @@ def apply(firewall):
             domain_action = 'stop'
     call(f'systemctl {domain_action} vyos-domain-resolver.service')
 
-    if firewall['geoip_updated']:
+    if firewall['geoip_sets']:
         # Call helper script to Update set contents
-        if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']:
-            print('Updating GeoIP. Please wait...')
-            geoip_update(firewall=firewall)
+        if 'name' in firewall['geoip_sets'] or 'ipv6_name' in firewall['geoip_sets']:
+            if firewall['geoip_updated'] or not geoip_refresh():
+                print('Updating GeoIP. Please wait...')
+                geoip_update(firewall)
 
     return None
 
diff --git a/src/conf_mode/policy_route.py b/src/conf_mode/policy_route.py
index 15d735f75..17f8e6bf9 100755
--- a/src/conf_mode/policy_route.py
+++ b/src/conf_mode/policy_route.py
@@ -21,7 +21,8 @@ from sys import exit
 
 from vyos.base import Warning
 from vyos.config import Config
-from vyos.configdiff import get_config_diff, Diff
+from vyos.configdict import node_changed
+from vyos.configdiff import Diff
 from vyos.template import render
 from vyos.utils.dict import dict_search_args
 from vyos.utils.dict import dict_search_recursive
@@ -30,7 +31,7 @@ from vyos.utils.process import run
 from vyos.utils.network import get_vrf_tableid
 from vyos.defaults import rt_global_table
 from vyos.defaults import rt_global_vrf
-from vyos.firewall import geoip_update
+from vyos.geoip import  geoip_refresh, geoip_update
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -46,42 +47,40 @@ valid_groups = [
     'interface_group'
 ]
 
-def geoip_updated(conf, policy):
-    diff = get_config_diff(conf)
-    node_diff = diff.get_child_nodes_diff(['policy'], expand_nodes=Diff.DELETE, recursive=True)
-
-    out = {
-        'name': [],
-        'ipv6_name': [],
-        'deleted_name': [],
-        'deleted_ipv6_name': []
-    }
+def geoip_updated(conf):
     updated = False
 
-    for key, path in dict_search_recursive(policy, 'geoip'):
-        set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}'
-        if (path[0] == 'route'):
-            out['name'].append(set_name)
-        elif (path[0] == 'route6'):
-            set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}'
-            out['ipv6_name'].append(set_name)
+    changes_v4 = node_changed(conf, ['policy', 'route'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
 
+    for _, path in dict_search_recursive(changes_v4, 'geoip'):
         updated = True
+        break
 
-    if 'delete' in node_diff:
-        for key, path in dict_search_recursive(node_diff['delete'], 'geoip'):
-            set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}'
-            if (path[0] == 'route'):
-                out['deleted_name'].append(set_name)
-            elif (path[0] == 'route6'):
-                set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}'
-                out['deleted_ipv6_name'].append(set_name)
+    if not updated:
+        changes_v6 = node_changed(conf, ['policy', 'route6'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
+
+        for _, path in dict_search_recursive(changes_v6, 'geoip'):
             updated = True
+            break
 
-    if updated:
-        return out
+    return updated
 
-    return False
+def geoip_sets(policy):
+    out = {'name': [], 'ipv6_name': []}
+
+    for _, path in dict_search_recursive(policy, 'geoip'):
+        if (path[0] == 'route'):
+            out['name'].append(f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}')
+        elif (path[0] == 'route6'):
+            out['ipv6_name'].append(f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}')
+
+    return out
 
 def get_config(config=None):
     if config:
@@ -100,7 +99,9 @@ def get_config(config=None):
     if 'dynamic_group' in policy['firewall_group']:
         del policy['firewall_group']['dynamic_group']
 
-    policy['geoip_updated'] = geoip_updated(conf, policy)
+    policy['geoip_sets'] = geoip_sets(policy)
+    policy['geoip_updated'] = geoip_updated(conf)
+
     return policy
 
 def verify_rule(policy, name, rule_conf, ipv6, rule_id):
@@ -244,11 +245,12 @@ def apply(policy):
 
     apply_table_marks(policy)
 
-    if policy['geoip_updated']:
+    if policy['geoip_sets']:
         # Call helper script to Update set contents
-        if 'name' in policy['geoip_updated'] or 'ipv6_name' in policy['geoip_updated']:
-            print('Updating GeoIP. Please wait...')
-            geoip_update(policy=policy)
+        if 'name' in policy['geoip_sets'] or 'ipv6_name' in policy['geoip_sets']:
+            if policy['geoip_updated'] or not geoip_refresh():
+                print('Updating GeoIP. Please wait...')
+                geoip_update(policy=policy)
 
     return None
 
diff --git a/src/helpers/geoip-update.py b/src/helpers/geoip-update.py
index 22d26e538..a6838c62d 100755
--- a/src/helpers/geoip-update.py
+++ b/src/helpers/geoip-update.py
@@ -18,7 +18,11 @@ import argparse
 import sys
 
 from vyos.configquery import ConfigTreeQuery
-from vyos.firewall import geoip_update
+from vyos.geoip import geoip_download_dbip
+from vyos.geoip import db_initialise
+from vyos.geoip import db_is_initialised
+from vyos.geoip import db_import_dbip_ranges
+from vyos.geoip import geoip_update
 
 def get_config(config=None):
     if config:
@@ -35,9 +39,27 @@ def get_config(config=None):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("--force", help="Force update", action="store_true")
+    parser.add_argument("--init", help="Initialise", action="store_true")
     args = parser.parse_args()
 
+    if args.init:
+        db_initialise()
+        db_import_dbip_ranges(delete_file=True)
+        sys.exit(0)
+
+    if not db_is_initialised():
+        db_initialise()
+
+    print('Dowloading latest DB-IP database...')
+    if not geoip_download_dbip():
+        print('Failed to download, aborting.')
+        sys.exit(1)
+
+    print('Extracting database...')
+    if not db_import_dbip_ranges(delete_file=True):
+        print('Failed to extract, aborting.')
+        sys.exit(1)
+
     firewall, policy = get_config()
-    if not geoip_update(firewall=firewall, policy=policy, force=args.force):
+    if not geoip_update(firewall=firewall, policy=policy):
         sys.exit(1)