Merge pull request #4949 from sarthurdev/geoip_refactor

geoip: T7926: T8049: Refactor geoip code and add MaxMind support

10 files changed, 438 insertions, 249 deletions

diff --git a/data/templates/firewall/nftables-geoip-update.j2 b/data/templates/firewall/nftables-geoip-update.j2
index d8f80d1f5..c51e4aafc 100644
--- a/data/templates/firewall/nftables-geoip-update.j2
+++ b/data/templates/firewall/nftables-geoip-update.j2
@@ -1,12 +1,12 @@
 #!/usr/sbin/nft -f
 
-{% if ipv4_sets is vyos_defined %}
-{%     for setname, ip_list in ipv4_sets.items() %}
+{% if firewall_sets.v4 is vyos_defined %}
+{%     for setname, ip_list in firewall_sets.v4.items() %}
 flush set ip vyos_filter {{ setname }}
 {%     endfor %}
 
 table ip vyos_filter {
-{%     for setname, ip_list in ipv4_sets.items() %}
+{%     for setname, ip_list in firewall_sets.v4.items() %}
     set {{ setname }} {
         type ipv4_addr
         flags interval
@@ -16,13 +16,13 @@ table ip vyos_filter {
 }
 {% endif %}
 
-{% if ipv6_sets is vyos_defined %}
-{%     for setname, ip_list in ipv6_sets.items() %}
+{% if firewall_sets.v6 is vyos_defined %}
+{%     for setname, ip_list in firewall_sets.v6.items() %}
 flush set ip6 vyos_filter {{ setname }}
 {%     endfor %}
 
 table ip6 vyos_filter {
-{%     for setname, ip_list in ipv6_sets.items() %}
+{%     for setname, ip_list in firewall_sets.v6.items() %}
     set {{ setname }} {
         type ipv6_addr
         flags interval
@@ -33,13 +33,13 @@ table ip6 vyos_filter {
 {% endif %}
 
 
-{% if ipv4_sets_policy is vyos_defined %}
-{%     for setname, ip_list in ipv4_sets_policy.items() %}
+{% if policy_sets.v4 is vyos_defined %}
+{%     for setname, ip_list in policy_sets.v4.items() %}
 flush set ip vyos_mangle {{ setname }}
 {%     endfor %}
 
 table ip vyos_mangle {
-{%     for setname, ip_list in ipv4_sets_policy.items() %}
+{%     for setname, ip_list in policy_sets.v4.items() %}
     set {{ setname }} {
         type ipv4_addr
         flags interval
@@ -49,13 +49,13 @@ table ip vyos_mangle {
 }
 {% endif %}
 
-{% if ipv6_sets_policy is vyos_defined %}
-{%     for setname, ip_list in ipv6_sets_policy.items() %}
+{% if policy_sets.v6 is vyos_defined %}
+{%     for setname, ip_list in policy_sets.v6.items() %}
 flush set ip6 vyos_mangle {{ setname }}
 {%     endfor %}
 
 table ip6 vyos_mangle {
-{%     for setname, ip_list in ipv6_sets_policy.items() %}
+{%     for setname, ip_list in policy_sets.v6.items() %}
     set {{ setname }} {
         type ipv6_addr
         flags interval
diff --git a/data/templates/firewall/nftables-policy.j2 b/data/templates/firewall/nftables-policy.j2
index 00d0e8a62..68f79e656 100644
--- a/data/templates/firewall/nftables-policy.j2
+++ b/data/templates/firewall/nftables-policy.j2
@@ -34,8 +34,8 @@ table ip vyos_mangle {
     }
 {%     endfor %}
 
-{%     if geoip_updated.name is vyos_defined %}
-{%         for setname in geoip_updated.name %}
+{%     if geoip_sets.name is vyos_defined %}
+{%         for setname in geoip_sets.name %}
     set {{ setname }} {
         type ipv4_addr
         flags interval
@@ -74,8 +74,8 @@ table ip6 vyos_mangle {
 {%         endif %}
     }
 {%     endfor %}
-{%     if geoip_updated.ipv6_name is vyos_defined %}
-{%         for setname in geoip_updated.ipv6_name %}
+{%     if geoip_sets.ipv6_name is vyos_defined %}
+{%         for setname in geoip_sets.ipv6_name %}
     set {{ setname }} {
         type ipv6_addr
         flags interval
diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2
index f810496f9..903fae330 100755
--- a/data/templates/firewall/nftables.j2
+++ b/data/templates/firewall/nftables.j2
@@ -172,8 +172,8 @@ table ip vyos_filter {
         flags interval
     }
 {%     endfor %}
-{%     if geoip_updated.name is vyos_defined %}
-{%         for setname in geoip_updated.name %}
+{%     if geoip_sets.name is vyos_defined %}
+{%         for setname in geoip_sets.name %}
     set {{ setname }} {
         type ipv4_addr
         flags interval
@@ -359,8 +359,8 @@ table ip6 vyos_filter {
         flags interval
     }
 {%     endfor %}
-{%     if geoip_updated.ipv6_name is vyos_defined %}
-{%         for setname in geoip_updated.ipv6_name %}
+{%     if geoip_sets.ipv6_name is vyos_defined %}
+{%         for setname in geoip_sets.ipv6_name %}
     set {{ setname }} {
         type ipv6_addr
         flags interval
diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i
index e19f3a7c5..7ec07100d 100644
--- a/interface-definitions/include/firewall/global-options.xml.i
+++ b/interface-definitions/include/firewall/global-options.xml.i
@@ -130,6 +130,49 @@
       </properties>
       <defaultValue>enable</defaultValue>
     </leafNode>
+    <node name="geoip">
+      <properties>
+        <help>GeoIP options</help>
+      </properties>
+      <children>
+        <leafNode name="provider">
+          <properties>
+            <help>GeoIP database provider</help>
+            <completionHelp>
+              <list>db-ip maxmind</list>
+            </completionHelp>
+            <valueHelp>
+              <format>db-ip</format>
+              <description>Use GeoIP database by DB-IP.com</description>
+            </valueHelp>
+            <valueHelp>
+              <format>maxmind</format>
+              <description>Use GeoIP database by MaxMind (Requires API key)</description>
+            </valueHelp>
+            <constraint>
+              <regex>(db-ip|maxmind)</regex>
+            </constraint>
+          </properties>
+          <defaultValue>db-ip</defaultValue>
+        </leafNode>
+        <leafNode name="maxmind-account-id">
+          <properties>
+            <help>Account ID for MaxMind GeoIP database</help>
+          </properties>
+        </leafNode>
+        <leafNode name="maxmind-license-key">
+          <properties>
+            <help>License key for MaxMind GeoIP database</help>
+          </properties>
+        </leafNode>
+        <leafNode name="maxmind-lite">
+          <properties>
+            <help>Use MaxMind GeoLite2 database</help>
+            <valueless/>
+          </properties>
+        </leafNode>
+      </children>
+    </node>
     <leafNode name="ip-src-route">
       <properties>
         <help>Policy for handling IPv4 packets with source route option</help>
diff --git a/op-mode-definitions/geoip.xml.in b/op-mode-definitions/geoip.xml.in
index 66ebfccca..0fc3d8df0 100644
--- a/op-mode-definitions/geoip.xml.in
+++ b/op-mode-definitions/geoip.xml.in
@@ -6,7 +6,7 @@
         <properties>
           <help>Update GeoIP database and firewall sets</help>
         </properties>
-        <command>${vyos_libexec_dir}/geoip-update.py --force</command>
+        <command>${vyos_libexec_dir}/geoip-update.py</command>
       </leafNode>
     </children>
   </node>
diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py
index b136b6fca..0025ed98a 100755
--- a/python/vyos/firewall.py
+++ b/python/vyos/firewall.py
@@ -12,24 +12,16 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import csv
-import gzip
-import os
 import re
 
-from pathlib import Path
 from socket import AF_INET
 from socket import AF_INET6
 from socket import getaddrinfo
-from time import strftime
 
-from vyos.remote import download
 from vyos.template import is_ipv4
-from vyos.template import render
 from vyos.utils.dict import dict_search_args
 from vyos.utils.dict import dict_search_recursive
 from vyos.utils.process import cmd
-from vyos.utils.process import run
 from vyos.utils.network import get_vrf_tableid
 from vyos.defaults import rt_global_table
 from vyos.defaults import rt_global_vrf
@@ -691,147 +683,3 @@ def parse_time(time):
         out_days = [f'"{day}"' for day in days if day[0] != '!']
         out.append(f'day {{{",".join(out_days)}}}')
     return " ".join(out)
-
-# GeoIP
-
-nftables_geoip_conf = '/run/nftables-geoip.conf'
-geoip_database = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz'
-geoip_lock_file = '/run/vyos-geoip.lock'
-
-def geoip_load_data(codes=[]):
-    data = None
-
-    if not os.path.exists(geoip_database):
-        return []
-
-    try:
-        with gzip.open(geoip_database, mode='rt') as csv_fh:
-            reader = csv.reader(csv_fh)
-            out = []
-            for start, end, code in reader:
-                if code.lower() in codes:
-                    out.append([start, end, code.lower()])
-            return out
-    except:
-        print('Error: Failed to open GeoIP database')
-    return []
-
-def geoip_download_data():
-    url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m"))
-    try:
-        dirname = os.path.dirname(geoip_database)
-        if not os.path.exists(dirname):
-            os.mkdir(dirname)
-
-        download(geoip_database, url)
-        print("Downloaded GeoIP database")
-        return True
-    except:
-        print("Error: Failed to download GeoIP database")
-    return False
-
-class GeoIPLock(object):
-    def __init__(self, file):
-        self.file = file
-
-    def __enter__(self):
-        if os.path.exists(self.file):
-            return False
-
-        Path(self.file).touch()
-        return True
-
-    def __exit__(self, exc_type, exc_value, tb):
-        os.unlink(self.file)
-
-def geoip_update(firewall=None, policy=None, force=False):
-    with GeoIPLock(geoip_lock_file) as lock:
-        if not lock:
-            print("Script is already running")
-            return False
-
-        if not firewall and not policy:
-            print("Firewall and policy are not configured")
-            return True
-
-        if not os.path.exists(geoip_database):
-            if not geoip_download_data():
-                return False
-        elif force:
-            geoip_download_data()
-
-        ipv4_codes = {}
-        ipv6_codes = {}
-
-        ipv4_sets = {}
-        ipv6_sets = {}
-
-        ipv4_codes_policy = {}
-        ipv6_codes_policy = {}
-
-        ipv4_sets_policy = {}
-        ipv6_sets_policy = {}
-
-        # Map country codes to set names
-        if firewall:
-            for codes, path in dict_search_recursive(firewall, 'country_code'):
-                set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
-                if ( path[0] == 'ipv4'):
-                    for code in codes:
-                        ipv4_codes.setdefault(code, []).append(set_name)
-                elif ( path[0] == 'ipv6' ):
-                    set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}'
-                    for code in codes:
-                        ipv6_codes.setdefault(code, []).append(set_name)
-
-        if policy:
-            for codes, path in dict_search_recursive(policy, 'country_code'):
-                set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}'
-                if ( path[0] == 'route'):
-                    for code in codes:
-                        ipv4_codes_policy.setdefault(code, []).append(set_name)
-                elif ( path[0] == 'route6' ):
-                    set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}'
-                    for code in codes:
-                        ipv6_codes_policy.setdefault(code, []).append(set_name)
-
-        if not ipv4_codes and not ipv6_codes and not ipv4_codes_policy and not ipv6_codes_policy:
-            if force:
-                print("GeoIP not in use by firewall and policy")
-            return True
-
-        geoip_data = geoip_load_data([*ipv4_codes, *ipv6_codes, *ipv4_codes_policy, *ipv6_codes_policy])
-
-        # Iterate IP blocks to assign to sets
-        for start, end, code in geoip_data:
-            ipv4 = is_ipv4(start)
-            if code in ipv4_codes and ipv4:
-                ip_range = f'{start}-{end}' if start != end else start
-                for setname in ipv4_codes[code]:
-                    ipv4_sets.setdefault(setname, []).append(ip_range)
-            if code in ipv4_codes_policy and ipv4:
-                ip_range = f'{start}-{end}' if start != end else start
-                for setname in ipv4_codes_policy[code]:
-                    ipv4_sets_policy.setdefault(setname, []).append(ip_range)
-            if code in ipv6_codes and not ipv4:
-                ip_range = f'{start}-{end}' if start != end else start
-                for setname in ipv6_codes[code]:
-                    ipv6_sets.setdefault(setname, []).append(ip_range)
-            if code in ipv6_codes_policy and not ipv4:
-                ip_range = f'{start}-{end}' if start != end else start
-                for setname in ipv6_codes_policy[code]:
-                    ipv6_sets_policy.setdefault(setname, []).append(ip_range)
-
-        render(nftables_geoip_conf, 'firewall/nftables-geoip-update.j2', {
-            'ipv4_sets': ipv4_sets,
-            'ipv6_sets': ipv6_sets,
-            'ipv4_sets_policy': ipv4_sets_policy,
-            'ipv6_sets_policy': ipv6_sets_policy,
-        })
-
-        result = run(f'nft --file {nftables_geoip_conf}')
-        if result != 0:
-            print('Error: GeoIP failed to update firewall/policy')
-            return False
-
-        return True
diff --git a/python/vyos/geoip.py b/python/vyos/geoip.py
new file mode 100644
index 000000000..9a2b40899
--- /dev/null
+++ b/python/vyos/geoip.py
@@ -0,0 +1,258 @@
+
+import csv
+import gzip
+import os
+import sqlite3
+import zipfile
+
+from io import TextIOWrapper
+from pathlib import Path
+from time import strftime
+
+from vyos.remote import download
+from vyos.template import is_ipv4, render
+from vyos.utils.dict import dict_search_recursive
+from vyos.utils.process import run
+
+nftables_geoip_conf = '/run/nftables-geoip.conf'
+dbip_database_raw = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz'
+mm_database_raw = '/usr/share/vyos-geoip/maxmind-country.zip'
+geoip_database_path = '/var/cache/vyos/geoip-lookup.db'
+geoip_lock_file = '/var/lock/vyos-geoip.lock'
+
+# Raw data
+
+def geoip_download_dbip():
+    url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m"))
+    try:
+        dirname = os.path.dirname(dbip_database_raw)
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+
+        download(dbip_database_raw, url)
+        return True
+    except:
+        return False
+
+def geoip_download_maxmind(account_id : str, license_key: str, lite : bool) -> bool:
+    db_str = 'GeoLite2' if lite else 'GeoIP2'
+    url = f'https://{account_id}:{license_key}@download.maxmind.com/geoip/databases/{db_str}-Country-CSV/download?suffix=zip'
+    try:
+        dirname = os.path.dirname(mm_database_raw)
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+
+        download(mm_database_raw, url)
+        return True
+    except:
+        return False
+
+# VyOS database
+
+def db_is_initialised():
+    if not os.path.exists(geoip_database_path):
+        return False
+
+    with sqlite3.connect(geoip_database_path) as conn:
+        cur = conn.cursor()
+        cur.execute("PRAGMA table_info(geoip_ranges);")
+        rows = cur.fetchall()
+        return len(rows) > 0
+
+def db_initialise():
+    dirname = os.path.dirname(geoip_database_path)
+    if not os.path.exists(dirname):
+        os.mkdir(dirname)
+
+    with sqlite3.connect(geoip_database_path) as conn:
+        cur = conn.cursor()
+        cur.execute("""
+                    CREATE TABLE IF NOT EXISTS geoip_ranges (
+                        country_code TEXT NOT NULL,
+                        range TEXT NOT NULL,
+                        version INT NOT NULL
+                    )
+                    """)
+        cur.execute('CREATE INDEX IF NOT EXISTS idx_cc_version ON geoip_ranges(country_code, version)')
+        conn.commit()
+
+def db_import_dbip_ranges(replace=True, delete_file=False):
+    if not os.path.exists(dbip_database_raw):
+        return False
+
+    if not os.path.exists(geoip_database_path):
+        return False
+
+    try:
+        with gzip.open(dbip_database_raw, mode='rt') as csv_fh:
+            reader = csv.reader(csv_fh)
+
+            with sqlite3.connect(geoip_database_path) as conn:
+                cur = conn.cursor()
+
+                if replace:
+                    cur.execute('DELETE FROM geoip_ranges')
+
+                for start, end, code in reader:
+                    version = 4 if is_ipv4(start) else 6
+                    cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, ?)', (code.lower(), f'{start}-{end}', version))
+                conn.commit()
+
+        if delete_file:
+            os.unlink(dbip_database_raw)
+
+        return True
+    except:
+        return False
+
+def db_import_maxmind_ranges(replace=True, delete_file=False):
+    if not os.path.exists(mm_database_raw):
+        return False
+
+    if not zipfile.is_zipfile(mm_database_raw):
+        return False
+
+    if not os.path.exists(geoip_database_path):
+        return False
+
+    try:
+        with zipfile.ZipFile(mm_database_raw, mode='r') as zip_fh:
+            directory = os.path.dirname(zip_fh.namelist()[0])
+            prefix = 'GeoLite2' if any(f.startswith('GeoLite2') for f in zip_fh.namelist()) else 'GeoIP2'
+
+            ipv4_file = f'{directory}/{prefix}-Country-Blocks-IPv4.csv'
+            ipv6_file = f'{directory}/{prefix}-Country-Blocks-IPv6.csv'
+            locations_file = f'{directory}/{prefix}-Country-Locations-en.csv'
+            locations_map = {}
+
+            with zip_fh.open(locations_file) as raw_csv_fh:
+                with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh:
+                    reader = csv.DictReader(csv_fh)
+
+                    for row in reader:
+                        id = row['geoname_id']
+                        locations_map[id] = row['country_iso_code']
+
+            with sqlite3.connect(geoip_database_path) as conn:
+                cur = conn.cursor()
+
+                if replace:
+                    cur.execute('DELETE FROM geoip_ranges')
+
+                with zip_fh.open(ipv4_file) as raw_csv_fh:
+                    with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh:
+                        reader = csv.DictReader(csv_fh)
+                        for row in reader:
+                            id = row['geoname_id']
+
+                            if not id or id not in locations_map:
+                                continue
+
+                            code = locations_map[id]
+                            cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 4)', (code.lower(), row['network']))
+
+                with zip_fh.open(ipv6_file) as raw_csv_fh:
+                    with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh:
+                        reader = csv.DictReader(csv_fh)
+                        for row in reader:
+                            id = row['geoname_id']
+
+                            if not id or id not in locations_map:
+                                continue
+
+                            code = locations_map[id]
+                            cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 6)', (code.lower(), row['network']))
+
+                conn.commit()
+
+        if delete_file:
+            os.unlink(mm_database_raw)
+
+        return True
+    except:
+        return False
+
+def db_return_ranges(codes, version):
+    out = []
+    with sqlite3.connect(geoip_database_path) as conn:
+        cur = conn.cursor()
+        ph = ','.join(['?'] * len(codes))
+        for row in cur.execute(f'SELECT range FROM geoip_ranges WHERE version = ? AND country_code IN ({ph})', [version, *codes]):
+            out.append(row[0])
+    return out
+
+# Update
+
+def geoip_refresh():
+    with GeoIPLock(geoip_lock_file) as lock:
+        if not lock:
+            return True
+
+        if not os.path.exists(nftables_geoip_conf):
+            return False
+
+        result = run(f'nft --file {nftables_geoip_conf}')
+        if result != 0:
+            return False
+
+        return True
+
+def geoip_update(firewall=None, policy=None):
+    with GeoIPLock(geoip_lock_file) as lock:
+        if not lock:
+            print("Script is already running")
+            return False
+
+        if not firewall and not policy:
+            print("Firewall and policy are not configured")
+            return True
+
+        if not os.path.exists(geoip_database_path):
+            print("Running one-time database initialisation")
+            db_initialise()
+            db_import_dbip_ranges()
+
+        firewall_sets = {'v4': {}, 'v6': {}}
+        policy_sets = {'v4': {}, 'v6': {}}
+
+        if firewall:
+            for codes, path in dict_search_recursive(firewall, 'country_code'):
+                version = 6 if path[0] == 'ipv6' else 4
+                vprefix = '6' if version == 6 else ''
+                set_name = f'GEOIP_CC{vprefix}_{path[1]}_{path[2]}_{path[4]}'
+                firewall_sets[f'v{version}'][set_name] = db_return_ranges(codes, version)
+
+        if policy:
+            for codes, path in dict_search_recursive(policy, 'country_code'):
+                version = 6 if path[0] == 'route6' else 4
+                vprefix = '6' if version == 6 else ''
+                set_name = f'GEOIP_CC{vprefix}_{path[0]}_{path[1]}_{path[3]}'
+                policy_sets[f'v{version}'][set_name] = db_return_ranges(codes, version)
+
+        render(nftables_geoip_conf, 'firewall/nftables-geoip-update.j2', {
+            'firewall_sets': firewall_sets,
+            'policy_sets': policy_sets
+        })
+
+        result = run(f'nft --file {nftables_geoip_conf}')
+        if result != 0:
+            print('Error: GeoIP failed to update firewall and/or policy')
+            return False
+
+        return True
+
+# Utility
+
+class GeoIPLock(object):
+    def __init__(self, file):
+        self.file = file
+
+    def __enter__(self):
+        if os.path.exists(self.file):
+            return False
+
+        Path(self.file).touch()
+        return True
+
+    def __exit__(self, exc_type, exc_value, tb):
+        os.unlink(self.file)
diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py
index a609e305c..02c9b2e3e 100755
--- a/src/conf_mode/firewall.py
+++ b/src/conf_mode/firewall.py
@@ -22,13 +22,13 @@ from glob import glob
 from sys import exit
 from vyos.base import Warning
 from vyos.config import Config
-from vyos.configdict import is_node_changed
-from vyos.configdiff import get_config_diff, Diff
+from vyos.configdict import is_node_changed, node_changed
+from vyos.configdiff import Diff
 from vyos.configdep import set_dependents, call_dependents
 from vyos.configverify import verify_interface_exists
 from vyos.ethtool import Ethtool
 from vyos.firewall import fqdn_config_parse
-from vyos.firewall import geoip_update
+from vyos.geoip import geoip_refresh, geoip_update
 from vyos.template import render
 from vyos.utils.dict import dict_search
 from vyos.utils.dict import dict_search_args
@@ -80,42 +80,29 @@ snmp_event_source = 1
 snmp_trap_mib = 'VYATTA-TRAP-MIB'
 snmp_trap_name = 'mgmtEventTrap'
 
-def geoip_updated(conf, firewall):
-    diff = get_config_diff(conf)
-    node_diff = diff.get_child_nodes_diff(['firewall'], expand_nodes=Diff.DELETE, recursive=True)
+def geoip_sets(firewall):
+    out = {'name': [], 'ipv6_name': []}
 
-    out = {
-        'name': [],
-        'ipv6_name': [],
-        'deleted_name': [],
-        'deleted_ipv6_name': []
-    }
-    updated = False
-
-    for key, path in dict_search_recursive(firewall, 'geoip'):
-        set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
+    for _, path in dict_search_recursive(firewall, 'geoip'):
         if (path[0] == 'ipv4'):
-            out['name'].append(set_name)
+            out['name'].append(f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}')
         elif (path[0] == 'ipv6'):
-            set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}'
-            out['ipv6_name'].append(set_name)
+            out['ipv6_name'].append(f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}')
 
-        updated = True
+    return out
 
-    if 'delete' in node_diff:
-        for key, path in dict_search_recursive(node_diff['delete'], 'geoip'):
-            set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
-            if (path[0] == 'ipv4'):
-                out['deleted_name'].append(set_name)
-            elif (path[0] == 'ipv6'):
-                set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}'
-                out['deleted_ipv6_name'].append(set_name)
-            updated = True
+def geoip_updated(conf):
+    changes = node_changed(conf, ['firewall'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
+    updated = False
 
-    if updated:
-        return out
+    for _, path in dict_search_recursive(changes, 'geoip'):
+        updated = True
+        break
 
-    return False
+    return updated
 
 def get_config(config=None):
     if config:
@@ -135,7 +122,8 @@ def get_config(config=None):
         # Update nat and policy-route as firewall groups were updated
         set_dependents('group_resync', conf)
 
-    firewall['geoip_updated'] = geoip_updated(conf, firewall)
+    firewall['geoip_sets'] = geoip_sets(firewall)
+    firewall['geoip_updated'] = geoip_updated(conf)
 
     fqdn_config_parse(firewall, 'firewall')
 
@@ -503,6 +491,12 @@ def verify(firewall):
                 for ifname in interfaces:
                     verify_hardware_offload(ifname)
 
+    if dict_search_args(firewall, 'global_options', 'geoip', 'provider') == 'maxmind':
+        geoip_options = dict_search_args(firewall, 'global_options', 'geoip')
+        required_keys = ['maxmind_account_id', 'maxmind_license_key']
+        if not all(key in geoip_options for key in required_keys):
+            raise ConfigError('MaxMind GeoIP provider requires maxmind-account-id and maxmind-license-key')
+
     if dict_search('global_options.state_policy', firewall) is not None:
         # Generate list of chains where conntrack is disabled
         conntrack_disabled_list = []
@@ -725,11 +719,12 @@ def apply(firewall):
             domain_action = 'stop'
     call(f'systemctl {domain_action} vyos-domain-resolver.service')
 
-    if firewall['geoip_updated']:
+    if firewall['geoip_sets']:
         # Call helper script to Update set contents
-        if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']:
-            print('Updating GeoIP. Please wait...')
-            geoip_update(firewall=firewall)
+        if 'name' in firewall['geoip_sets'] or 'ipv6_name' in firewall['geoip_sets']:
+            if firewall['geoip_updated'] or not geoip_refresh():
+                print('Updating GeoIP. Please wait...')
+                geoip_update(firewall)
 
     return None
 
diff --git a/src/conf_mode/policy_route.py b/src/conf_mode/policy_route.py
index 15d735f75..17f8e6bf9 100755
--- a/src/conf_mode/policy_route.py
+++ b/src/conf_mode/policy_route.py
@@ -21,7 +21,8 @@ from sys import exit
 
 from vyos.base import Warning
 from vyos.config import Config
-from vyos.configdiff import get_config_diff, Diff
+from vyos.configdict import node_changed
+from vyos.configdiff import Diff
 from vyos.template import render
 from vyos.utils.dict import dict_search_args
 from vyos.utils.dict import dict_search_recursive
@@ -30,7 +31,7 @@ from vyos.utils.process import run
 from vyos.utils.network import get_vrf_tableid
 from vyos.defaults import rt_global_table
 from vyos.defaults import rt_global_vrf
-from vyos.firewall import geoip_update
+from vyos.geoip import  geoip_refresh, geoip_update
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -46,42 +47,40 @@ valid_groups = [
     'interface_group'
 ]
 
-def geoip_updated(conf, policy):
-    diff = get_config_diff(conf)
-    node_diff = diff.get_child_nodes_diff(['policy'], expand_nodes=Diff.DELETE, recursive=True)
-
-    out = {
-        'name': [],
-        'ipv6_name': [],
-        'deleted_name': [],
-        'deleted_ipv6_name': []
-    }
+def geoip_updated(conf):
     updated = False
 
-    for key, path in dict_search_recursive(policy, 'geoip'):
-        set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}'
-        if (path[0] == 'route'):
-            out['name'].append(set_name)
-        elif (path[0] == 'route6'):
-            set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}'
-            out['ipv6_name'].append(set_name)
+    changes_v4 = node_changed(conf, ['policy', 'route'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
 
+    for _, path in dict_search_recursive(changes_v4, 'geoip'):
         updated = True
+        break
 
-    if 'delete' in node_diff:
-        for key, path in dict_search_recursive(node_diff['delete'], 'geoip'):
-            set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}'
-            if (path[0] == 'route'):
-                out['deleted_name'].append(set_name)
-            elif (path[0] == 'route6'):
-                set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}'
-                out['deleted_ipv6_name'].append(set_name)
+    if not updated:
+        changes_v6 = node_changed(conf, ['policy', 'route6'],
+                                 key_mangling=('-', '_'),
+                                 recursive=True,
+                                 expand_nodes=Diff.ADD | Diff.DELETE)
+
+        for _, path in dict_search_recursive(changes_v6, 'geoip'):
             updated = True
+            break
 
-    if updated:
-        return out
+    return updated
 
-    return False
+def geoip_sets(policy):
+    out = {'name': [], 'ipv6_name': []}
+
+    for _, path in dict_search_recursive(policy, 'geoip'):
+        if (path[0] == 'route'):
+            out['name'].append(f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}')
+        elif (path[0] == 'route6'):
+            out['ipv6_name'].append(f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}')
+
+    return out
 
 def get_config(config=None):
     if config:
@@ -100,7 +99,9 @@ def get_config(config=None):
     if 'dynamic_group' in policy['firewall_group']:
         del policy['firewall_group']['dynamic_group']
 
-    policy['geoip_updated'] = geoip_updated(conf, policy)
+    policy['geoip_sets'] = geoip_sets(policy)
+    policy['geoip_updated'] = geoip_updated(conf)
+
     return policy
 
 def verify_rule(policy, name, rule_conf, ipv6, rule_id):
@@ -244,11 +245,12 @@ def apply(policy):
 
     apply_table_marks(policy)
 
-    if policy['geoip_updated']:
+    if policy['geoip_sets']:
         # Call helper script to Update set contents
-        if 'name' in policy['geoip_updated'] or 'ipv6_name' in policy['geoip_updated']:
-            print('Updating GeoIP. Please wait...')
-            geoip_update(policy=policy)
+        if 'name' in policy['geoip_sets'] or 'ipv6_name' in policy['geoip_sets']:
+            if policy['geoip_updated'] or not geoip_refresh():
+                print('Updating GeoIP. Please wait...')
+                geoip_update(policy=policy)
 
     return None
 
diff --git a/src/helpers/geoip-update.py b/src/helpers/geoip-update.py
index 22d26e538..8a414267e 100755
--- a/src/helpers/geoip-update.py
+++ b/src/helpers/geoip-update.py
@@ -18,7 +18,13 @@ import argparse
 import sys
 
 from vyos.configquery import ConfigTreeQuery
-from vyos.firewall import geoip_update
+from vyos.geoip import geoip_download_dbip
+from vyos.geoip import geoip_download_maxmind
+from vyos.geoip import db_initialise
+from vyos.geoip import db_is_initialised
+from vyos.geoip import db_import_dbip_ranges
+from vyos.geoip import db_import_maxmind_ranges
+from vyos.geoip import geoip_update
 
 def get_config(config=None):
     if config:
@@ -27,6 +33,8 @@ def get_config(config=None):
         conf = ConfigTreeQuery()
 
     return (
+        conf.get_config_dict(['firewall', 'global-options', 'geoip'], key_mangling=('-', '_'), get_first_key=True,
+                                    no_tag_node_value_mangle=True, with_defaults=True),
         conf.get_config_dict(['firewall'], key_mangling=('-', '_'), get_first_key=True,
                                     no_tag_node_value_mangle=True) if conf.exists(['firewall']) else None,
         conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True,
@@ -35,9 +43,44 @@ def get_config(config=None):
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()
-    parser.add_argument("--force", help="Force update", action="store_true")
+    parser.add_argument("--init", help="Initialise", action="store_true")
     args = parser.parse_args()
 
-    firewall, policy = get_config()
-    if not geoip_update(firewall=firewall, policy=policy, force=args.force):
+    if args.init:
+        db_initialise()
+        db_import_dbip_ranges(delete_file=True)
+        sys.exit(0)
+
+    options, firewall, policy = get_config()
+
+    if not db_is_initialised():
+        db_initialise()
+
+    if options['provider'] == 'db-ip':
+        print('Dowloading latest DB-IP database...')
+        if not geoip_download_dbip():
+            print('Failed to download, aborting.')
+            sys.exit(1)
+
+        print('Extracting database...')
+        if not db_import_dbip_ranges(delete_file=True):
+            print('Failed to extract, aborting.')
+            sys.exit(1)
+
+    elif options['provider'] == 'maxmind':
+        account_id = options['maxmind_account_id']
+        license_key = options['maxmind_license_key']
+        lite = 'maxmind_lite' in options
+
+        print('Dowloading latest MaxMind database...')
+        if not geoip_download_maxmind(account_id, license_key, lite):
+            print('Failed to download, aborting.')
+            sys.exit(1)
+
+        print('Extracting database...')
+        if not db_import_maxmind_ranges(delete_file=True):
+            print('Failed to extract, aborting.')
+            sys.exit(1)
+
+    if not geoip_update(firewall=firewall, policy=policy):
         sys.exit(1)