From 7f6f94370ec04ce48e7a19880a74ba0c25f7bfb5 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Thu, 2 Oct 2025 17:47:10 +0200 Subject: geoip: T7926: Refactor geoip handling * Move core logic to separate vyos.geoip module * Use a sqlite database for storing and querying address ranges by country * Remove downloaded geoip ranges once loaded into sqlite db * No longer rebuild geoip sets on each commit unless necessary * Allows for extensibility using other geoip data vendors --- src/conf_mode/firewall.py | 63 ++++++++++++++++--------------------- src/conf_mode/policy_route.py | 72 ++++++++++++++++++++++--------------------- src/helpers/geoip-update.py | 28 +++++++++++++++-- 3 files changed, 88 insertions(+), 75 deletions(-) (limited to 'src') diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index a609e305c..18c250d08 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -22,13 +22,13 @@ from glob import glob from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdict import is_node_changed -from vyos.configdiff import get_config_diff, Diff +from vyos.configdict import is_node_changed, node_changed +from vyos.configdiff import Diff from vyos.configdep import set_dependents, call_dependents from vyos.configverify import verify_interface_exists from vyos.ethtool import Ethtool from vyos.firewall import fqdn_config_parse -from vyos.firewall import geoip_update +from vyos.geoip import geoip_refresh, geoip_update from vyos.template import render from vyos.utils.dict import dict_search from vyos.utils.dict import dict_search_args @@ -80,42 +80,29 @@ snmp_event_source = 1 snmp_trap_mib = 'VYATTA-TRAP-MIB' snmp_trap_name = 'mgmtEventTrap' -def geoip_updated(conf, firewall): - diff = get_config_diff(conf) - node_diff = diff.get_child_nodes_diff(['firewall'], expand_nodes=Diff.DELETE, recursive=True) +def geoip_sets(firewall): + out = {'name': [], 'ipv6_name': []} - out = { - 'name': [], - 'ipv6_name': [], - 'deleted_name': [], - 'deleted_ipv6_name': [] - } - updated = False - - for key, path in dict_search_recursive(firewall, 'geoip'): - set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}' + for _, path in dict_search_recursive(firewall, 'geoip'): if (path[0] == 'ipv4'): - out['name'].append(set_name) + out['name'].append(f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}') elif (path[0] == 'ipv6'): - set_name = f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}' - out['ipv6_name'].append(set_name) + out['ipv6_name'].append(f'GEOIP_CC6_{path[1]}_{path[2]}_{path[4]}') - updated = True + return out - if 'delete' in node_diff: - for key, path in dict_search_recursive(node_diff['delete'], 'geoip'): - set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}' - if (path[0] == 'ipv4'): - out['deleted_name'].append(set_name) - elif (path[0] == 'ipv6'): - set_name = f'GEOIP_CC_{path[1]}_{path[2]}_{path[4]}' - out['deleted_ipv6_name'].append(set_name) - updated = True +def geoip_updated(conf): + changes = node_changed(conf, ['firewall'], + key_mangling=('-', '_'), + recursive=True, + expand_nodes=Diff.ADD | Diff.DELETE) + updated = False - if updated: - return out + for _, path in dict_search_recursive(changes, 'geoip'): + updated = True + break - return False + return updated def get_config(config=None): if config: @@ -135,7 +122,8 @@ def get_config(config=None): # Update nat and policy-route as firewall groups were updated set_dependents('group_resync', conf) - firewall['geoip_updated'] = geoip_updated(conf, firewall) + firewall['geoip_sets'] = geoip_sets(firewall) + firewall['geoip_updated'] = geoip_updated(conf) fqdn_config_parse(firewall, 'firewall') @@ -725,11 +713,12 @@ def apply(firewall): domain_action = 'stop' call(f'systemctl {domain_action} vyos-domain-resolver.service') - if firewall['geoip_updated']: + if firewall['geoip_sets']: # Call helper script to Update set contents - if 'name' in firewall['geoip_updated'] or 'ipv6_name' in firewall['geoip_updated']: - print('Updating GeoIP. Please wait...') - geoip_update(firewall=firewall) + if 'name' in firewall['geoip_sets'] or 'ipv6_name' in firewall['geoip_sets']: + if firewall['geoip_updated'] or not geoip_refresh(): + print('Updating GeoIP. Please wait...') + geoip_update(firewall) return None diff --git a/src/conf_mode/policy_route.py b/src/conf_mode/policy_route.py index 15d735f75..17f8e6bf9 100755 --- a/src/conf_mode/policy_route.py +++ b/src/conf_mode/policy_route.py @@ -21,7 +21,8 @@ from sys import exit from vyos.base import Warning from vyos.config import Config -from vyos.configdiff import get_config_diff, Diff +from vyos.configdict import node_changed +from vyos.configdiff import Diff from vyos.template import render from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive @@ -30,7 +31,7 @@ from vyos.utils.process import run from vyos.utils.network import get_vrf_tableid from vyos.defaults import rt_global_table from vyos.defaults import rt_global_vrf -from vyos.firewall import geoip_update +from vyos.geoip import geoip_refresh, geoip_update from vyos import ConfigError from vyos import airbag airbag.enable() @@ -46,42 +47,40 @@ valid_groups = [ 'interface_group' ] -def geoip_updated(conf, policy): - diff = get_config_diff(conf) - node_diff = diff.get_child_nodes_diff(['policy'], expand_nodes=Diff.DELETE, recursive=True) - - out = { - 'name': [], - 'ipv6_name': [], - 'deleted_name': [], - 'deleted_ipv6_name': [] - } +def geoip_updated(conf): updated = False - for key, path in dict_search_recursive(policy, 'geoip'): - set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}' - if (path[0] == 'route'): - out['name'].append(set_name) - elif (path[0] == 'route6'): - set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}' - out['ipv6_name'].append(set_name) + changes_v4 = node_changed(conf, ['policy', 'route'], + key_mangling=('-', '_'), + recursive=True, + expand_nodes=Diff.ADD | Diff.DELETE) + for _, path in dict_search_recursive(changes_v4, 'geoip'): updated = True + break - if 'delete' in node_diff: - for key, path in dict_search_recursive(node_diff['delete'], 'geoip'): - set_name = f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}' - if (path[0] == 'route'): - out['deleted_name'].append(set_name) - elif (path[0] == 'route6'): - set_name = f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}' - out['deleted_ipv6_name'].append(set_name) + if not updated: + changes_v6 = node_changed(conf, ['policy', 'route6'], + key_mangling=('-', '_'), + recursive=True, + expand_nodes=Diff.ADD | Diff.DELETE) + + for _, path in dict_search_recursive(changes_v6, 'geoip'): updated = True + break - if updated: - return out + return updated - return False +def geoip_sets(policy): + out = {'name': [], 'ipv6_name': []} + + for _, path in dict_search_recursive(policy, 'geoip'): + if (path[0] == 'route'): + out['name'].append(f'GEOIP_CC_{path[0]}_{path[1]}_{path[3]}') + elif (path[0] == 'route6'): + out['ipv6_name'].append(f'GEOIP_CC6_{path[0]}_{path[1]}_{path[3]}') + + return out def get_config(config=None): if config: @@ -100,7 +99,9 @@ def get_config(config=None): if 'dynamic_group' in policy['firewall_group']: del policy['firewall_group']['dynamic_group'] - policy['geoip_updated'] = geoip_updated(conf, policy) + policy['geoip_sets'] = geoip_sets(policy) + policy['geoip_updated'] = geoip_updated(conf) + return policy def verify_rule(policy, name, rule_conf, ipv6, rule_id): @@ -244,11 +245,12 @@ def apply(policy): apply_table_marks(policy) - if policy['geoip_updated']: + if policy['geoip_sets']: # Call helper script to Update set contents - if 'name' in policy['geoip_updated'] or 'ipv6_name' in policy['geoip_updated']: - print('Updating GeoIP. Please wait...') - geoip_update(policy=policy) + if 'name' in policy['geoip_sets'] or 'ipv6_name' in policy['geoip_sets']: + if policy['geoip_updated'] or not geoip_refresh(): + print('Updating GeoIP. Please wait...') + geoip_update(policy=policy) return None diff --git a/src/helpers/geoip-update.py b/src/helpers/geoip-update.py index 22d26e538..a6838c62d 100755 --- a/src/helpers/geoip-update.py +++ b/src/helpers/geoip-update.py @@ -18,7 +18,11 @@ import argparse import sys from vyos.configquery import ConfigTreeQuery -from vyos.firewall import geoip_update +from vyos.geoip import geoip_download_dbip +from vyos.geoip import db_initialise +from vyos.geoip import db_is_initialised +from vyos.geoip import db_import_dbip_ranges +from vyos.geoip import geoip_update def get_config(config=None): if config: @@ -35,9 +39,27 @@ def get_config(config=None): if __name__ == '__main__': parser = argparse.ArgumentParser() - parser.add_argument("--force", help="Force update", action="store_true") + parser.add_argument("--init", help="Initialise", action="store_true") args = parser.parse_args() + if args.init: + db_initialise() + db_import_dbip_ranges(delete_file=True) + sys.exit(0) + + if not db_is_initialised(): + db_initialise() + + print('Dowloading latest DB-IP database...') + if not geoip_download_dbip(): + print('Failed to download, aborting.') + sys.exit(1) + + print('Extracting database...') + if not db_import_dbip_ranges(delete_file=True): + print('Failed to extract, aborting.') + sys.exit(1) + firewall, policy = get_config() - if not geoip_update(firewall=firewall, policy=policy, force=args.force): + if not geoip_update(firewall=firewall, policy=policy): sys.exit(1) -- cgit v1.2.3 From 8a329ef98eb120cc91aac6f7da6cd61dbc1ab950 Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Thu, 6 Nov 2025 11:14:08 +0100 Subject: geoip: T8049: Add MaxMind database support --- .../include/firewall/global-options.xml.i | 43 ++++++++++ python/vyos/geoip.py | 95 ++++++++++++++++++++-- src/conf_mode/firewall.py | 6 ++ src/helpers/geoip-update.py | 39 +++++++-- 4 files changed, 168 insertions(+), 15 deletions(-) (limited to 'src') diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i index e19f3a7c5..7ec07100d 100644 --- a/interface-definitions/include/firewall/global-options.xml.i +++ b/interface-definitions/include/firewall/global-options.xml.i @@ -130,6 +130,49 @@ enable + + + GeoIP options + + + + + GeoIP database provider + + db-ip maxmind + + + db-ip + Use GeoIP database by DB-IP.com + + + maxmind + Use GeoIP database by MaxMind (Requires API key) + + + (db-ip|maxmind) + + + db-ip + + + + Account ID for MaxMind GeoIP database + + + + + License key for MaxMind GeoIP database + + + + + Use MaxMind GeoLite2 database + + + + + Policy for handling IPv4 packets with source route option diff --git a/python/vyos/geoip.py b/python/vyos/geoip.py index a942b888b..9a2b40899 100644 --- a/python/vyos/geoip.py +++ b/python/vyos/geoip.py @@ -3,7 +3,9 @@ import csv import gzip import os import sqlite3 +import zipfile +from io import TextIOWrapper from pathlib import Path from time import strftime @@ -13,7 +15,8 @@ from vyos.utils.dict import dict_search_recursive from vyos.utils.process import run nftables_geoip_conf = '/run/nftables-geoip.conf' -geoip_database_raw = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz' +dbip_database_raw = '/usr/share/vyos-geoip/dbip-country-lite.csv.gz' +mm_database_raw = '/usr/share/vyos-geoip/maxmind-country.zip' geoip_database_path = '/var/cache/vyos/geoip-lookup.db' geoip_lock_file = '/var/lock/vyos-geoip.lock' @@ -22,11 +25,24 @@ geoip_lock_file = '/var/lock/vyos-geoip.lock' def geoip_download_dbip(): url = 'https://download.db-ip.com/free/dbip-country-lite-{}.csv.gz'.format(strftime("%Y-%m")) try: - dirname = os.path.dirname(geoip_database_raw) + dirname = os.path.dirname(dbip_database_raw) if not os.path.exists(dirname): os.mkdir(dirname) - download(geoip_database_raw, url) + download(dbip_database_raw, url) + return True + except: + return False + +def geoip_download_maxmind(account_id : str, license_key: str, lite : bool) -> bool: + db_str = 'GeoLite2' if lite else 'GeoIP2' + url = f'https://{account_id}:{license_key}@download.maxmind.com/geoip/databases/{db_str}-Country-CSV/download?suffix=zip' + try: + dirname = os.path.dirname(mm_database_raw) + if not os.path.exists(dirname): + os.mkdir(dirname) + + download(mm_database_raw, url) return True except: return False @@ -61,14 +77,14 @@ def db_initialise(): conn.commit() def db_import_dbip_ranges(replace=True, delete_file=False): - if not os.path.exists(geoip_database_raw): + if not os.path.exists(dbip_database_raw): return False if not os.path.exists(geoip_database_path): return False try: - with gzip.open(geoip_database_raw, mode='rt') as csv_fh: + with gzip.open(dbip_database_raw, mode='rt') as csv_fh: reader = csv.reader(csv_fh) with sqlite3.connect(geoip_database_path) as conn: @@ -83,7 +99,74 @@ def db_import_dbip_ranges(replace=True, delete_file=False): conn.commit() if delete_file: - os.unlink(geoip_database_raw) + os.unlink(dbip_database_raw) + + return True + except: + return False + +def db_import_maxmind_ranges(replace=True, delete_file=False): + if not os.path.exists(mm_database_raw): + return False + + if not zipfile.is_zipfile(mm_database_raw): + return False + + if not os.path.exists(geoip_database_path): + return False + + try: + with zipfile.ZipFile(mm_database_raw, mode='r') as zip_fh: + directory = os.path.dirname(zip_fh.namelist()[0]) + prefix = 'GeoLite2' if any(f.startswith('GeoLite2') for f in zip_fh.namelist()) else 'GeoIP2' + + ipv4_file = f'{directory}/{prefix}-Country-Blocks-IPv4.csv' + ipv6_file = f'{directory}/{prefix}-Country-Blocks-IPv6.csv' + locations_file = f'{directory}/{prefix}-Country-Locations-en.csv' + locations_map = {} + + with zip_fh.open(locations_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + + for row in reader: + id = row['geoname_id'] + locations_map[id] = row['country_iso_code'] + + with sqlite3.connect(geoip_database_path) as conn: + cur = conn.cursor() + + if replace: + cur.execute('DELETE FROM geoip_ranges') + + with zip_fh.open(ipv4_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + for row in reader: + id = row['geoname_id'] + + if not id or id not in locations_map: + continue + + code = locations_map[id] + cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 4)', (code.lower(), row['network'])) + + with zip_fh.open(ipv6_file) as raw_csv_fh: + with TextIOWrapper(raw_csv_fh, encoding='utf-8') as csv_fh: + reader = csv.DictReader(csv_fh) + for row in reader: + id = row['geoname_id'] + + if not id or id not in locations_map: + continue + + code = locations_map[id] + cur.execute('INSERT INTO geoip_ranges (country_code, range, version) VALUES (?, ?, 6)', (code.lower(), row['network'])) + + conn.commit() + + if delete_file: + os.unlink(mm_database_raw) return True except: diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 18c250d08..02c9b2e3e 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -491,6 +491,12 @@ def verify(firewall): for ifname in interfaces: verify_hardware_offload(ifname) + if dict_search_args(firewall, 'global_options', 'geoip', 'provider') == 'maxmind': + geoip_options = dict_search_args(firewall, 'global_options', 'geoip') + required_keys = ['maxmind_account_id', 'maxmind_license_key'] + if not all(key in geoip_options for key in required_keys): + raise ConfigError('MaxMind GeoIP provider requires maxmind-account-id and maxmind-license-key') + if dict_search('global_options.state_policy', firewall) is not None: # Generate list of chains where conntrack is disabled conntrack_disabled_list = [] diff --git a/src/helpers/geoip-update.py b/src/helpers/geoip-update.py index a6838c62d..8a414267e 100755 --- a/src/helpers/geoip-update.py +++ b/src/helpers/geoip-update.py @@ -19,9 +19,11 @@ import sys from vyos.configquery import ConfigTreeQuery from vyos.geoip import geoip_download_dbip +from vyos.geoip import geoip_download_maxmind from vyos.geoip import db_initialise from vyos.geoip import db_is_initialised from vyos.geoip import db_import_dbip_ranges +from vyos.geoip import db_import_maxmind_ranges from vyos.geoip import geoip_update def get_config(config=None): @@ -31,6 +33,8 @@ def get_config(config=None): conf = ConfigTreeQuery() return ( + conf.get_config_dict(['firewall', 'global-options', 'geoip'], key_mangling=('-', '_'), get_first_key=True, + no_tag_node_value_mangle=True, with_defaults=True), conf.get_config_dict(['firewall'], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) if conf.exists(['firewall']) else None, conf.get_config_dict(['policy'], key_mangling=('-', '_'), get_first_key=True, @@ -47,19 +51,36 @@ if __name__ == '__main__': db_import_dbip_ranges(delete_file=True) sys.exit(0) + options, firewall, policy = get_config() + if not db_is_initialised(): db_initialise() - print('Dowloading latest DB-IP database...') - if not geoip_download_dbip(): - print('Failed to download, aborting.') - sys.exit(1) + if options['provider'] == 'db-ip': + print('Dowloading latest DB-IP database...') + if not geoip_download_dbip(): + print('Failed to download, aborting.') + sys.exit(1) - print('Extracting database...') - if not db_import_dbip_ranges(delete_file=True): - print('Failed to extract, aborting.') - sys.exit(1) + print('Extracting database...') + if not db_import_dbip_ranges(delete_file=True): + print('Failed to extract, aborting.') + sys.exit(1) + + elif options['provider'] == 'maxmind': + account_id = options['maxmind_account_id'] + license_key = options['maxmind_license_key'] + lite = 'maxmind_lite' in options + + print('Dowloading latest MaxMind database...') + if not geoip_download_maxmind(account_id, license_key, lite): + print('Failed to download, aborting.') + sys.exit(1) + + print('Extracting database...') + if not db_import_maxmind_ranges(delete_file=True): + print('Failed to extract, aborting.') + sys.exit(1) - firewall, policy = get_config() if not geoip_update(firewall=firewall, policy=policy): sys.exit(1) -- cgit v1.2.3