diff options
Diffstat (limited to 'src/services')
29 files changed, 853 insertions, 135 deletions
diff --git a/src/services/api/background.py b/src/services/api/background.py new file mode 100644 index 000000000..2b25af307 --- /dev/null +++ b/src/services/api/background.py @@ -0,0 +1,179 @@ +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see <http://www.gnu.org/licenses/>. + +import time +import functools +from collections import deque +from enum import Enum +from threading import Lock +from typing import Any +from typing import Callable +from typing import Optional +from uuid import uuid4 + +from fastapi import BackgroundTasks +from pydantic import BaseModel +from pydantic import StrictStr +from pydantic import StrictInt + + +def _ts(): + """Return current Unix timestamp (seconds since epoch)""" + return int(time.time()) + + +class BackgroundOpStatus(str, Enum): + queued = 'queued' + running = 'running' + succeeded = 'succeeded' + failed = 'failed' + + @property + def is_completed(self): + """True if the operation is in a terminal state (succeeded/failed)""" + return self in (BackgroundOpStatus.succeeded, BackgroundOpStatus.failed) + + +class BackgroundOpRecord(BaseModel): + """Metadata and outcome for a single background operation""" + + op_id: StrictStr + created_at: StrictInt + started_at: Optional[StrictInt] = None + finished_at: Optional[StrictInt] = None + status: BackgroundOpStatus = BackgroundOpStatus.queued + result: Optional[Any] = None + error: Optional[StrictStr] = None + + +class BackgroundOpError(Exception): + """Raised when a background operation cannot be enqueued/executed""" + + pass + + +class BackgroundOpManager: + """ + In-memory FIFO operation queue. + + Uses BackgroundTasks to schedule a `drain()` call after the response, + so `enqueue()` is fast and non-blocking for the client. + """ + + DEFAULT_MAX_QUEUE_SIZE = 128 + + def __init__(self, max_queue_size: int = DEFAULT_MAX_QUEUE_SIZE): + # max number of queued (pending) operations allowed at a time + self._max_queue_size = max_queue_size + + # FIFO queue of operation IDs waiting to be executed + self._queue = deque() + self._jobs = {} + self._workers = {} + + # protects _queue/_jobs/_workers/_drain_scheduled from concurrent access + self._mx = Lock() + + # whether a drain task has already been scheduled via BackgroundTasks + self._drain_scheduled = False + + def enqueue( + self, + background_tasks: BackgroundTasks, + func: Callable, + *args, + **kwargs, + ) -> BackgroundOpRecord: + """Enqueue a function for background execution and return its record""" + + assert isinstance(background_tasks, BackgroundTasks) + assert callable(func), '`func` argument should be function or lambda' + + with self._mx: + if len(self._queue) >= self._max_queue_size: + raise BackgroundOpError( + f'Background operation queue is full ({self._max_queue_size})' + ) + + op_id = str(uuid4()) + record = BackgroundOpRecord(op_id=op_id, created_at=_ts()) + + self._jobs[op_id] = record + # store the callable for later execution (outside the lock) + self._workers[op_id] = functools.partial(func, *args, **kwargs) + self._queue.append(op_id) + + if not self._drain_scheduled: + # schedule a single drain() call after the current response + background_tasks.add_task(self.drain) + self._drain_scheduled = True + + # Best-effort pruning: keep history bounded by dropping oldest completed records + if len(self._jobs) > self._max_queue_size: + oldest = min(self._jobs.values(), key=lambda record: record.created_at) + if oldest.status.is_completed: + del self._jobs[oldest.op_id] + + return record + + def drain(self): + """Run queued operations sequentially until the queue is empty""" + + while True: + with self._mx: + if not self._queue: + # allow future enqueue() calls to schedule the next drain() + self._drain_scheduled = False + return + + op_id = self._queue.popleft() + record = self._jobs[op_id] + func = self._workers.pop(op_id) + + record.status = BackgroundOpStatus.running + record.started_at = _ts() + + # execute outside the lock to avoid blocking enqueues/status reads + result = error = status = None + try: + result = func() + except Exception as e: # noqa: BLE001 + status = BackgroundOpStatus.failed + error = str(e) + else: + status = BackgroundOpStatus.succeeded + + with self._mx: + record.result = result + record.error = error + record.status = status + record.finished_at = _ts() + + def get_record(self, op_id: str) -> BackgroundOpRecord | None: + """Return a deep copy of a single record""" + + with self._mx: + record = self._jobs.get(op_id) + return record.copy(deep=True) if record else None + + def get_records(self) -> list: + """Return deep copies of all records, sorted oldest-first by created_at""" + + with self._mx: + records = [record.copy(deep=True) for record in self._jobs.values()] + + # stable-ish ordering (oldest first) + records.sort(key=lambda record: record.created_at) + return records diff --git a/src/services/api/graphql/README.graphql b/src/services/api/graphql/README.graphql index 1133d79ed..0f43ac356 100644 --- a/src/services/api/graphql/README.graphql +++ b/src/services/api/graphql/README.graphql @@ -64,7 +64,7 @@ save to /config/config.boot; to save to an alternative path, specify fileName. Similarly, using an analogous 'endpoint' (meaning the form of the request -and resolver; the actual enpoint for all GraphQL requests is +and resolver; the actual endpoint for all GraphQL requests is https://hostname/graphql), one can load an arbitrary config file from a path. diff --git a/src/services/api/graphql/bindings.py b/src/services/api/graphql/bindings.py index ebf745f32..7380dbb5f 100644 --- a/src/services/api/graphql/bindings.py +++ b/src/services/api/graphql/bindings.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/generate/generate_schema.py b/src/services/api/graphql/generate/generate_schema.py index dd5e7ea56..bb36a4c04 100755 --- a/src/services/api/graphql/generate/generate_schema.py +++ b/src/services/api/graphql/generate/generate_schema.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/src/services/api/graphql/generate/schema_from_composite.py b/src/services/api/graphql/generate/schema_from_composite.py index 06e74032d..9a07f88fe 100755 --- a/src/services/api/graphql/generate/schema_from_composite.py +++ b/src/services/api/graphql/generate/schema_from_composite.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022-2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # # -# A utility to generate GraphQL schema defintions from typing information of +# A utility to generate GraphQL schema definitions from typing information of # composite functions comprising several requests. import os diff --git a/src/services/api/graphql/generate/schema_from_config_session.py b/src/services/api/graphql/generate/schema_from_config_session.py index 1d5ff1e53..bfa4bc006 100755 --- a/src/services/api/graphql/generate/schema_from_config_session.py +++ b/src/services/api/graphql/generate/schema_from_config_session.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022-2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # # -# A utility to generate GraphQL schema defintions from typing information of +# A utility to generate GraphQL schema definitions from typing information of # (wrappers of) native configsession functions. import os diff --git a/src/services/api/graphql/generate/schema_from_op_mode.py b/src/services/api/graphql/generate/schema_from_op_mode.py index ab7cb691f..618ea2e61 100755 --- a/src/services/api/graphql/generate/schema_from_op_mode.py +++ b/src/services/api/graphql/generate/schema_from_op_mode.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022-2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -15,7 +15,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. # # -# A utility to generate GraphQL schema defintions from standardized op-mode +# A utility to generate GraphQL schema definitions from standardized op-mode # scripts. import os diff --git a/src/services/api/graphql/graphql/auth_token_mutation.py b/src/services/api/graphql/graphql/auth_token_mutation.py index c74364603..a8020d149 100644 --- a/src/services/api/graphql/graphql/auth_token_mutation.py +++ b/src/services/api/graphql/graphql/auth_token_mutation.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/graphql/directives.py b/src/services/api/graphql/graphql/directives.py index 3927aee58..037f09204 100644 --- a/src/services/api/graphql/graphql/directives.py +++ b/src/services/api/graphql/graphql/directives.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/graphql/mutations.py b/src/services/api/graphql/graphql/mutations.py index 0b391c070..c979d06e8 100644 --- a/src/services/api/graphql/graphql/mutations.py +++ b/src/services/api/graphql/graphql/mutations.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/graphql/queries.py b/src/services/api/graphql/graphql/queries.py index 9303fe909..3a8d12344 100644 --- a/src/services/api/graphql/graphql/queries.py +++ b/src/services/api/graphql/graphql/queries.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/libs/key_auth.py b/src/services/api/graphql/libs/key_auth.py index ffd7f32b2..dc3322fea 100644 --- a/src/services/api/graphql/libs/key_auth.py +++ b/src/services/api/graphql/libs/key_auth.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/libs/op_mode.py b/src/services/api/graphql/libs/op_mode.py index 86e38eae6..fa726264c 100644 --- a/src/services/api/graphql/libs/op_mode.py +++ b/src/services/api/graphql/libs/op_mode.py @@ -1,4 +1,4 @@ -# Copyright 2022-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/libs/token_auth.py b/src/services/api/graphql/libs/token_auth.py index 4f743a096..73c52bdf0 100644 --- a/src/services/api/graphql/libs/token_auth.py +++ b/src/services/api/graphql/libs/token_auth.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/routers.py b/src/services/api/graphql/routers.py index ed3ee1e8c..c6886ba1c 100644 --- a/src/services/api/graphql/routers.py +++ b/src/services/api/graphql/routers.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -32,7 +32,7 @@ def graphql_init(app: 'FastAPI'): state = SessionState() - # import after initializaion of state + # import after initialization of state from .bindings import generate_schema schema = generate_schema() diff --git a/src/services/api/graphql/session/composite/system_status.py b/src/services/api/graphql/session/composite/system_status.py index 516a4eff6..1674b2c2b 100755 --- a/src/services/api/graphql/session/composite/system_status.py +++ b/src/services/api/graphql/session/composite/system_status.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as diff --git a/src/services/api/graphql/session/override/remove_firewall_address_group_members.py b/src/services/api/graphql/session/override/remove_firewall_address_group_members.py index b91932e14..9f39465a1 100644 --- a/src/services/api/graphql/session/override/remove_firewall_address_group_members.py +++ b/src/services/api/graphql/session/override/remove_firewall_address_group_members.py @@ -1,4 +1,4 @@ -# Copyright 2021 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/graphql/session/session.py b/src/services/api/graphql/session/session.py index 619534f43..e4725e752 100644 --- a/src/services/api/graphql/session/session.py +++ b/src/services/api/graphql/session/session.py @@ -1,4 +1,4 @@ -# Copyright 2021-2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/api/rest/models.py b/src/services/api/rest/models.py index dda50010f..bfea17344 100644 --- a/src/services/api/rest/models.py +++ b/src/services/api/rest/models.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -26,6 +26,7 @@ from typing import Self from pydantic import BaseModel from pydantic import StrictStr +from pydantic import StrictInt from pydantic import field_validator from pydantic import model_validator from fastapi.responses import HTMLResponse @@ -47,7 +48,7 @@ def success(data): # Pydantic models for validation # Pydantic will cast when possible, so use StrictStr validators added as # needed for additional constraints -# json_schema_extra adds anotations to OpenAPI to add examples +# json_schema_extra adds annotations to OpenAPI to add examples class ApiModel(BaseModel): @@ -71,6 +72,8 @@ class BaseConfigureModel(BasePathModel): class ConfigureModel(ApiModel, BaseConfigureModel): + confirm_time: StrictInt = 0 + class Config: json_schema_extra = { 'example': { @@ -81,8 +84,12 @@ class ConfigureModel(ApiModel, BaseConfigureModel): } +class ConfirmModel(ApiModel): + op: StrictStr + class ConfigureListModel(ApiModel): commands: List[BaseConfigureModel] + confirm_time: StrictInt = 0 class Config: json_schema_extra = { @@ -134,13 +141,17 @@ class RetrieveModel(ApiModel): class ConfigFileModel(ApiModel): op: StrictStr file: StrictStr = None + string: StrictStr = None + confirm_time: StrictInt = 0 + destructive: bool = False class Config: json_schema_extra = { 'example': { 'key': 'id_key', - 'op': 'save | load', + 'op': 'save | load | merge | confirm', 'file': 'filename', + 'string': 'config_string' } } @@ -251,6 +262,20 @@ class RebootModel(ApiModel): } +class RenewModel(ApiModel): + op: StrictStr + path: List[StrictStr] + + class Config: + json_schema_extra = { + 'example': { + 'key': 'id_key', + 'op': 'renew', + 'path': ['op', 'mode', 'path'], + } + } + + class ResetModel(ApiModel): op: StrictStr path: List[StrictStr] diff --git a/src/services/api/rest/routers.py b/src/services/api/rest/routers.py index e52c77fda..fe67d4612 100644 --- a/src/services/api/rest/routers.py +++ b/src/services/api/rest/routers.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -18,6 +18,7 @@ # pylint: disable=wildcard-import,unused-wildcard-import # pylint: disable=broad-exception-caught +import asyncio import json import copy import logging @@ -28,12 +29,14 @@ from typing import Callable from typing import TYPE_CHECKING from fastapi import Depends +from fastapi import Query from fastapi import Request from fastapi import Response from fastapi import HTTPException from fastapi import APIRouter from fastapi import BackgroundTasks from fastapi.routing import APIRoute +from fastapi.concurrency import run_in_threadpool from starlette.datastructures import FormData from starlette.formparsers import FormParser from starlette.formparsers import MultiPartParser @@ -45,12 +48,15 @@ from vyos.configtree import ConfigTree from vyos.configdiff import get_config_diff from vyos.configsession import ConfigSessionError +from ..background import BackgroundOpManager +from ..background import BackgroundOpError from ..session import SessionState from .models import success from .models import error from .models import responses from .models import ApiModel from .models import ConfigureModel +from .models import ConfirmModel from .models import ConfigureListModel from .models import ConfigSectionModel from .models import ConfigSectionListModel @@ -66,6 +72,7 @@ from .models import GenerateModel from .models import ShowModel from .models import RebootModel from .models import ResetModel +from .models import RenewModel from .models import ImportPkiModel from .models import PoweroffModel from .models import TracerouteModel @@ -79,6 +86,7 @@ LOG = logging.getLogger('http_api.routers') lock = Lock() +asynclock = asyncio.Lock() def check_auth(key_list, key): key_id = None @@ -99,7 +107,7 @@ def auth_required(data: ApiModel): # override Request and APIRoute classes in order to convert form request to json; -# do all explicit validation here, for backwards compatability of error messages; +# do all explicit validation here, for backwards compatibility of error messages; # the explicit validation may be dropped, if desired, in favor of native # validation by FastAPI/Pydantic, as is used for application/json requests class MultipartRequest(Request): @@ -227,7 +235,7 @@ class MultipartRequest(Request): 400, f"Malformed command '{0}': 'path' field must be a list of strings", ) - if endpoint in ('/configure'): + if endpoint in ('/configure',): if not c['path']: self.form_err = ( 400, @@ -238,7 +246,7 @@ class MultipartRequest(Request): 400, f"Malformed command '{c}': 'value' field must be a string", ) - if endpoint in ('/configure-section'): + if endpoint in ('/configure-section',): if 'section' not in c and 'config' not in c: self.form_err = ( 400, @@ -290,6 +298,10 @@ router = APIRouter( self_ref_msg = 'Requested HTTP API server configuration change; commit will be called in the background' +# Global background-op manager used by the REST API to run long config commits after the response +background_op_manager = BackgroundOpManager() + + def call_commit(s: SessionState): try: s.session.commit() @@ -301,24 +313,71 @@ def call_commit(s: SessionState): LOG.warning(f'ConfigSessionError: {e}') -def _configure_op( +def call_commit_confirm(s: SessionState): + env = s.session.get_session_env() + env['IN_COMMIT_CONFIRM'] = 't' + try: + s.session.commit() + s.session.commit_confirm(minutes=s.confirm_time) + except ConfigSessionError as e: + s.session.discard() + if s.debug: + LOG.warning(f'ConfigSessionError:\n {traceback.format_exc()}') + else: + LOG.warning(f'ConfigSessionError: {e}') + finally: + del env['IN_COMMIT_CONFIRM'] + + +def run_commit(s: SessionState): + try: + out = s.session.commit() + return out, None + except Exception as e: + return None, e + + +def run_commit_confirm(s: SessionState): + env = s.session.get_session_env() + env['IN_COMMIT_CONFIRM'] = 't' + try: + out_c = s.session.commit() + out_cc = s.session.commit_confirm(minutes=s.confirm_time) + out = out_c + '\n' + out_cc + return out, None + except Exception as e: + return None, e + finally: + del env['IN_COMMIT_CONFIRM'] + + +def _execute_configure_op( data: Union[ + ConfirmModel, ConfigureModel, ConfigureListModel, ConfigSectionModel, ConfigSectionListModel, ConfigSectionTreeModel, ], - _request: Request, - background_tasks: BackgroundTasks, + background_tasks: BackgroundTasks | None = None, ): # pylint: disable=too-many-branches,too-many-locals,too-many-nested-blocks,too-many-statements # pylint: disable=consider-using-with + # True when invoked by the background operation + # runner (no FastAPI BackgroundTasks context passed in) + is_background_job = background_tasks is None + state = SessionState() session = state.session env = session.get_session_env() + # A non-zero confirm_time will start commit-confirm timer on commit + confirm_time = 0 + if isinstance(data, (ConfigureModel, ConfigureListModel, ConfigFileModel)): + confirm_time = data.confirm_time + # Allow users to pass just one command if not isinstance(data, (ConfigureListModel, ConfigSectionListModel)): data = [data] @@ -338,10 +397,18 @@ def _configure_op( try: for c in data: op = c.op - if not isinstance(c, BaseConfigSectionTreeModel): + op_error = ConfigSessionError(f"'{op}' is not a valid operation") + + if not isinstance(c, (ConfirmModel, BaseConfigSectionTreeModel)): path = c.path - if isinstance(c, BaseConfigureModel): + if isinstance(c, ConfirmModel): + if op == 'confirm': + msg = session.confirm() + else: + raise op_error + + elif isinstance(c, BaseConfigureModel): if c.value: value = c.value else: @@ -354,8 +421,8 @@ def _configure_op( section = c.section elif isinstance(c, BaseConfigSectionTreeModel): - mask = c.mask - config = c.config + mask_dict = c.mask + config_dict = c.config if isinstance(c, BaseConfigureModel): if op == 'set': @@ -369,7 +436,7 @@ def _configure_op( elif op == 'comment': session.comment(path, value=value) else: - raise ConfigSessionError(f"'{op}' is not a valid operation") + raise op_error elif isinstance(c, BaseConfigSectionModel): if op == 'set': @@ -377,26 +444,50 @@ def _configure_op( elif op == 'load': session.load_section(path, section) else: - raise ConfigSessionError(f"'{op}' is not a valid operation") + raise op_error elif isinstance(c, BaseConfigSectionTreeModel): if op == 'set': - session.set_section_tree(config) + session.set_section_tree(config_dict) elif op == 'load': - session.load_section_tree(mask, config) + config_tree = config.get_config_tree() + session.load_section_tree(config_tree, mask_dict, config_dict) else: - raise ConfigSessionError(f"'{op}' is not a valid operation") + raise op_error # end for + config = Config(session_env=env) d = get_config_diff(config) - if d.is_node_changed(['service', 'https']): - background_tasks.add_task(call_commit, state) - msg = self_ref_msg + state.confirm_time = confirm_time if confirm_time else 0 + + if not d.is_node_changed(['service', 'https']): + if confirm_time: + out, err = run_commit_confirm(state) + if err: + raise err + msg = msg + out if msg else out + else: + out, err = run_commit(state) + if err: + raise err + msg = msg + out if msg else out else: - # capture non-fatal warnings - out = session.commit() - msg = out if out else msg + if is_background_job: + # If already running as a background job, commit synchronously here + if confirm_time: + call_commit_confirm(state) + else: + call_commit(state) + else: + # Otherwise schedule the commit to run after the HTTP response + if confirm_time: + background_tasks.add_task(call_commit_confirm, state) + else: + background_tasks.add_task(call_commit, state) + + out = self_ref_msg + msg = msg + out if msg else out LOG.info(f"Configuration modified via HTTP API using key '{state.id}'") except ConfigSessionError as e: @@ -411,16 +502,68 @@ def _configure_op( status = 500 # Don't give the details away to the outer world - error_msg = 'An internal error occured. Check the logs for details.' + error_msg = 'An internal error occurred. Check the logs for details.' finally: + if 'IN_COMMIT_CONFIRM' in env: + del env['IN_COMMIT_CONFIRM'] lock.release() + # Background jobs return raw success text or raise on failure; + # the API wrapper formats HTTP responses and returns it + if is_background_job: + if status == 200: + return msg + else: + raise RuntimeError(error_msg) + if status != 200: return error(status, error_msg) return success(msg) +async def _configure_op( + data: Union[ + ConfirmModel, + ConfigureModel, + ConfigureListModel, + ConfigSectionModel, + ConfigSectionListModel, + ConfigSectionTreeModel, + ], + background_tasks: BackgroundTasks, + in_background: bool = False, +): + """ + API wrapper for configure operations. + + If `in_background=True`: enqueue the whole configure + workflow and return an operation record immediately. + Otherwise: run the configure workflow in a threadpool + and return the normal API response. + """ + + if in_background: + try: + # Enqueue and return an operation handle that + # can be polled via `/retrieve/background-operations` + record = background_op_manager.enqueue( + background_tasks, + _execute_configure_op, + data, + ) + except BackgroundOpError as e: + return error(500, str(e)) + + return success({'operation': record.model_dump()}) + + return await run_in_threadpool( + _execute_configure_op, + data, + background_tasks=background_tasks, + ) + + def create_path_import_pki_no_prompt(path): correct_paths = ['ca', 'certificate', 'key-pair'] if path[1] not in correct_paths: @@ -431,21 +574,23 @@ def create_path_import_pki_no_prompt(path): @router.post('/configure') -def configure_op( - data: Union[ConfigureModel, ConfigureListModel], +async def configure_op( + data: Union[ConfigureModel, ConfigureListModel, ConfirmModel], request: Request, background_tasks: BackgroundTasks, + in_background: bool = Query(False), ): - return _configure_op(data, request, background_tasks) + return await _configure_op(data, background_tasks, in_background) @router.post('/configure-section') -def configure_section_op( +async def configure_section_op( data: Union[ConfigSectionModel, ConfigSectionListModel, ConfigSectionTreeModel], request: Request, background_tasks: BackgroundTasks, + in_background: bool = Query(False), ): - return _configure_op(data, request, background_tasks) + return await _configure_op(data, background_tasks, in_background) @router.post('/retrieve') @@ -487,49 +632,98 @@ async def retrieve_op(data: RetrieveModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) +@router.post('/retrieve/background-operations') +async def retrieve_background_operations( + op_id: str = Query(None), +): + if op_id: + # Return only that record + record = background_op_manager.get_record(op_id) + records = [record] if record else [] + else: + # Return the full in-memory operation history (oldest first) + records = background_op_manager.get_records() + + result = { + 'operations': [record.model_dump() for record in records], + } + + return success(result) + + @router.post('/config-file') -def config_file_op(data: ConfigFileModel, background_tasks: BackgroundTasks): +async def config_file_op(data: ConfigFileModel, background_tasks: BackgroundTasks): state = SessionState() session = state.session env = session.get_session_env() op = data.op msg = None - try: - if op == 'save': - if data.file: - path = data.file - else: - path = '/config/config.boot' - msg = session.save_config(path) - elif op == 'load': - if data.file: - path = data.file - else: - return error(400, 'Missing required field "file"') + # A non-zero confirm_time will start commit-confirm timer on commit + confirm_time = data.confirm_time + + # Serialize config operations without blocking the event loop + async with asynclock: + try: + if op == 'save': + path = data.file or '/config/config.boot' + msg = session.save_config(path) + + elif op in ('load', 'merge'): + if data.file: + path = data.file + elif data.string: + path = '/tmp/config.file' + with open(path, 'w') as f: + f.write(data.string) + else: + return error(400, 'Missing required field "file | string"') + + match op: + case 'load': + session.migrate_and_load_config(path) + case 'merge': + session.merge_config(path, destructive=data.destructive) - session.migrate_and_load_config(path) + config = Config(session_env=env) + d = get_config_diff(config) - config = Config(session_env=env) - d = get_config_diff(config) + state.confirm_time = confirm_time if confirm_time else 0 - if d.is_node_changed(['service', 'https']): - background_tasks.add_task(call_commit, state) - msg = self_ref_msg + if not d.is_node_changed(['service', 'https']): + if confirm_time: + out, err = await run_in_threadpool(run_commit_confirm, state) + else: + out, err = await run_in_threadpool(run_commit, state) + + if err: + raise err + msg = (msg or '') + (out or '') + else: + if confirm_time: + background_tasks.add_task(call_commit_confirm, state) + else: + background_tasks.add_task(call_commit, state) + out = self_ref_msg + msg = (msg or '') + (out or '') + elif op == 'confirm': + msg = session.confirm() else: - session.commit() - else: - return error(400, f"'{op}' is not a valid operation") - except ConfigSessionError as e: - return error(400, str(e)) - except Exception: - LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(400, f"'{op}' is not a valid operation") + + except ConfigSessionError as e: + return error(400, str(e)) + except Exception: + LOG.critical(traceback.format_exc()) + return error(500, 'An internal error occurred. Check the logs for details.') + finally: + if 'IN_COMMIT_CONFIRM' in env: + del env['IN_COMMIT_CONFIRM'] return success(msg) @@ -554,7 +748,7 @@ def image_op(data: ImageModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) @@ -587,7 +781,7 @@ def container_image_op(data: ContainerImageModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) @@ -603,13 +797,14 @@ def generate_op(data: GenerateModel): try: if op == 'generate': res = session.generate(path) + session.commit() else: return error(400, f"'{op}' is not a valid operation") except ConfigSessionError as e: return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) @@ -631,7 +826,7 @@ def show_op(data: ShowModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) @@ -653,10 +848,30 @@ def reboot_op(data: RebootModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) +@router.post('/renew') +def renew_op(data: RenewModel): + state = SessionState() + session = state.session + + op = data.op + path = data.path + + try: + if op == 'renew': + res = session.renew(path) + else: + return error(400, f"'{op}' is not a valid operation") + except ConfigSessionError as e: + return error(400, str(e)) + except Exception: + LOG.critical(traceback.format_exc()) + return error(500, 'An internal error occurred. Check the logs for details.') + + return success(res) @router.post('/reset') def reset_op(data: ResetModel): @@ -675,7 +890,7 @@ def reset_op(data: ResetModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) @@ -715,7 +930,7 @@ def import_pki(data: ImportPkiModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') finally: lock.release() @@ -739,7 +954,7 @@ def poweroff_op(data: PoweroffModel): return error(400, str(e)) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) diff --git a/src/services/api/session.py b/src/services/api/session.py index ad3ef660c..c25a444e9 100644 --- a/src/services/api/session.py +++ b/src/services/api/session.py @@ -1,4 +1,4 @@ -# Copyright 2024 VyOS maintainers and contributors <maintainers@vyos.io> +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public diff --git a/src/services/vyos-commitd b/src/services/vyos-commitd index 8dbd39058..b8e430b93 100755 --- a/src/services/vyos-commitd +++ b/src/services/vyos-commitd @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -42,6 +42,7 @@ from vyos.defaults import directories from vyos.utils.boot import boot_configuration_complete from vyos.configsource import ConfigSourceCache from vyos.configsource import ConfigSourceError +from vyos.configdiff import get_commit_scripts from vyos.config import Config from vyos.frrender import FRRender from vyos.frrender import get_frrender_dict @@ -72,8 +73,9 @@ class Session: # pylint: disable=too-many-instance-attributes session_id: str = '' - named_active: str = None - named_proposed: str = None + session_pid: int = None + sudo_user: str = None + user: str = None dry_run: bool = False atomic: bool = False background: bool = False @@ -229,14 +231,30 @@ def initialization(session: Session) -> Session: config = Config(config_source=configsource) + # required by protobuf schema; non-existence will raise early error + if session.session_pid: + os.environ['SESSION_PID'] = str(session.session_pid) + + # required by protobuf schema; may be empty string + if session.sudo_user: + os.environ['SUDO_USER'] = session.sudo_user + + # required by protobuf schema; may be empty string + if session.user: + os.environ['USER'] = session.user + dependent_func: dict[str, list[typing.Callable]] = {} setattr(config, 'dependent_func', dependent_func) + commit_scripts = get_commit_scripts(config) + logger.debug(f'commit_scripts: {commit_scripts}') + scripts_called = [] setattr(config, 'scripts_called', scripts_called) - dry_run = False - setattr(config, 'dry_run', dry_run) + dry_run = session.dry_run + config.set_bool_attr('dry_run', dry_run) + logger.debug(f'commit dry_run is {dry_run}') session.config = config @@ -249,11 +267,16 @@ def run_script(script_name: str, config: Config, args: list) -> tuple[bool, str] script = conf_mode_scripts[script_name] script.argv = args config.set_level([]) + dry_run = config.get_bool_attr('dry_run') try: c = script.get_config(config) script.verify(c) - script.generate(c) - script.apply(c) + if not dry_run: + script.generate(c) + script.apply(c) + else: + if hasattr(script, 'call_dependents'): + script.call_dependents() except ConfigError as e: logger.error(e) return False, str(e) @@ -265,6 +288,38 @@ def run_script(script_name: str, config: Config, args: list) -> tuple[bool, str] return True, '' +def call_frr_render(frr, config): + # pylint: disable=redefined-outer-name + def _call_frr_render(frr, config): + # pylint: disable=broad-exception-caught + try: + tmp = get_frrender_dict(config) + if frr.generate(tmp): + # only apply a new FRR configuration if anything changed + # in comparison to the previous applied configuration + frr.apply() + + except ConfigError as e: + logger.error(e) + return False, str(e) + except Exception: + tb = traceback.format_exc() + logger.error(tb) + return False, tb + + return True, '' + + with redirect_stdout(io.StringIO()) as o: + result, err_out = _call_frr_render(frr, config) + amb_out = o.getvalue() + o.close() + + out = amb_out + err_out + logger.info(out) + + return result, out + + def process_call_data(call: Call, config: Config, last: bool = False) -> None: # pylint: disable=too-many-locals @@ -296,8 +351,6 @@ def process_call_data(call: Call, config: Config, last: bool = False) -> None: out = amb_out + err_out - call.set_reply(success, out) - logger.info(f'[{script_name}] {out}') if last: @@ -305,11 +358,11 @@ def process_call_data(call: Call, config: Config, last: bool = False) -> None: logger.debug(f'scripts_called: {scripts_called}') if last and success: - tmp = get_frrender_dict(config) - if frr.generate(tmp): - # only apply a new FRR configuration if anything changed - # in comparison to the previous applied configuration - frr.apply() + s, o = call_frr_render(frr, config) + success = s + out = out + o + + call.set_reply(success, out) def process_session_data(session: Session) -> Session: diff --git a/src/services/vyos-configd b/src/services/vyos-configd index 28acccd2c..2f060dc82 100755 --- a/src/services/vyos-configd +++ b/src/services/vyos-configd @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -33,6 +33,7 @@ from enum import Enum import zmq from vyos.defaults import directories +from vyos.defaults import vyos_configd_socket_path from vyos.utils.boot import boot_configuration_complete from vyos.configsource import ConfigSourceString from vyos.configsource import ConfigSourceError @@ -57,7 +58,7 @@ if debug: else: logger.setLevel(logging.INFO) -SOCKET_PATH = 'ipc:///run/vyos-configd.sock' +SOCKET_PATH = vyos_configd_socket_path MAX_MSG_SIZE = 65535 PAD_MSG_SIZE = 6 @@ -68,6 +69,7 @@ class Response(Enum): ERROR_COMMIT = 2 ERROR_DAEMON = 4 PASS = 8 + ERROR_COMMIT_APPLY = 16 vyos_conf_scripts_dir = directories['conf_mode'] @@ -142,8 +144,6 @@ def run_script(script_name, config, args) -> tuple[Response, str]: try: c = script.get_config(config) script.verify(c) - script.generate(c) - script.apply(c) except ConfigError as e: logger.error(e) return Response.ERROR_COMMIT, str(e) @@ -152,6 +152,17 @@ def run_script(script_name, config, args) -> tuple[Response, str]: logger.error(tb) return Response.ERROR_COMMIT, tb + try: + script.generate(c) + script.apply(c) + except ConfigError as e: + logger.error(e) + return Response.ERROR_COMMIT_APPLY, str(e) + except Exception: + tb = traceback.format_exc() + logger.error(tb) + return Response.ERROR_COMMIT_APPLY, tb + return Response.SUCCESS, '' @@ -263,6 +274,39 @@ def process_node_data(config, data, _last: bool = False) -> tuple[Response, str] out = amb_out + err_out + logger.info(f'[{script_name}] {out}') + + return result, out + + +def call_frr_render(frr, config): + def _call_frr_render(frr, config): + # pylint: disable=broad-exception-caught + try: + tmp = get_frrender_dict(config) + if frr.generate(tmp): + # only apply a new FRR configuration if anything changed + # in comparison to the previous applied configuration + frr.apply() + + except ConfigError as e: + logger.error(e) + return Response.ERROR_COMMIT_APPLY, str(e) + except Exception: + tb = traceback.format_exc() + logger.error(tb) + return Response.ERROR_COMMIT_APPLY, tb + + return Response.SUCCESS, '' + + with redirect_stdout(io.StringIO()) as o: + result, err_out = _call_frr_render(frr, config) + amb_out = o.getvalue() + o.close() + + out = amb_out + err_out + logger.info(out) + return result, out @@ -335,17 +379,16 @@ if __name__ == '__main__': config = initialization(socket) elif message['type'] == 'node': res, out = process_node_data(config, message['data'], message['last']) - send_result(socket, res, out) if message['last'] and config: scripts_called = getattr(config, 'scripts_called', []) logger.debug(f'scripts_called: {scripts_called}') if res == Response.SUCCESS: - tmp = get_frrender_dict(config) - if frr.generate(tmp): - # only apply a new FRR configuration if anything changed - # in comparison to the previous applied configuration - frr.apply() + r, o = call_frr_render(frr, config) + res = r + out = out + o + + send_result(socket, res, out) else: logger.critical(f'Unexpected message: {message}') diff --git a/src/services/vyos-conntrack-logger b/src/services/vyos-conntrack-logger index 9c31b465f..6e0733291 100755 --- a/src/services/vyos-conntrack-logger +++ b/src/services/vyos-conntrack-logger @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -15,10 +15,8 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. import argparse -import grp import logging import multiprocessing -import os import queue import signal import socket diff --git a/src/services/vyos-domain-resolver b/src/services/vyos-domain-resolver index aba5ba9db..e1a52c93a 100755 --- a/src/services/vyos-domain-resolver +++ b/src/services/vyos-domain-resolver @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -28,7 +28,7 @@ from vyos.utils.commit import commit_in_progress from vyos.utils.dict import dict_search_args from vyos.utils.kernel import WIREGUARD_REKEY_AFTER_TIME from vyos.utils.file import makedir, chmod_775, write_file, read_file -from vyos.utils.network import is_valid_ipv4_address_or_range +from vyos.utils.network import is_valid_ipv4_address_or_range, is_valid_ipv6_address_or_range from vyos.utils.process import cmd from vyos.utils.process import run from vyos.xml_ref import get_defaults @@ -48,6 +48,7 @@ ipv4_tables = { 'ip vyos_mangle', 'ip vyos_filter', 'ip vyos_nat', + 'ip vyos_wanloadbalance', 'ip raw' } @@ -92,12 +93,14 @@ def resolve(domains, ipv6=False): for domain in domains: resolved = fqdn_resolve(domain, ipv6=ipv6) + cache_key = f'{domain}_ipv6' if ipv6 else domain + if resolved and cache: - domain_state[domain] = resolved + domain_state[cache_key] = resolved elif not resolved: - if domain not in domain_state: + if cache_key not in domain_state: continue - resolved = domain_state[domain] + resolved = domain_state[cache_key] ip_list = ip_list | resolved return ip_list @@ -141,10 +144,11 @@ def update_remote_group(config): for set_name, remote_config in remote_groups.items(): if 'url' not in remote_config: continue - nft_set_name = f'R_{set_name}' + nft_ip_set_name = f'R_{set_name}' + nft_ip6_set_name = f'R6_{set_name}' # Create list file if necessary - list_file = os.path.join(firewall_config_dir, f"{nft_set_name}.txt") + list_file = os.path.join(firewall_config_dir, f"{nft_ip_set_name}.txt") if not os.path.exists(list_file): write_file(list_file, '', user="root", group="vyattacfg", mode=0o644) @@ -157,16 +161,32 @@ def update_remote_group(config): # Read list file ip_list = [] + ip6_list = [] + invalid_list = [] for line in read_file(list_file).splitlines(): line_first_word = line.strip().partition(' ')[0] if is_valid_ipv4_address_or_range(line_first_word): ip_list.append(line_first_word) + elif is_valid_ipv6_address_or_range(line_first_word): + ip6_list.append(line_first_word) + else: + if line_first_word[0].isalnum(): + invalid_list.append(line_first_word) - # Load tables + # Load ip tables for table in ipv4_tables: - if (table, nft_set_name) in valid_sets: - conf_lines += nft_output(table, nft_set_name, ip_list) + if (table, nft_ip_set_name) in valid_sets: + conf_lines += nft_output(table, nft_ip_set_name, ip_list) + + # Load ip6 tables + for table in ipv6_tables: + if (table, nft_ip6_set_name) in valid_sets: + conf_lines += nft_output(table, nft_ip6_set_name, ip6_list) + + invalid_str = ", ".join(invalid_list) + if invalid_str: + logger.info(f'Invalid address for set {set_name}: {invalid_str}') count += 1 diff --git a/src/services/vyos-hostsd b/src/services/vyos-hostsd index 1ba90471e..89742b431 100755 --- a/src/services/vyos-hostsd +++ b/src/services/vyos-hostsd @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2023 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -233,10 +233,7 @@ # } import os -import sys -import time import json -import signal import traceback import re import logging @@ -245,7 +242,6 @@ import zmq from voluptuous import Schema, MultipleInvalid, Required, Any from collections import OrderedDict from vyos.utils.file import makedir -from vyos.utils.permission import chown from vyos.utils.permission import chmod_755 from vyos.utils.process import popen from vyos.utils.process import process_named_running diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index be3dd5051..94697ed4d 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -1,6 +1,6 @@ #!/usr/share/vyos-http-api-tools/bin/python3 # -# Copyright (C) 2019-2024 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -99,7 +99,7 @@ def info(q: Annotated[InfoQueryParams, Query()]): res.update(banner=banner) except Exception: LOG.critical(traceback.format_exc()) - return error(500, 'An internal error occured. Check the logs for details.') + return error(500, 'An internal error occurred. Check the logs for details.') return success(res) diff --git a/src/services/vyos-netlinkd b/src/services/vyos-netlinkd new file mode 100755 index 000000000..2b158e05d --- /dev/null +++ b/src/services/vyos-netlinkd @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +# +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import re +import sys +import syslog +import signal +import select + +from pyroute2 import IPRoute # pylint: disable = no-name-in-module +from pyroute2 import NetlinkError # pylint: disable = no-name-in-module +from pyroute2.netlink.rtnl import RTMGRP_LINK +from time import sleep +from typing import Optional + +from vyos.configquery import op_mode_config_dict +from vyos.ifconfig import Section +from vyos.utils.boot import boot_configuration_complete +from vyos.utils.commit import commit_in_progress2 +from vyos.utils.dict import dict_search +from vyos.utils.process import cmd +from vyos.utils.process import is_systemd_service_active +from vyos.utils.process import stop_systemd_unit + +running = True + +# compile regex once during startup for fast match +IFACE_RE = re.compile(r"^(?:eth|br|bond|wlan)") + +def match_iface(ifname: str) -> bool: + """ Helper function returning true if interface name is a match for further + processing (e.g. restart of DHCP(v6) client) + """ + return IFACE_RE.match(ifname) is not None + +def sigterm_handler(signo, frame): + global running + running = False + sig = signal.Signals(signo) + syslog.syslog(syslog.LOG_INFO, f'Received signal {sig.name} - shutting down...') + +def _handle_dhcp_events(operstate: Optional[str], ifname: str) -> None: + systemdV4_service = f'dhclient@{ifname}.service' + systemdV6_service = f'dhcp6c@{ifname}.service' + + # Only handle explicit UP/DOWN state transitions; ignore other kernel states. + if operstate not in ['UP', 'DOWN']: + return None + + if operstate == 'DOWN': + # Interface moved state to down + if is_systemd_service_active(systemdV4_service): + syslog.syslog(syslog.LOG_DEBUG, f'Stopping {systemdV4_service}...') + stop_systemd_unit(systemdV4_service, raise_on_failure=False) + if is_systemd_service_active(systemdV6_service): + syslog.syslog(syslog.LOG_DEBUG, f'Stopping {systemdV6_service}...') + stop_systemd_unit(systemdV6_service, raise_on_failure=False) + + elif operstate == 'UP': + v6_restart = False + interface_path = Section.get_config_path(ifname, delimiter='.') + + config_dict = op_mode_config_dict( + ['interfaces'], key_mangling=('-', '_'), get_first_key=True + ) + + if tmp := dict_search(f'{interface_path}.address', config_dict): + # Always (re-)start the DHCP(v6) client service. If the DHCP(v6) client + # is already running - which could happen if the interface is re- + # configured in operational down state, it will have an exponential backoff + # time increasing while not receiving a DHCP(v6) reply. + # + # To make the interface instantly available, and as for a DHCP(v6) lease + # we will re-start the service and thus cancel the backoff time. + if 'dhcp' in tmp: + syslog.syslog(syslog.LOG_DEBUG, f'Restarting {systemdV4_service}...') + cmd(f'systemctl restart {systemdV4_service}') + if 'dhcpv6' in tmp: + v6_restart = True + + if dict_search(f'{interface_path}.dhcpv6_options.pd', config_dict): + v6_restart = True + + if v6_restart: + syslog.syslog(syslog.LOG_DEBUG, f'Restarting {systemdV6_service}...') + cmd(f'systemctl restart {systemdV6_service}') + + return None + +def main(): + syslog.openlog(ident="vyos-netlinkd", + logoption=syslog.LOG_PID, + facility=syslog.LOG_DAEMON) + syslog.syslog(syslog.LOG_INFO, "VyOS Netlink listener daemon started.") + + # Subscribe to link notifications only (not routes/rules/neigh/addr/...). + ipr = IPRoute() + try: + # newer pyroute2 versions support bind group in IPRoute() constructor + ipr.bind(groups=RTMGRP_LINK) + syslog.syslog(syslog.LOG_INFO, + 'IPRoute.bind() using groups=RTMGRP_LINK RTNL subscription') + except TypeError: + syslog.syslog(syslog.LOG_WARNING, + 'IPRoute.bind() has no groups= support; using default RTNL subscriptions', + ) + ipr.bind() + fd = ipr.fileno() + + global running + while running: + if not boot_configuration_complete(): + syslog.syslog(syslog.LOG_INFO, 'System bootup not yet finished...') + sleep(5) + continue + + try: + # Wait for up to 1 second for a netlink message + rlist, _, _ = select.select([fd], [], [], 1.0) + if not rlist: + # timeout - retry + continue + + # Check if a config commit is in progress before processing any + # messages. This avoids blocking per-message and reduces unnecessary + # calls to commit_in_progress2() + if commit_in_progress2(): + syslog.syslog(syslog.LOG_DEBUG, + 'Config commit in progress, skipping netlink events') + sleep(1) + continue + + # Receive and process any messages + for message in ipr.get(): + # Parse NETLINK message + match message['event']: + # Message received during interface creation or modification + # e.g. link up/down. + case 'RTM_NEWLINK': + attrs = dict(message.get('attrs', [])) + ifname = attrs.get('IFLA_IFNAME', None) + mac = attrs.get('IFLA_ADDRESS', '<unknown>') + operstate = attrs.get('IFLA_OPERSTATE', None) + syslog.syslog(syslog.LOG_DEBUG, + f'RTM_NEWLINK -> {ifname}, state={operstate}, mac={mac}') + + # Bail out early - no interface name in the message + if not ifname: + continue + # Bail out early - not interested in interface type + if not match_iface(ifname): + continue + + _handle_dhcp_events(operstate, ifname) + + # Deletion of a network link which has been previously added to the kernel + case 'RTM_DELLINK': + pass + case _: + pass + + except NetlinkError as e: + syslog.syslog(syslog.LOG_ERR, f'Netlink error: {e}') + except Exception as e: + syslog.syslog(syslog.LOG_ERR, f'Unhandled exception: {e}') + except KeyboardInterrupt: + break + + ipr.close() + syslog.syslog(syslog.LOG_INFO, 'Netlink listener daemon stopped.') + sys.exit(0) + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, sigterm_handler) + signal.signal(signal.SIGINT, sigterm_handler) + main() diff --git a/src/services/vyos-network-event-logger b/src/services/vyos-network-event-logger index 840ff3cda..699c57b1a 100644 --- a/src/services/vyos-network-event-logger +++ b/src/services/vyos-network-event-logger @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2025 VyOS maintainers and contributors +# Copyright VyOS maintainers and contributors <maintainers@vyos.io> # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -732,7 +732,7 @@ class RouteFormatter(BaseMSGFormatter): message += self._format_rta_pref(msg.get_attr("RTA_PREF")) if msg.get_attr('RTA_TTL_PROPAGATE') is not None: - message += f' ttl-propogate {"enabled" if msg.get_attr("RTA_TTL_PROPAGATE") else "disabled"}' + message += f' ttl-propagate {"enabled" if msg.get_attr("RTA_TTL_PROPAGATE") else "disabled"}' if msg.get_attr('RTA_MULTIPATH') is not None: _tmp = self._format_rta_multipath( |