http-api: T9453: wait for the API server to be ready after a reload

A commit that changes an already running HTTP API only signals the server
and returns at once. The server then restarts uvicorn asynchronously, and
until it listens again requests through nginx fail. A fixed one-second
sleep was meant to cover this but is too short on slow systems, making
the https smoketest fail intermittently and API clients see errors right
after a commit.

The server now creates a readiness marker once it accepts requests. The
commit clears the marker before reloading and waits for it to reappear,
warning if that takes longer than 30 seconds.

1 files changed, 8 insertions, 3 deletions

diff --git a/src/conf_mode/service_https.py b/src/conf_mode/service_https.py
index 10e0afe49..1a9ab0b75 100755
--- a/src/conf_mode/service_https.py
+++ b/src/conf_mode/service_https.py
@@ -19,7 +19,6 @@ import socket
 import sys
 import json
 
-from time import sleep
 
 from vyos.base import Warning
 from vyos.config import Config
@@ -30,6 +29,7 @@ from vyos.configverify import verify_pki_ca_certificate
 from vyos.configverify import verify_pki_dh_parameters
 from vyos.configdiff import get_config_diff
 from vyos.defaults import api_config_state
+from vyos.defaults import api_ready_state
 from vyos.pki import encode_certificate
 from vyos.pki import find_chain
 from vyos.pki import load_certificate
@@ -42,6 +42,7 @@ from vyos.utils.process import is_systemd_service_active
 from vyos.utils.network import check_port_availability
 from vyos.utils.network import is_listen_port_bind_service
 from vyos.utils.file import write_file
+from vyos.utils.misc import wait_for
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -292,9 +293,13 @@ def apply(https):
         return
 
     if 'api' in https:
+        # A reload is only signalled; the server restarts asynchronously and
+        # nginx answers 502 until it listens again. Wait for it.
+        if os.path.exists(api_ready_state):
+            os.unlink(api_ready_state)
         call(f'systemctl reload-or-restart {http_api_service_name}')
-        # Let uvicorn settle before (possibly) restarting nginx
-        sleep(1)
+        if not wait_for(os.path.exists, api_ready_state, interval=0.1, timeout=30):
+            Warning('HTTP API server did not become ready within 30 seconds')
     elif is_systemd_service_active(http_api_service_name):
         call(f'systemctl stop {http_api_service_name}')