Merge pull request #5550 from c-po/T9453-http-api-reload-readyrolling

http-api: T9453: wait for the API server to be ready after a reload

2 files changed, 19 insertions, 3 deletions

diff --git a/src/conf_mode/service_https.py b/src/conf_mode/service_https.py
index 10e0afe49..1a9ab0b75 100755
--- a/src/conf_mode/service_https.py
+++ b/src/conf_mode/service_https.py
@@ -19,7 +19,6 @@ import socket
 import sys
 import json
 
-from time import sleep
 
 from vyos.base import Warning
 from vyos.config import Config
@@ -30,6 +29,7 @@ from vyos.configverify import verify_pki_ca_certificate
 from vyos.configverify import verify_pki_dh_parameters
 from vyos.configdiff import get_config_diff
 from vyos.defaults import api_config_state
+from vyos.defaults import api_ready_state
 from vyos.pki import encode_certificate
 from vyos.pki import find_chain
 from vyos.pki import load_certificate
@@ -42,6 +42,7 @@ from vyos.utils.process import is_systemd_service_active
 from vyos.utils.network import check_port_availability
 from vyos.utils.network import is_listen_port_bind_service
 from vyos.utils.file import write_file
+from vyos.utils.misc import wait_for
 from vyos import ConfigError
 from vyos import airbag
 airbag.enable()
@@ -292,9 +293,13 @@ def apply(https):
         return
 
     if 'api' in https:
+        # A reload is only signalled; the server restarts asynchronously and
+        # nginx answers 502 until it listens again. Wait for it.
+        if os.path.exists(api_ready_state):
+            os.unlink(api_ready_state)
         call(f'systemctl reload-or-restart {http_api_service_name}')
-        # Let uvicorn settle before (possibly) restarting nginx
-        sleep(1)
+        if not wait_for(os.path.exists, api_ready_state, interval=0.1, timeout=30):
+            Warning('HTTP API server did not become ready within 30 seconds')
     elif is_systemd_service_active(http_api_service_name):
         call(f'systemctl stop {http_api_service_name}')
 
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index 6a420f499..c839dbd88 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -31,6 +31,7 @@ from uvicorn import Server as UvicornServer
 
 from vyos.configsession import ConfigSession
 from vyos.defaults import api_config_state
+from vyos.defaults import api_ready_state
 from vyos.utils.file import read_file
 from vyos.version import get_version
 
@@ -120,12 +121,20 @@ class ApiServer(UvicornServer):
     def install_signal_handlers(self):
         pass
 
+    async def startup(self, sockets=None):
+        await super().startup(sockets=sockets)
+        # Tells service_https.py that the socket accepts requests again
+        if self.started:
+            open(api_ready_state, 'w').close()
+
 
 def reload_handler(signum, frame):
     # pylint: disable=global-statement
 
     global server
     LOG.debug('Reload signal received...')
+    if os.path.exists(api_ready_state):
+        os.unlink(api_ready_state)
     if server is not None:
         server.handle_exit(signum, frame)
         server = None
@@ -139,6 +148,8 @@ def shutdown_handler(signum, frame):
 
     global shutdown
     LOG.debug('Shutdown signal received...')
+    if os.path.exists(api_ready_state):
+        os.unlink(api_ready_state)
     server.handle_exit(signum, frame)
     LOG.info('Server shutdown...')
     shutdown = True