summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorJohn Estabrook <jestabro@vyos.io>2021-03-26 11:25:44 -0500
committerJohn Estabrook <jestabro@vyos.io>2021-03-26 14:29:13 -0500
commit0125fff200efe3259aa25953e7505f69679261f8 (patch)
tree4dbd0282bbf0073e4eb35a184dd92897ff4842ca /src
parent137f50a1f00073842364ee373f98e1e215671c6b (diff)
downloadvyos-1x-0125fff200efe3259aa25953e7505f69679261f8.tar.gz
vyos-1x-0125fff200efe3259aa25953e7505f69679261f8.zip
http api: T3412: use FastAPI as web framework; support application/json
Replace the Flask micro-framework with FastAPI, in order to support extensions to the API and OpenAPI 3.* generation. This change will remain backwards compatible with previous versions. Notably, the multipart forms version of requests remain supported; in addition application/json requests are now natively supported.
Diffstat (limited to 'src')
-rwxr-xr-xsrc/services/vyos-http-api-server571
-rw-r--r--src/systemd/vyos-http-api.service3
2 files changed, 388 insertions, 186 deletions
diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server
index 703628558..8069d7146 100755
--- a/src/services/vyos-http-api-server
+++ b/src/services/vyos-http-api-server
@@ -1,6 +1,6 @@
-#!/usr/bin/env python3
+#!/usr/share/vyos-http-api-tools/bin/python3
#
-# Copyright (C) 2019 VyOS maintainers and contributors
+# Copyright (C) 2019-2021 VyOS maintainers and contributors
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 or later as
@@ -19,25 +19,37 @@
import os
import sys
import grp
+import copy
import json
+import logging
import traceback
import threading
-import signal
+from typing import List, Union, Callable, Dict
-import vyos.config
-
-from flask import Flask, request
-from waitress import serve
+import uvicorn
+from fastapi import FastAPI, Depends, Request, Response, HTTPException
+from fastapi.responses import HTMLResponse
+from fastapi.exceptions import RequestValidationError
+from fastapi.routing import APIRoute
+from pydantic import BaseModel, StrictStr, validator
-from functools import wraps
+import vyos.config
from vyos.configsession import ConfigSession, ConfigSessionError
-
DEFAULT_CONFIG_FILE = '/etc/vyos/http-api.conf'
CFG_GROUP = 'vyattacfg'
-app = Flask(__name__)
+debug = True
+
+logger = logging.getLogger(__name__)
+logs_handler = logging.StreamHandler()
+logger.addHandler(logs_handler)
+
+if debug:
+ logger.setLevel(logging.DEBUG)
+else:
+ logger.setLevel(logging.INFO)
# Giant lock!
lock = threading.Lock()
@@ -56,55 +68,310 @@ def check_auth(key_list, key):
def error(code, msg):
resp = {"success": False, "error": msg, "data": None}
- return json.dumps(resp), code
+ resp = json.dumps(resp)
+ return HTMLResponse(resp, status_code=code)
def success(data):
resp = {"success": True, "data": data, "error": None}
- return json.dumps(resp)
-
-def get_command(f):
- @wraps(f)
- def decorated_function(*args, **kwargs):
- cmd = request.form.get("data")
- if not cmd:
- return error(400, "Non-empty data field is required")
- try:
- cmd = json.loads(cmd)
- except Exception as e:
- return error(400, "Failed to parse JSON: {0}".format(e))
- return f(cmd, *args, **kwargs)
-
- return decorated_function
-
-def auth_required(f):
- @wraps(f)
- def decorated_function(*args, **kwargs):
- key = request.form.get("key")
- api_keys = app.config['vyos_keys']
- id = check_auth(api_keys, key)
- if not id:
- return error(401, "Valid API key is required")
- return f(*args, **kwargs)
-
- return decorated_function
-
-@app.route('/configure', methods=['POST'])
-@get_command
-@auth_required
-def configure_op(commands):
- session = app.config['vyos_session']
+ resp = json.dumps(resp)
+ return HTMLResponse(resp)
+
+# Pydantic models for validation
+# Pydantic will cast when possible, so use StrictStr
+# validators added as needed for additional constraints
+# schema_extra adds anotations to OpenAPI, to add examples
+
+class ApiModel(BaseModel):
+ key: StrictStr
+
+class BaseConfigureModel(BaseModel):
+ op: StrictStr
+ path: List[StrictStr]
+ value: StrictStr = None
+
+ @validator("path", pre=True, always=True)
+ def check_non_empty(cls, path):
+ assert len(path) > 0
+ return path
+
+class ConfigureModel(ApiModel):
+ op: StrictStr
+ path: List[StrictStr]
+ value: StrictStr = None
+
+ @validator("path", pre=True, always=True)
+ def check_non_empty(cls, path):
+ assert len(path) > 0
+ return path
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "set | delete | comment",
+ "path": ['config', 'mode', 'path'],
+ }
+ }
+
+class ConfigureListModel(ApiModel):
+ commands: List[BaseConfigureModel]
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "commands": "list of commands",
+ }
+ }
+
+class RetrieveModel(ApiModel):
+ op: StrictStr
+ path: List[StrictStr]
+ configFormat: StrictStr = None
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "returnValue | returnValues | exists | showConfig",
+ "path": ['config', 'mode', 'path'],
+ "configFormat": "json (default) | json_ast | raw",
+
+ }
+ }
+
+class ConfigFileModel(ApiModel):
+ op: StrictStr
+ file: StrictStr = None
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "save | load",
+ "file": "filename",
+ }
+ }
+
+class ImageModel(ApiModel):
+ op: StrictStr
+ url: StrictStr = None
+ name: StrictStr = None
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "add | delete",
+ "url": "imagelocation",
+ "name": "imagename",
+ }
+ }
+
+class GenerateModel(ApiModel):
+ op: StrictStr
+ path: List[StrictStr]
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "generate",
+ "path": ["op", "mode", "path"],
+ }
+ }
+
+class ShowModel(ApiModel):
+ op: StrictStr
+ path: List[StrictStr]
+
+ class Config:
+ schema_extra = {
+ "example": {
+ "key": "id_key",
+ "op": "show",
+ "path": ["op", "mode", "path"],
+ }
+ }
+
+class Success(BaseModel):
+ success: bool
+ data: Union[str, bool, Dict]
+ error: str
+
+class Error(BaseModel):
+ success: bool = False
+ data: Union[str, bool, Dict]
+ error: str
+
+responses = {
+ 200: {'model': Success},
+ 400: {'model': Error},
+ 422: {'model': Error, 'description': 'Validation Error'},
+ 500: {'model': Error}
+}
+
+def auth_required(data: ApiModel):
+ key = data.key
+ api_keys = app.state.vyos_keys
+ id = check_auth(api_keys, key)
+ if not id:
+ raise HTTPException(status_code=401, detail="Valid API key is required")
+ app.state.vyos_id = id
+
+# override Request and APIRoute classes in order to convert form request to json;
+# do all explicit validation here, for backwards compatability of error messages;
+# the explicit validation may be dropped, if desired, in favor of native
+# validation by FastAPI/Pydantic, as is used for application/json requests
+class MultipartRequest(Request):
+ ERR_MISSING_KEY = False
+ ERR_MISSING_DATA = False
+ ERR_NOT_JSON = False
+ ERR_NOT_DICT = False
+ ERR_NO_OP = False
+ ERR_NO_PATH = False
+ ERR_EMPTY_PATH = False
+ ERR_PATH_NOT_LIST = False
+ ERR_VALUE_NOT_STRING = False
+ ERR_PATH_NOT_LIST_OF_STR = False
+ offending_command = {}
+ exception = None
+ async def body(self) -> bytes:
+ if not hasattr(self, "_body"):
+ forms = {}
+ merge = {}
+ body = await super().body()
+ self._body = body
+
+ form_data = await self.form()
+ if form_data:
+ logger.debug("processing form data")
+ for k, v in form_data.multi_items():
+ forms[k] = v
+
+ if 'data' not in forms:
+ self.ERR_MISSING_DATA = True
+ else:
+ try:
+ tmp = json.loads(forms['data'])
+ except json.JSONDecodeError as e:
+ self.ERR_NOT_JSON = True
+ self.exception = e
+ tmp = {}
+ if isinstance(tmp, list):
+ merge['commands'] = tmp
+ else:
+ merge = tmp
+
+ if 'commands' in merge:
+ cmds = merge['commands']
+ else:
+ cmds = copy.deepcopy(merge)
+ cmds = [cmds]
+
+ for c in cmds:
+ if not isinstance(c, dict):
+ self.ERR_NOT_DICT = True
+ self.offending_command = c
+ elif 'op' not in c:
+ self.ERR_NO_OP = True
+ self.offending_command = c
+ elif 'path' not in c:
+ self.ERR_NO_PATH = True
+ self.offending_command = c
+ elif not c['path']:
+ self.ERR_EMPTY_PATH = True
+ self.offending_command = c
+ elif not isinstance(c['path'], list):
+ self.ERR_PATH_NOT_LIST = True
+ self.offending_command = c
+ elif not all(isinstance(el, str) for el in c['path']):
+ self.ERR_PATH_NOT_LIST_OF_STR = True
+ self.offending_command = c
+ elif 'value' in c and not isinstance(c['value'], str):
+ self.ERR_VALUE_NOT_STRING = True
+ self.offending_command = c
+
+ if 'key' not in forms and 'key' not in merge:
+ self.ERR_MISSING_KEY = True
+ if 'key' in forms and 'key' not in merge:
+ merge['key'] = forms['key']
+
+ new_body = json.dumps(merge)
+ new_body = new_body.encode()
+ self._body = new_body
+
+ return self._body
+
+class MultipartRoute(APIRoute):
+ def get_route_handler(self) -> Callable:
+ original_route_handler = super().get_route_handler()
+
+ async def custom_route_handler(request: Request) -> Response:
+ request = MultipartRequest(request.scope, request.receive)
+ endpoint = request.url.path
+ try:
+ response: Response = await original_route_handler(request)
+ except HTTPException as e:
+ return error(e.status_code, e.detail)
+ except Exception as e:
+ if request.ERR_MISSING_KEY:
+ return error(422, "Valid API key is required")
+ if request.ERR_MISSING_DATA:
+ return error(422, "Non-empty data field is required")
+ if request.ERR_NOT_JSON:
+ return error(400, "Failed to parse JSON: {0}".format(request.exception))
+ if endpoint == '/configure':
+ if request.ERR_NOT_DICT:
+ return error(400, "Malformed command \"{0}\": any command must be a dict".format(json.dumps(request.offending_command)))
+ if request.ERR_NO_OP:
+ return error(400, "Malformed command \"{0}\": missing \"op\" field".format(json.dumps(request.offending_command)))
+ if request.ERR_NO_PATH:
+ return error(400, "Malformed command \"{0}\": missing \"path\" field".format(json.dumps(request.offending_command)))
+ if request.ERR_EMPTY_PATH:
+ return error(400, "Malformed command \"{0}\": empty path".format(json.dumps(request.offending_command)))
+ if request.ERR_PATH_NOT_LIST:
+ return error(400, "Malformed command \"{0}\": \"path\" field must be a list".format(json.dumps(request.offending_command)))
+ if request.ERR_VALUE_NOT_STRING:
+ return error(400, "Malformed command \"{0}\": \"value\" field must be a string".format(json.dumps(request.offending_command)))
+ if request.ERR_PATH_NOT_LIST_OF_STR:
+ return error(400, "Malformed command \"{0}\": \"path\" field must be a list of strings".format(json.dumps(request.offending_command)))
+ if endpoint in ('/retrieve','/generate','/show'):
+ if request.ERR_NO_OP or request.ERR_NO_PATH:
+ return error(400, "Missing required field. \"op\" and \"path\" fields are required")
+ if endpoint in ('/config-file', '/image'):
+ if request.ERR_NO_OP:
+ return error(400, "Missing required field \"op\"")
+
+ raise e
+
+ return response
+
+ return custom_route_handler
+
+app = FastAPI(debug=True,
+ title="VyOS API",
+ version="0.1.0",
+ responses={**responses},
+ dependencies=[Depends(auth_required)])
+
+app.router.route_class = MultipartRoute
+
+@app.exception_handler(RequestValidationError)
+async def validation_exception_handler(request, exc):
+ return error(400, str(exc.errors()[0]))
+
+@app.post('/configure')
+def configure_op(data: Union[ConfigureModel, ConfigureListModel]):
+ session = app.state.vyos_session
env = session.get_session_env()
config = vyos.config.Config(session_env=env)
- strict_field = request.form.get("strict")
- if strict_field == "true":
- strict = True
- else:
- strict = False
-
# Allow users to pass just one command
- if not isinstance(commands, list):
- commands = [commands]
+ if not isinstance(data, ConfigureListModel):
+ data = [data]
+ else:
+ data = data.commands
# We don't want multiple people/apps to be able to commit at once,
# or modify the shared session while someone else is doing the same,
@@ -114,53 +381,25 @@ def configure_op(commands):
status = 200
error_msg = None
try:
- for c in commands:
- # What we've got may not even be a dict
- if not isinstance(c, dict):
- raise ConfigSessionError("Malformed command \"{0}\": any command must be a dict".format(json.dumps(c)))
-
- # Missing op or path is a show stopper
- if not ('op' in c):
- raise ConfigSessionError("Malformed command \"{0}\": missing \"op\" field".format(json.dumps(c)))
- if not ('path' in c):
- raise ConfigSessionError("Malformed command \"{0}\": missing \"path\" field".format(json.dumps(c)))
-
- # Missing value is fine, substitute for empty string
- if 'value' in c:
- value = c['value']
- else:
- value = ""
-
- op = c['op']
- path = c['path']
-
- if not path:
- raise ConfigSessionError("Malformed command \"{0}\": empty path".format(json.dumps(c)))
-
- # Type checking
- if not isinstance(path, list):
- raise ConfigSessionError("Malformed command \"{0}\": \"path\" field must be a list".format(json.dumps(c)))
+ for c in data:
+ op = c.op
+ path = c.path
- if not isinstance(value, str):
- raise ConfigSessionError("Malformed command \"{0}\": \"value\" field must be a string".format(json.dumps(c)))
-
- # Account for the case when value field is present and set to null
- if not value:
+ if c.value:
+ value = c.value
+ else:
value = ""
- # For vyos.configsessios calls that have no separate value arguments,
+ # For vyos.configsession calls that have no separate value arguments,
# and for type checking too
- try:
- cfg_path = " ".join(path + [value]).strip()
- except TypeError:
- raise ConfigSessionError("Malformed command \"{0}\": \"path\" field must be a list of strings".format(json.dumps(c)))
+ cfg_path = " ".join(path + [value]).strip()
if op == 'set':
# XXX: it would be nice to do a strict check for "path already exists",
# but there's probably no way to do that
session.set(path, value=value)
elif op == 'delete':
- if strict and not config.exists(cfg_path):
+ if app.state.vyos_strict and not config.exists(cfg_path):
raise ConfigSessionError("Cannot delete [{0}]: path/value does not exist".format(cfg_path))
session.delete(path, value=value)
elif op == 'comment':
@@ -169,16 +408,16 @@ def configure_op(commands):
raise ConfigSessionError("\"{0}\" is not a valid operation".format(op))
# end for
session.commit()
- print("Configuration modified via HTTP API using key \"{0}\"".format(id))
+ logger.info(f"Configuration modified via HTTP API using key '{app.state.vyos_id}'")
except ConfigSessionError as e:
session.discard()
status = 400
- if app.config['vyos_debug']:
- print(traceback.format_exc(), file=sys.stderr)
+ if app.state.vyos_debug:
+ logger.critical(f"ConfigSessionError:\n {traceback.format_exc()}")
error_msg = str(e)
except Exception as e:
session.discard()
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
status = 500
# Don't give the details away to the outer world
@@ -188,22 +427,17 @@ def configure_op(commands):
if status != 200:
return error(status, error_msg)
- else:
- return success(None)
-@app.route('/retrieve', methods=['POST'])
-@get_command
-@auth_required
-def retrieve_op(command):
- session = app.config['vyos_session']
+ return success(None)
+
+@app.post("/retrieve")
+def retrieve_op(data: RetrieveModel):
+ session = app.state.vyos_session
env = session.get_session_env()
config = vyos.config.Config(session_env=env)
- try:
- op = command['op']
- path = " ".join(command['path'])
- except KeyError:
- return error(400, "Missing required field. \"op\" and \"path\" fields are required")
+ op = data.op
+ path = " ".join(data.path)
try:
if op == 'returnValue':
@@ -214,10 +448,10 @@ def retrieve_op(command):
res = config.exists(path)
elif op == 'showConfig':
config_format = 'json'
- if 'configFormat' in command:
- config_format = command['configFormat']
+ if data.configFormat:
+ config_format = data.configFormat
- res = session.show_config(path=command['path'])
+ res = session.show_config(path=data.path)
if config_format == 'json':
config_tree = vyos.configtree.ConfigTree(res)
res = json.loads(config_tree.to_json())
@@ -233,33 +467,28 @@ def retrieve_op(command):
except ConfigSessionError as e:
return error(400, str(e))
except Exception as e:
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
return error(500, "An internal error occured. Check the logs for details.")
return success(res)
-@app.route('/config-file', methods=['POST'])
-@get_command
-@auth_required
-def config_file_op(command):
- session = app.config['vyos_session']
+@app.post('/config-file')
+def config_file_op(data: ConfigFileModel):
+ session = app.state.vyos_session
- try:
- op = command['op']
- except KeyError:
- return error(400, "Missing required field \"op\"")
+ op = data.op
try:
if op == 'save':
- try:
- path = command['file']
- except KeyError:
+ if data.file:
+ path = data.file
+ else:
path = '/config/config.boot'
res = session.save_config(path)
elif op == 'load':
- try:
- path = command['file']
- except KeyError:
+ if data.file:
+ path = data.file
+ else:
return error(400, "Missing required field \"file\"")
res = session.migrate_and_load_config(path)
res = session.commit()
@@ -268,33 +497,28 @@ def config_file_op(command):
except ConfigSessionError as e:
return error(400, str(e))
except Exception as e:
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
return error(500, "An internal error occured. Check the logs for details.")
return success(res)
-@app.route('/image', methods=['POST'])
-@get_command
-@auth_required
-def image_op(command):
- session = app.config['vyos_session']
+@app.post('/image')
+def image_op(data: ImageModel):
+ session = app.state.vyos_session
- try:
- op = command['op']
- except KeyError:
- return error(400, "Missing required field \"op\"")
+ op = data.op
try:
if op == 'add':
- try:
- url = command['url']
- except KeyError:
+ if data.url:
+ url = data.url
+ else:
return error(400, "Missing required field \"url\"")
res = session.install_image(url)
elif op == 'delete':
- try:
- name = command['name']
- except KeyError:
+ if data.name:
+ name = data.name
+ else:
return error(400, "Missing required field \"name\"")
res = session.remove_image(name)
else:
@@ -302,26 +526,17 @@ def image_op(command):
except ConfigSessionError as e:
return error(400, str(e))
except Exception as e:
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
return error(500, "An internal error occured. Check the logs for details.")
return success(res)
+@app.post('/generate')
+def generate_op(data: GenerateModel):
+ session = app.state.vyos_session
-@app.route('/generate', methods=['POST'])
-@get_command
-@auth_required
-def generate_op(command):
- session = app.config['vyos_session']
-
- try:
- op = command['op']
- path = command['path']
- except KeyError:
- return error(400, "Missing required field. \"op\" and \"path\" fields are required")
-
- if not isinstance(path, list):
- return error(400, "Malformed command: \"path\" field must be a list of strings")
+ op = data.op
+ path = data.path
try:
if op == 'generate':
@@ -331,25 +546,17 @@ def generate_op(command):
except ConfigSessionError as e:
return error(400, str(e))
except Exception as e:
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
return error(500, "An internal error occured. Check the logs for details.")
return success(res)
-@app.route('/show', methods=['POST'])
-@get_command
-@auth_required
-def show_op(command):
- session = app.config['vyos_session']
+@app.post('/show')
+def show_op(data: ShowModel):
+ session = app.state.vyos_session
- try:
- op = command['op']
- path = command['path']
- except KeyError:
- return error(400, "Missing required field. \"op\" and \"path\" fields are required")
-
- if not isinstance(path, list):
- return error(400, "Malformed command: \"path\" field must be a list of strings")
+ op = data.op
+ path = data.path
try:
if op == 'show':
@@ -359,14 +566,11 @@ def show_op(command):
except ConfigSessionError as e:
return error(400, str(e))
except Exception as e:
- print(traceback.format_exc(), file=sys.stderr)
+ logger.critical(traceback.format_exc())
return error(500, "An internal error occured. Check the logs for details.")
return success(res)
-def shutdown():
- raise KeyboardInterrupt
-
if __name__ == '__main__':
# systemd's user and group options don't work, do it by hand here,
# else no one else will be able to commit
@@ -380,21 +584,20 @@ if __name__ == '__main__':
try:
server_config = load_server_config()
except Exception as e:
- print("Failed to load the HTTP API server config: {0}".format(e))
+ logger.critical("Failed to load the HTTP API server config: {0}".format(e))
session = ConfigSession(os.getpid())
- app.config['vyos_session'] = session
- app.config['vyos_keys'] = server_config['api_keys']
- app.config['vyos_debug'] = server_config['debug']
-
- def sig_handler(signum, frame):
- shutdown()
+ app.state.vyos_session = session
+ app.state.vyos_keys = server_config['api_keys']
- signal.signal(signal.SIGTERM, sig_handler)
+ app.state.vyos_debug = True if server_config['debug'] == 'true' else False
+ app.state.vyos_strict = True if server_config['strict'] == 'true' else False
try:
- serve(app, host=server_config["listen_address"],
- port=server_config["port"])
+ uvicorn.run(app, host=server_config["listen_address"],
+ port=int(server_config["port"]),
+ proxy_headers=True)
except OSError as e:
- print(f"OSError {e}")
+ logger.critical(f"OSError {e}")
+ sys.exit(1)
diff --git a/src/systemd/vyos-http-api.service b/src/systemd/vyos-http-api.service
index 4fa68b4ff..ba5df5984 100644
--- a/src/systemd/vyos-http-api.service
+++ b/src/systemd/vyos-http-api.service
@@ -5,9 +5,8 @@ Requires=vyos-router.service
[Service]
ExecStartPre=/usr/libexec/vyos/init/vyos-config
-ExecStart=/usr/bin/python3 -u /usr/libexec/vyos/services/vyos-http-api-server
+ExecStart=/usr/libexec/vyos/services/vyos-http-api-server
Type=idle
-KillMode=process
SyslogIdentifier=vyos-http-api
SyslogFacility=daemon