From b5c80d310527223b93e4133ac2f4c8c063c70a98 Mon Sep 17 00:00:00 2001 From: Christian Poessinger Date: Fri, 18 Dec 2020 17:13:13 +0100 Subject: xdp: T2666: switch to example code provided by xdp-tutorial --- src/xdp/common/Makefile | 23 +++ src/xdp/common/README.org | 8 + src/xdp/common/common.mk | 103 ++++++++++ src/xdp/common/common_defines.h | 38 ++++ src/xdp/common/common_libbpf.c | 162 +++++++++++++++ src/xdp/common/common_libbpf.h | 24 +++ src/xdp/common/common_params.c | 197 ++++++++++++++++++ src/xdp/common/common_params.h | 22 ++ src/xdp/common/common_user_bpf_xdp.c | 380 +++++++++++++++++++++++++++++++++++ src/xdp/common/common_user_bpf_xdp.h | 20 ++ src/xdp/common/parsing_helpers.h | 244 ++++++++++++++++++++++ src/xdp/common/rewrite_helpers.h | 144 +++++++++++++ src/xdp/common/xdp_stats_kern.h | 44 ++++ src/xdp/common/xdp_stats_kern_user.h | 19 ++ 14 files changed, 1428 insertions(+) create mode 100644 src/xdp/common/Makefile create mode 100644 src/xdp/common/README.org create mode 100644 src/xdp/common/common.mk create mode 100644 src/xdp/common/common_defines.h create mode 100644 src/xdp/common/common_libbpf.c create mode 100644 src/xdp/common/common_libbpf.h create mode 100644 src/xdp/common/common_params.c create mode 100644 src/xdp/common/common_params.h create mode 100644 src/xdp/common/common_user_bpf_xdp.c create mode 100644 src/xdp/common/common_user_bpf_xdp.h create mode 100644 src/xdp/common/parsing_helpers.h create mode 100644 src/xdp/common/rewrite_helpers.h create mode 100644 src/xdp/common/xdp_stats_kern.h create mode 100644 src/xdp/common/xdp_stats_kern_user.h (limited to 'src/xdp/common') diff --git a/src/xdp/common/Makefile b/src/xdp/common/Makefile new file mode 100644 index 000000000..2502027e9 --- /dev/null +++ b/src/xdp/common/Makefile @@ -0,0 +1,23 @@ +# SPDX-License-Identifier: (GPL-2.0) +CC := gcc + +all: common_params.o common_user_bpf_xdp.o common_libbpf.o + +CFLAGS := -g -Wall + +CFLAGS += -I../include/ +# TODO: Do we need to make libbpf from this make file too? + +common_params.o: common_params.c common_params.h + $(CC) $(CFLAGS) -c -o $@ $< + +common_user_bpf_xdp.o: common_user_bpf_xdp.c common_user_bpf_xdp.h + $(CC) $(CFLAGS) -c -o $@ $< + +common_libbpf.o: common_libbpf.c common_libbpf.h + $(CC) $(CFLAGS) -c -o $@ $< + +.PHONY: clean + +clean: + rm -f *.o diff --git a/src/xdp/common/README.org b/src/xdp/common/README.org new file mode 100644 index 000000000..561fdbced --- /dev/null +++ b/src/xdp/common/README.org @@ -0,0 +1,8 @@ +# -*- fill-column: 76; -*- +#+TITLE: Common files +#+OPTIONS: ^:nil + +This directory contains code that is common between the different +assignments. This reduce code duplication in each tutorial assignment, and +allow us to hideaway code that is irrelevant or have been seen/introduced in +earlier assignments. diff --git a/src/xdp/common/common.mk b/src/xdp/common/common.mk new file mode 100644 index 000000000..89aac68ef --- /dev/null +++ b/src/xdp/common/common.mk @@ -0,0 +1,103 @@ +# Common Makefile parts for BPF-building with libbpf +# -------------------------------------------------- +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# +# This file should be included from your Makefile like: +# COMMON_DIR = ../common/ +# include $(COMMON_DIR)/common.mk +# +# It is expected that you define the variables: +# XDP_TARGETS and USER_TARGETS +# as a space-separated list +# +LLC ?= llc +CLANG ?= clang +CC ?= gcc + +XDP_C = ${XDP_TARGETS:=.c} +XDP_OBJ = ${XDP_C:.c=.o} +USER_C := ${USER_TARGETS:=.c} +USER_OBJ := ${USER_C:.c=.o} + +# Expect this is defined by including Makefile, but define if not +COMMON_DIR ?= ../common/ + +COPY_LOADER ?= +LOADER_DIR ?= $(COMMON_DIR)/../utils + +# Extend if including Makefile already added some +COMMON_OBJS += $(COMMON_DIR)/common_params.o $(COMMON_DIR)/common_user_bpf_xdp.o + +# Create expansions for dependencies +COMMON_H := ${COMMON_OBJS:.o=.h} + +EXTRA_DEPS += + +# BPF-prog kern and userspace shares struct via header file: +KERN_USER_H ?= $(wildcard common_kern_user.h) + +CFLAGS ?= -g -I../include/ +BPF_CFLAGS ?= -I../include/ + +LIBS = -l:libbpf.a -lelf -lz $(USER_LIBS) + +all: llvm-check $(USER_TARGETS) $(XDP_OBJ) $(COPY_LOADER) $(COPY_STATS) + +.PHONY: clean $(CLANG) $(LLC) + +clean: + $(MAKE) -C $(COMMON_DIR) clean + rm -f $(USER_TARGETS) $(XDP_OBJ) $(USER_OBJ) $(COPY_LOADER) $(COPY_STATS) + rm -f *.ll + rm -f *~ + +ifdef COPY_LOADER +$(COPY_LOADER): $(LOADER_DIR)/${COPY_LOADER:=.c} $(COMMON_H) + make -C $(LOADER_DIR) $(COPY_LOADER) + cp $(LOADER_DIR)/$(COPY_LOADER) $(COPY_LOADER) +endif + +ifdef COPY_STATS +$(COPY_STATS): $(LOADER_DIR)/${COPY_STATS:=.c} $(COMMON_H) + make -C $(LOADER_DIR) $(COPY_STATS) + cp $(LOADER_DIR)/$(COPY_STATS) $(COPY_STATS) +# Needing xdp_stats imply depending on header files: +EXTRA_DEPS += $(COMMON_DIR)/xdp_stats_kern.h $(COMMON_DIR)/xdp_stats_kern_user.h +endif + +# For build dependency on this file, if it gets updated +COMMON_MK = $(COMMON_DIR)/common.mk + +llvm-check: $(CLANG) $(LLC) + @for TOOL in $^ ; do \ + if [ ! $$(command -v $${TOOL} 2>/dev/null) ]; then \ + echo "*** ERROR: Cannot find tool $${TOOL}" ;\ + exit 1; \ + else true; fi; \ + done + +# Create dependency: detect if C-file change and touch H-file, to trigger +# target $(COMMON_OBJS) +$(COMMON_H): %.h: %.c + touch $@ + +# Detect if any of common obj changed and create dependency on .h-files +$(COMMON_OBJS): %.o: %.h + make -C $(COMMON_DIR) + +$(USER_TARGETS): %: %.c Makefile $(COMMON_MK) $(COMMON_OBJS) $(KERN_USER_H) $(EXTRA_DEPS) + $(CC) -Wall $(CFLAGS) $(LDFLAGS) -o $@ $(COMMON_OBJS) \ + $< $(LIBS) + +$(XDP_OBJ): %.o: %.c Makefile $(COMMON_MK) $(KERN_USER_H) $(EXTRA_DEPS) + $(CLANG) -S \ + -target bpf \ + -D __BPF_TRACING__ \ + $(BPF_CFLAGS) \ + -Wall \ + -Wno-unused-value \ + -Wno-pointer-sign \ + -Wno-compare-distinct-pointer-types \ + -Werror \ + -O2 -emit-llvm -c -g -o ${@:.o=.ll} $< + $(LLC) -march=bpf -filetype=obj -o $@ ${@:.o=.ll} diff --git a/src/xdp/common/common_defines.h b/src/xdp/common/common_defines.h new file mode 100644 index 000000000..2986d2d67 --- /dev/null +++ b/src/xdp/common/common_defines.h @@ -0,0 +1,38 @@ +#ifndef __COMMON_DEFINES_H +#define __COMMON_DEFINES_H + +#include +#include +#include + +struct config { + __u32 xdp_flags; + int ifindex; + char *ifname; + char ifname_buf[IF_NAMESIZE]; + int redirect_ifindex; + char *redirect_ifname; + char redirect_ifname_buf[IF_NAMESIZE]; + bool do_unload; + bool reuse_maps; + char pin_dir[512]; + char filename[512]; + char progsec[32]; + char src_mac[18]; + char dest_mac[18]; + __u16 xsk_bind_flags; + int xsk_if_queue; + bool xsk_poll_mode; +}; + +/* Defined in common_params.o */ +extern int verbose; + +/* Exit return codes */ +#define EXIT_OK 0 /* == EXIT_SUCCESS (stdlib.h) man exit(3) */ +#define EXIT_FAIL 1 /* == EXIT_FAILURE (stdlib.h) man exit(3) */ +#define EXIT_FAIL_OPTION 2 +#define EXIT_FAIL_XDP 30 +#define EXIT_FAIL_BPF 40 + +#endif /* __COMMON_DEFINES_H */ diff --git a/src/xdp/common/common_libbpf.c b/src/xdp/common/common_libbpf.c new file mode 100644 index 000000000..5788ecd9e --- /dev/null +++ b/src/xdp/common/common_libbpf.c @@ -0,0 +1,162 @@ +/* Common function that with time should be moved to libbpf */ + +#include +#include + +#include +#include + +#include "common_libbpf.h" + +/* From: include/linux/err.h */ +#define MAX_ERRNO 4095 +#define IS_ERR_VALUE(x) ((x) >= (unsigned long)-MAX_ERRNO) +static inline bool IS_ERR_OR_NULL(const void *ptr) +{ + return (!ptr) || IS_ERR_VALUE((unsigned long)ptr); +} + +#define pr_warning printf + +/* As close as possible to libbpf bpf_prog_load_xattr(), with the + * difference of handling pinned maps. + */ +int bpf_prog_load_xattr_maps(const struct bpf_prog_load_attr_maps *attr, + struct bpf_object **pobj, int *prog_fd) +{ + struct bpf_object_open_attr open_attr = { + .file = attr->file, + .prog_type = attr->prog_type, + }; + struct bpf_program *prog, *first_prog = NULL; + enum bpf_attach_type expected_attach_type; + enum bpf_prog_type prog_type; + struct bpf_object *obj; + struct bpf_map *map; + int err; + int i; + + if (!attr) + return -EINVAL; + if (!attr->file) + return -EINVAL; + + + obj = bpf_object__open_xattr(&open_attr); + if (IS_ERR_OR_NULL(obj)) + return -ENOENT; + + bpf_object__for_each_program(prog, obj) { + /* + * If type is not specified, try to guess it based on + * section name. + */ + prog_type = attr->prog_type; + // Was: prog->prog_ifindex = attr->ifindex; + bpf_program__set_ifindex(prog, attr->ifindex); + + expected_attach_type = attr->expected_attach_type; +#if 0 /* Use internal libbpf variables */ + if (prog_type == BPF_PROG_TYPE_UNSPEC) { + err = bpf_program__identify_section(prog, &prog_type, + &expected_attach_type); + if (err < 0) { + bpf_object__close(obj); + return -EINVAL; + } + } +#endif + + bpf_program__set_type(prog, prog_type); + bpf_program__set_expected_attach_type(prog, + expected_attach_type); + + if (!first_prog) + first_prog = prog; + } + + /* Reset attr->pinned_maps.map_fd to identify successful file load */ + for (i = 0; i < attr->nr_pinned_maps; i++) + attr->pinned_maps[i].map_fd = -1; + + bpf_map__for_each(map, obj) { + const char* mapname = bpf_map__name(map); + + if (!bpf_map__is_offload_neutral(map)) + bpf_map__set_ifindex(map, attr->ifindex); + /* Was: map->map_ifindex = attr->ifindex; */ + + for (i = 0; i < attr->nr_pinned_maps; i++) { + struct bpf_pinned_map *pin_map = &attr->pinned_maps[i]; + int fd; + + if (strcmp(mapname, pin_map->name) != 0) + continue; + + /* Matched, try opening pinned file */ + fd = bpf_obj_get(pin_map->filename); + if (fd > 0) { + /* Use FD from pinned map as replacement */ + bpf_map__reuse_fd(map, fd); + /* TODO: Might want to set internal map "name" + * if opened pinned map didn't, to allow + * bpf_object__find_map_fd_by_name() to work. + */ + pin_map->map_fd = fd; + continue; + } + /* Could not open pinned filename map, then this prog + * should then pin the map, BUT this can only happen + * after bpf_object__load(). + */ + } + } + + if (!first_prog) { + pr_warning("object file doesn't contain bpf program\n"); + bpf_object__close(obj); + return -ENOENT; + } + + err = bpf_object__load(obj); + if (err) { + bpf_object__close(obj); + return -EINVAL; + } + + /* Pin the maps that were not loaded via pinned filename */ + bpf_map__for_each(map, obj) { + const char* mapname = bpf_map__name(map); + + for (i = 0; i < attr->nr_pinned_maps; i++) { + struct bpf_pinned_map *pin_map = &attr->pinned_maps[i]; + int err; + + if (strcmp(mapname, pin_map->name) != 0) + continue; + + /* Matched, check if map is already loaded */ + if (pin_map->map_fd != -1) + continue; + + /* Needs to be pinned */ + err = bpf_map__pin(map, pin_map->filename); + if (err) + continue; + pin_map->map_fd = bpf_map__fd(map); + } + } + + /* Help user if requested map name that doesn't exist */ + for (i = 0; i < attr->nr_pinned_maps; i++) { + struct bpf_pinned_map *pin_map = &attr->pinned_maps[i]; + + if (pin_map->map_fd < 0) + pr_warning("%s() requested mapname:%s not seen\n", + __func__, pin_map->name); + } + + *pobj = obj; + *prog_fd = bpf_program__fd(first_prog); + return 0; +} diff --git a/src/xdp/common/common_libbpf.h b/src/xdp/common/common_libbpf.h new file mode 100644 index 000000000..4754bd8ca --- /dev/null +++ b/src/xdp/common/common_libbpf.h @@ -0,0 +1,24 @@ +/* Common function that with time should be moved to libbpf */ +#ifndef __COMMON_LIBBPF_H +#define __COMMON_LIBBPF_H + +struct bpf_pinned_map { + const char *name; + const char *filename; + int map_fd; +}; + +/* bpf_prog_load_attr extended */ +struct bpf_prog_load_attr_maps { + const char *file; + enum bpf_prog_type prog_type; + enum bpf_attach_type expected_attach_type; + int ifindex; + int nr_pinned_maps; + struct bpf_pinned_map *pinned_maps; +}; + +int bpf_prog_load_xattr_maps(const struct bpf_prog_load_attr_maps *attr, + struct bpf_object **pobj, int *prog_fd); + +#endif /* __COMMON_LIBBPF_H */ diff --git a/src/xdp/common/common_params.c b/src/xdp/common/common_params.c new file mode 100644 index 000000000..642d56d92 --- /dev/null +++ b/src/xdp/common/common_params.c @@ -0,0 +1,197 @@ +#include +#include +#include +#include +#include +#include +#include + +#include +#include /* XDP_FLAGS_* depend on kernel-headers installed */ +#include + +#include "common_params.h" + +int verbose = 1; + +#define BUFSIZE 30 + +void _print_options(const struct option_wrapper *long_options, bool required) +{ + int i, pos; + char buf[BUFSIZE]; + + for (i = 0; long_options[i].option.name != 0; i++) { + if (long_options[i].required != required) + continue; + + if (long_options[i].option.val > 64) /* ord('A') = 65 */ + printf(" -%c,", long_options[i].option.val); + else + printf(" "); + pos = snprintf(buf, BUFSIZE, " --%s", long_options[i].option.name); + if (long_options[i].metavar) + snprintf(&buf[pos], BUFSIZE-pos, " %s", long_options[i].metavar); + printf("%-22s", buf); + printf(" %s", long_options[i].help); + printf("\n"); + } +} + +void usage(const char *prog_name, const char *doc, + const struct option_wrapper *long_options, bool full) +{ + printf("Usage: %s [options]\n", prog_name); + + if (!full) { + printf("Use --help (or -h) to see full option list.\n"); + return; + } + + printf("\nDOCUMENTATION:\n %s\n", doc); + printf("Required options:\n"); + _print_options(long_options, true); + printf("\n"); + printf("Other options:\n"); + _print_options(long_options, false); + printf("\n"); +} + +int option_wrappers_to_options(const struct option_wrapper *wrapper, + struct option **options) +{ + int i, num; + struct option *new_options; + for (i = 0; wrapper[i].option.name != 0; i++) {} + num = i; + + new_options = malloc(sizeof(struct option) * num); + if (!new_options) + return -1; + for (i = 0; i < num; i++) { + memcpy(&new_options[i], &wrapper[i], sizeof(struct option)); + } + + *options = new_options; + return 0; +} + +void parse_cmdline_args(int argc, char **argv, + const struct option_wrapper *options_wrapper, + struct config *cfg, const char *doc) +{ + struct option *long_options; + bool full_help = false; + int longindex = 0; + char *dest; + int opt; + + if (option_wrappers_to_options(options_wrapper, &long_options)) { + fprintf(stderr, "Unable to malloc()\n"); + exit(EXIT_FAIL_OPTION); + } + + /* Parse commands line args */ + while ((opt = getopt_long(argc, argv, "hd:r:L:R:ASNFUMQ:czpq", + long_options, &longindex)) != -1) { + switch (opt) { + case 'd': + if (strlen(optarg) >= IF_NAMESIZE) { + fprintf(stderr, "ERR: --dev name too long\n"); + goto error; + } + cfg->ifname = (char *)&cfg->ifname_buf; + strncpy(cfg->ifname, optarg, IF_NAMESIZE); + cfg->ifindex = if_nametoindex(cfg->ifname); + if (cfg->ifindex == 0) { + fprintf(stderr, + "ERR: --dev name unknown err(%d):%s\n", + errno, strerror(errno)); + goto error; + } + break; + case 'r': + if (strlen(optarg) >= IF_NAMESIZE) { + fprintf(stderr, "ERR: --redirect-dev name too long\n"); + goto error; + } + cfg->redirect_ifname = (char *)&cfg->redirect_ifname_buf; + strncpy(cfg->redirect_ifname, optarg, IF_NAMESIZE); + cfg->redirect_ifindex = if_nametoindex(cfg->redirect_ifname); + if (cfg->redirect_ifindex == 0) { + fprintf(stderr, + "ERR: --redirect-dev name unknown err(%d):%s\n", + errno, strerror(errno)); + goto error; + } + break; + case 'A': + cfg->xdp_flags &= ~XDP_FLAGS_MODES; /* Clear flags */ + break; + case 'S': + cfg->xdp_flags &= ~XDP_FLAGS_MODES; /* Clear flags */ + cfg->xdp_flags |= XDP_FLAGS_SKB_MODE; /* Set flag */ + cfg->xsk_bind_flags &= XDP_ZEROCOPY; + cfg->xsk_bind_flags |= XDP_COPY; + break; + case 'N': + cfg->xdp_flags &= ~XDP_FLAGS_MODES; /* Clear flags */ + cfg->xdp_flags |= XDP_FLAGS_DRV_MODE; /* Set flag */ + break; + case 3: /* --offload-mode */ + cfg->xdp_flags &= ~XDP_FLAGS_MODES; /* Clear flags */ + cfg->xdp_flags |= XDP_FLAGS_HW_MODE; /* Set flag */ + break; + case 'F': + cfg->xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; + break; + case 'M': + cfg->reuse_maps = true; + break; + case 'U': + cfg->do_unload = true; + break; + case 'p': + cfg->xsk_poll_mode = true; + break; + case 'q': + verbose = false; + break; + case 'Q': + cfg->xsk_if_queue = atoi(optarg); + break; + case 1: /* --filename */ + dest = (char *)&cfg->filename; + strncpy(dest, optarg, sizeof(cfg->filename)); + break; + case 2: /* --progsec */ + dest = (char *)&cfg->progsec; + strncpy(dest, optarg, sizeof(cfg->progsec)); + break; + case 'L': /* --src-mac */ + dest = (char *)&cfg->src_mac; + strncpy(dest, optarg, sizeof(cfg->src_mac)); + break; + case 'R': /* --dest-mac */ + dest = (char *)&cfg->dest_mac; + strncpy(dest, optarg, sizeof(cfg->dest_mac)); + case 'c': + cfg->xsk_bind_flags &= XDP_ZEROCOPY; + cfg->xsk_bind_flags |= XDP_COPY; + break; + case 'z': + cfg->xsk_bind_flags &= XDP_COPY; + cfg->xsk_bind_flags |= XDP_ZEROCOPY; + break; + case 'h': + full_help = true; + /* fall-through */ + error: + default: + usage(argv[0], doc, options_wrapper, full_help); + free(long_options); + exit(EXIT_FAIL_OPTION); + } + } + free(long_options); +} diff --git a/src/xdp/common/common_params.h b/src/xdp/common/common_params.h new file mode 100644 index 000000000..6f64c82e3 --- /dev/null +++ b/src/xdp/common/common_params.h @@ -0,0 +1,22 @@ +/* This common_user.h is used by userspace programs */ +#ifndef __COMMON_PARAMS_H +#define __COMMON_PARAMS_H + +#include +#include "common_defines.h" + +struct option_wrapper { + struct option option; + char *help; + char *metavar; + bool required; +}; + +void usage(const char *prog_name, const char *doc, + const struct option_wrapper *long_options, bool full); + +void parse_cmdline_args(int argc, char **argv, + const struct option_wrapper *long_options, + struct config *cfg, const char *doc); + +#endif /* __COMMON_PARAMS_H */ diff --git a/src/xdp/common/common_user_bpf_xdp.c b/src/xdp/common/common_user_bpf_xdp.c new file mode 100644 index 000000000..e7ef77174 --- /dev/null +++ b/src/xdp/common/common_user_bpf_xdp.c @@ -0,0 +1,380 @@ +#include /* bpf_get_link_xdp_id + bpf_set_link_xdp_id */ +#include /* strerror */ +#include /* IF_NAMESIZE */ +#include /* exit(3) */ +#include + +#include +#include + +#include /* Need XDP flags */ +#include + +#include "common_defines.h" + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +int xdp_link_attach(int ifindex, __u32 xdp_flags, int prog_fd) +{ + int err; + + /* libbpf provide the XDP net_device link-level hook attach helper */ + err = bpf_set_link_xdp_fd(ifindex, prog_fd, xdp_flags); + if (err == -EEXIST && !(xdp_flags & XDP_FLAGS_UPDATE_IF_NOEXIST)) { + /* Force mode didn't work, probably because a program of the + * opposite type is loaded. Let's unload that and try loading + * again. + */ + + __u32 old_flags = xdp_flags; + + xdp_flags &= ~XDP_FLAGS_MODES; + xdp_flags |= (old_flags & XDP_FLAGS_SKB_MODE) ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE; + err = bpf_set_link_xdp_fd(ifindex, -1, xdp_flags); + if (!err) + err = bpf_set_link_xdp_fd(ifindex, prog_fd, old_flags); + } + if (err < 0) { + fprintf(stderr, "ERR: " + "ifindex(%d) link set xdp fd failed (%d): %s\n", + ifindex, -err, strerror(-err)); + + switch (-err) { + case EBUSY: + case EEXIST: + fprintf(stderr, "Hint: XDP already loaded on device" + " use --force to swap/replace\n"); + break; + case EOPNOTSUPP: + fprintf(stderr, "Hint: Native-XDP not supported" + " use --skb-mode or --auto-mode\n"); + break; + default: + break; + } + return EXIT_FAIL_XDP; + } + + return EXIT_OK; +} + +int xdp_link_detach(int ifindex, __u32 xdp_flags, __u32 expected_prog_id) +{ + __u32 curr_prog_id; + int err; + + err = bpf_get_link_xdp_id(ifindex, &curr_prog_id, xdp_flags); + if (err) { + fprintf(stderr, "ERR: get link xdp id failed (err=%d): %s\n", + -err, strerror(-err)); + return EXIT_FAIL_XDP; + } + + if (!curr_prog_id) { + if (verbose) + printf("INFO: %s() no curr XDP prog on ifindex:%d\n", + __func__, ifindex); + return EXIT_OK; + } + + if (expected_prog_id && curr_prog_id != expected_prog_id) { + fprintf(stderr, "ERR: %s() " + "expected prog ID(%d) no match(%d), not removing\n", + __func__, expected_prog_id, curr_prog_id); + return EXIT_FAIL; + } + + if ((err = bpf_set_link_xdp_fd(ifindex, -1, xdp_flags)) < 0) { + fprintf(stderr, "ERR: %s() link set xdp failed (err=%d): %s\n", + __func__, err, strerror(-err)); + return EXIT_FAIL_XDP; + } + + if (verbose) + printf("INFO: %s() removed XDP prog ID:%d on ifindex:%d\n", + __func__, curr_prog_id, ifindex); + + return EXIT_OK; +} + +struct bpf_object *load_bpf_object_file(const char *filename, int ifindex) +{ + int first_prog_fd = -1; + struct bpf_object *obj; + int err; + + /* This struct allow us to set ifindex, this features is used for + * hardware offloading XDP programs (note this sets libbpf + * bpf_program->prog_ifindex and foreach bpf_map->map_ifindex). + */ + struct bpf_prog_load_attr prog_load_attr = { + .prog_type = BPF_PROG_TYPE_XDP, + .ifindex = ifindex, + }; + prog_load_attr.file = filename; + + /* Use libbpf for extracting BPF byte-code from BPF-ELF object, and + * loading this into the kernel via bpf-syscall + */ + err = bpf_prog_load_xattr(&prog_load_attr, &obj, &first_prog_fd); + if (err) { + fprintf(stderr, "ERR: loading BPF-OBJ file(%s) (%d): %s\n", + filename, err, strerror(-err)); + return NULL; + } + + /* Notice how a pointer to a libbpf bpf_object is returned */ + return obj; +} + +static struct bpf_object *open_bpf_object(const char *file, int ifindex) +{ + int err; + struct bpf_object *obj; + struct bpf_map *map; + struct bpf_program *prog, *first_prog = NULL; + + struct bpf_object_open_attr open_attr = { + .file = file, + .prog_type = BPF_PROG_TYPE_XDP, + }; + + obj = bpf_object__open_xattr(&open_attr); + if (IS_ERR_OR_NULL(obj)) { + err = -PTR_ERR(obj); + fprintf(stderr, "ERR: opening BPF-OBJ file(%s) (%d): %s\n", + file, err, strerror(-err)); + return NULL; + } + + bpf_object__for_each_program(prog, obj) { + bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); + bpf_program__set_ifindex(prog, ifindex); + if (!first_prog) + first_prog = prog; + } + + bpf_object__for_each_map(map, obj) { + if (!bpf_map__is_offload_neutral(map)) + bpf_map__set_ifindex(map, ifindex); + } + + if (!first_prog) { + fprintf(stderr, "ERR: file %s contains no programs\n", file); + return NULL; + } + + return obj; +} + +static int reuse_maps(struct bpf_object *obj, const char *path) +{ + struct bpf_map *map; + + if (!obj) + return -ENOENT; + + if (!path) + return -EINVAL; + + bpf_object__for_each_map(map, obj) { + int len, err; + int pinned_map_fd; + char buf[PATH_MAX]; + + len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); + if (len < 0) { + return -EINVAL; + } else if (len >= PATH_MAX) { + return -ENAMETOOLONG; + } + + pinned_map_fd = bpf_obj_get(buf); + if (pinned_map_fd < 0) + return pinned_map_fd; + + err = bpf_map__reuse_fd(map, pinned_map_fd); + if (err) + return err; + } + + return 0; +} + +struct bpf_object *load_bpf_object_file_reuse_maps(const char *file, + int ifindex, + const char *pin_dir) +{ + int err; + struct bpf_object *obj; + + obj = open_bpf_object(file, ifindex); + if (!obj) { + fprintf(stderr, "ERR: failed to open object %s\n", file); + return NULL; + } + + err = reuse_maps(obj, pin_dir); + if (err) { + fprintf(stderr, "ERR: failed to reuse maps for object %s, pin_dir=%s\n", + file, pin_dir); + return NULL; + } + + err = bpf_object__load(obj); + if (err) { + fprintf(stderr, "ERR: loading BPF-OBJ file(%s) (%d): %s\n", + file, err, strerror(-err)); + return NULL; + } + + return obj; +} + +struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg) +{ + struct bpf_program *bpf_prog; + struct bpf_object *bpf_obj; + int offload_ifindex = 0; + int prog_fd = -1; + int err; + + /* If flags indicate hardware offload, supply ifindex */ + if (cfg->xdp_flags & XDP_FLAGS_HW_MODE) + offload_ifindex = cfg->ifindex; + + /* Load the BPF-ELF object file and get back libbpf bpf_object */ + if (cfg->reuse_maps) + bpf_obj = load_bpf_object_file_reuse_maps(cfg->filename, + offload_ifindex, + cfg->pin_dir); + else + bpf_obj = load_bpf_object_file(cfg->filename, offload_ifindex); + if (!bpf_obj) { + fprintf(stderr, "ERR: loading file: %s\n", cfg->filename); + exit(EXIT_FAIL_BPF); + } + /* At this point: All XDP/BPF programs from the cfg->filename have been + * loaded into the kernel, and evaluated by the verifier. Only one of + * these gets attached to XDP hook, the others will get freed once this + * process exit. + */ + + if (cfg->progsec[0]) + /* Find a matching BPF prog section name */ + bpf_prog = bpf_object__find_program_by_title(bpf_obj, cfg->progsec); + else + /* Find the first program */ + bpf_prog = bpf_program__next(NULL, bpf_obj); + + if (!bpf_prog) { + fprintf(stderr, "ERR: couldn't find a program in ELF section '%s'\n", cfg->progsec); + exit(EXIT_FAIL_BPF); + } + + strncpy(cfg->progsec, bpf_program__title(bpf_prog, false), sizeof(cfg->progsec)); + + prog_fd = bpf_program__fd(bpf_prog); + if (prog_fd <= 0) { + fprintf(stderr, "ERR: bpf_program__fd failed\n"); + exit(EXIT_FAIL_BPF); + } + + /* At this point: BPF-progs are (only) loaded by the kernel, and prog_fd + * is our select file-descriptor handle. Next step is attaching this FD + * to a kernel hook point, in this case XDP net_device link-level hook. + */ + err = xdp_link_attach(cfg->ifindex, cfg->xdp_flags, prog_fd); + if (err) + exit(err); + + return bpf_obj; +} + +#define XDP_UNKNOWN XDP_REDIRECT + 1 +#ifndef XDP_ACTION_MAX +#define XDP_ACTION_MAX (XDP_UNKNOWN + 1) +#endif + +static const char *xdp_action_names[XDP_ACTION_MAX] = { + [XDP_ABORTED] = "XDP_ABORTED", + [XDP_DROP] = "XDP_DROP", + [XDP_PASS] = "XDP_PASS", + [XDP_TX] = "XDP_TX", + [XDP_REDIRECT] = "XDP_REDIRECT", + [XDP_UNKNOWN] = "XDP_UNKNOWN", +}; + +const char *action2str(__u32 action) +{ + if (action < XDP_ACTION_MAX) + return xdp_action_names[action]; + return NULL; +} + +int check_map_fd_info(const struct bpf_map_info *info, + const struct bpf_map_info *exp) +{ + if (exp->key_size && exp->key_size != info->key_size) { + fprintf(stderr, "ERR: %s() " + "Map key size(%d) mismatch expected size(%d)\n", + __func__, info->key_size, exp->key_size); + return EXIT_FAIL; + } + if (exp->value_size && exp->value_size != info->value_size) { + fprintf(stderr, "ERR: %s() " + "Map value size(%d) mismatch expected size(%d)\n", + __func__, info->value_size, exp->value_size); + return EXIT_FAIL; + } + if (exp->max_entries && exp->max_entries != info->max_entries) { + fprintf(stderr, "ERR: %s() " + "Map max_entries(%d) mismatch expected size(%d)\n", + __func__, info->max_entries, exp->max_entries); + return EXIT_FAIL; + } + if (exp->type && exp->type != info->type) { + fprintf(stderr, "ERR: %s() " + "Map type(%d) mismatch expected type(%d)\n", + __func__, info->type, exp->type); + return EXIT_FAIL; + } + + return 0; +} + +int open_bpf_map_file(const char *pin_dir, + const char *mapname, + struct bpf_map_info *info) +{ + char filename[PATH_MAX]; + int err, len, fd; + __u32 info_len = sizeof(*info); + + len = snprintf(filename, PATH_MAX, "%s/%s", pin_dir, mapname); + if (len < 0) { + fprintf(stderr, "ERR: constructing full mapname path\n"); + return -1; + } + + fd = bpf_obj_get(filename); + if (fd < 0) { + fprintf(stderr, + "WARN: Failed to open bpf map file:%s err(%d):%s\n", + filename, errno, strerror(errno)); + return fd; + } + + if (info) { + err = bpf_obj_get_info_by_fd(fd, info, &info_len); + if (err) { + fprintf(stderr, "ERR: %s() can't get info - %s\n", + __func__, strerror(errno)); + return EXIT_FAIL_BPF; + } + } + + return fd; +} diff --git a/src/xdp/common/common_user_bpf_xdp.h b/src/xdp/common/common_user_bpf_xdp.h new file mode 100644 index 000000000..4f8aa51d4 --- /dev/null +++ b/src/xdp/common/common_user_bpf_xdp.h @@ -0,0 +1,20 @@ +/* Common BPF/XDP functions used by userspace side programs */ +#ifndef __COMMON_USER_BPF_XDP_H +#define __COMMON_USER_BPF_XDP_H + +int xdp_link_attach(int ifindex, __u32 xdp_flags, int prog_fd); +int xdp_link_detach(int ifindex, __u32 xdp_flags, __u32 expected_prog_id); + +struct bpf_object *load_bpf_object_file(const char *filename, int ifindex); +struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg); + +const char *action2str(__u32 action); + +int check_map_fd_info(const struct bpf_map_info *info, + const struct bpf_map_info *exp); + +int open_bpf_map_file(const char *pin_dir, + const char *mapname, + struct bpf_map_info *info); + +#endif /* __COMMON_USER_BPF_XDP_H */ diff --git a/src/xdp/common/parsing_helpers.h b/src/xdp/common/parsing_helpers.h new file mode 100644 index 000000000..c0837e88f --- /dev/null +++ b/src/xdp/common/parsing_helpers.h @@ -0,0 +1,244 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-clause) */ +/* + * This file contains parsing functions that are used in the packetXX XDP + * programs. The functions are marked as __always_inline, and fully defined in + * this header file to be included in the BPF program. + * + * Each helper parses a packet header, including doing bounds checking, and + * returns the type of its contents if successful, and -1 otherwise. + * + * For Ethernet and IP headers, the content type is the type of the payload + * (h_proto for Ethernet, nexthdr for IPv6), for ICMP it is the ICMP type field. + * All return values are in host byte order. + * + * The versions of the functions included here are slightly expanded versions of + * the functions in the packet01 lesson. For instance, the Ethernet header + * parsing has support for parsing VLAN tags. + */ + +#ifndef __PARSING_HELPERS_H +#define __PARSING_HELPERS_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Header cursor to keep track of current parsing position */ +struct hdr_cursor { + void *pos; +}; + +/* + * struct vlan_hdr - vlan header + * @h_vlan_TCI: priority and VLAN ID + * @h_vlan_encapsulated_proto: packet type ID or len + */ +struct vlan_hdr { + __be16 h_vlan_TCI; + __be16 h_vlan_encapsulated_proto; +}; + +/* + * Struct icmphdr_common represents the common part of the icmphdr and icmp6hdr + * structures. + */ +struct icmphdr_common { + __u8 type; + __u8 code; + __sum16 cksum; +}; + +/* Allow users of header file to redefine VLAN max depth */ +#ifndef VLAN_MAX_DEPTH +#define VLAN_MAX_DEPTH 4 +#endif + +static __always_inline int proto_is_vlan(__u16 h_proto) +{ + return !!(h_proto == bpf_htons(ETH_P_8021Q) || + h_proto == bpf_htons(ETH_P_8021AD)); +} + +/* Notice, parse_ethhdr() will skip VLAN tags, by advancing nh->pos and returns + * next header EtherType, BUT the ethhdr pointer supplied still points to the + * Ethernet header. Thus, caller can look at eth->h_proto to see if this was a + * VLAN tagged packet. + */ +static __always_inline int parse_ethhdr(struct hdr_cursor *nh, void *data_end, + struct ethhdr **ethhdr) +{ + struct ethhdr *eth = nh->pos; + int hdrsize = sizeof(*eth); + struct vlan_hdr *vlh; + __u16 h_proto; + int i; + + /* Byte-count bounds check; check if current pointer + size of header + * is after data_end. + */ + if (nh->pos + hdrsize > data_end) + return -1; + + nh->pos += hdrsize; + *ethhdr = eth; + vlh = nh->pos; + h_proto = eth->h_proto; + + /* Use loop unrolling to avoid the verifier restriction on loops; + * support up to VLAN_MAX_DEPTH layers of VLAN encapsulation. + */ + #pragma unroll + for (i = 0; i < VLAN_MAX_DEPTH; i++) { + if (!proto_is_vlan(h_proto)) + break; + + if (vlh + 1 > data_end) + break; + + h_proto = vlh->h_vlan_encapsulated_proto; + vlh++; + } + + nh->pos = vlh; + return h_proto; /* network-byte-order */ +} + +static __always_inline int parse_ip6hdr(struct hdr_cursor *nh, + void *data_end, + struct ipv6hdr **ip6hdr) +{ + struct ipv6hdr *ip6h = nh->pos; + + /* Pointer-arithmetic bounds check; pointer +1 points to after end of + * thing being pointed to. We will be using this style in the remainder + * of the tutorial. + */ + if (ip6h + 1 > data_end) + return -1; + + nh->pos = ip6h + 1; + *ip6hdr = ip6h; + + return ip6h->nexthdr; +} + +static __always_inline int parse_iphdr(struct hdr_cursor *nh, + void *data_end, + struct iphdr **iphdr) +{ + struct iphdr *iph = nh->pos; + int hdrsize; + + if (iph + 1 > data_end) + return -1; + + hdrsize = iph->ihl * 4; + + /* Variable-length IPv4 header, need to use byte-based arithmetic */ + if (nh->pos + hdrsize > data_end) + return -1; + + nh->pos += hdrsize; + *iphdr = iph; + + return iph->protocol; +} + +static __always_inline int parse_icmp6hdr(struct hdr_cursor *nh, + void *data_end, + struct icmp6hdr **icmp6hdr) +{ + struct icmp6hdr *icmp6h = nh->pos; + + if (icmp6h + 1 > data_end) + return -1; + + nh->pos = icmp6h + 1; + *icmp6hdr = icmp6h; + + return icmp6h->icmp6_type; +} + +static __always_inline int parse_icmphdr(struct hdr_cursor *nh, + void *data_end, + struct icmphdr **icmphdr) +{ + struct icmphdr *icmph = nh->pos; + + if (icmph + 1 > data_end) + return -1; + + nh->pos = icmph + 1; + *icmphdr = icmph; + + return icmph->type; +} + +static __always_inline int parse_icmphdr_common(struct hdr_cursor *nh, + void *data_end, + struct icmphdr_common **icmphdr) +{ + struct icmphdr_common *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + nh->pos = h + 1; + *icmphdr = h; + + return h->type; +} + +/* + * parse_udphdr: parse the udp header and return the length of the udp payload + */ +static __always_inline int parse_udphdr(struct hdr_cursor *nh, + void *data_end, + struct udphdr **udphdr) +{ + int len; + struct udphdr *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + nh->pos = h + 1; + *udphdr = h; + + len = bpf_ntohs(h->len) - sizeof(struct udphdr); + if (len < 0) + return -1; + + return len; +} + +/* + * parse_tcphdr: parse and return the length of the tcp header + */ +static __always_inline int parse_tcphdr(struct hdr_cursor *nh, + void *data_end, + struct tcphdr **tcphdr) +{ + int len; + struct tcphdr *h = nh->pos; + + if (h + 1 > data_end) + return -1; + + len = h->doff * 4; + if ((void *) h + len > data_end) + return -1; + + nh->pos = h + 1; + *tcphdr = h; + + return len; +} + +#endif /* __PARSING_HELPERS_H */ diff --git a/src/xdp/common/rewrite_helpers.h b/src/xdp/common/rewrite_helpers.h new file mode 100644 index 000000000..a5d3e671d --- /dev/null +++ b/src/xdp/common/rewrite_helpers.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: (GPL-2.0-or-later OR BSD-2-clause) */ +/* + * This file contains functions that are used in the packetXX XDP programs to + * manipulate on packets data. The functions are marked as __always_inline, and + * fully defined in this header file to be included in the BPF program. + */ + +#ifndef __REWRITE_HELPERS_H +#define __REWRITE_HELPERS_H + +#include +#include +#include +#include + +#include +#include + +/* Pops the outermost VLAN tag off the packet. Returns the popped VLAN ID on + * success or negative errno on failure. + */ +static __always_inline int vlan_tag_pop(struct xdp_md *ctx, struct ethhdr *eth) +{ + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr eth_cpy; + struct vlan_hdr *vlh; + __be16 h_proto; + int vlid; + + if (!proto_is_vlan(eth->h_proto)) + return -1; + + /* Careful with the parenthesis here */ + vlh = (void *)(eth + 1); + + /* Still need to do bounds checking */ + if (vlh + 1 > data_end) + return -1; + + /* Save vlan ID for returning, h_proto for updating Ethernet header */ + vlid = bpf_ntohs(vlh->h_vlan_TCI); + h_proto = vlh->h_vlan_encapsulated_proto; + + /* Make a copy of the outer Ethernet header before we cut it off */ + __builtin_memcpy(ð_cpy, eth, sizeof(eth_cpy)); + + /* Actually adjust the head pointer */ + if (bpf_xdp_adjust_head(ctx, (int)sizeof(*vlh))) + return -1; + + /* Need to re-evaluate data *and* data_end and do new bounds checking + * after adjusting head + */ + eth = (void *)(long)ctx->data; + data_end = (void *)(long)ctx->data_end; + if (eth + 1 > data_end) + return -1; + + /* Copy back the old Ethernet header and update the proto type */ + __builtin_memcpy(eth, ð_cpy, sizeof(*eth)); + eth->h_proto = h_proto; + + return vlid; +} + +/* Pushes a new VLAN tag after the Ethernet header. Returns 0 on success, + * -1 on failure. + */ +static __always_inline int vlan_tag_push(struct xdp_md *ctx, + struct ethhdr *eth, int vlid) +{ + void *data_end = (void *)(long)ctx->data_end; + struct ethhdr eth_cpy; + struct vlan_hdr *vlh; + + /* First copy the original Ethernet header */ + __builtin_memcpy(ð_cpy, eth, sizeof(eth_cpy)); + + /* Then add space in front of the packet */ + if (bpf_xdp_adjust_head(ctx, 0 - (int)sizeof(*vlh))) + return -1; + + /* Need to re-evaluate data_end and data after head adjustment, and + * bounds check, even though we know there is enough space (as we + * increased it). + */ + data_end = (void *)(long)ctx->data_end; + eth = (void *)(long)ctx->data; + + if (eth + 1 > data_end) + return -1; + + /* Copy back Ethernet header in the right place, populate VLAN tag with + * ID and proto, and set outer Ethernet header to VLAN type. + */ + __builtin_memcpy(eth, ð_cpy, sizeof(*eth)); + + vlh = (void *)(eth + 1); + + if (vlh + 1 > data_end) + return -1; + + vlh->h_vlan_TCI = bpf_htons(vlid); + vlh->h_vlan_encapsulated_proto = eth->h_proto; + + eth->h_proto = bpf_htons(ETH_P_8021Q); + return 0; +} + +/* + * Swaps destination and source MAC addresses inside an Ethernet header + */ +static __always_inline void swap_src_dst_mac(struct ethhdr *eth) +{ + __u8 h_tmp[ETH_ALEN]; + + __builtin_memcpy(h_tmp, eth->h_source, ETH_ALEN); + __builtin_memcpy(eth->h_source, eth->h_dest, ETH_ALEN); + __builtin_memcpy(eth->h_dest, h_tmp, ETH_ALEN); +} + +/* + * Swaps destination and source IPv6 addresses inside an IPv6 header + */ +static __always_inline void swap_src_dst_ipv6(struct ipv6hdr *ipv6) +{ + struct in6_addr tmp = ipv6->saddr; + + ipv6->saddr = ipv6->daddr; + ipv6->daddr = tmp; +} + +/* + * Swaps destination and source IPv4 addresses inside an IPv4 header + */ +static __always_inline void swap_src_dst_ipv4(struct iphdr *iphdr) +{ + __be32 tmp = iphdr->saddr; + + iphdr->saddr = iphdr->daddr; + iphdr->daddr = tmp; +} + +#endif /* __REWRITE_HELPERS_H */ diff --git a/src/xdp/common/xdp_stats_kern.h b/src/xdp/common/xdp_stats_kern.h new file mode 100644 index 000000000..4e08551a0 --- /dev/null +++ b/src/xdp/common/xdp_stats_kern.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Used *ONLY* by BPF-prog running kernel side. */ +#ifndef __XDP_STATS_KERN_H +#define __XDP_STATS_KERN_H + +/* Data record type 'struct datarec' is defined in common/xdp_stats_kern_user.h, + * programs using this header must first include that file. + */ +#ifndef __XDP_STATS_KERN_USER_H +#warning "You forgot to #include <../common/xdp_stats_kern_user.h>" +#include <../common/xdp_stats_kern_user.h> +#endif + +/* Keeps stats per (enum) xdp_action */ +struct bpf_map_def SEC("maps") xdp_stats_map = { + .type = BPF_MAP_TYPE_PERCPU_ARRAY, + .key_size = sizeof(__u32), + .value_size = sizeof(struct datarec), + .max_entries = XDP_ACTION_MAX, +}; + +static __always_inline +__u32 xdp_stats_record_action(struct xdp_md *ctx, __u32 action) +{ + if (action >= XDP_ACTION_MAX) + return XDP_ABORTED; + + /* Lookup in kernel BPF-side return pointer to actual data record */ + struct datarec *rec = bpf_map_lookup_elem(&xdp_stats_map, &action); + if (!rec) + return XDP_ABORTED; + + /* BPF_MAP_TYPE_PERCPU_ARRAY returns a data record specific to current + * CPU and XDP hooks runs under Softirq, which makes it safe to update + * without atomic operations. + */ + rec->rx_packets++; + rec->rx_bytes += (ctx->data_end - ctx->data); + + return action; +} + +#endif /* __XDP_STATS_KERN_H */ diff --git a/src/xdp/common/xdp_stats_kern_user.h b/src/xdp/common/xdp_stats_kern_user.h new file mode 100644 index 000000000..d7b8d05e6 --- /dev/null +++ b/src/xdp/common/xdp_stats_kern_user.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* Used by BPF-prog kernel side BPF-progs and userspace programs, + * for sharing xdp_stats common struct and DEFINEs. + */ +#ifndef __XDP_STATS_KERN_USER_H +#define __XDP_STATS_KERN_USER_H + +/* This is the data record stored in the map */ +struct datarec { + __u64 rx_packets; + __u64 rx_bytes; +}; + +#ifndef XDP_ACTION_MAX +#define XDP_ACTION_MAX (XDP_REDIRECT + 1) +#endif + +#endif /* __XDP_STATS_KERN_USER_H */ -- cgit v1.2.3