#include <bpf/libbpf.h> /* bpf_get_link_xdp_id + bpf_set_link_xdp_id */ #include <string.h> /* strerror */ #include <net/if.h> /* IF_NAMESIZE */ #include <stdlib.h> /* exit(3) */ #include <errno.h> #include <bpf/bpf.h> #include <bpf/libbpf.h> #include <linux/if_link.h> /* Need XDP flags */ #include <linux/err.h> #include "common_defines.h" #ifndef PATH_MAX #define PATH_MAX 4096 #endif int xdp_link_attach(int ifindex, __u32 xdp_flags, int prog_fd) { int err; /* libbpf provide the XDP net_device link-level hook attach helper */ err = bpf_xdp_attach(ifindex, prog_fd, xdp_flags, NULL); if (err == -EEXIST && !(xdp_flags & XDP_FLAGS_UPDATE_IF_NOEXIST)) { /* Force mode didn't work, probably because a program of the * opposite type is loaded. Let's unload that and try loading * again. */ __u32 old_flags = xdp_flags; xdp_flags &= ~XDP_FLAGS_MODES; xdp_flags |= (old_flags & XDP_FLAGS_SKB_MODE) ? XDP_FLAGS_DRV_MODE : XDP_FLAGS_SKB_MODE; err = bpf_xdp_detach(ifindex, xdp_flags, NULL); if (!err) err = bpf_xdp_attach(ifindex, prog_fd, old_flags, NULL); } if (err < 0) { fprintf(stderr, "ERR: " "ifindex(%d) link set xdp fd failed (%d): %s\n", ifindex, -err, strerror(-err)); switch (-err) { case EBUSY: case EEXIST: fprintf(stderr, "Hint: XDP already loaded on device" " use --force to swap/replace\n"); break; case EOPNOTSUPP: fprintf(stderr, "Hint: Native-XDP not supported" " use --skb-mode or --auto-mode\n"); break; default: break; } return EXIT_FAIL_XDP; } return EXIT_OK; } int xdp_link_detach(int ifindex, __u32 xdp_flags, __u32 expected_prog_id) { __u32 curr_prog_id; int err; err = bpf_xdp_query_id(ifindex, xdp_flags, &curr_prog_id); if (err) { fprintf(stderr, "ERR: get link xdp id failed (err=%d): %s\n", -err, strerror(-err)); return EXIT_FAIL_XDP; } if (!curr_prog_id) { if (verbose) printf("INFO: %s() no curr XDP prog on ifindex:%d\n", __func__, ifindex); return EXIT_OK; } if (expected_prog_id && curr_prog_id != expected_prog_id) { fprintf(stderr, "ERR: %s() " "expected prog ID(%d) no match(%d), not removing\n", __func__, expected_prog_id, curr_prog_id); return EXIT_FAIL; } if ((err = bpf_xdp_detach(ifindex, xdp_flags, NULL)) < 0) { fprintf(stderr, "ERR: %s() link set xdp failed (err=%d): %s\n", __func__, err, strerror(-err)); return EXIT_FAIL_XDP; } if (verbose) printf("INFO: %s() removed XDP prog ID:%d on ifindex:%d\n", __func__, curr_prog_id, ifindex); return EXIT_OK; } struct bpf_object *load_bpf_object_file(const char *filename, int ifindex) { int first_prog_fd = -1; struct bpf_object *obj; int err; /* This struct allow us to set ifindex, this features is used for * hardware offloading XDP programs (note this sets libbpf * bpf_program->prog_ifindex and foreach bpf_map->map_ifindex). */ struct bpf_program *prog; obj = bpf_object__open_file(filename, NULL); if (libbpf_get_error(obj)) return NULL; prog = bpf_object__next_program(obj, NULL); bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); bpf_program__set_ifindex(prog, ifindex); /* Use libbpf for extracting BPF byte-code from BPF-ELF object, and * loading this into the kernel via bpf-syscall */ err = bpf_object__load(obj); if (err) { fprintf(stderr, "ERR: loading BPF-OBJ file(%s) (%d): %s\n", filename, err, strerror(-err)); return NULL; } first_prog_fd = bpf_program__fd(prog); /* Notice how a pointer to a libbpf bpf_object is returned */ return obj; } static struct bpf_object *open_bpf_object(const char *file, int ifindex) { int err; struct bpf_object *obj; struct bpf_map *map; struct bpf_program *prog, *first_prog = NULL; obj = bpf_object__open_file(file, NULL); if (libbpf_get_error(obj)) return NULL; prog = bpf_object__next_program(obj, NULL); bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); err = bpf_object__load(obj); if (IS_ERR_OR_NULL(obj)) { err = -PTR_ERR(obj); fprintf(stderr, "ERR: opening BPF-OBJ file(%s) (%d): %s\n", file, err, strerror(-err)); return NULL; } bpf_object__for_each_program(prog, obj) { bpf_program__set_type(prog, BPF_PROG_TYPE_XDP); bpf_program__set_ifindex(prog, ifindex); if (!first_prog) first_prog = prog; } bpf_object__for_each_map(map, obj) { if (bpf_map__type(map) != BPF_MAP_TYPE_PERF_EVENT_ARRAY) bpf_map__set_ifindex(map, ifindex); } if (!first_prog) { fprintf(stderr, "ERR: file %s contains no programs\n", file); return NULL; } return obj; } static int reuse_maps(struct bpf_object *obj, const char *path) { struct bpf_map *map; if (!obj) return -ENOENT; if (!path) return -EINVAL; bpf_object__for_each_map(map, obj) { int len, err; int pinned_map_fd; char buf[PATH_MAX]; len = snprintf(buf, PATH_MAX, "%s/%s", path, bpf_map__name(map)); if (len < 0) { return -EINVAL; } else if (len >= PATH_MAX) { return -ENAMETOOLONG; } pinned_map_fd = bpf_obj_get(buf); if (pinned_map_fd < 0) return pinned_map_fd; err = bpf_map__reuse_fd(map, pinned_map_fd); if (err) return err; } return 0; } struct bpf_object *load_bpf_object_file_reuse_maps(const char *file, int ifindex, const char *pin_dir) { int err; struct bpf_object *obj; obj = open_bpf_object(file, ifindex); if (!obj) { fprintf(stderr, "ERR: failed to open object %s\n", file); return NULL; } err = reuse_maps(obj, pin_dir); if (err) { fprintf(stderr, "ERR: failed to reuse maps for object %s, pin_dir=%s\n", file, pin_dir); return NULL; } err = bpf_object__load(obj); if (err) { fprintf(stderr, "ERR: loading BPF-OBJ file(%s) (%d): %s\n", file, err, strerror(-err)); return NULL; } return obj; } struct bpf_object *load_bpf_and_xdp_attach(struct config *cfg) { struct bpf_program *bpf_prog; struct bpf_object *bpf_obj; int offload_ifindex = 0; int prog_fd = -1; int err; /* If flags indicate hardware offload, supply ifindex */ if (cfg->xdp_flags & XDP_FLAGS_HW_MODE) offload_ifindex = cfg->ifindex; /* Load the BPF-ELF object file and get back libbpf bpf_object */ if (cfg->reuse_maps) bpf_obj = load_bpf_object_file_reuse_maps(cfg->filename, offload_ifindex, cfg->pin_dir); else bpf_obj = load_bpf_object_file(cfg->filename, offload_ifindex); if (!bpf_obj) { fprintf(stderr, "ERR: loading file: %s\n", cfg->filename); exit(EXIT_FAIL_BPF); } /* At this point: All XDP/BPF programs from the cfg->filename have been * loaded into the kernel, and evaluated by the verifier. Only one of * these gets attached to XDP hook, the others will get freed once this * process exit. */ if (cfg->progsec[0]) /* Find a matching BPF prog section name */ bpf_prog = bpf_object__find_program_by_name(bpf_obj, cfg->progsec); else /* Find the first program */ bpf_prog = bpf_object__next_program(bpf_obj, NULL); if (!bpf_prog) { fprintf(stderr, "ERR: couldn't find a program in ELF section '%s'\n", cfg->progsec); exit(EXIT_FAIL_BPF); } strncpy(cfg->progsec, bpf_program__section_name(bpf_prog), sizeof(cfg->progsec)); prog_fd = bpf_program__fd(bpf_prog); if (prog_fd <= 0) { fprintf(stderr, "ERR: bpf_program__fd failed\n"); exit(EXIT_FAIL_BPF); } /* At this point: BPF-progs are (only) loaded by the kernel, and prog_fd * is our select file-descriptor handle. Next step is attaching this FD * to a kernel hook point, in this case XDP net_device link-level hook. */ err = xdp_link_attach(cfg->ifindex, cfg->xdp_flags, prog_fd); if (err) exit(err); return bpf_obj; } #define XDP_UNKNOWN XDP_REDIRECT + 1 #ifndef XDP_ACTION_MAX #define XDP_ACTION_MAX (XDP_UNKNOWN + 1) #endif static const char *xdp_action_names[XDP_ACTION_MAX] = { [XDP_ABORTED] = "XDP_ABORTED", [XDP_DROP] = "XDP_DROP", [XDP_PASS] = "XDP_PASS", [XDP_TX] = "XDP_TX", [XDP_REDIRECT] = "XDP_REDIRECT", [XDP_UNKNOWN] = "XDP_UNKNOWN", }; const char *action2str(__u32 action) { if (action < XDP_ACTION_MAX) return xdp_action_names[action]; return NULL; } int check_map_fd_info(const struct bpf_map_info *info, const struct bpf_map_info *exp) { if (exp->key_size && exp->key_size != info->key_size) { fprintf(stderr, "ERR: %s() " "Map key size(%d) mismatch expected size(%d)\n", __func__, info->key_size, exp->key_size); return EXIT_FAIL; } if (exp->value_size && exp->value_size != info->value_size) { fprintf(stderr, "ERR: %s() " "Map value size(%d) mismatch expected size(%d)\n", __func__, info->value_size, exp->value_size); return EXIT_FAIL; } if (exp->max_entries && exp->max_entries != info->max_entries) { fprintf(stderr, "ERR: %s() " "Map max_entries(%d) mismatch expected size(%d)\n", __func__, info->max_entries, exp->max_entries); return EXIT_FAIL; } if (exp->type && exp->type != info->type) { fprintf(stderr, "ERR: %s() " "Map type(%d) mismatch expected type(%d)\n", __func__, info->type, exp->type); return EXIT_FAIL; } return 0; } int open_bpf_map_file(const char *pin_dir, const char *mapname, struct bpf_map_info *info) { char filename[PATH_MAX]; int err, len, fd; __u32 info_len = sizeof(*info); len = snprintf(filename, PATH_MAX, "%s/%s", pin_dir, mapname); if (len < 0) { fprintf(stderr, "ERR: constructing full mapname path\n"); return -1; } fd = bpf_obj_get(filename); if (fd < 0) { fprintf(stderr, "WARN: Failed to open bpf map file:%s err(%d):%s\n", filename, errno, strerror(errno)); return fd; } if (info) { err = bpf_obj_get_info_by_fd(fd, info, &info_len); if (err) { fprintf(stderr, "ERR: %s() can't get info - %s\n", __func__, strerror(errno)); return EXIT_FAIL_BPF; } } return fd; }