summaryrefslogtreecommitdiffstats
path: root/tools/lib
diff options
context:
space:
mode:
Diffstat (limited to 'tools/lib')
-rw-r--r--tools/lib/bpf/Makefile18
-rw-r--r--tools/lib/bpf/bpf.c17
-rw-r--r--tools/lib/bpf/bpf.h17
-rw-r--r--tools/lib/bpf/bpf_helpers.h11
-rw-r--r--tools/lib/bpf/btf.c48
-rw-r--r--tools/lib/bpf/btf.h29
-rw-r--r--tools/lib/bpf/btf_dump.c115
-rw-r--r--tools/lib/bpf/libbpf.c1741
-rw-r--r--tools/lib/bpf/libbpf.h107
-rw-r--r--tools/lib/bpf/libbpf.map16
-rw-r--r--tools/lib/bpf/libbpf.pc.template2
-rw-r--r--tools/lib/bpf/libbpf_common.h40
-rw-r--r--tools/lib/bpf/libbpf_internal.h21
13 files changed, 1764 insertions, 418 deletions
diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile
index defae23a0169..d4790121adf4 100644
--- a/tools/lib/bpf/Makefile
+++ b/tools/lib/bpf/Makefile
@@ -56,8 +56,8 @@ ifndef VERBOSE
endif
FEATURE_USER = .libbpf
-FEATURE_TESTS = libelf libelf-mmap bpf reallocarray
-FEATURE_DISPLAY = libelf bpf
+FEATURE_TESTS = libelf libelf-mmap zlib bpf reallocarray
+FEATURE_DISPLAY = libelf zlib bpf
INCLUDES = -I. -I$(srctree)/tools/include -I$(srctree)/tools/arch/$(ARCH)/include/uapi -I$(srctree)/tools/include/uapi
FEATURE_CHECK_CFLAGS-bpf = $(INCLUDES)
@@ -147,6 +147,7 @@ TAGS_PROG := $(if $(shell which etags 2>/dev/null),etags,ctags)
GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \
cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \
+ sed 's/\[.*\]//' | \
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \
sort -u | wc -l)
VERSIONED_SYM_COUNT = $(shell readelf -s --wide $(OUTPUT)libbpf.so | \
@@ -159,7 +160,7 @@ all: fixdep
all_cmd: $(CMD_TARGETS) check
-$(BPF_IN_SHARED): force elfdep bpfdep bpf_helper_defs.h
+$(BPF_IN_SHARED): force elfdep zdep bpfdep bpf_helper_defs.h
@(test -f ../../include/uapi/linux/bpf.h -a -f ../../../include/uapi/linux/bpf.h && ( \
(diff -B ../../include/uapi/linux/bpf.h ../../../include/uapi/linux/bpf.h >/dev/null) || \
echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/bpf.h' differs from latest version at 'include/uapi/linux/bpf.h'" >&2 )) || true
@@ -177,7 +178,7 @@ $(BPF_IN_SHARED): force elfdep bpfdep bpf_helper_defs.h
echo "Warning: Kernel ABI header at 'tools/include/uapi/linux/if_xdp.h' differs from latest version at 'include/uapi/linux/if_xdp.h'" >&2 )) || true
$(Q)$(MAKE) $(build)=libbpf OUTPUT=$(SHARED_OBJDIR) CFLAGS="$(CFLAGS) $(SHLIB_FLAGS)"
-$(BPF_IN_STATIC): force elfdep bpfdep bpf_helper_defs.h
+$(BPF_IN_STATIC): force elfdep zdep bpfdep bpf_helper_defs.h
$(Q)$(MAKE) $(build)=libbpf OUTPUT=$(STATIC_OBJDIR)
bpf_helper_defs.h: $(srctree)/tools/include/uapi/linux/bpf.h
@@ -189,7 +190,7 @@ $(OUTPUT)libbpf.so: $(OUTPUT)libbpf.so.$(LIBBPF_VERSION)
$(OUTPUT)libbpf.so.$(LIBBPF_VERSION): $(BPF_IN_SHARED)
$(QUIET_LINK)$(CC) $(LDFLAGS) \
--shared -Wl,-soname,libbpf.so.$(LIBBPF_MAJOR_VERSION) \
- -Wl,--version-script=$(VERSION_SCRIPT) $^ -lelf -o $@
+ -Wl,--version-script=$(VERSION_SCRIPT) $^ -lelf -lz -o $@
@ln -sf $(@F) $(OUTPUT)libbpf.so
@ln -sf $(@F) $(OUTPUT)libbpf.so.$(LIBBPF_MAJOR_VERSION)
@@ -213,6 +214,7 @@ check_abi: $(OUTPUT)libbpf.so
"versioned in $(VERSION_SCRIPT)." >&2; \
readelf -s --wide $(BPF_IN_SHARED) | \
cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \
+ sed 's/\[.*\]//' | \
awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}'| \
sort -u > $(OUTPUT)libbpf_global_syms.tmp; \
readelf -s --wide $(OUTPUT)libbpf.so | \
@@ -249,6 +251,7 @@ install_headers: bpf_helper_defs.h
$(call do_install,libbpf.h,$(prefix)/include/bpf,644); \
$(call do_install,btf.h,$(prefix)/include/bpf,644); \
$(call do_install,libbpf_util.h,$(prefix)/include/bpf,644); \
+ $(call do_install,libbpf_common.h,$(prefix)/include/bpf,644); \
$(call do_install,xsk.h,$(prefix)/include/bpf,644); \
$(call do_install,bpf_helpers.h,$(prefix)/include/bpf,644); \
$(call do_install,bpf_helper_defs.h,$(prefix)/include/bpf,644); \
@@ -277,12 +280,15 @@ clean:
-PHONY += force elfdep bpfdep cscope tags
+PHONY += force elfdep zdep bpfdep cscope tags
force:
elfdep:
@if [ "$(feature-libelf)" != "1" ]; then echo "No libelf found"; exit 1 ; fi
+zdep:
+ @if [ "$(feature-zlib)" != "1" ]; then echo "No zlib found"; exit 1 ; fi
+
bpfdep:
@if [ "$(feature-bpf)" != "1" ]; then echo "BPF API too old"; exit 1 ; fi
diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c
index 98596e15390f..a787d53699c8 100644
--- a/tools/lib/bpf/bpf.c
+++ b/tools/lib/bpf/bpf.c
@@ -467,13 +467,28 @@ int bpf_obj_get(const char *pathname)
int bpf_prog_attach(int prog_fd, int target_fd, enum bpf_attach_type type,
unsigned int flags)
{
+ DECLARE_LIBBPF_OPTS(bpf_prog_attach_opts, opts,
+ .flags = flags,
+ );
+
+ return bpf_prog_attach_xattr(prog_fd, target_fd, type, &opts);
+}
+
+int bpf_prog_attach_xattr(int prog_fd, int target_fd,
+ enum bpf_attach_type type,
+ const struct bpf_prog_attach_opts *opts)
+{
union bpf_attr attr;
+ if (!OPTS_VALID(opts, bpf_prog_attach_opts))
+ return -EINVAL;
+
memset(&attr, 0, sizeof(attr));
attr.target_fd = target_fd;
attr.attach_bpf_fd = prog_fd;
attr.attach_type = type;
- attr.attach_flags = flags;
+ attr.attach_flags = OPTS_GET(opts, flags, 0);
+ attr.replace_bpf_fd = OPTS_GET(opts, replace_prog_fd, 0);
return sys_bpf(BPF_PROG_ATTACH, &attr, sizeof(attr));
}
diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h
index 3c791fa8e68e..f0ab8519986e 100644
--- a/tools/lib/bpf/bpf.h
+++ b/tools/lib/bpf/bpf.h
@@ -28,14 +28,12 @@
#include <stddef.h>
#include <stdint.h>
+#include "libbpf_common.h"
+
#ifdef __cplusplus
extern "C" {
#endif
-#ifndef LIBBPF_API
-#define LIBBPF_API __attribute__((visibility("default")))
-#endif
-
struct bpf_create_map_attr {
const char *name;
enum bpf_map_type map_type;
@@ -128,8 +126,19 @@ LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key);
LIBBPF_API int bpf_map_freeze(int fd);
LIBBPF_API int bpf_obj_pin(int fd, const char *pathname);
LIBBPF_API int bpf_obj_get(const char *pathname);
+
+struct bpf_prog_attach_opts {
+ size_t sz; /* size of this struct for forward/backward compatibility */
+ unsigned int flags;
+ int replace_prog_fd;
+};
+#define bpf_prog_attach_opts__last_field replace_prog_fd
+
LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd,
enum bpf_attach_type type, unsigned int flags);
+LIBBPF_API int bpf_prog_attach_xattr(int prog_fd, int attachable_fd,
+ enum bpf_attach_type type,
+ const struct bpf_prog_attach_opts *opts);
LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type);
LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd,
enum bpf_attach_type type);
diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h
index 0c7d28292898..f69cc208778a 100644
--- a/tools/lib/bpf/bpf_helpers.h
+++ b/tools/lib/bpf/bpf_helpers.h
@@ -25,6 +25,9 @@
#ifndef __always_inline
#define __always_inline __attribute__((always_inline))
#endif
+#ifndef __weak
+#define __weak __attribute__((weak))
+#endif
/*
* Helper structure used by eBPF C program
@@ -44,4 +47,12 @@ enum libbpf_pin_type {
LIBBPF_PIN_BY_NAME,
};
+enum libbpf_tristate {
+ TRI_NO = 0,
+ TRI_YES = 1,
+ TRI_MODULE = 2,
+};
+
+#define __kconfig __attribute__((section(".kconfig")))
+
#endif
diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c
index 88efa2bb7137..5f04f56e1eb6 100644
--- a/tools/lib/bpf/btf.c
+++ b/tools/lib/bpf/btf.c
@@ -278,6 +278,45 @@ done:
return nelems * size;
}
+int btf__align_of(const struct btf *btf, __u32 id)
+{
+ const struct btf_type *t = btf__type_by_id(btf, id);
+ __u16 kind = btf_kind(t);
+
+ switch (kind) {
+ case BTF_KIND_INT:
+ case BTF_KIND_ENUM:
+ return min(sizeof(void *), t->size);
+ case BTF_KIND_PTR:
+ return sizeof(void *);
+ case BTF_KIND_TYPEDEF:
+ case BTF_KIND_VOLATILE:
+ case BTF_KIND_CONST:
+ case BTF_KIND_RESTRICT:
+ return btf__align_of(btf, t->type);
+ case BTF_KIND_ARRAY:
+ return btf__align_of(btf, btf_array(t)->type);
+ case BTF_KIND_STRUCT:
+ case BTF_KIND_UNION: {
+ const struct btf_member *m = btf_members(t);
+ __u16 vlen = btf_vlen(t);
+ int i, max_align = 1, align;
+
+ for (i = 0; i < vlen; i++, m++) {
+ align = btf__align_of(btf, m->type);
+ if (align <= 0)
+ return align;
+ max_align = max(max_align, align);
+ }
+
+ return max_align;
+ }
+ default:
+ pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
+ return 0;
+ }
+}
+
int btf__resolve_type(const struct btf *btf, __u32 type_id)
{
const struct btf_type *t;
@@ -539,6 +578,12 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
return -ENOENT;
}
+ /* .extern datasec size and var offsets were set correctly during
+ * extern collection step, so just skip straight to sorting variables
+ */
+ if (t->size)
+ goto sort_vars;
+
ret = bpf_object__section_size(obj, name, &size);
if (ret || !size || (t->size && t->size != size)) {
pr_debug("Invalid size for section %s: %u bytes\n", name, size);
@@ -575,7 +620,8 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf,
vsi->offset = off;
}
- qsort(t + 1, vars, sizeof(*vsi), compare_vsi_off);
+sort_vars:
+ qsort(btf_var_secinfos(t), vars, sizeof(*vsi), compare_vsi_off);
return 0;
}
diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h
index d9ac73a02cde..8d73f7f5551f 100644
--- a/tools/lib/bpf/btf.h
+++ b/tools/lib/bpf/btf.h
@@ -8,14 +8,12 @@
#include <linux/btf.h>
#include <linux/types.h>
+#include "libbpf_common.h"
+
#ifdef __cplusplus
extern "C" {
#endif
-#ifndef LIBBPF_API
-#define LIBBPF_API __attribute__((visibility("default")))
-#endif
-
#define BTF_ELF_SEC ".BTF"
#define BTF_EXT_ELF_SEC ".BTF.ext"
#define MAPS_ELF_SEC ".maps"
@@ -79,6 +77,7 @@ LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf,
__u32 id);
LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id);
LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id);
+LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id);
LIBBPF_API int btf__fd(const struct btf *btf);
LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size);
LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset);
@@ -127,6 +126,28 @@ LIBBPF_API void btf_dump__free(struct btf_dump *d);
LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id);
+struct btf_dump_emit_type_decl_opts {
+ /* size of this struct, for forward/backward compatiblity */
+ size_t sz;
+ /* optional field name for type declaration, e.g.:
+ * - struct my_struct <FNAME>
+ * - void (*<FNAME>)(int)
+ * - char (*<FNAME>)[123]
+ */
+ const char *field_name;
+ /* extra indentation level (in number of tabs) to emit for multi-line
+ * type declarations (e.g., anonymous struct); applies for lines
+ * starting from the second one (first line is assumed to have
+ * necessary indentation already
+ */
+ int indent_level;
+};
+#define btf_dump_emit_type_decl_opts__last_field indent_level
+
+LIBBPF_API int
+btf_dump__emit_type_decl(struct btf_dump *d, __u32 id,
+ const struct btf_dump_emit_type_decl_opts *opts);
+
/*
* A set of helpers for easier BTF types handling
*/
diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c
index cb126d8fcf75..e95f7710f210 100644
--- a/tools/lib/bpf/btf_dump.c
+++ b/tools/lib/bpf/btf_dump.c
@@ -116,6 +116,8 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...)
va_end(args);
}
+static int btf_dump_mark_referenced(struct btf_dump *d);
+
struct btf_dump *btf_dump__new(const struct btf *btf,
const struct btf_ext *btf_ext,
const struct btf_dump_opts *opts,
@@ -137,18 +139,39 @@ struct btf_dump *btf_dump__new(const struct btf *btf,
if (IS_ERR(d->type_names)) {
err = PTR_ERR(d->type_names);
d->type_names = NULL;
- btf_dump__free(d);
- return ERR_PTR(err);
}
d->ident_names = hashmap__new(str_hash_fn, str_equal_fn, NULL);
if (IS_ERR(d->ident_names)) {
err = PTR_ERR(d->ident_names);
d->ident_names = NULL;
- btf_dump__free(d);
- return ERR_PTR(err);
+ goto err;
+ }
+ d->type_states = calloc(1 + btf__get_nr_types(d->btf),
+ sizeof(d->type_states[0]));
+ if (!d->type_states) {
+ err = -ENOMEM;
+ goto err;
+ }
+ d->cached_names = calloc(1 + btf__get_nr_types(d->btf),
+ sizeof(d->cached_names[0]));
+ if (!d->cached_names) {
+ err = -ENOMEM;
+ goto err;
}
+ /* VOID is special */
+ d->type_states[0].order_state = ORDERED;
+ d->type_states[0].emit_state = EMITTED;
+
+ /* eagerly determine referenced types for anon enums */
+ err = btf_dump_mark_referenced(d);
+ if (err)
+ goto err;
+
return d;
+err:
+ btf_dump__free(d);
+ return ERR_PTR(err);
}
void btf_dump__free(struct btf_dump *d)
@@ -175,7 +198,6 @@ void btf_dump__free(struct btf_dump *d)
free(d);
}
-static int btf_dump_mark_referenced(struct btf_dump *d);
static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr);
static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id);
@@ -202,27 +224,6 @@ int btf_dump__dump_type(struct btf_dump *d, __u32 id)
if (id > btf__get_nr_types(d->btf))
return -EINVAL;
- /* type states are lazily allocated, as they might not be needed */
- if (!d->type_states) {
- d->type_states = calloc(1 + btf__get_nr_types(d->btf),
- sizeof(d->type_states[0]));
- if (!d->type_states)
- return -ENOMEM;
- d->cached_names = calloc(1 + btf__get_nr_types(d->btf),
- sizeof(d->cached_names[0]));
- if (!d->cached_names)
- return -ENOMEM;
-
- /* VOID is special */
- d->type_states[0].order_state = ORDERED;
- d->type_states[0].emit_state = EMITTED;
-
- /* eagerly determine referenced types for anon enums */
- err = btf_dump_mark_referenced(d);
- if (err)
- return err;
- }
-
d->emit_queue_cnt = 0;
err = btf_dump_order_type(d, id, false);
if (err < 0)
@@ -752,41 +753,6 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id)
}
}
-static int btf_align_of(const struct btf *btf, __u32 id)
-{
- const struct btf_type *t = btf__type_by_id(btf, id);
- __u16 kind = btf_kind(t);
-
- switch (kind) {
- case BTF_KIND_INT:
- case BTF_KIND_ENUM:
- return min(sizeof(void *), t->size);
- case BTF_KIND_PTR:
- return sizeof(void *);
- case BTF_KIND_TYPEDEF:
- case BTF_KIND_VOLATILE:
- case BTF_KIND_CONST:
- case BTF_KIND_RESTRICT:
- return btf_align_of(btf, t->type);
- case BTF_KIND_ARRAY:
- return btf_align_of(btf, btf_array(t)->type);
- case BTF_KIND_STRUCT:
- case BTF_KIND_UNION: {
- const struct btf_member *m = btf_members(t);
- __u16 vlen = btf_vlen(t);
- int i, align = 1;
-
- for (i = 0; i < vlen; i++, m++)
- align = max(align, btf_align_of(btf, m->type));
-
- return align;
- }
- default:
- pr_warn("unsupported BTF_KIND:%u\n", btf_kind(t));
- return 1;
- }
-}
-
static bool btf_is_struct_packed(const struct btf *btf, __u32 id,
const struct btf_type *t)
{
@@ -794,18 +760,18 @@ static bool btf_is_struct_packed(const struct btf *btf, __u32 id,
int align, i, bit_sz;
__u16 vlen;
- align = btf_align_of(btf, id);
+ align = btf__align_of(btf, id);
/* size of a non-packed struct has to be a multiple of its alignment*/
- if (t->size % align)
+ if (align && t->size % align)
return true;
m = btf_members(t);
vlen = btf_vlen(t);
/* all non-bitfield fields have to be naturally aligned */
for (i = 0; i < vlen; i++, m++) {
- align = btf_align_of(btf, m->type);
+ align = btf__align_of(btf, m->type);
bit_sz = btf_member_bitfield_size(t, i);
- if (bit_sz == 0 && m->offset % (8 * align) != 0)
+ if (align && bit_sz == 0 && m->offset % (8 * align) != 0)
return true;
}
@@ -889,7 +855,7 @@ static void btf_dump_emit_struct_def(struct btf_dump *d,
fname = btf_name_of(d, m->name_off);
m_sz = btf_member_bitfield_size(t, i);
m_off = btf_member_bit_offset(t, i);
- align = packed ? 1 : btf_align_of(d->btf, m->type);
+ align = packed ? 1 : btf__align_of(d->btf, m->type);
btf_dump_emit_bit_padding(d, off, m_off, m_sz, align, lvl + 1);
btf_dump_printf(d, "\n%s", pfx(lvl + 1));
@@ -907,7 +873,7 @@ static void btf_dump_emit_struct_def(struct btf_dump *d,
/* pad at the end, if necessary */
if (is_struct) {
- align = packed ? 1 : btf_align_of(d->btf, id);
+ align = packed ? 1 : btf__align_of(d->btf, id);
btf_dump_emit_bit_padding(d, off, t->size * 8, 0, align,
lvl + 1);
}
@@ -1051,6 +1017,21 @@ static int btf_dump_push_decl_stack_id(struct btf_dump *d, __u32 id)
* of a stack frame. Some care is required to "pop" stack frames after
* processing type declaration chain.
*/
+int btf_dump__emit_type_decl(struct btf_dump *d, __u32 id,
+ const struct btf_dump_emit_type_decl_opts *opts)
+{
+ const char *fname;
+ int lvl;
+
+ if (!OPTS_VALID(opts, btf_dump_emit_type_decl_opts))
+ return -EINVAL;
+
+ fname = OPTS_GET(opts, field_name, NULL);
+ lvl = OPTS_GET(opts, indent_level, 0);
+ btf_dump_emit_type_decl(d, id, fname, lvl);
+ return 0;
+}
+
static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id,
const char *fname, int lvl)
{
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 3f09772192f1..7513165b104f 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -18,6 +18,7 @@
#include <stdarg.h>
#include <libgen.h>
#include <inttypes.h>
+#include <limits.h>
#include <string.h>
#include <unistd.h>
#include <endian.h>
@@ -41,9 +42,11 @@
#include <sys/types.h>
#include <sys/vfs.h>
#include <sys/utsname.h>
+#include <sys/resource.h>
#include <tools/libc_compat.h>
#include <libelf.h>
#include <gelf.h>
+#include <zlib.h>
#include "libbpf.h"
#include "bpf.h"
@@ -99,14 +102,33 @@ void libbpf_print(enum libbpf_print_level level, const char *format, ...)
va_end(args);
}
-#define STRERR_BUFSIZE 128
+static void pr_perm_msg(int err)
+{
+ struct rlimit limit;
+ char buf[100];
+
+ if (err != -EPERM || geteuid() != 0)
+ return;
+
+ err = getrlimit(RLIMIT_MEMLOCK, &limit);
+ if (err)
+ return;
+
+ if (limit.rlim_cur == RLIM_INFINITY)
+ return;
+
+ if (limit.rlim_cur < 1024)
+ snprintf(buf, sizeof(buf), "%zu bytes", (size_t)limit.rlim_cur);
+ else if (limit.rlim_cur < 1024*1024)
+ snprintf(buf, sizeof(buf), "%.1f KiB", (double)limit.rlim_cur / 1024);
+ else
+ snprintf(buf, sizeof(buf), "%.1f MiB", (double)limit.rlim_cur / (1024*1024));
-#define CHECK_ERR(action, err, out) do { \
- err = action; \
- if (err) \
- goto out; \
-} while (0)
+ pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",
+ buf);
+}
+#define STRERR_BUFSIZE 128
/* Copied from tools/perf/util/util.h */
#ifndef zfree
@@ -146,6 +168,20 @@ struct bpf_capabilities {
__u32 array_mmap:1;
};
+enum reloc_type {
+ RELO_LD64,
+ RELO_CALL,
+ RELO_DATA,
+ RELO_EXTERN,
+};
+
+struct reloc_desc {
+ enum reloc_type type;
+ int insn_idx;
+ int map_idx;
+ int sym_off;
+};
+
/*
* bpf_prog should be a better name but it has been used in
* linux/filter.h.
@@ -164,16 +200,7 @@ struct bpf_program {
size_t insns_cnt, main_prog_cnt;
enum bpf_prog_type type;
- struct reloc_desc {
- enum {
- RELO_LD64,
- RELO_CALL,
- RELO_DATA,
- } type;
- int insn_idx;
- int map_idx;
- int sym_off;
- } *reloc_desc;
+ struct reloc_desc *reloc_desc;
int nr_reloc;
int log_level;
@@ -202,22 +229,29 @@ struct bpf_program {
__u32 prog_flags;
};
+#define DATA_SEC ".data"
+#define BSS_SEC ".bss"
+#define RODATA_SEC ".rodata"
+#define KCONFIG_SEC ".kconfig"
+
enum libbpf_map_type {
LIBBPF_MAP_UNSPEC,
LIBBPF_MAP_DATA,
LIBBPF_MAP_BSS,
LIBBPF_MAP_RODATA,
+ LIBBPF_MAP_KCONFIG,
};
static const char * const libbpf_type_to_btf_name[] = {
- [LIBBPF_MAP_DATA] = ".data",
- [LIBBPF_MAP_BSS] = ".bss",
- [LIBBPF_MAP_RODATA] = ".rodata",
+ [LIBBPF_MAP_DATA] = DATA_SEC,
+ [LIBBPF_MAP_BSS] = BSS_SEC,
+ [LIBBPF_MAP_RODATA] = RODATA_SEC,
+ [LIBBPF_MAP_KCONFIG] = KCONFIG_SEC,
};
struct bpf_map {
- int fd;
char *name;
+ int fd;
int sec_idx;
size_t sec_offset;
int map_ifindex;
@@ -228,14 +262,32 @@ struct bpf_map {
void *priv;
bpf_map_clear_priv_t clear_priv;
enum libbpf_map_type libbpf_type;
+ void *mmaped;
char *pin_path;
bool pinned;
bool reused;
};
-struct bpf_secdata {
- void *rodata;
- void *data;
+enum extern_type {
+ EXT_UNKNOWN,
+ EXT_CHAR,
+ EXT_BOOL,
+ EXT_INT,
+ EXT_TRISTATE,
+ EXT_CHAR_ARR,
+};
+
+struct extern_desc {
+ const char *name;
+ int sym_idx;
+ int btf_id;
+ enum extern_type type;
+ int sz;
+ int align;
+ int data_off;
+ bool is_signed;
+ bool is_weak;
+ bool is_set;
};
static LIST_HEAD(bpf_objects_list);
@@ -250,7 +302,11 @@ struct bpf_object {
struct bpf_map *maps;
size_t nr_maps;
size_t maps_cap;
- struct bpf_secdata sections;
+
+ char *kconfig;
+ struct extern_desc *externs;
+ int nr_extern;
+ int kconfig_map_idx;
bool loaded;
bool has_pseudo_calls;
@@ -279,6 +335,7 @@ struct bpf_object {
int maps_shndx;
int btf_maps_shndx;
int text_shndx;
+ int symbols_shndx;
int data_shndx;
int rodata_shndx;
int bss_shndx;
@@ -550,6 +607,7 @@ static struct bpf_object *bpf_object__new(const char *path,
obj->efile.data_shndx = -1;
obj->efile.rodata_shndx = -1;
obj->efile.bss_shndx = -1;
+ obj->kconfig_map_idx = -1;
obj->kern_version = get_kernel_version();
obj->loaded = false;
@@ -748,13 +806,13 @@ int bpf_object__section_size(const struct bpf_object *obj, const char *name,
*size = 0;
if (!name) {
return -EINVAL;
- } else if (!strcmp(name, ".data")) {
+ } else if (!strcmp(name, DATA_SEC)) {
if (obj->efile.data)
*size = obj->efile.data->d_size;
- } else if (!strcmp(name, ".bss")) {
+ } else if (!strcmp(name, BSS_SEC)) {
if (obj->efile.bss)
*size = obj->efile.bss->d_size;
- } else if (!strcmp(name, ".rodata")) {
+ } else if (!strcmp(name, RODATA_SEC)) {
if (obj->efile.rodata)
*size = obj->efile.rodata->d_size;
} else {
@@ -835,13 +893,38 @@ static struct bpf_map *bpf_object__add_map(struct bpf_object *obj)
return &obj->maps[obj->nr_maps++];
}
+static size_t bpf_map_mmap_sz(const struct bpf_map *map)
+{
+ long page_sz = sysconf(_SC_PAGE_SIZE);
+ size_t map_sz;
+
+ map_sz = roundup(map->def.value_size, 8) * map->def.max_entries;
+ map_sz = roundup(map_sz, page_sz);
+ return map_sz;
+}
+
+static char *internal_map_name(struct bpf_object *obj,
+ enum libbpf_map_type type)
+{
+ char map_name[BPF_OBJ_NAME_LEN];
+ const char *sfx = libbpf_type_to_btf_name[type];
+ int sfx_len = max((size_t)7, strlen(sfx));
+ int pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1,
+ strlen(obj->name));
+
+ snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,
+ sfx_len, libbpf_type_to_btf_name[type]);
+
+ return strdup(map_name);
+}
+
static int
bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
- int sec_idx, Elf_Data *data, void **data_buff)
+ int sec_idx, void *data, size_t data_sz)
{
- char map_name[BPF_OBJ_NAME_LEN];
struct bpf_map_def *def;
struct bpf_map *map;
+ int err;
map = bpf_object__add_map(obj);
if (IS_ERR(map))
@@ -850,9 +933,7 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
map->libbpf_type = type;
map->sec_idx = sec_idx;
map->sec_offset = 0;
- snprintf(map_name, sizeof(map_name), "%.8s%.7s", obj->name,
- libbpf_type_to_btf_name[type]);
- map->name = strdup(map_name);
+ map->name = internal_map_name(obj, type);
if (!map->name) {
pr_warn("failed to alloc map name\n");
return -ENOMEM;
@@ -861,25 +942,29 @@ bpf_object__init_internal_map(struct bpf_object *obj, enum libbpf_map_type type,
def = &map->def;
def->type = BPF_MAP_TYPE_ARRAY;
def->key_size = sizeof(int);
- def->value_size = data->d_size;
+ def->value_size = data_sz;
def->max_entries = 1;
- def->map_flags = type == LIBBPF_MAP_RODATA ? BPF_F_RDONLY_PROG : 0;
- if (obj->caps.array_mmap)
- def->map_flags |= BPF_F_MMAPABLE;
+ def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG
+ ? BPF_F_RDONLY_PROG : 0;
+ def->map_flags |= BPF_F_MMAPABLE;
pr_debug("map '%s' (global data): at sec_idx %d, offset %zu, flags %x.\n",
- map_name, map->sec_idx, map->sec_offset, def->map_flags);
+ map->name, map->sec_idx, map->sec_offset, def->map_flags);
- if (data_buff) {
- *data_buff = malloc(data->d_size);
- if (!*data_buff) {
- zfree(&map->name);
- pr_warn("failed to alloc map content buffer\n");
- return -ENOMEM;
- }
- memcpy(*data_buff, data->d_buf, data->d_size);
+ map->mmaped = mmap(NULL, bpf_map_mmap_sz(map), PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_ANONYMOUS, -1, 0);
+ if (map->mmaped == MAP_FAILED) {
+ err = -errno;
+ map->mmaped = NULL;
+ pr_warn("failed to alloc map '%s' content buffer: %d\n",
+ map->name, err);
+ zfree(&map->name);
+ return err;
}
+ if (data)
+ memcpy(map->mmaped, data, data_sz);
+
pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);
return 0;
}
@@ -888,37 +973,332 @@ static int bpf_object__init_global_data_maps(struct bpf_object *obj)
{
int err;
- if (!obj->caps.global_data)
- return 0;
/*
* Populate obj->maps with libbpf internal maps.
*/
if (obj->efile.data_shndx >= 0) {
err = bpf_object__init_internal_map(obj, LIBBPF_MAP_DATA,
obj->efile.data_shndx,
- obj->efile.data,
- &obj->sections.data);
+ obj->efile.data->d_buf,
+ obj->efile.data->d_size);
if (err)
return err;
}
if (obj->efile.rodata_shndx >= 0) {
err = bpf_object__init_internal_map(obj, LIBBPF_MAP_RODATA,
obj->efile.rodata_shndx,
- obj->efile.rodata,
- &obj->sections.rodata);
+ obj->efile.rodata->d_buf,
+ obj->efile.rodata->d_size);
if (err)
return err;
}
if (obj->efile.bss_shndx >= 0) {
err = bpf_object__init_internal_map(obj, LIBBPF_MAP_BSS,
obj->efile.bss_shndx,
- obj->efile.bss, NULL);
+ NULL,
+ obj->efile.bss->d_size);
if (err)
return err;
}
return 0;
}
+
+static struct extern_desc *find_extern_by_name(const struct bpf_object *obj,
+ const void *name)
+{
+ int i;
+
+ for (i = 0; i < obj->nr_extern; i++) {
+ if (strcmp(obj->externs[i].name, name) == 0)
+ return &obj->externs[i];
+ }
+ return NULL;
+}
+
+static int set_ext_value_tri(struct extern_desc *ext, void *ext_val,
+ char value)
+{
+ switch (ext->type) {
+ case EXT_BOOL:
+ if (value == 'm') {
+ pr_warn("extern %s=%c should be tristate or char\n",
+ ext->name, value);
+ return -EINVAL;
+ }
+ *(bool *)ext_val = value == 'y' ? true : false;
+ break;
+ case EXT_TRISTATE:
+ if (value == 'y')
+ *(enum libbpf_tristate *)ext_val = TRI_YES;
+ else if (value == 'm')
+ *(enum libbpf_tristate *)ext_val = TRI_MODULE;
+ else /* value == 'n' */
+ *(enum libbpf_tristate *)ext_val = TRI_NO;
+ break;
+ case EXT_CHAR:
+ *(char *)ext_val = value;
+ break;
+ case EXT_UNKNOWN:
+ case EXT_INT:
+ case EXT_CHAR_ARR:
+ default:
+ pr_warn("extern %s=%c should be bool, tristate, or char\n",
+ ext->name, value);
+ return -EINVAL;
+ }
+ ext->is_set = true;
+ return 0;
+}
+
+static int set_ext_value_str(struct extern_desc *ext, char *ext_val,
+ const char *value)
+{
+ size_t len;
+
+ if (ext->type != EXT_CHAR_ARR) {
+ pr_warn("extern %s=%s should char array\n", ext->name, value);
+ return -EINVAL;
+ }
+
+ len = strlen(value);
+ if (value[len - 1] != '"') {
+ pr_warn("extern '%s': invalid string config '%s'\n",
+ ext->name, value);
+ return -EINVAL;
+ }
+
+ /* strip quotes */
+ len -= 2;
+ if (len >= ext->sz) {
+ pr_warn("extern '%s': long string config %s of (%zu bytes) truncated to %d bytes\n",
+ ext->name, value, len, ext->sz - 1);
+ len = ext->sz - 1;
+ }
+ memcpy(ext_val, value + 1, len);
+ ext_val[len] = '\0';
+ ext->is_set = true;
+ return 0;
+}
+
+static int parse_u64(const char *value, __u64 *res)
+{
+ char *value_end;
+ int err;
+
+ errno = 0;
+ *res = strtoull(value, &value_end, 0);
+ if (errno) {
+ err = -errno;
+ pr_warn("failed to parse '%s' as integer: %d\n", value, err);
+ return err;
+ }
+ if (*value_end) {
+ pr_warn("failed to parse '%s' as integer completely\n", value);
+ return -EINVAL;
+ }
+ return 0;
+}
+
+static bool is_ext_value_in_range(const struct extern_desc *ext, __u64 v)
+{
+ int bit_sz = ext->sz * 8;
+
+ if (ext->sz == 8)
+ return true;
+
+ /* Validate that value stored in u64 fits in integer of `ext->sz`
+ * bytes size without any loss of information. If the target integer
+ * is signed, we rely on the following limits of integer type of
+ * Y bits and subsequent transformation:
+ *
+ * -2^(Y-1) <= X <= 2^(Y-1) - 1
+ * 0 <= X + 2^(Y-1) <= 2^Y - 1
+ * 0 <= X + 2^(Y-1) < 2^Y
+ *
+ * For unsigned target integer, check that all the (64 - Y) bits are
+ * zero.
+ */
+ if (ext->is_signed)
+ return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);
+ else
+ return (v >> bit_sz) == 0;
+}
+
+static int set_ext_value_num(struct extern_desc *ext, void *ext_val,
+ __u64 value)
+{
+ if (ext->type != EXT_INT && ext->type != EXT_CHAR) {
+ pr_warn("extern %s=%llu should be integer\n",
+ ext->name, (unsigned long long)value);
+ return -EINVAL;
+ }
+ if (!is_ext_value_in_range(ext, value)) {
+ pr_warn("extern %s=%llu value doesn't fit in %d bytes\n",
+ ext->name, (unsigned long long)value, ext->sz);
+ return -ERANGE;
+ }
+ switch (ext->sz) {
+ case 1: *(__u8 *)ext_val = value; break;
+ case 2: *(__u16 *)ext_val = value; break;
+ case 4: *(__u32 *)ext_val = value; break;
+ case 8: *(__u64 *)ext_val = value; break;
+ default:
+ return -EINVAL;
+ }
+ ext->is_set = true;
+ return 0;
+}
+
+static int bpf_object__process_kconfig_line(struct bpf_object *obj,
+ char *buf, void *data)
+{
+ struct extern_desc *ext;
+ char *sep, *value;
+ int len, err = 0;
+ void *ext_val;
+ __u64 num;
+
+ if (strncmp(buf, "CONFIG_", 7))
+ return 0;
+
+ sep = strchr(buf, '=');
+ if (!sep) {
+ pr_warn("failed to parse '%s': no separator\n", buf);
+ return -EINVAL;
+ }
+
+ /* Trim ending '\n' */
+ len = strlen(buf);
+ if (buf[len - 1] == '\n')
+ buf[len - 1] = '\0';
+ /* Split on '=' and ensure that a value is present. */
+ *sep = '\0';
+ if (!sep[1]) {
+ *sep = '=';
+ pr_warn("failed to parse '%s': no value\n", buf);
+ return -EINVAL;
+ }
+
+ ext = find_extern_by_name(obj, buf);
+ if (!ext || ext->is_set)
+ return 0;
+
+ ext_val = data + ext->data_off;
+ value = sep + 1;
+
+ switch (*value) {
+ case 'y': case 'n': case 'm':
+ err = set_ext_value_tri(ext, ext_val, *value);
+ break;
+ case '"':
+ err = set_ext_value_str(ext, ext_val, value);
+ break;
+ default:
+ /* assume integer */
+ err = parse_u64(value