build(deps): bump github.com/cilium/ebpf from 0.11.0 to 0.12.0

Bumps [github.com/cilium/ebpf](https://github.com/cilium/ebpf) from 0.11.0 to 0.12.0.
- [Release notes](https://github.com/cilium/ebpf/releases)
- [Commits](https://github.com/cilium/ebpf/compare/v0.11.0...v0.12.0)

---
updated-dependencies:
- dependency-name: github.com/cilium/ebpf
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
This commit is contained in:
dependabot[bot]
2023-10-12 04:42:12 +00:00
committed by GitHub
parent 0274ca2580
commit aec0dc7dcd
47 changed files with 1134 additions and 710 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ go 1.20
require (
github.com/checkpoint-restore/go-criu/v6 v6.3.0
github.com/cilium/ebpf v0.11.0
github.com/cilium/ebpf v0.12.0
github.com/containerd/console v1.0.3
github.com/coreos/go-systemd/v22 v22.5.0
github.com/cyphar/filepath-securejoin v0.2.4
+2 -2
View File
@@ -1,8 +1,8 @@
github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
github.com/checkpoint-restore/go-criu/v6 v6.3.0 h1:mIdrSO2cPNWQY1truPg6uHLXyKHk3Z5Odx4wjKOASzA=
github.com/checkpoint-restore/go-criu/v6 v6.3.0/go.mod h1:rrRTN/uSwY2X+BPRl/gkulo9gsKOSAeVp9/K2tv7xZI=
github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
github.com/cilium/ebpf v0.11.0/go.mod h1:WE7CZAnqOL2RouJ4f1uyNhqr2P4CCvXFIqdRDUgWsVs=
github.com/cilium/ebpf v0.12.0 h1:oQEuIQIXgYhe1v7sYUG0P9vtJTYZLLdA6tiQmrOB1mo=
github.com/cilium/ebpf v0.12.0/go.mod h1:u9H29/Iq+8cy70YqI6p5pfADkFl3vdnV2qXDg5JL0Zo=
github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
+6
View File
@@ -4,6 +4,9 @@ BasedOnStyle: LLVM
AlignAfterOpenBracket: DontAlign
AlignConsecutiveAssignments: true
AlignEscapedNewlines: DontAlign
# mkdocs annotations in source code are written as trailing comments
# and alignment pushes these really far away from the content.
AlignTrailingComments: false
AlwaysBreakBeforeMultilineStrings: true
AlwaysBreakTemplateDeclarations: false
AllowAllParametersOfDeclarationOnNextLine: false
@@ -16,4 +19,7 @@ UseTab: ForContinuationAndIndentation
ColumnLimit: 1000
# Go compiler comments need to stay unindented.
CommentPragmas: '^go:.*'
# linux/bpf.h needs to be included before bpf/bpf_helpers.h for types like __u64
# and sorting makes this impossible.
SortIncludes: false
...
-13
View File
@@ -1,15 +1,7 @@
---
issues:
exclude-rules:
# syscall param structs will have unused fields in Go code.
- path: syscall.*.go
linters:
- structcheck
linters:
disable-all: true
enable:
- errcheck
- goimports
- gosimple
- govet
@@ -19,8 +11,3 @@ linters:
- typecheck
- unused
- gofmt
# Could be enabled later:
# - gocyclo
# - maligned
# - gosec
+20 -21
View File
@@ -1,9 +1,9 @@
# The development version of clang is distributed as the 'clang' binary,
# while stable/released versions have a version number attached.
# Pin the default clang to a stable version.
CLANG ?= clang-14
STRIP ?= llvm-strip-14
OBJCOPY ?= llvm-objcopy-14
CLANG ?= clang-17
STRIP ?= llvm-strip-17
OBJCOPY ?= llvm-objcopy-17
CFLAGS := -O2 -g -Wall -Werror $(CFLAGS)
CI_KERNEL_URL ?= https://github.com/cilium/ci-kernels/raw/master/
@@ -21,12 +21,9 @@ CONTAINER_RUN_ARGS ?= $(if $(filter ${CONTAINER_ENGINE}, podman), --log-driver=n
IMAGE := $(shell cat ${REPODIR}/testdata/docker/IMAGE)
VERSION := $(shell cat ${REPODIR}/testdata/docker/VERSION)
# clang <8 doesn't tag relocs properly (STT_NOTYPE)
# clang 9 is the first version emitting BTF
TARGETS := \
testdata/loader-clang-7 \
testdata/loader-clang-9 \
testdata/loader-clang-11 \
testdata/loader-clang-14 \
testdata/loader-$(CLANG) \
testdata/manyprogs \
testdata/btf_map_init \
@@ -36,6 +33,7 @@ TARGETS := \
testdata/invalid_btf_map_init \
testdata/strings \
testdata/freplace \
testdata/fentry_fexit \
testdata/iproute2_map_compat \
testdata/map_spin_lock \
testdata/subprog_reloc \
@@ -45,6 +43,7 @@ TARGETS := \
testdata/kfunc \
testdata/invalid-kfunc \
testdata/kfunc-kmod \
testdata/constants \
btf/testdata/relocs \
btf/testdata/relocs_read \
btf/testdata/relocs_read_tgt \
@@ -56,22 +55,26 @@ TARGETS := \
# Build all ELF binaries using a containerized LLVM toolchain.
container-all:
+${CONTAINER_ENGINE} run --rm -ti ${CONTAINER_RUN_ARGS} \
+${CONTAINER_ENGINE} run --rm -t ${CONTAINER_RUN_ARGS} \
-v "${REPODIR}":/ebpf -w /ebpf --env MAKEFLAGS \
--env CFLAGS="-fdebug-prefix-map=/ebpf=." \
--env HOME="/tmp" \
--env BPF2GO_CC="$(CLANG)" \
--env BPF2GO_FLAGS="-fdebug-prefix-map=/ebpf=. $(CFLAGS)" \
"${IMAGE}:${VERSION}" \
make all
# (debug) Drop the user into a shell inside the container as root.
# Set BPF2GO_ envs to make 'make generate' just work.
container-shell:
${CONTAINER_ENGINE} run --rm -ti \
-v "${REPODIR}":/ebpf -w /ebpf \
--env BPF2GO_CC="$(CLANG)" \
--env BPF2GO_FLAGS="-fdebug-prefix-map=/ebpf=. $(CFLAGS)" \
"${IMAGE}:${VERSION}"
clean:
-$(RM) testdata/*.elf
-$(RM) btf/testdata/*.elf
find "$(CURDIR)" -name "*.elf" -delete
find "$(CURDIR)" -name "*.o" -delete
format:
find . -type f -name "*.c" | xargs clang-format -i
@@ -80,9 +83,6 @@ all: format $(addsuffix -el.elf,$(TARGETS)) $(addsuffix -eb.elf,$(TARGETS)) gene
ln -srf testdata/loader-$(CLANG)-el.elf testdata/loader-el.elf
ln -srf testdata/loader-$(CLANG)-eb.elf testdata/loader-eb.elf
# $BPF_CLANG is used in go:generate invocations.
generate: export BPF_CLANG := $(CLANG)
generate: export BPF_CFLAGS := $(CFLAGS)
generate:
go generate ./...
@@ -103,13 +103,12 @@ testdata/loader-%-eb.elf: testdata/loader.c
$(STRIP) -g $@
.PHONY: generate-btf
generate-btf: KERNEL_VERSION?=5.19
generate-btf: KERNEL_VERSION?=6.1.29
generate-btf:
$(eval TMP := $(shell mktemp -d))
curl -fL "$(CI_KERNEL_URL)/linux-$(KERNEL_VERSION).bz" -o "$(TMP)/bzImage"
/lib/modules/$(uname -r)/build/scripts/extract-vmlinux "$(TMP)/bzImage" > "$(TMP)/vmlinux"
curl -fL "$(CI_KERNEL_URL)/linux-$(KERNEL_VERSION)-amd64.tgz" -o "$(TMP)/linux.tgz"
tar xvf "$(TMP)/linux.tgz" -C "$(TMP)" --strip-components=2 ./boot/vmlinuz ./lib/modules
/lib/modules/$(shell uname -r)/build/scripts/extract-vmlinux "$(TMP)/vmlinuz" > "$(TMP)/vmlinux"
$(OBJCOPY) --dump-section .BTF=/dev/stdout "$(TMP)/vmlinux" /dev/null | gzip > "btf/testdata/vmlinux.btf.gz"
curl -fL "$(CI_KERNEL_URL)/linux-$(KERNEL_VERSION)-selftests-bpf.tgz" -o "$(TMP)/selftests.tgz"
tar -xf "$(TMP)/selftests.tgz" --to-stdout tools/testing/selftests/bpf/bpf_testmod/bpf_testmod.ko | \
$(OBJCOPY) --dump-section .BTF="btf/testdata/btf_testmod.btf" - /dev/null
find "$(TMP)/modules" -type f -name bpf_testmod.ko -exec $(OBJCOPY) --dump-section .BTF="btf/testdata/btf_testmod.btf" {} /dev/null \;
$(RM) -r "$(TMP)"
+1 -1
View File
@@ -2,7 +2,7 @@
[![PkgGoDev](https://pkg.go.dev/badge/github.com/cilium/ebpf)](https://pkg.go.dev/github.com/cilium/ebpf)
![HoneyGopher](.github/images/cilium-ebpf.png)
![HoneyGopher](docs/ebpf/ebpf-go.png)
ebpf-go is a pure Go library that provides utilities for loading, compiling, and
debugging eBPF programs. It has minimal external dependencies and is intended to
+2 -2
View File
@@ -1,6 +1,6 @@
package asm
//go:generate stringer -output alu_string.go -type=Source,Endianness,ALUOp
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output alu_string.go -type=Source,Endianness,ALUOp
// Source of ALU / ALU64 / Branch operations
//
@@ -75,7 +75,7 @@ const (
Xor ALUOp = 0xa0
// Mov - move value from one place to another
Mov ALUOp = 0xb0
// ArSh - arithmatic shift
// ArSh - arithmetic shift
ArSh ALUOp = 0xc0
// Swap - endian conversions
Swap ALUOp = 0xd0
+1 -1
View File
@@ -1,6 +1,6 @@
package asm
//go:generate stringer -output func_string.go -type=BuiltinFunc
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output func_string.go -type=BuiltinFunc
// BuiltinFunc is a built-in eBPF function.
type BuiltinFunc int32
+1 -1
View File
@@ -1,6 +1,6 @@
package asm
//go:generate stringer -output jump_string.go -type=JumpOp
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output jump_string.go -type=JumpOp
// JumpOp affect control flow.
//
+1 -1
View File
@@ -1,6 +1,6 @@
package asm
//go:generate stringer -output load_store_string.go -type=Mode,Size
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output load_store_string.go -type=Mode,Size
// Mode for load and store operations
//
+1 -1
View File
@@ -5,7 +5,7 @@ import (
"strings"
)
//go:generate stringer -output opcode_string.go -type=Class
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output opcode_string.go -type=Class
// Class of operations
//
+1 -169
View File
@@ -14,7 +14,6 @@ import (
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/unix"
)
const btfMagic = 0xeB9F
@@ -47,51 +46,13 @@ type Spec struct {
// Includes all struct flavors and types with the same name.
namedTypes map[essentialName][]Type
// String table from ELF, may be nil.
// String table from ELF.
strings *stringTable
// Byte order of the ELF we decoded the spec from, may be nil.
byteOrder binary.ByteOrder
}
var btfHeaderLen = binary.Size(&btfHeader{})
type btfHeader struct {
Magic uint16
Version uint8
Flags uint8
HdrLen uint32
TypeOff uint32
TypeLen uint32
StringOff uint32
StringLen uint32
}
// typeStart returns the offset from the beginning of the .BTF section
// to the start of its type entries.
func (h *btfHeader) typeStart() int64 {
return int64(h.HdrLen + h.TypeOff)
}
// stringStart returns the offset from the beginning of the .BTF section
// to the start of its string table.
func (h *btfHeader) stringStart() int64 {
return int64(h.HdrLen + h.StringOff)
}
// newSpec creates a Spec containing only Void.
func newSpec() *Spec {
return &Spec{
[]Type{(*Void)(nil)},
map[Type]TypeID{(*Void)(nil): 0},
0,
make(map[essentialName][]Type),
nil,
nil,
}
}
// LoadSpec opens file and calls LoadSpecFromReader on it.
func LoadSpec(file string) (*Spec, error) {
fh, err := os.Open(file)
@@ -240,10 +201,6 @@ func loadRawSpec(btf io.ReaderAt, bo binary.ByteOrder, base *Spec) (*Spec, error
return nil, fmt.Errorf("can't use split BTF as base")
}
if base.strings == nil {
return nil, fmt.Errorf("parse split BTF: base must be loaded from an ELF")
}
baseStrings = base.strings
firstTypeID, err = base.nextTypeID()
@@ -399,37 +356,6 @@ func findVMLinux() (*internal.SafeELFFile, error) {
return nil, fmt.Errorf("no BTF found for kernel version %s: %w", release, internal.ErrNotSupported)
}
// parseBTFHeader parses the header of the .BTF section.
func parseBTFHeader(r io.Reader, bo binary.ByteOrder) (*btfHeader, error) {
var header btfHeader
if err := binary.Read(r, bo, &header); err != nil {
return nil, fmt.Errorf("can't read header: %v", err)
}
if header.Magic != btfMagic {
return nil, fmt.Errorf("incorrect magic value %v", header.Magic)
}
if header.Version != 1 {
return nil, fmt.Errorf("unexpected version %v", header.Version)
}
if header.Flags != 0 {
return nil, fmt.Errorf("unsupported flags %v", header.Flags)
}
remainder := int64(header.HdrLen) - int64(binary.Size(&header))
if remainder < 0 {
return nil, errors.New("header length shorter than btfHeader size")
}
if _, err := io.CopyN(internal.DiscardZeroes{}, r, remainder); err != nil {
return nil, fmt.Errorf("header padding: %v", err)
}
return &header, nil
}
func guessRawBTFByteOrder(r io.ReaderAt) binary.ByteOrder {
buf := new(bufio.Reader)
for _, bo := range []binary.ByteOrder{
@@ -773,97 +699,3 @@ func (iter *TypesIterator) Next() bool {
iter.index++
return true
}
// haveBTF attempts to load a BTF blob containing an Int. It should pass on any
// kernel that supports BPF_BTF_LOAD.
var haveBTF = internal.NewFeatureTest("BTF", "4.18", func() error {
// 0-length anonymous integer
err := probeBTF(&Int{})
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
return internal.ErrNotSupported
}
return err
})
// haveMapBTF attempts to load a minimal BTF blob containing a Var. It is
// used as a proxy for .bss, .data and .rodata map support, which generally
// come with a Var and Datasec. These were introduced in Linux 5.2.
var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", "5.2", func() error {
if err := haveBTF(); err != nil {
return err
}
v := &Var{
Name: "a",
Type: &Pointer{(*Void)(nil)},
}
err := probeBTF(v)
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
// Treat both EINVAL and EPERM as not supported: creating the map may still
// succeed without Btf* attrs.
return internal.ErrNotSupported
}
return err
})
// haveProgBTF attempts to load a BTF blob containing a Func and FuncProto. It
// is used as a proxy for ext_info (func_info) support, which depends on
// Func(Proto) by definition.
var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", "5.0", func() error {
if err := haveBTF(); err != nil {
return err
}
fn := &Func{
Name: "a",
Type: &FuncProto{Return: (*Void)(nil)},
}
err := probeBTF(fn)
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
return internal.ErrNotSupported
}
return err
})
var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", "5.6", func() error {
if err := haveProgBTF(); err != nil {
return err
}
fn := &Func{
Name: "a",
Type: &FuncProto{Return: (*Void)(nil)},
Linkage: GlobalFunc,
}
err := probeBTF(fn)
if errors.Is(err, unix.EINVAL) {
return internal.ErrNotSupported
}
return err
})
func probeBTF(typ Type) error {
b, err := NewBuilder([]Type{typ})
if err != nil {
return err
}
buf, err := b.Marshal(nil, nil)
if err != nil {
return err
}
fd, err := sys.BtfLoad(&sys.BtfLoadAttr{
Btf: sys.NewSlicePointer(buf),
BtfSize: uint32(len(buf)),
})
if err == nil {
fd.Close()
}
return err
}
+61 -1
View File
@@ -2,12 +2,15 @@ package btf
import (
"encoding/binary"
"errors"
"fmt"
"io"
"unsafe"
"github.com/cilium/ebpf/internal"
)
//go:generate stringer -linecomment -output=btf_types_string.go -type=FuncLinkage,VarLinkage,btfKind
//go:generate go run golang.org/x/tools/cmd/stringer@latest -linecomment -output=btf_types_string.go -type=FuncLinkage,VarLinkage,btfKind
// btfKind describes a Type.
type btfKind uint8
@@ -69,6 +72,63 @@ const (
btfTypeKindFlagMask = 1
)
var btfHeaderLen = binary.Size(&btfHeader{})
type btfHeader struct {
Magic uint16
Version uint8
Flags uint8
HdrLen uint32
TypeOff uint32
TypeLen uint32
StringOff uint32
StringLen uint32
}
// typeStart returns the offset from the beginning of the .BTF section
// to the start of its type entries.
func (h *btfHeader) typeStart() int64 {
return int64(h.HdrLen + h.TypeOff)
}
// stringStart returns the offset from the beginning of the .BTF section
// to the start of its string table.
func (h *btfHeader) stringStart() int64 {
return int64(h.HdrLen + h.StringOff)
}
// parseBTFHeader parses the header of the .BTF section.
func parseBTFHeader(r io.Reader, bo binary.ByteOrder) (*btfHeader, error) {
var header btfHeader
if err := binary.Read(r, bo, &header); err != nil {
return nil, fmt.Errorf("can't read header: %v", err)
}
if header.Magic != btfMagic {
return nil, fmt.Errorf("incorrect magic value %v", header.Magic)
}
if header.Version != 1 {
return nil, fmt.Errorf("unexpected version %v", header.Version)
}
if header.Flags != 0 {
return nil, fmt.Errorf("unsupported flags %v", header.Flags)
}
remainder := int64(header.HdrLen) - int64(binary.Size(&header))
if remainder < 0 {
return nil, errors.New("header length shorter than btfHeader size")
}
if _, err := io.CopyN(internal.DiscardZeroes{}, r, remainder); err != nil {
return nil, fmt.Errorf("header padding: %v", err)
}
return &header, nil
}
var btfTypeLen = binary.Size(btfType{})
// btfType is equivalent to struct btf_type in Documentation/bpf/btf.rst.
+101 -40
View File
@@ -16,9 +16,9 @@ import (
// ExtInfos contains ELF section metadata.
type ExtInfos struct {
// The slices are sorted by offset in ascending order.
funcInfos map[string][]funcInfo
lineInfos map[string][]lineInfo
relocationInfos map[string][]coreRelocationInfo
funcInfos map[string]FuncInfos
lineInfos map[string]LineInfos
relocationInfos map[string]CORERelocationInfos
}
// loadExtInfosFromELF parses ext infos from the .BTF.ext section in an ELF.
@@ -34,11 +34,11 @@ func loadExtInfosFromELF(file *internal.SafeELFFile, spec *Spec) (*ExtInfos, err
return nil, fmt.Errorf("compressed ext_info is not supported")
}
return loadExtInfos(section.ReaderAt, file.ByteOrder, spec, spec.strings)
return loadExtInfos(section.ReaderAt, file.ByteOrder, spec)
}
// loadExtInfos parses bare ext infos.
func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *stringTable) (*ExtInfos, error) {
func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec) (*ExtInfos, error) {
// Open unbuffered section reader. binary.Read() calls io.ReadFull on
// the header structs, resulting in one syscall per header.
headerRd := io.NewSectionReader(r, 0, math.MaxInt64)
@@ -53,12 +53,12 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin
}
buf := internal.NewBufferedSectionReader(r, extHeader.funcInfoStart(), int64(extHeader.FuncInfoLen))
btfFuncInfos, err := parseFuncInfos(buf, bo, strings)
btfFuncInfos, err := parseFuncInfos(buf, bo, spec.strings)
if err != nil {
return nil, fmt.Errorf("parsing BTF function info: %w", err)
}
funcInfos := make(map[string][]funcInfo, len(btfFuncInfos))
funcInfos := make(map[string]FuncInfos, len(btfFuncInfos))
for section, bfis := range btfFuncInfos {
funcInfos[section], err = newFuncInfos(bfis, spec)
if err != nil {
@@ -67,14 +67,14 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin
}
buf = internal.NewBufferedSectionReader(r, extHeader.lineInfoStart(), int64(extHeader.LineInfoLen))
btfLineInfos, err := parseLineInfos(buf, bo, strings)
btfLineInfos, err := parseLineInfos(buf, bo, spec.strings)
if err != nil {
return nil, fmt.Errorf("parsing BTF line info: %w", err)
}
lineInfos := make(map[string][]lineInfo, len(btfLineInfos))
lineInfos := make(map[string]LineInfos, len(btfLineInfos))
for section, blis := range btfLineInfos {
lineInfos[section], err = newLineInfos(blis, strings)
lineInfos[section], err = newLineInfos(blis, spec.strings)
if err != nil {
return nil, fmt.Errorf("section %s: line infos: %w", section, err)
}
@@ -86,14 +86,14 @@ func loadExtInfos(r io.ReaderAt, bo binary.ByteOrder, spec *Spec, strings *strin
var btfCORERelos map[string][]bpfCORERelo
buf = internal.NewBufferedSectionReader(r, extHeader.coreReloStart(coreHeader), int64(coreHeader.COREReloLen))
btfCORERelos, err = parseCORERelos(buf, bo, strings)
btfCORERelos, err = parseCORERelos(buf, bo, spec.strings)
if err != nil {
return nil, fmt.Errorf("parsing CO-RE relocation info: %w", err)
}
coreRelos := make(map[string][]coreRelocationInfo, len(btfCORERelos))
coreRelos := make(map[string]CORERelocationInfos, len(btfCORERelos))
for section, brs := range btfCORERelos {
coreRelos[section], err = newRelocationInfos(brs, spec, strings)
coreRelos[section], err = newRelocationInfos(brs, spec, spec.strings)
if err != nil {
return nil, fmt.Errorf("section %s: CO-RE relocations: %w", section, err)
}
@@ -111,21 +111,31 @@ func (ei *ExtInfos) Assign(insns asm.Instructions, section string) {
lineInfos := ei.lineInfos[section]
reloInfos := ei.relocationInfos[section]
AssignMetadataToInstructions(insns, funcInfos, lineInfos, reloInfos)
}
// Assign per-instruction metadata to the instructions in insns.
func AssignMetadataToInstructions(
insns asm.Instructions,
funcInfos FuncInfos,
lineInfos LineInfos,
reloInfos CORERelocationInfos,
) {
iter := insns.Iterate()
for iter.Next() {
if len(funcInfos) > 0 && funcInfos[0].offset == iter.Offset {
*iter.Ins = WithFuncMetadata(*iter.Ins, funcInfos[0].fn)
funcInfos = funcInfos[1:]
if len(funcInfos.infos) > 0 && funcInfos.infos[0].offset == iter.Offset {
*iter.Ins = WithFuncMetadata(*iter.Ins, funcInfos.infos[0].fn)
funcInfos.infos = funcInfos.infos[1:]
}
if len(lineInfos) > 0 && lineInfos[0].offset == iter.Offset {
*iter.Ins = iter.Ins.WithSource(lineInfos[0].line)
lineInfos = lineInfos[1:]
if len(lineInfos.infos) > 0 && lineInfos.infos[0].offset == iter.Offset {
*iter.Ins = iter.Ins.WithSource(lineInfos.infos[0].line)
lineInfos.infos = lineInfos.infos[1:]
}
if len(reloInfos) > 0 && reloInfos[0].offset == iter.Offset {
iter.Ins.Metadata.Set(coreRelocationMeta{}, reloInfos[0].relo)
reloInfos = reloInfos[1:]
if len(reloInfos.infos) > 0 && reloInfos.infos[0].offset == iter.Offset {
iter.Ins.Metadata.Set(coreRelocationMeta{}, reloInfos.infos[0].relo)
reloInfos.infos = reloInfos.infos[1:]
}
}
}
@@ -323,6 +333,11 @@ func parseExtInfoRecordSize(r io.Reader, bo binary.ByteOrder) (uint32, error) {
return recordSize, nil
}
// FuncInfos contains a sorted list of func infos.
type FuncInfos struct {
infos []funcInfo
}
// The size of a FuncInfo in BTF wire format.
var FuncInfoSize = uint32(binary.Size(bpfFuncInfo{}))
@@ -359,21 +374,38 @@ func newFuncInfo(fi bpfFuncInfo, spec *Spec) (*funcInfo, error) {
}, nil
}
func newFuncInfos(bfis []bpfFuncInfo, spec *Spec) ([]funcInfo, error) {
fis := make([]funcInfo, 0, len(bfis))
func newFuncInfos(bfis []bpfFuncInfo, spec *Spec) (FuncInfos, error) {
fis := FuncInfos{
infos: make([]funcInfo, 0, len(bfis)),
}
for _, bfi := range bfis {
fi, err := newFuncInfo(bfi, spec)
if err != nil {
return nil, fmt.Errorf("offset %d: %w", bfi.InsnOff, err)
return FuncInfos{}, fmt.Errorf("offset %d: %w", bfi.InsnOff, err)
}
fis = append(fis, *fi)
fis.infos = append(fis.infos, *fi)
}
sort.Slice(fis, func(i, j int) bool {
return fis[i].offset <= fis[j].offset
sort.Slice(fis.infos, func(i, j int) bool {
return fis.infos[i].offset <= fis.infos[j].offset
})
return fis, nil
}
// LoadFuncInfos parses btf func info in wire format.
func LoadFuncInfos(reader io.Reader, bo binary.ByteOrder, recordNum uint32, spec *Spec) (FuncInfos, error) {
fis, err := parseFuncInfoRecords(
reader,
bo,
FuncInfoSize,
recordNum,
)
if err != nil {
return FuncInfos{}, fmt.Errorf("parsing BTF func info: %w", err)
}
return newFuncInfos(fis, spec)
}
// marshal into the BTF wire format.
func (fi *funcInfo) marshal(w *bytes.Buffer, b *Builder) error {
id, err := b.Add(fi.fn)
@@ -480,6 +512,11 @@ func (li *Line) String() string {
return li.line
}
// LineInfos contains a sorted list of line infos.
type LineInfos struct {
infos []lineInfo
}
type lineInfo struct {
line *Line
offset asm.RawInstructionOffset
@@ -500,6 +537,21 @@ type bpfLineInfo struct {
LineCol uint32
}
// LoadLineInfos parses btf line info in wire format.
func LoadLineInfos(reader io.Reader, bo binary.ByteOrder, recordNum uint32, spec *Spec) (LineInfos, error) {
lis, err := parseLineInfoRecords(
reader,
bo,
LineInfoSize,
recordNum,
)
if err != nil {
return LineInfos{}, fmt.Errorf("parsing BTF line info: %w", err)
}
return newLineInfos(lis, spec.strings)
}
func newLineInfo(li bpfLineInfo, strings *stringTable) (*lineInfo, error) {
line, err := strings.Lookup(li.LineOff)
if err != nil {
@@ -525,17 +577,19 @@ func newLineInfo(li bpfLineInfo, strings *stringTable) (*lineInfo, error) {
}, nil
}
func newLineInfos(blis []bpfLineInfo, strings *stringTable) ([]lineInfo, error) {
lis := make([]lineInfo, 0, len(blis))
func newLineInfos(blis []bpfLineInfo, strings *stringTable) (LineInfos, error) {
lis := LineInfos{
infos: make([]lineInfo, 0, len(blis)),
}
for _, bli := range blis {
li, err := newLineInfo(bli, strings)
if err != nil {
return nil, fmt.Errorf("offset %d: %w", bli.InsnOff, err)
return LineInfos{}, fmt.Errorf("offset %d: %w", bli.InsnOff, err)
}
lis = append(lis, *li)
lis.infos = append(lis.infos, *li)
}
sort.Slice(lis, func(i, j int) bool {
return lis[i].offset <= lis[j].offset
sort.Slice(lis.infos, func(i, j int) bool {
return lis.infos[i].offset <= lis.infos[j].offset
})
return lis, nil
}
@@ -661,6 +715,11 @@ func CORERelocationMetadata(ins *asm.Instruction) *CORERelocation {
return relo
}
// CORERelocationInfos contains a sorted list of co:re relocation infos.
type CORERelocationInfos struct {
infos []coreRelocationInfo
}
type coreRelocationInfo struct {
relo *CORERelocation
offset asm.RawInstructionOffset
@@ -693,17 +752,19 @@ func newRelocationInfo(relo bpfCORERelo, spec *Spec, strings *stringTable) (*cor
}, nil
}
func newRelocationInfos(brs []bpfCORERelo, spec *Spec, strings *stringTable) ([]coreRelocationInfo, error) {
rs := make([]coreRelocationInfo, 0, len(brs))
func newRelocationInfos(brs []bpfCORERelo, spec *Spec, strings *stringTable) (CORERelocationInfos, error) {
rs := CORERelocationInfos{
infos: make([]coreRelocationInfo, 0, len(brs)),
}
for _, br := range brs {
relo, err := newRelocationInfo(br, spec, strings)
if err != nil {
return nil, fmt.Errorf("offset %d: %w", br.InsnOff, err)
return CORERelocationInfos{}, fmt.Errorf("offset %d: %w", br.InsnOff, err)
}
rs = append(rs, *relo)
rs.infos = append(rs.infos, *relo)
}
sort.Slice(rs, func(i, j int) bool {
return rs[i].offset < rs[j].offset
sort.Slice(rs.infos, func(i, j int) bool {
return rs.infos[i].offset < rs.infos[j].offset
})
return rs, nil
}
+123
View File
@@ -0,0 +1,123 @@
package btf
import (
"errors"
"math"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/unix"
)
// haveBTF attempts to load a BTF blob containing an Int. It should pass on any
// kernel that supports BPF_BTF_LOAD.
var haveBTF = internal.NewFeatureTest("BTF", "4.18", func() error {
// 0-length anonymous integer
err := probeBTF(&Int{})
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
return internal.ErrNotSupported
}
return err
})
// haveMapBTF attempts to load a minimal BTF blob containing a Var. It is
// used as a proxy for .bss, .data and .rodata map support, which generally
// come with a Var and Datasec. These were introduced in Linux 5.2.
var haveMapBTF = internal.NewFeatureTest("Map BTF (Var/Datasec)", "5.2", func() error {
if err := haveBTF(); err != nil {
return err
}
v := &Var{
Name: "a",
Type: &Pointer{(*Void)(nil)},
}
err := probeBTF(v)
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
// Treat both EINVAL and EPERM as not supported: creating the map may still
// succeed without Btf* attrs.
return internal.ErrNotSupported
}
return err
})
// haveProgBTF attempts to load a BTF blob containing a Func and FuncProto. It
// is used as a proxy for ext_info (func_info) support, which depends on
// Func(Proto) by definition.
var haveProgBTF = internal.NewFeatureTest("Program BTF (func/line_info)", "5.0", func() error {
if err := haveBTF(); err != nil {
return err
}
fn := &Func{
Name: "a",
Type: &FuncProto{Return: (*Void)(nil)},
}
err := probeBTF(fn)
if errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM) {
return internal.ErrNotSupported
}
return err
})
var haveFuncLinkage = internal.NewFeatureTest("BTF func linkage", "5.6", func() error {
if err := haveProgBTF(); err != nil {
return err
}
fn := &Func{
Name: "a",
Type: &FuncProto{Return: (*Void)(nil)},
Linkage: GlobalFunc,
}
err := probeBTF(fn)
if errors.Is(err, unix.EINVAL) {
return internal.ErrNotSupported
}
return err
})
var haveEnum64 = internal.NewFeatureTest("ENUM64", "6.0", func() error {
if err := haveBTF(); err != nil {
return err
}
enum := &Enum{
Size: 8,
Values: []EnumValue{
{"TEST", math.MaxUint32 + 1},
},
}
err := probeBTF(enum)
if errors.Is(err, unix.EINVAL) {
return internal.ErrNotSupported
}
return err
})
func probeBTF(typ Type) error {
b, err := NewBuilder([]Type{typ})
if err != nil {
return err
}
buf, err := b.Marshal(nil, nil)
if err != nil {
return err
}
fd, err := sys.BtfLoad(&sys.BtfLoadAttr{
Btf: sys.NewSlicePointer(buf),
BtfSize: uint32(len(buf)),
})
if err == nil {
fd.Close()
}
return err
}
+7 -1
View File
@@ -77,7 +77,13 @@ func (gf *GoFormatter) writeTypeDecl(name string, typ Type) error {
gf.w.WriteString("; const ( ")
for _, ev := range e.Values {
id := gf.enumIdentifier(name, ev.Name)
fmt.Fprintf(&gf.w, "%s %s = %d; ", id, name, ev.Value)
var value any
if e.Signed {
value = int64(ev.Value)
} else {
value = ev.Value
}
fmt.Fprintf(&gf.w, "%s %s = %d; ", id, name, value)
}
gf.w.WriteString(")")
+70 -17
View File
@@ -18,6 +18,8 @@ type MarshalOptions struct {
Order binary.ByteOrder
// Remove function linkage information for compatibility with <5.6 kernels.
StripFuncLinkage bool
// Replace Enum64 with a placeholder for compatibility with <6.0 kernels.
ReplaceEnum64 bool
}
// KernelMarshalOptions will generate BTF suitable for the current kernel.
@@ -25,6 +27,7 @@ func KernelMarshalOptions() *MarshalOptions {
return &MarshalOptions{
Order: internal.NativeEndian,
StripFuncLinkage: haveFuncLinkage() != nil,
ReplaceEnum64: haveEnum64() != nil,
}
}
@@ -328,21 +331,13 @@ func (e *encoder) deflateType(typ Type) (err error) {
raw.data, err = e.convertMembers(&raw.btfType, v.Members)
case *Union:
raw.SetKind(kindUnion)
raw.SetSize(v.Size)
raw.data, err = e.convertMembers(&raw.btfType, v.Members)
err = e.deflateUnion(&raw, v)
case *Enum:
raw.SetSize(v.size())
raw.SetVlen(len(v.Values))
raw.SetSigned(v.Signed)
if v.has64BitValues() {
raw.SetKind(kindEnum64)
raw.data, err = e.deflateEnum64Values(v.Values)
if v.Size == 8 {
err = e.deflateEnum64(&raw, v)
} else {
raw.SetKind(kindEnum)
raw.data, err = e.deflateEnumValues(v.Values)
err = e.deflateEnum(&raw, v)
}
case *Fwd:
@@ -415,6 +410,13 @@ func (e *encoder) deflateType(typ Type) (err error) {
return raw.Marshal(e.buf, e.Order)
}
func (e *encoder) deflateUnion(raw *rawType, union *Union) (err error) {
raw.SetKind(kindUnion)
raw.SetSize(union.Size)
raw.data, err = e.convertMembers(&raw.btfType, union.Members)
return
}
func (e *encoder) convertMembers(header *btfType, members []Member) ([]btfMember, error) {
bms := make([]btfMember, 0, len(members))
isBitfield := false
@@ -443,16 +445,32 @@ func (e *encoder) convertMembers(header *btfType, members []Member) ([]btfMember
return bms, nil
}
func (e *encoder) deflateEnumValues(values []EnumValue) ([]btfEnum, error) {
bes := make([]btfEnum, 0, len(values))
for _, value := range values {
func (e *encoder) deflateEnum(raw *rawType, enum *Enum) (err error) {
raw.SetKind(kindEnum)
raw.SetSize(enum.Size)
raw.SetVlen(len(enum.Values))
// Signedness appeared together with ENUM64 support.
raw.SetSigned(enum.Signed && !e.ReplaceEnum64)
raw.data, err = e.deflateEnumValues(enum)
return
}
func (e *encoder) deflateEnumValues(enum *Enum) ([]btfEnum, error) {
bes := make([]btfEnum, 0, len(enum.Values))
for _, value := range enum.Values {
nameOff, err := e.strings.Add(value.Name)
if err != nil {
return nil, err
}
if value.Value > math.MaxUint32 {
return nil, fmt.Errorf("value of enum %q exceeds 32 bits", value.Name)
if enum.Signed {
if signedValue := int64(value.Value); signedValue < math.MinInt32 || signedValue > math.MaxInt32 {
return nil, fmt.Errorf("value %d of enum %q exceeds 32 bits", signedValue, value.Name)
}
} else {
if value.Value > math.MaxUint32 {
return nil, fmt.Errorf("value %d of enum %q exceeds 32 bits", value.Value, value.Name)
}
}
bes = append(bes, btfEnum{
@@ -464,6 +482,41 @@ func (e *encoder) deflateEnumValues(values []EnumValue) ([]btfEnum, error) {
return bes, nil
}
func (e *encoder) deflateEnum64(raw *rawType, enum *Enum) (err error) {
if e.ReplaceEnum64 {
// Replace the ENUM64 with a union of fields with the correct size.
// This matches libbpf behaviour on purpose.
placeholder := &Int{
"enum64_placeholder",
enum.Size,
Unsigned,
}
if enum.Signed {
placeholder.Encoding = Signed
}
if err := e.allocateID(placeholder); err != nil {
return fmt.Errorf("add enum64 placeholder: %w", err)
}
members := make([]Member, 0, len(enum.Values))
for _, v := range enum.Values {
members = append(members, Member{
Name: v.Name,
Type: placeholder,
})
}
return e.deflateUnion(raw, &Union{enum.Name, enum.Size, members})
}
raw.SetKind(kindEnum64)
raw.SetSize(enum.Size)
raw.SetVlen(len(enum.Values))
raw.SetSigned(enum.Signed)
raw.data, err = e.deflateEnum64Values(enum.Values)
return
}
func (e *encoder) deflateEnum64Values(values []EnumValue) ([]btfEnum64, error) {
bes := make([]btfEnum64, 0, len(values))
for _, value := range values {
+3 -22
View File
@@ -9,6 +9,7 @@ import (
"strings"
"golang.org/x/exp/maps"
"golang.org/x/exp/slices"
)
type stringTable struct {
@@ -83,8 +84,8 @@ func (st *stringTable) Lookup(offset uint32) (string, error) {
}
func (st *stringTable) lookup(offset uint32) (string, error) {
i := search(st.offsets, offset)
if i == len(st.offsets) || st.offsets[i] != offset {
i, found := slices.BinarySearch(st.offsets, offset)
if !found {
return "", fmt.Errorf("offset %d isn't start of a string", offset)
}
@@ -110,26 +111,6 @@ func (st *stringTable) Num() int {
return len(st.strings)
}
// search is a copy of sort.Search specialised for uint32.
//
// Licensed under https://go.dev/LICENSE
func search(ints []uint32, needle uint32) int {
// Define f(-1) == false and f(n) == true.
// Invariant: f(i-1) == false, f(j) == true.
i, j := 0, len(ints)
for i < j {
h := int(uint(i+j) >> 1) // avoid overflow when computing h
// i ≤ h < j
if !(ints[h] >= needle) {
i = h + 1 // preserves f(i-1) == false
} else {
j = h // preserves f(j) == true
}
}
// i == j, f(i-1) == false, and f(j) (= f(i)) == true => answer is i.
return i
}
// stringTableBuilder builds BTF string tables.
type stringTableBuilder struct {
length uint32
-15
View File
@@ -278,21 +278,6 @@ func (e *Enum) copy() Type {
return &cpy
}
// has64BitValues returns true if the Enum contains a value larger than 32 bits.
// Kernels before 6.0 have enum values that overrun u32 replaced with zeroes.
//
// 64-bit enums have their Enum.Size attributes correctly set to 8, but if we
// use the size attribute as a heuristic during BTF marshaling, we'll emit
// ENUM64s to kernels that don't support them.
func (e *Enum) has64BitValues() bool {
for _, v := range e.Values {
if v.Value > math.MaxUint32 {
return true
}
}
return false
}
// FwdKind is the type of forward declaration.
type FwdKind int
+43 -27
View File
@@ -11,6 +11,7 @@ import (
"github.com/cilium/ebpf/btf"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/kconfig"
"github.com/cilium/ebpf/internal/sysenc"
)
// CollectionOptions control loading a collection into the kernel.
@@ -175,12 +176,12 @@ func (cs *CollectionSpec) RewriteConstants(consts map[string]interface{}) error
return fmt.Errorf("section %s: offset %d(+%d) for variable %s is out of bounds", name, v.Offset, v.Size, vname)
}
b, err := marshalBytes(replacement, int(v.Size))
b, err := sysenc.Marshal(replacement, int(v.Size))
if err != nil {
return fmt.Errorf("marshaling constant replacement %s: %w", vname, err)
}
copy(cpy[v.Offset:v.Offset+v.Size], b)
b.CopyTo(cpy[v.Offset : v.Offset+v.Size])
replaced[vname] = true
}
@@ -308,7 +309,7 @@ func (cs *CollectionSpec) LoadAndAssign(to interface{}, opts *CollectionOptions)
}
// Populate the requested maps. Has a chance of lazy-loading other dependent maps.
if err := loader.populateMaps(); err != nil {
if err := loader.populateDeferredMaps(); err != nil {
return err
}
@@ -388,7 +389,7 @@ func NewCollectionWithOptions(spec *CollectionSpec, opts CollectionOptions) (*Co
// Maps can contain Program and Map stubs, so populate them after
// all Maps and Programs have been successfully loaded.
if err := loader.populateMaps(); err != nil {
if err := loader.populateDeferredMaps(); err != nil {
return nil, err
}
@@ -470,6 +471,15 @@ func (cl *collectionLoader) loadMap(mapName string) (*Map, error) {
return nil, fmt.Errorf("map %s: %w", mapName, err)
}
// Finalize 'scalar' maps that don't refer to any other eBPF resources
// potentially pending creation. This is needed for frozen maps like .rodata
// that need to be finalized before invoking the verifier.
if !mapSpec.Type.canStoreMapOrProgram() {
if err := m.finalize(mapSpec); err != nil {
return nil, fmt.Errorf("finalizing map %s: %w", mapName, err)
}
}
cl.maps[mapName] = m
return m, nil
}
@@ -527,44 +537,50 @@ func (cl *collectionLoader) loadProgram(progName string) (*Program, error) {
return prog, nil
}
func (cl *collectionLoader) populateMaps() error {
// populateDeferredMaps iterates maps holding programs or other maps and loads
// any dependencies. Populates all maps in cl and freezes them if specified.
func (cl *collectionLoader) populateDeferredMaps() error {
for mapName, m := range cl.maps {
mapSpec, ok := cl.coll.Maps[mapName]
if !ok {
return fmt.Errorf("missing map spec %s", mapName)
}
// Scalar maps without Map or Program references are finalized during
// creation. Don't finalize them again.
if !mapSpec.Type.canStoreMapOrProgram() {
continue
}
mapSpec = mapSpec.Copy()
// MapSpecs that refer to inner maps or programs within the same
// CollectionSpec do so using strings. These strings are used as the key
// to look up the respective object in the Maps or Programs fields.
// Resolve those references to actual Map or Program resources that
// have been loaded into the kernel.
if mapSpec.Type.canStoreMap() || mapSpec.Type.canStoreProgram() {
mapSpec = mapSpec.Copy()
for i, kv := range mapSpec.Contents {
objName, ok := kv.Value.(string)
if !ok {
continue
}
for i, kv := range mapSpec.Contents {
objName, ok := kv.Value.(string)
if !ok {
continue
switch t := mapSpec.Type; {
case t.canStoreProgram():
// loadProgram is idempotent and could return an existing Program.
prog, err := cl.loadProgram(objName)
if err != nil {
return fmt.Errorf("loading program %s, for map %s: %w", objName, mapName, err)
}
mapSpec.Contents[i] = MapKV{kv.Key, prog}
switch t := mapSpec.Type; {
case t.canStoreProgram():
// loadProgram is idempotent and could return an existing Program.
prog, err := cl.loadProgram(objName)
if err != nil {
return fmt.Errorf("loading program %s, for map %s: %w", objName, mapName, err)
}
mapSpec.Contents[i] = MapKV{kv.Key, prog}
case t.canStoreMap():
// loadMap is idempotent and could return an existing Map.
innerMap, err := cl.loadMap(objName)
if err != nil {
return fmt.Errorf("loading inner map %s, for map %s: %w", objName, mapName, err)
}
mapSpec.Contents[i] = MapKV{kv.Key, innerMap}
case t.canStoreMap():
// loadMap is idempotent and could return an existing Map.
innerMap, err := cl.loadMap(objName)
if err != nil {
return fmt.Errorf("loading inner map %s, for map %s: %w", objName, mapName, err)
}
mapSpec.Contents[i] = MapKV{kv.Key, innerMap}
}
}
+6 -11
View File
@@ -81,6 +81,8 @@ func LoadCollectionSpecFromReader(rd io.ReaderAt) (*CollectionSpec, error) {
// Collect all the sections we're interested in. This includes relocations
// which we parse later.
//
// Keep the documentation at docs/ebpf/loading/elf-sections.md up-to-date.
for i, sec := range f.Sections {
idx := elf.SectionIndex(i)
@@ -694,10 +696,6 @@ func (ec *elfCode) loadMaps() error {
spec.Extra = bytes.NewReader(extra)
}
if err := spec.clampPerfEventArraySize(); err != nil {
return fmt.Errorf("map %s: %w", mapName, err)
}
ec.maps[mapName] = &spec
}
}
@@ -752,7 +750,7 @@ func (ec *elfCode) loadBTFMaps() error {
}
// Each Var representing a BTF map definition contains a Struct.
mapStruct, ok := v.Type.(*btf.Struct)
mapStruct, ok := btf.UnderlyingType(v.Type).(*btf.Struct)
if !ok {
return fmt.Errorf("expected struct, got %s", v.Type)
}
@@ -762,10 +760,6 @@ func (ec *elfCode) loadBTFMaps() error {
return fmt.Errorf("map %v: %w", name, err)
}
if err := mapSpec.clampPerfEventArraySize(); err != nil {
return fmt.Errorf("map %v: %w", name, err)
}
ec.maps[name] = mapSpec
}
@@ -785,7 +779,7 @@ func (ec *elfCode) loadBTFMaps() error {
// mapSpecFromBTF produces a MapSpec based on a btf.Struct def representing
// a BTF map definition. The name and spec arguments will be copied to the
// resulting MapSpec, and inner must be true on any resursive invocations.
// resulting MapSpec, and inner must be true on any recursive invocations.
func mapSpecFromBTF(es *elfSection, vs *btf.VarSecinfo, def *btf.Struct, spec *btf.Spec, name string, inner bool) (*MapSpec, error) {
var (
key, value btf.Type
@@ -1150,7 +1144,7 @@ func (ec *elfCode) loadKconfigSection() error {
KeySize: uint32(4),
ValueSize: ds.Size,
MaxEntries: 1,
Flags: unix.BPF_F_RDONLY_PROG | unix.BPF_F_MMAPABLE,
Flags: unix.BPF_F_RDONLY_PROG,
Freeze: true,
Key: &btf.Int{Size: 4},
Value: ds,
@@ -1268,6 +1262,7 @@ func getProgType(sectionName string) (ProgramType, AttachType, uint32, string) {
{"seccomp", SocketFilter, AttachNone, 0},
{"kprobe.multi", Kprobe, AttachTraceKprobeMulti, 0},
{"kretprobe.multi", Kprobe, AttachTraceKprobeMulti, 0},
// Document all prefixes in docs/ebpf/concepts/elf-sections.md.
}
for _, t := range types {
+84 -4
View File
@@ -101,6 +101,11 @@ type ProgramInfo struct {
maps []MapID
insns []byte
lineInfos []byte
numLineInfos uint32
funcInfos []byte
numFuncInfos uint32
}
func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) {
@@ -128,10 +133,13 @@ func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) {
// Start with a clean struct for the second call, otherwise we may get EFAULT.
var info2 sys.ProgInfo
makeSecondCall := false
if info.NrMapIds > 0 {
pi.maps = make([]MapID, info.NrMapIds)
info2.NrMapIds = info.NrMapIds
info2.MapIds = sys.NewPointer(unsafe.Pointer(&pi.maps[0]))
makeSecondCall = true
} else if haveProgramInfoMapIDs() == nil {
// This program really has no associated maps.
pi.maps = make([]MapID, 0)
@@ -150,9 +158,28 @@ func newProgramInfoFromFd(fd *sys.FD) (*ProgramInfo, error) {
pi.insns = make([]byte, info.XlatedProgLen)
info2.XlatedProgLen = info.XlatedProgLen
info2.XlatedProgInsns = sys.NewSlicePointer(pi.insns)
makeSecondCall = true
}
if info.NrMapIds > 0 || info.XlatedProgLen > 0 {
if info.NrLineInfo > 0 {
pi.lineInfos = make([]byte, btf.LineInfoSize*info.NrLineInfo)
info2.LineInfo = sys.NewSlicePointer(pi.lineInfos)
info2.LineInfoRecSize = btf.LineInfoSize
info2.NrLineInfo = info.NrLineInfo
pi.numLineInfos = info.NrLineInfo
makeSecondCall = true
}
if info.NrFuncInfo > 0 {
pi.funcInfos = make([]byte, btf.FuncInfoSize*info.NrFuncInfo)
info2.FuncInfo = sys.NewSlicePointer(pi.funcInfos)
info2.FuncInfoRecSize = btf.FuncInfoSize
info2.NrFuncInfo = info.NrFuncInfo
pi.numFuncInfos = info.NrFuncInfo
makeSecondCall = true
}
if makeSecondCall {
if err := sys.ObjInfo(fd, &info2); err != nil {
return nil, err
}
@@ -245,7 +272,13 @@ func (pi *ProgramInfo) Runtime() (time.Duration, bool) {
//
// The first instruction is marked as a symbol using the Program's name.
//
// Available from 4.13. Requires CAP_BPF or equivalent.
// If available, the instructions will be annotated with metadata from the
// BTF. This includes line information and function information. Reading
// this metadata requires CAP_SYS_ADMIN or equivalent. If capability is
// unavailable, the instructions will be returned without metadata.
//
// Available from 4.13. Requires CAP_BPF or equivalent for plain instructions.
// Requires CAP_SYS_ADMIN for instructions with metadata.
func (pi *ProgramInfo) Instructions() (asm.Instructions, error) {
// If the calling process is not BPF-capable or if the kernel doesn't
// support getting xlated instructions, the field will be zero.
@@ -259,8 +292,55 @@ func (pi *ProgramInfo) Instructions() (asm.Instructions, error) {
return nil, fmt.Errorf("unmarshaling instructions: %w", err)
}
// Tag the first instruction with the name of the program, if available.
insns[0] = insns[0].WithSymbol(pi.Name)
if pi.btf != 0 {
btfh, err := btf.NewHandleFromID(pi.btf)
if err != nil {
// Getting a BTF handle requires CAP_SYS_ADMIN, if not available we get an -EPERM.
// Ignore it and fall back to instructions without metadata.
if !errors.Is(err, unix.EPERM) {
return nil, fmt.Errorf("unable to get BTF handle: %w", err)
}
}
// If we have a BTF handle, we can use it to assign metadata to the instructions.
if btfh != nil {
defer btfh.Close()
spec, err := btfh.Spec(nil)
if err != nil {
return nil, fmt.Errorf("unable to get BTF spec: %w", err)
}
lineInfos, err := btf.LoadLineInfos(
bytes.NewReader(pi.lineInfos),
internal.NativeEndian,
pi.numLineInfos,
spec,
)
if err != nil {
return nil, fmt.Errorf("parse line info: %w", err)
}
funcInfos, err := btf.LoadFuncInfos(
bytes.NewReader(pi.funcInfos),
internal.NativeEndian,
pi.numFuncInfos,
spec,
)
if err != nil {
return nil, fmt.Errorf("parse func info: %w", err)
}
btf.AssignMetadataToInstructions(insns, funcInfos, lineInfos, btf.CORERelocationInfos{})
}
}
fn := btf.FuncMetadata(&insns[0])
name := pi.Name
if fn != nil {
name = fn.Name
}
insns[0] = insns[0].WithSymbol(name)
return insns, nil
}
+1 -1
View File
@@ -6,7 +6,7 @@ import "encoding/binary"
// NativeEndian is set to either binary.BigEndian or binary.LittleEndian,
// depending on the host's endianness.
var NativeEndian binary.ByteOrder = binary.BigEndian
var NativeEndian = binary.BigEndian
// ClangEndian is set to either "el" or "eb" depending on the host's endianness.
const ClangEndian = "eb"
+1 -1
View File
@@ -6,7 +6,7 @@ import "encoding/binary"
// NativeEndian is set to either binary.BigEndian or binary.LittleEndian,
// depending on the host's endianness.
var NativeEndian binary.ByteOrder = binary.LittleEndian
var NativeEndian = binary.LittleEndian
// ClangEndian is set to either "el" or "eb" depending on the host's endianness.
const ClangEndian = "el"
+2 -2
View File
@@ -11,7 +11,7 @@ import (
// ENOTSUPP is a Linux internal error code that has leaked into UAPI.
//
// It is not the same as ENOTSUP or EOPNOTSUPP.
var ENOTSUPP = syscall.Errno(524)
const ENOTSUPP = syscall.Errno(524)
// BPF wraps SYS_BPF.
//
@@ -123,7 +123,7 @@ type TypeID uint32
// MapFlags control map behaviour.
type MapFlags uint32
//go:generate stringer -type MapFlags
//go:generate go run golang.org/x/tools/cmd/stringer@latest -type MapFlags
const (
BPF_F_NO_PREALLOC MapFlags = 1 << iota
+28 -16
View File
@@ -59,7 +59,8 @@ const (
BPF_SK_REUSEPORT_SELECT_OR_MIGRATE AttachType = 40
BPF_PERF_EVENT AttachType = 41
BPF_TRACE_KPROBE_MULTI AttachType = 42
__MAX_BPF_ATTACH_TYPE AttachType = 43
BPF_LSM_CGROUP AttachType = 43
__MAX_BPF_ATTACH_TYPE AttachType = 44
)
type Cmd uint32
@@ -311,7 +312,13 @@ const (
BPF_FUNC_dynptr_read FunctionId = 201
BPF_FUNC_dynptr_write FunctionId = 202
BPF_FUNC_dynptr_data FunctionId = 203
__BPF_FUNC_MAX_ID FunctionId = 204
BPF_FUNC_tcp_raw_gen_syncookie_ipv4 FunctionId = 204
BPF_FUNC_tcp_raw_gen_syncookie_ipv6 FunctionId = 205
BPF_FUNC_tcp_raw_check_syncookie_ipv4 FunctionId = 206
BPF_FUNC_tcp_raw_check_syncookie_ipv6 FunctionId = 207
BPF_FUNC_ktime_get_tai_ns FunctionId = 208
BPF_FUNC_user_ringbuf_drain FunctionId = 209
__BPF_FUNC_MAX_ID FunctionId = 210
)
type HdrStartOff uint32
@@ -371,6 +378,7 @@ const (
BPF_MAP_TYPE_INODE_STORAGE MapType = 28
BPF_MAP_TYPE_TASK_STORAGE MapType = 29
BPF_MAP_TYPE_BLOOM_FILTER MapType = 30
BPF_MAP_TYPE_USER_RINGBUF MapType = 31
)
type ProgType uint32
@@ -413,10 +421,11 @@ const (
type RetCode uint32
const (
BPF_OK RetCode = 0
BPF_DROP RetCode = 2
BPF_REDIRECT RetCode = 7
BPF_LWT_REROUTE RetCode = 128
BPF_OK RetCode = 0
BPF_DROP RetCode = 2
BPF_REDIRECT RetCode = 7
BPF_LWT_REROUTE RetCode = 128
BPF_FLOW_DISSECTOR_CONTINUE RetCode = 129
)
type SkAction uint32
@@ -476,7 +485,7 @@ type LinkInfo struct {
Id LinkID
ProgId uint32
_ [4]byte
Extra [16]uint8
Extra [32]uint8
}
type MapInfo struct {
@@ -521,10 +530,10 @@ type ProgInfo struct {
JitedFuncLens uint64
BtfId BTFID
FuncInfoRecSize uint32
FuncInfo uint64
FuncInfo Pointer
NrFuncInfo uint32
NrLineInfo uint32
LineInfo uint64
LineInfo Pointer
JitedLineInfo uint64
NrJitedLineInfo uint32
LineInfoRecSize uint32
@@ -535,6 +544,8 @@ type ProgInfo struct {
RunCnt uint64
RecursionMisses uint64
VerifiedInsns uint32
AttachBtfObjId BTFID
AttachBtfId TypeID
_ [4]byte
}
@@ -1034,13 +1045,14 @@ func ProgLoad(attr *ProgLoadAttr) (*FD, error) {
}
type ProgQueryAttr struct {
TargetFd uint32
AttachType AttachType
QueryFlags uint32
AttachFlags uint32
ProgIds Pointer
ProgCount uint32
_ [4]byte
TargetFd uint32
AttachType AttachType
QueryFlags uint32
AttachFlags uint32
ProgIds Pointer
ProgCount uint32
_ [4]byte
ProgAttachFlags uint64
}
func ProgQuery(attr *ProgQueryAttr) error {
+77
View File
@@ -0,0 +1,77 @@
package sysenc
import (
"unsafe"
"github.com/cilium/ebpf/internal/sys"
)
type Buffer struct {
ptr unsafe.Pointer
// Size of the buffer. syscallPointerOnly if created from UnsafeBuffer or when using
// zero-copy unmarshaling.
size int
}
const syscallPointerOnly = -1
func newBuffer(buf []byte) Buffer {
if len(buf) == 0 {
return Buffer{}
}
return Buffer{unsafe.Pointer(&buf[0]), len(buf)}
}
// UnsafeBuffer constructs a Buffer for zero-copy unmarshaling.
//
// [Pointer] is the only valid method to call on such a Buffer.
// Use [SyscallBuffer] instead if possible.
func UnsafeBuffer(ptr unsafe.Pointer) Buffer {
return Buffer{ptr, syscallPointerOnly}
}
// SyscallOutput prepares a Buffer for a syscall to write into.
//
// The buffer may point at the underlying memory of dst, in which case [Unmarshal]
// becomes a no-op.
//
// The contents of the buffer are undefined and may be non-zero.
func SyscallOutput(dst any, size int) Buffer {
if dstBuf := unsafeBackingMemory(dst); len(dstBuf) == size {
buf := newBuffer(dstBuf)
buf.size = syscallPointerOnly
return buf
}
return newBuffer(make([]byte, size))
}
// CopyTo copies the buffer into dst.
//
// Returns the number of copied bytes.
func (b Buffer) CopyTo(dst []byte) int {
return copy(dst, b.unsafeBytes())
}
// Pointer returns the location where a syscall should write.
func (b Buffer) Pointer() sys.Pointer {
// NB: This deliberately ignores b.length to support zero-copy
// marshaling / unmarshaling using unsafe.Pointer.
return sys.NewPointer(b.ptr)
}
// Unmarshal the buffer into the provided value.
func (b Buffer) Unmarshal(data any) error {
if b.size == syscallPointerOnly {
return nil
}
return Unmarshal(data, b.unsafeBytes())
}
func (b Buffer) unsafeBytes() []byte {
if b.size == syscallPointerOnly {
return nil
}
return unsafe.Slice((*byte)(b.ptr), b.size)
}
+3
View File
@@ -0,0 +1,3 @@
// Package sysenc provides efficient conversion of Go values to system
// call interfaces.
package sysenc
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found at https://go.dev/LICENSE.
package sysenc
import (
"reflect"
"sync"
)
var hasUnexportedFieldsCache sync.Map // map[reflect.Type]bool
func hasUnexportedFields(typ reflect.Type) bool {
switch typ.Kind() {
case reflect.Slice, reflect.Array, reflect.Pointer:
return hasUnexportedFields(typ.Elem())
case reflect.Struct:
if unexported, ok := hasUnexportedFieldsCache.Load(typ); ok {
return unexported.(bool)
}
unexported := false
for i, n := 0, typ.NumField(); i < n; i++ {
field := typ.Field(i)
// Package binary allows _ fields but always writes zeroes into them.
if (!field.IsExported() && field.Name != "_") || hasUnexportedFields(field.Type) {
unexported = true
break
}
}
hasUnexportedFieldsCache.Store(typ, unexported)
return unexported
default:
// NB: It's not clear what this means for Chan and so on.
return false
}
}
+163
View File
@@ -0,0 +1,163 @@
package sysenc
import (
"bytes"
"encoding"
"encoding/binary"
"errors"
"fmt"
"reflect"
"sync"
"unsafe"
"github.com/cilium/ebpf/internal"
)
// Marshal turns data into a byte slice using the system's native endianness.
//
// If possible, avoids allocations by directly using the backing memory
// of data. This means that the variable must not be modified for the lifetime
// of the returned [Buffer].
//
// Returns an error if the data can't be turned into a byte slice according to
// the behaviour of [binary.Write].
func Marshal(data any, size int) (Buffer, error) {
if data == nil {
return Buffer{}, errors.New("can't marshal a nil value")
}
var buf []byte
var err error
switch value := data.(type) {
case encoding.BinaryMarshaler:
buf, err = value.MarshalBinary()
case string:
buf = unsafe.Slice(unsafe.StringData(value), len(value))
case []byte:
buf = value
case int16:
buf = internal.NativeEndian.AppendUint16(make([]byte, 0, 2), uint16(value))
case uint16:
buf = internal.NativeEndian.AppendUint16(make([]byte, 0, 2), value)
case int32:
buf = internal.NativeEndian.AppendUint32(make([]byte, 0, 4), uint32(value))
case uint32:
buf = internal.NativeEndian.AppendUint32(make([]byte, 0, 4), value)
case int64:
buf = internal.NativeEndian.AppendUint64(make([]byte, 0, 8), uint64(value))
case uint64:
buf = internal.NativeEndian.AppendUint64(make([]byte, 0, 8), value)
default:
if buf := unsafeBackingMemory(data); len(buf) == size {
return newBuffer(buf), nil
}
wr := internal.NewBuffer(make([]byte, 0, size))
defer internal.PutBuffer(wr)
err = binary.Write(wr, internal.NativeEndian, value)
buf = wr.Bytes()
}
if err != nil {
return Buffer{}, err
}
if len(buf) != size {
return Buffer{}, fmt.Errorf("%T doesn't marshal to %d bytes", data, size)
}
return newBuffer(buf), nil
}
var bytesReaderPool = sync.Pool{
New: func() interface{} {
return new(bytes.Reader)
},
}
// Unmarshal a byte slice in the system's native endianness into data.
//
// Returns an error if buf can't be unmarshalled according to the behaviour
// of [binary.Read].
func Unmarshal(data interface{}, buf []byte) error {
switch value := data.(type) {
case encoding.BinaryUnmarshaler:
return value.UnmarshalBinary(buf)
case *string:
*value = string(buf)
return nil
default:
if dataBuf := unsafeBackingMemory(data); len(dataBuf) == len(buf) {
copy(dataBuf, buf)
return nil
}
rd := bytesReaderPool.Get().(*bytes.Reader)
defer bytesReaderPool.Put(rd)
rd.Reset(buf)
return binary.Read(rd, internal.NativeEndian, value)
}
}
// unsafeBackingMemory returns the backing memory of data if it can be used
// instead of calling into package binary.
//
// Returns nil if the value is not a pointer or a slice, or if it contains
// padding or unexported fields.
func unsafeBackingMemory(data any) []byte {
if data == nil {
return nil
}
value := reflect.ValueOf(data)
var valueSize int
switch value.Kind() {
case reflect.Pointer:
if value.IsNil() {
return nil
}
if elemType := value.Type().Elem(); elemType.Kind() != reflect.Slice {
valueSize = int(elemType.Size())
break
}
// We're dealing with a pointer to a slice. Dereference and
// handle it like a regular slice.
value = value.Elem()
fallthrough
case reflect.Slice:
valueSize = int(value.Type().Elem().Size()) * value.Len()
default:
// Prevent Value.UnsafePointer from panicking.
return nil
}
// Some nil pointer types currently crash binary.Size. Call it after our own
// code so that the panic isn't reachable.
// See https://github.com/golang/go/issues/60892
if size := binary.Size(data); size == -1 || size != valueSize {
// The type contains padding or unsupported types.
return nil
}
if hasUnexportedFields(reflect.TypeOf(data)) {
return nil
}
// Reinterpret the pointer as a byte slice. This violates the unsafe.Pointer
// rules because it's very unlikely that the source data has "an equivalent
// memory layout". However, we can make it safe-ish because of the
// following reasons:
// - There is no alignment mismatch since we cast to a type with an
// alignment of 1.
// - There are no pointers in the source type so we don't upset the GC.
// - The length is verified at runtime.
return unsafe.Slice((*byte)(value.UnsafePointer()), valueSize)
}
+1 -1
View File
@@ -20,7 +20,7 @@ var (
ErrInvalidMaxActive = errors.New("can only set maxactive on kretprobes")
)
//go:generate stringer -type=ProbeType -linecomment
//go:generate go run golang.org/x/tools/cmd/stringer@latest -type=ProbeType -linecomment
type ProbeType uint8
+2
View File
@@ -85,6 +85,8 @@ const (
BPF_FS_MAGIC = linux.BPF_FS_MAGIC
TRACEFS_MAGIC = linux.TRACEFS_MAGIC
DEBUGFS_MAGIC = linux.DEBUGFS_MAGIC
BPF_RB_NO_WAKEUP = linux.BPF_RB_NO_WAKEUP
BPF_RB_FORCE_WAKEUP = linux.BPF_RB_FORCE_WAKEUP
)
type Statfs_t = linux.Statfs_t
+2
View File
@@ -89,6 +89,8 @@ const (
BPF_FS_MAGIC
TRACEFS_MAGIC
DEBUGFS_MAGIC
BPF_RB_NO_WAKEUP
BPF_RB_FORCE_WAKEUP
)
type Statfs_t struct {
+3 -4
View File
@@ -25,10 +25,6 @@ type IterOptions struct {
// AttachIter attaches a BPF seq_file iterator.
func AttachIter(opts IterOptions) (*Iter, error) {
if err := haveBPFLink(); err != nil {
return nil, err
}
progFd := opts.Program.FD()
if progFd < 0 {
return nil, fmt.Errorf("invalid program: %s", sys.ErrClosedFd)
@@ -52,6 +48,9 @@ func AttachIter(opts IterOptions) (*Iter, error) {
fd, err := sys.LinkCreateIter(&attr)
if err != nil {
if haveFeatErr := haveBPFLink(); haveFeatErr != nil {
return nil, haveFeatErr
}
return nil, fmt.Errorf("can't link iterator: %w", err)
}
+4 -4
View File
@@ -82,10 +82,6 @@ func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Lin
return nil, fmt.Errorf("Cookies must be exactly Symbols or Addresses in length: %w", errInvalidInput)
}
if err := haveBPFLinkKprobeMulti(); err != nil {
return nil, err
}
attr := &sys.LinkCreateKprobeMultiAttr{
ProgFd: uint32(prog.FD()),
AttachType: sys.BPF_TRACE_KPROBE_MULTI,
@@ -113,7 +109,11 @@ func kprobeMulti(prog *ebpf.Program, opts KprobeMultiOptions, flags uint32) (Lin
if errors.Is(err, unix.EINVAL) {
return nil, fmt.Errorf("%w (missing kernel symbol or prog's AttachType not AttachTraceKprobeMulti?)", err)
}
if err != nil {
if haveFeatErr := haveBPFLinkKprobeMulti(); haveFeatErr != nil {
return nil, haveFeatErr
}
return nil, err
}
+7 -8
View File
@@ -25,10 +25,6 @@ type RawAttachProgramOptions struct {
// You should use one of the higher level abstractions available in this
// package if possible.
func RawAttachProgram(opts RawAttachProgramOptions) error {
if err := haveProgAttach(); err != nil {
return err
}
var replaceFd uint32
if opts.Replace != nil {
replaceFd = uint32(opts.Replace.FD())
@@ -43,8 +39,12 @@ func RawAttachProgram(opts RawAttachProgramOptions) error {
}
if err := sys.ProgAttach(&attr); err != nil {
if haveFeatErr := haveProgAttach(); haveFeatErr != nil {
return haveFeatErr
}
return fmt.Errorf("can't attach program: %w", err)
}
return nil
}
@@ -59,16 +59,15 @@ type RawDetachProgramOptions struct {
// You should use one of the higher level abstractions available in this
// package if possible.
func RawDetachProgram(opts RawDetachProgramOptions) error {
if err := haveProgAttach(); err != nil {
return err
}
attr := sys.ProgDetachAttr{
TargetFd: uint32(opts.Target),
AttachBpfFd: uint32(opts.Program.FD()),
AttachType: uint32(opts.Attach),
}
if err := sys.ProgDetach(&attr); err != nil {
if haveFeatErr := haveProgAttach(); haveFeatErr != nil {
return haveFeatErr
}
return fmt.Errorf("can't detach program: %w", err)
}
+7 -4
View File
@@ -60,9 +60,11 @@ var haveProgAttachReplace = internal.NewFeatureTest("BPF_PROG_ATTACH atomic repl
asm.Return(),
},
})
if err != nil {
return internal.ErrNotSupported
}
defer prog.Close()
// We know that we have BPF_PROG_ATTACH since we can load CGroupSKB programs.
@@ -113,11 +115,12 @@ var haveProgQuery = internal.NewFeatureTest("BPF_PROG_QUERY", "4.15", func() err
}
err := sys.ProgQuery(&attr)
if errors.Is(err, unix.EINVAL) {
return internal.ErrNotSupported
}
if errors.Is(err, unix.EBADF) {
return nil
}
return err
if err != nil {
return ErrNotSupported
}
return errors.New("syscall succeeded unexpectedly")
})
+4 -1
View File
@@ -18,9 +18,12 @@ var (
uprobeRefCtrOffsetShift = 32
haveRefCtrOffsetPMU = internal.NewFeatureTest("RefCtrOffsetPMU", "4.20", func() error {
_, err := os.Stat(uprobeRefCtrOffsetPMUPath)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return internal.ErrNotSupported
}
if err != nil {
return err
}
return nil
})
+19 -9
View File
@@ -40,10 +40,12 @@ func (hs handles) fdArray() []int32 {
return fda
}
func (hs handles) close() {
for _, h := range hs {
h.Close()
func (hs *handles) Close() error {
var errs []error
for _, h := range *hs {
errs = append(errs, h.Close())
}
return errors.Join(errs...)
}
// splitSymbols splits insns into subsections delimited by Symbol Instructions.
@@ -231,7 +233,13 @@ func fixupAndValidate(insns asm.Instructions) error {
// fixupKfuncs loops over all instructions in search for kfunc calls.
// If at least one is found, the current kernels BTF and module BTFis are searched to set Instruction.Constant
// and Instruction.Offset to the correct values.
func fixupKfuncs(insns asm.Instructions) (handles, error) {
func fixupKfuncs(insns asm.Instructions) (_ handles, err error) {
closeOnError := func(c io.Closer) {
if err != nil {
c.Close()
}
}
iter := insns.Iterate()
for iter.Next() {
ins := iter.Ins
@@ -250,6 +258,8 @@ fixups:
}
fdArray := make(handles, 0)
defer closeOnError(&fdArray)
for {
ins := iter.Ins
@@ -276,6 +286,11 @@ fixups:
return nil, err
}
idx, err := fdArray.add(module)
if err != nil {
return nil, err
}
if err := btf.CheckTypeCompatibility(kfm.Type, target.(*btf.Func).Type); err != nil {
return nil, &incompatibleKfuncError{kfm.Name, err}
}
@@ -285,11 +300,6 @@ fixups:
return nil, err
}
idx, err := fdArray.add(module)
if err != nil {
return nil, err
}
ins.Constant = int64(id)
ins.Offset = int16(idx)
+142 -140
View File
@@ -15,6 +15,7 @@ import (
"github.com/cilium/ebpf/btf"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/sysenc"
"github.com/cilium/ebpf/internal/unix"
)
@@ -102,26 +103,55 @@ func (ms *MapSpec) Copy() *MapSpec {
return &cpy
}
func (ms *MapSpec) clampPerfEventArraySize() error {
if ms.Type != PerfEventArray {
return nil
// fixupMagicFields fills fields of MapSpec which are usually
// left empty in ELF or which depend on runtime information.
//
// The method doesn't modify Spec, instead returning a copy.
// The copy is only performed if fixups are necessary, so callers mustn't mutate
// the returned spec.
func (spec *MapSpec) fixupMagicFields() (*MapSpec, error) {
switch spec.Type {
case ArrayOfMaps, HashOfMaps:
if spec.ValueSize != 0 && spec.ValueSize != 4 {
return nil, errors.New("ValueSize must be zero or four for map of map")
}
spec = spec.Copy()
spec.ValueSize = 4
case PerfEventArray:
if spec.KeySize != 0 && spec.KeySize != 4 {
return nil, errors.New("KeySize must be zero or four for perf event array")
}
if spec.ValueSize != 0 && spec.ValueSize != 4 {
return nil, errors.New("ValueSize must be zero or four for perf event array")
}
spec = spec.Copy()
spec.KeySize = 4
spec.ValueSize = 4
n, err := internal.PossibleCPUs()
if err != nil {
return nil, fmt.Errorf("fixup perf event array: %w", err)
}
if n := uint32(n); spec.MaxEntries == 0 || spec.MaxEntries > n {
// MaxEntries should be zero most of the time, but there is code
// out there which hardcodes large constants. Clamp the number
// of entries to the number of CPUs at most. Allow creating maps with
// less than n items since some kernel selftests relied on this
// behaviour in the past.
spec.MaxEntries = n
}
}
n, err := internal.PossibleCPUs()
if err != nil {
return fmt.Errorf("perf event array: %w", err)
}
if n := uint32(n); ms.MaxEntries > n {
ms.MaxEntries = n
}
return nil
return spec, nil
}
// dataSection returns the contents and BTF Datasec descriptor of the spec.
func (ms *MapSpec) dataSection() ([]byte, *btf.Datasec, error) {
if ms.Value == nil {
return nil, nil, errMapNoBTFValue
}
@@ -155,6 +185,11 @@ type MapKV struct {
//
// Returns an error wrapping [ErrMapIncompatible] otherwise.
func (ms *MapSpec) Compatible(m *Map) error {
ms, err := ms.fixupMagicFields()
if err != nil {
return err
}
switch {
case m.typ != ms.Type:
return fmt.Errorf("expected type %v, got %v: %w", ms.Type, m.typ, ErrMapIncompatible)
@@ -165,8 +200,7 @@ func (ms *MapSpec) Compatible(m *Map) error {
case m.valueSize != ms.ValueSize:
return fmt.Errorf("expected value size %v, got %v: %w", ms.ValueSize, m.valueSize, ErrMapIncompatible)
case !(ms.Type == PerfEventArray && ms.MaxEntries == 0) &&
m.maxEntries != ms.MaxEntries:
case m.maxEntries != ms.MaxEntries:
return fmt.Errorf("expected max entries %v, got %v: %w", ms.MaxEntries, m.maxEntries, ErrMapIncompatible)
// BPF_F_RDONLY_PROG is set unconditionally for devmaps. Explicitly allow
@@ -350,60 +384,9 @@ func (spec *MapSpec) createMap(inner *sys.FD, opts MapOptions) (_ *Map, err erro
}
}
switch spec.Type {
case ArrayOfMaps, HashOfMaps:
if err := haveNestedMaps(); err != nil {
return nil, err
}
if spec.ValueSize != 0 && spec.ValueSize != 4 {
return nil, errors.New("ValueSize must be zero or four for map of map")
}
spec = spec.Copy()
spec.ValueSize = 4
case PerfEventArray:
if spec.KeySize != 0 && spec.KeySize != 4 {
return nil, errors.New("KeySize must be zero or four for perf event array")
}
if spec.ValueSize != 0 && spec.ValueSize != 4 {
return nil, errors.New("ValueSize must be zero or four for perf event array")
}
spec = spec.Copy()
spec.KeySize = 4
spec.ValueSize = 4
if spec.MaxEntries == 0 {
n, err := internal.PossibleCPUs()
if err != nil {
return nil, fmt.Errorf("perf event array: %w", err)
}
spec.MaxEntries = uint32(n)
}
}
if spec.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze {
if err := haveMapMutabilityModifiers(); err != nil {
return nil, fmt.Errorf("map create: %w", err)
}
}
if spec.Flags&unix.BPF_F_MMAPABLE > 0 {
if err := haveMmapableMaps(); err != nil {
return nil, fmt.Errorf("map create: %w", err)
}
}
if spec.Flags&unix.BPF_F_INNER_MAP > 0 {
if err := haveInnerMaps(); err != nil {
return nil, fmt.Errorf("map create: %w", err)
}
}
if spec.Flags&unix.BPF_F_NO_PREALLOC > 0 {
if err := haveNoPreallocMaps(); err != nil {
return nil, fmt.Errorf("map create: %w", err)
}
spec, err = spec.fixupMagicFields()
if err != nil {
return nil, err
}
attr := sys.MapCreateAttr{
@@ -440,38 +423,72 @@ func (spec *MapSpec) createMap(inner *sys.FD, opts MapOptions) (_ *Map, err erro
}
fd, err := sys.MapCreate(&attr)
// Some map types don't support BTF k/v in earlier kernel versions.
// Remove BTF metadata and retry map creation.
if (errors.Is(err, sys.ENOTSUPP) || errors.Is(err, unix.EINVAL)) && attr.BtfFd != 0 {
attr.BtfFd, attr.BtfKeyTypeId, attr.BtfValueTypeId = 0, 0, 0
fd, err = sys.MapCreate(&attr)
}
if err != nil {
if errors.Is(err, unix.EPERM) {
return nil, fmt.Errorf("map create: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
}
if errors.Is(err, unix.EINVAL) && attr.MaxEntries == 0 {
return nil, fmt.Errorf("map create: %w (MaxEntries may be incorrectly set to zero)", err)
}
if errors.Is(err, unix.EINVAL) && spec.Type == UnspecifiedMap {
return nil, fmt.Errorf("map create: cannot use type %s", UnspecifiedMap)
}
if attr.BtfFd == 0 {
return nil, fmt.Errorf("map create: %w (without BTF k/v)", err)
}
return nil, fmt.Errorf("map create: %w", err)
return nil, handleMapCreateError(attr, spec, err)
}
defer closeOnError(fd)
defer closeOnError(fd)
m, err := newMap(fd, spec.Name, spec.Type, spec.KeySize, spec.ValueSize, spec.MaxEntries, spec.Flags)
if err != nil {
return nil, fmt.Errorf("map create: %w", err)
}
return m, nil
}
func handleMapCreateError(attr sys.MapCreateAttr, spec *MapSpec, err error) error {
if errors.Is(err, unix.EPERM) {
return fmt.Errorf("map create: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
}
if errors.Is(err, unix.EINVAL) && spec.MaxEntries == 0 {
return fmt.Errorf("map create: %w (MaxEntries may be incorrectly set to zero)", err)
}
if errors.Is(err, unix.EINVAL) && spec.Type == UnspecifiedMap {
return fmt.Errorf("map create: cannot use type %s", UnspecifiedMap)
}
if errors.Is(err, unix.EINVAL) && spec.Flags&unix.BPF_F_NO_PREALLOC > 0 {
return fmt.Errorf("map create: %w (noPrealloc flag may be incompatible with map type %s)", err, spec.Type)
}
switch spec.Type {
case ArrayOfMaps, HashOfMaps:
if haveFeatErr := haveNestedMaps(); haveFeatErr != nil {
return fmt.Errorf("map create: %w", haveFeatErr)
}
}
if spec.Flags&(unix.BPF_F_RDONLY_PROG|unix.BPF_F_WRONLY_PROG) > 0 || spec.Freeze {
if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil {
return fmt.Errorf("map create: %w", haveFeatErr)
}
}
if spec.Flags&unix.BPF_F_MMAPABLE > 0 {
if haveFeatErr := haveMmapableMaps(); haveFeatErr != nil {
return fmt.Errorf("map create: %w", haveFeatErr)
}
}
if spec.Flags&unix.BPF_F_INNER_MAP > 0 {
if haveFeatErr := haveInnerMaps(); haveFeatErr != nil {
return fmt.Errorf("map create: %w", haveFeatErr)
}
}
if spec.Flags&unix.BPF_F_NO_PREALLOC > 0 {
if haveFeatErr := haveNoPreallocMaps(); haveFeatErr != nil {
return fmt.Errorf("map create: %w", haveFeatErr)
}
}
if attr.BtfFd == 0 {
return fmt.Errorf("map create: %w (without BTF k/v)", err)
}
return fmt.Errorf("map create: %w", err)
}
// newMap allocates and returns a new Map structure.
// Sets the fullValueSize on per-CPU maps.
func newMap(fd *sys.FD, name string, typ MapType, keySize, valueSize, maxEntries, flags uint32) (*Map, error) {
@@ -568,8 +585,8 @@ func (m *Map) LookupWithFlags(key, valueOut interface{}, flags MapLookupFlags) e
return m.lookupPerCPU(key, valueOut, flags)
}
valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize)
if err := m.lookup(key, valuePtr, flags); err != nil {
valueBytes := makeMapSyscallOutput(valueOut, m.fullValueSize)
if err := m.lookup(key, valueBytes.Pointer(), flags); err != nil {
return err
}
@@ -595,8 +612,8 @@ func (m *Map) LookupAndDeleteWithFlags(key, valueOut interface{}, flags MapLooku
return m.lookupAndDeletePerCPU(key, valueOut, flags)
}
valuePtr, valueBytes := makeBuffer(valueOut, m.fullValueSize)
if err := m.lookupAndDelete(key, valuePtr, flags); err != nil {
valueBytes := makeMapSyscallOutput(valueOut, m.fullValueSize)
if err := m.lookupAndDelete(key, valueBytes.Pointer(), flags); err != nil {
return err
}
return m.unmarshalValue(valueOut, valueBytes)
@@ -764,13 +781,13 @@ func (m *Map) Delete(key interface{}) error {
//
// Returns ErrKeyNotExist if there is no next key.
func (m *Map) NextKey(key, nextKeyOut interface{}) error {
nextKeyPtr, nextKeyBytes := makeBuffer(nextKeyOut, int(m.keySize))
nextKeyBytes := makeMapSyscallOutput(nextKeyOut, int(m.keySize))
if err := m.nextKey(key, nextKeyPtr); err != nil {
if err := m.nextKey(key, nextKeyBytes.Pointer()); err != nil {
return err
}
if err := m.unmarshalKey(nextKeyOut, nextKeyBytes); err != nil {
if err := nextKeyBytes.Unmarshal(nextKeyOut); err != nil {
return fmt.Errorf("can't unmarshal next key: %w", err)
}
return nil
@@ -941,14 +958,14 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut
keyPtr := sys.NewSlicePointer(keyBuf)
valueBuf := make([]byte, count*int(m.fullValueSize))
valuePtr := sys.NewSlicePointer(valueBuf)
nextPtr, nextBuf := makeBuffer(nextKeyOut, int(m.keySize))
nextBuf := makeMapSyscallOutput(nextKeyOut, int(m.keySize))
attr := sys.MapLookupBatchAttr{
MapFd: m.fd.Uint(),
Keys: keyPtr,
Values: valuePtr,
Count: uint32(count),
OutBatch: nextPtr,
OutBatch: nextBuf.Pointer(),
}
if opts != nil {
@@ -958,7 +975,7 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut
var err error
if startKey != nil {
attr.InBatch, err = marshalPtr(startKey, int(m.keySize))
attr.InBatch, err = marshalMapSyscallInput(startKey, int(m.keySize))
if err != nil {
return 0, err
}
@@ -970,15 +987,15 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut
return 0, sysErr
}
err = m.unmarshalKey(nextKeyOut, nextBuf)
err = nextBuf.Unmarshal(nextKeyOut)
if err != nil {
return 0, err
}
err = unmarshalBytes(keysOut, keyBuf)
err = sysenc.Unmarshal(keysOut, keyBuf)
if err != nil {
return 0, err
}
err = unmarshalBytes(valuesOut, valueBuf)
err = sysenc.Unmarshal(valuesOut, valueBuf)
if err != nil {
return 0, err
}
@@ -991,9 +1008,6 @@ func (m *Map) batchLookup(cmd sys.Cmd, startKey, nextKeyOut, keysOut, valuesOut
// "keys" and "values" must be of type slice, a pointer
// to a slice or buffer will not work.
func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, error) {
if err := haveBatchAPI(); err != nil {
return 0, err
}
if m.typ.hasPerCPUValue() {
return 0, ErrNotSupported
}
@@ -1013,11 +1027,11 @@ func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, er
if count != valuesValue.Len() {
return 0, fmt.Errorf("keys and values must be the same length")
}
keyPtr, err := marshalPtr(keys, count*int(m.keySize))
keyPtr, err := marshalMapSyscallInput(keys, count*int(m.keySize))
if err != nil {
return 0, err
}
valuePtr, err = marshalPtr(values, count*int(m.valueSize))
valuePtr, err = marshalMapSyscallInput(values, count*int(m.valueSize))
if err != nil {
return 0, err
}
@@ -1035,6 +1049,9 @@ func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, er
err = sys.MapUpdateBatch(&attr)
if err != nil {
if haveFeatErr := haveBatchAPI(); haveFeatErr != nil {
return 0, haveFeatErr
}
return int(attr.Count), fmt.Errorf("batch update: %w", wrapMapError(err))
}
@@ -1044,9 +1061,6 @@ func (m *Map) BatchUpdate(keys, values interface{}, opts *BatchOptions) (int, er
// BatchDelete batch deletes entries in the map by keys.
// "keys" must be of type slice, a pointer to a slice or buffer will not work.
func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) {
if err := haveBatchAPI(); err != nil {
return 0, err
}
if m.typ.hasPerCPUValue() {
return 0, ErrNotSupported
}
@@ -1055,7 +1069,7 @@ func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) {
return 0, fmt.Errorf("keys must be a slice")
}
count := keysValue.Len()
keyPtr, err := marshalPtr(keys, count*int(m.keySize))
keyPtr, err := marshalMapSyscallInput(keys, count*int(m.keySize))
if err != nil {
return 0, fmt.Errorf("cannot marshal keys: %v", err)
}
@@ -1072,6 +1086,9 @@ func (m *Map) BatchDelete(keys interface{}, opts *BatchOptions) (int, error) {
}
if err = sys.MapDeleteBatch(&attr); err != nil {
if haveFeatErr := haveBatchAPI(); haveFeatErr != nil {
return 0, haveFeatErr
}
return int(attr.Count), fmt.Errorf("batch delete: %w", wrapMapError(err))
}
@@ -1176,15 +1193,14 @@ func (m *Map) IsPinned() bool {
//
// It makes no changes to kernel-side restrictions.
func (m *Map) Freeze() error {
if err := haveMapMutabilityModifiers(); err != nil {
return fmt.Errorf("can't freeze map: %w", err)
}
attr := sys.MapFreezeAttr{
MapFd: m.fd.Uint(),
}
if err := sys.MapFreeze(&attr); err != nil {
if haveFeatErr := haveMapMutabilityModifiers(); haveFeatErr != nil {
return fmt.Errorf("can't freeze map: %w", haveFeatErr)
}
return fmt.Errorf("can't freeze map: %w", err)
}
return nil
@@ -1217,16 +1233,7 @@ func (m *Map) marshalKey(data interface{}) (sys.Pointer, error) {
return sys.Pointer{}, errors.New("can't use nil as key of map")
}
return marshalPtr(data, int(m.keySize))
}
func (m *Map) unmarshalKey(data interface{}, buf []byte) error {
if buf == nil {
// This is from a makeBuffer call, nothing do do here.
return nil
}
return unmarshalBytes(data, buf)
return marshalMapSyscallInput(data, int(m.keySize))
}
func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) {
@@ -1249,7 +1256,7 @@ func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) {
buf, err = marshalProgram(value, int(m.valueSize))
default:
return marshalPtr(data, int(m.valueSize))
return marshalMapSyscallInput(data, int(m.valueSize))
}
if err != nil {
@@ -1259,16 +1266,7 @@ func (m *Map) marshalValue(data interface{}) (sys.Pointer, error) {
return sys.NewSlicePointer(buf), nil
}
func (m *Map) unmarshalValue(value interface{}, buf []byte) error {
if buf == nil {
// This is from a makeBuffer call, nothing do do here.
return nil
}
if m.typ.hasPerCPUValue() {
return unmarshalPerCPUValue(value, int(m.valueSize), buf)
}
func (m *Map) unmarshalValue(value any, buf sysenc.Buffer) error {
switch value := value.(type) {
case **Map:
if !m.typ.canStoreMap() {
@@ -1315,7 +1313,7 @@ func (m *Map) unmarshalValue(value interface{}, buf []byte) error {
return errors.New("require pointer to *Program")
}
return unmarshalBytes(value, buf)
return buf.Unmarshal(value)
}
// LoadPinnedMap loads a Map from a BPF file.
@@ -1337,12 +1335,11 @@ func LoadPinnedMap(fileName string, opts *LoadPinOptions) (*Map, error) {
}
// unmarshalMap creates a map from a map ID encoded in host endianness.
func unmarshalMap(buf []byte) (*Map, error) {
if len(buf) != 4 {
return nil, errors.New("map id requires 4 byte value")
func unmarshalMap(buf sysenc.Buffer) (*Map, error) {
var id uint32
if err := buf.Unmarshal(&id); err != nil {
return nil, err
}
id := internal.NativeEndian.Uint32(buf)
return NewMapFromID(MapID(id))
}
@@ -1438,7 +1435,12 @@ func (mi *MapIterator) Next(keyOut, valueOut interface{}) bool {
return false
}
mi.err = mi.target.unmarshalKey(keyOut, nextKey)
if ptr, ok := keyOut.(unsafe.Pointer); ok {
copy(unsafe.Slice((*byte)(ptr), len(nextKey)), nextKey)
} else {
mi.err = sysenc.Unmarshal(keyOut, nextKey)
}
return mi.err == nil
}
+21 -141
View File
@@ -1,166 +1,53 @@
package ebpf
import (
"bytes"
"encoding"
"encoding/binary"
"errors"
"fmt"
"reflect"
"runtime"
"sync"
"unsafe"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/sysenc"
)
// marshalPtr converts an arbitrary value into a pointer suitable
// marshalMapSyscallInput converts an arbitrary value into a pointer suitable
// to be passed to the kernel.
//
// As an optimization, it returns the original value if it is an
// unsafe.Pointer.
func marshalPtr(data interface{}, length int) (sys.Pointer, error) {
func marshalMapSyscallInput(data any, length int) (sys.Pointer, error) {
if ptr, ok := data.(unsafe.Pointer); ok {
return sys.NewPointer(ptr), nil
}
buf, err := marshalBytes(data, length)
buf, err := sysenc.Marshal(data, length)
if err != nil {
return sys.Pointer{}, err
}
return sys.NewSlicePointer(buf), nil
return buf.Pointer(), nil
}
// marshalBytes converts an arbitrary value into a byte buffer.
//
// Prefer using Map.marshalKey and Map.marshalValue if possible, since
// those have special cases that allow more types to be encoded.
//
// Returns an error if the given value isn't representable in exactly
// length bytes.
func marshalBytes(data interface{}, length int) (buf []byte, err error) {
if data == nil {
return nil, errors.New("can't marshal a nil value")
}
switch value := data.(type) {
case encoding.BinaryMarshaler:
buf, err = value.MarshalBinary()
case string:
buf = []byte(value)
case []byte:
buf = value
case unsafe.Pointer:
err = errors.New("can't marshal from unsafe.Pointer")
case Map, *Map, Program, *Program:
err = fmt.Errorf("can't marshal %T", value)
default:
wr := internal.NewBuffer(make([]byte, 0, length))
defer internal.PutBuffer(wr)
err = binary.Write(wr, internal.NativeEndian, value)
if err != nil {
err = fmt.Errorf("encoding %T: %v", value, err)
}
buf = wr.Bytes()
}
if err != nil {
return nil, err
}
if len(buf) != length {
return nil, fmt.Errorf("%T doesn't marshal to %d bytes", data, length)
}
return buf, nil
}
func makeBuffer(dst interface{}, length int) (sys.Pointer, []byte) {
func makeMapSyscallOutput(dst any, length int) sysenc.Buffer {
if ptr, ok := dst.(unsafe.Pointer); ok {
return sys.NewPointer(ptr), nil
return sysenc.UnsafeBuffer(ptr)
}
buf := make([]byte, length)
return sys.NewSlicePointer(buf), buf
}
var bytesReaderPool = sync.Pool{
New: func() interface{} {
return new(bytes.Reader)
},
}
// unmarshalBytes converts a byte buffer into an arbitrary value.
//
// Prefer using Map.unmarshalKey and Map.unmarshalValue if possible, since
// those have special cases that allow more types to be encoded.
//
// The common int32 and int64 types are directly handled to avoid
// unnecessary heap allocations as happening in the default case.
func unmarshalBytes(data interface{}, buf []byte) error {
switch value := data.(type) {
case unsafe.Pointer:
dst := unsafe.Slice((*byte)(value), len(buf))
copy(dst, buf)
runtime.KeepAlive(value)
return nil
case Map, *Map, Program, *Program:
return fmt.Errorf("can't unmarshal into %T", value)
case encoding.BinaryUnmarshaler:
return value.UnmarshalBinary(buf)
case *string:
*value = string(buf)
return nil
case *[]byte:
*value = buf
return nil
case *int32:
if len(buf) < 4 {
return errors.New("int32 requires 4 bytes")
}
*value = int32(internal.NativeEndian.Uint32(buf))
return nil
case *uint32:
if len(buf) < 4 {
return errors.New("uint32 requires 4 bytes")
}
*value = internal.NativeEndian.Uint32(buf)
return nil
case *int64:
if len(buf) < 8 {
return errors.New("int64 requires 8 bytes")
}
*value = int64(internal.NativeEndian.Uint64(buf))
return nil
case *uint64:
if len(buf) < 8 {
return errors.New("uint64 requires 8 bytes")
}
*value = internal.NativeEndian.Uint64(buf)
return nil
case string:
return errors.New("require pointer to string")
case []byte:
return errors.New("require pointer to []byte")
default:
rd := bytesReaderPool.Get().(*bytes.Reader)
rd.Reset(buf)
defer bytesReaderPool.Put(rd)
if err := binary.Read(rd, internal.NativeEndian, value); err != nil {
return fmt.Errorf("decoding %T: %v", value, err)
}
return nil
_, ok := dst.(encoding.BinaryUnmarshaler)
if ok {
return sysenc.SyscallOutput(nil, length)
}
return sysenc.SyscallOutput(dst, length)
}
// marshalPerCPUValue encodes a slice containing one value per
// possible CPU into a buffer of bytes.
//
// Values are initialized to zero if the slice has less elements than CPUs.
//
// slice must have a type like []elementType.
func marshalPerCPUValue(slice interface{}, elemLength int) (sys.Pointer, error) {
func marshalPerCPUValue(slice any, elemLength int) (sys.Pointer, error) {
sliceType := reflect.TypeOf(slice)
if sliceType.Kind() != reflect.Slice {
return sys.Pointer{}, errors.New("per-CPU value requires slice")
@@ -182,13 +69,13 @@ func marshalPerCPUValue(slice interface{}, elemLength int) (sys.Pointer, error)
for i := 0; i < sliceLen; i++ {
elem := sliceValue.Index(i).Interface()
elemBytes, err := marshalBytes(elem, elemLength)
elemBytes, err := sysenc.Marshal(elem, elemLength)
if err != nil {
return sys.Pointer{}, err
}
offset := i * alignedElemLength
copy(buf[offset:offset+elemLength], elemBytes)
elemBytes.CopyTo(buf[offset : offset+elemLength])
}
return sys.NewSlicePointer(buf), nil
@@ -197,8 +84,8 @@ func marshalPerCPUValue(slice interface{}, elemLength int) (sys.Pointer, error)
// unmarshalPerCPUValue decodes a buffer into a slice containing one value per
// possible CPU.
//
// valueOut must have a type like *[]elementType
func unmarshalPerCPUValue(slicePtr interface{}, elemLength int, buf []byte) error {
// slicePtr must be a pointer to a slice.
func unmarshalPerCPUValue(slicePtr any, elemLength int, buf []byte) error {
slicePtrType := reflect.TypeOf(slicePtr)
if slicePtrType.Kind() != reflect.Ptr || slicePtrType.Elem().Kind() != reflect.Slice {
return fmt.Errorf("per-cpu value requires pointer to slice")
@@ -218,12 +105,9 @@ func unmarshalPerCPUValue(slicePtr interface{}, elemLength int, buf []byte) erro
sliceElemType = sliceElemType.Elem()
}
step := len(buf) / possibleCPUs
if step < elemLength {
return fmt.Errorf("per-cpu element length is larger than available data")
}
stride := internal.Align(elemLength, 8)
for i := 0; i < possibleCPUs; i++ {
var elem interface{}
var elem any
if sliceElemIsPointer {
newElem := reflect.New(sliceElemType)
slice.Index(i).Set(newElem)
@@ -232,16 +116,12 @@ func unmarshalPerCPUValue(slicePtr interface{}, elemLength int, buf []byte) erro
elem = slice.Index(i).Addr().Interface()
}
// Make a copy, since unmarshal can hold on to itemBytes
elemBytes := make([]byte, elemLength)
copy(elemBytes, buf[:elemLength])
err := unmarshalBytes(elem, elemBytes)
err := sysenc.Unmarshal(elem, buf[:elemLength])
if err != nil {
return fmt.Errorf("cpu %d: %w", i, err)
}
buf = buf[step:]
buf = buf[stride:]
}
reflect.ValueOf(slicePtr).Elem().Set(slice)
+4
View File
@@ -0,0 +1,4 @@
[build]
base = "docs/"
publish = "site/"
command = "mkdocs build"
+15 -7
View File
@@ -16,6 +16,7 @@ import (
"github.com/cilium/ebpf/btf"
"github.com/cilium/ebpf/internal"
"github.com/cilium/ebpf/internal/sys"
"github.com/cilium/ebpf/internal/sysenc"
"github.com/cilium/ebpf/internal/unix"
)
@@ -277,7 +278,7 @@ func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, er
if err != nil {
return nil, fmt.Errorf("fixing up kfuncs: %w", err)
}
defer handles.close()
defer handles.Close()
if len(handles) > 0 {
fdArray := handles.fdArray()
@@ -763,14 +764,14 @@ retry:
return attr.Retval, total, nil
}
func unmarshalProgram(buf []byte) (*Program, error) {
if len(buf) != 4 {
return nil, errors.New("program id requires 4 byte value")
func unmarshalProgram(buf sysenc.Buffer) (*Program, error) {
var id uint32
if err := buf.Unmarshal(&id); err != nil {
return nil, err
}
// Looking up an entry in a nested map or prog array returns an id,
// not an fd.
id := internal.NativeEndian.Uint32(buf)
return NewProgramFromID(ProgramID(id))
}
@@ -921,7 +922,12 @@ func findProgramTargetInKernel(name string, progType ProgramType, attachType Att
}
id, err := spec.TypeID(target)
return module, id, err
if err != nil {
module.Close()
return nil, 0, err
}
return module, id, nil
}
// findTargetInKernel attempts to find a named type in the current kernel.
@@ -999,7 +1005,9 @@ func findTargetInProgram(prog *Program, name string, progType ProgramType, attac
var typeName string
switch (match{progType, attachType}) {
case match{Extension, AttachNone}:
case match{Extension, AttachNone},
match{Tracing, AttachTraceFEntry},
match{Tracing, AttachTraceFExit}:
typeName = name
default:
return 0, errUnrecognizedAttachType
+35 -9
View File
@@ -14,6 +14,21 @@ set -euo pipefail
script="$(realpath "$0")"
readonly script
quote_env() {
for var in "$@"; do
if [ -v "$var" ]; then
printf "%s=%q " "$var" "${!var}"
fi
done
}
declare -a preserved_env=(
PATH
CI_MAX_KERNEL_VERSION
TEST_SEED
KERNEL_VERSION
)
# This script is a bit like a Matryoshka doll since it keeps re-executing itself
# in various different contexts:
#
@@ -51,11 +66,11 @@ if [[ "${1:-}" = "--exec-vm" ]]; then
fi
for ((i = 0; i < 3; i++)); do
if ! $sudo virtme-run --kimg "${input}/bzImage" --memory 768M --pwd \
if ! $sudo virtme-run --kimg "${input}/boot/vmlinuz" --memory 768M --pwd \
--rwdir="${testdir}=${testdir}" \
--rodir=/run/input="${input}" \
--rwdir=/run/output="${output}" \
--script-sh "PATH=\"$PATH\" CI_MAX_KERNEL_VERSION="${CI_MAX_KERNEL_VERSION:-}" \"$script\" --exec-test $cmd" \
--script-sh "$(quote_env "${preserved_env[@]}") \"$script\" --exec-test $cmd" \
--kopt possible_cpus=2; then # need at least two CPUs for some tests
exit 23
fi
@@ -85,8 +100,8 @@ elif [[ "${1:-}" = "--exec-test" ]]; then
export KERNEL_SELFTESTS="/run/input/bpf"
fi
if [[ -f "/run/input/bpf/bpf_testmod/bpf_testmod.ko" ]]; then
insmod "/run/input/bpf/bpf_testmod/bpf_testmod.ko"
if [[ -d "/run/input/lib/modules" ]]; then
find /run/input/lib/modules -type f -name bpf_testmod.ko -exec insmod {} \;
fi
dmesg --clear
@@ -114,6 +129,9 @@ fetch() {
return $ret
}
machine="$(uname -m)"
readonly machine
if [[ -f "${1}" ]]; then
readonly kernel="${1}"
cp "${1}" "${input}/bzImage"
@@ -121,16 +139,24 @@ else
# LINUX_VERSION_CODE test compares this to discovered value.
export KERNEL_VERSION="${1}"
readonly kernel="linux-${1}.bz"
readonly selftests="linux-${1}-selftests-bpf.tgz"
if [ "${machine}" = "x86_64" ]; then
readonly kernel="linux-${1}-amd64.tgz"
readonly selftests="linux-${1}-amd64-selftests-bpf.tgz"
elif [ "${machine}" = "aarch64" ]; then
readonly kernel="linux-${1}-arm64.tgz"
readonly selftests=""
else
echo "Arch ${machine} is not supported"
exit 1
fi
fetch "${kernel}"
cp "${tmp_dir}/${kernel}" "${input}/bzImage"
tar xf "${tmp_dir}/${kernel}" -C "${input}"
if fetch "${selftests}"; then
if [ -n "${selftests}" ] && fetch "${selftests}"; then
echo "Decompressing selftests"
mkdir "${input}/bpf"
tar --strip-components=4 -xf "${tmp_dir}/${selftests}" -C "${input}/bpf"
tar --strip-components=5 -xf "${tmp_dir}/${selftests}" -C "${input}/bpf"
else
echo "No selftests found, disabling"
fi
+5 -6
View File
@@ -119,6 +119,7 @@ var haveInnerMaps = internal.NewFeatureTest("inner maps", "5.10", func() error {
MaxEntries: 1,
MapFlags: unix.BPF_F_INNER_MAP,
})
if err != nil {
return internal.ErrNotSupported
}
@@ -135,6 +136,7 @@ var haveNoPreallocMaps = internal.NewFeatureTest("prealloc maps", "4.6", func()
MaxEntries: 1,
MapFlags: unix.BPF_F_NO_PREALLOC,
})
if err != nil {
return internal.ErrNotSupported
}
@@ -223,8 +225,8 @@ var haveBatchAPI = internal.NewFeatureTest("map batch api", "5.6", func() error
keys := []uint32{1, 2}
values := []uint32{3, 4}
kp, _ := marshalPtr(keys, 8)
vp, _ := marshalPtr(values, 8)
kp, _ := marshalMapSyscallInput(keys, 8)
vp, _ := marshalMapSyscallInput(values, 8)
err = sys.MapUpdateBatch(&sys.MapUpdateBatchAttr{
MapFd: fd.Uint(),
@@ -265,11 +267,8 @@ var haveBPFToBPFCalls = internal.NewFeatureTest("bpf2bpf calls", "4.16", func()
}
fd, err := progLoad(insns, SocketFilter, "MIT")
if errors.Is(err, unix.EINVAL) {
return internal.ErrNotSupported
}
if err != nil {
return err
return internal.ErrNotSupported
}
_ = fd.Close()
return nil
+9 -3
View File
@@ -5,7 +5,7 @@ import (
"github.com/cilium/ebpf/internal/unix"
)
//go:generate stringer -output types_string.go -type=MapType,ProgramType,PinType
//go:generate go run golang.org/x/tools/cmd/stringer@latest -output types_string.go -type=MapType,ProgramType,PinType
// MapType indicates the type map structure
// that will be initialized in the kernel.
@@ -44,7 +44,7 @@ const (
// if an skb is from a socket belonging to a specific cgroup
CGroupArray
// LRUHash - This allows you to create a small hash structure that will purge the
// least recently used items rather than thow an error when you run out of memory
// least recently used items rather than throw an error when you run out of memory
LRUHash
// LRUCPUHash - This is NOT like PerCPUHash, this structure is shared among the CPUs,
// it has more to do with including the CPU id with the LRU calculation so that if a
@@ -102,6 +102,12 @@ func (mt MapType) hasPerCPUValue() bool {
return mt == PerCPUHash || mt == PerCPUArray || mt == LRUCPUHash || mt == PerCPUCGroupStorage
}
// canStoreMapOrProgram returns true if the Map stores references to another Map
// or Program.
func (mt MapType) canStoreMapOrProgram() bool {
return mt.canStoreMap() || mt.canStoreProgram()
}
// canStoreMap returns true if the map type accepts a map fd
// for update and returns a map id for lookup.
func (mt MapType) canStoreMap() bool {
@@ -158,7 +164,7 @@ const (
// Will cause invalid argument (EINVAL) at program load time if set incorrectly.
type AttachType uint32
//go:generate stringer -type AttachType -trimprefix Attach
//go:generate go run golang.org/x/tools/cmd/stringer@latest -type AttachType -trimprefix Attach
// AttachNone is an alias for AttachCGroupInetIngress for readability reasons.
const AttachNone AttachType = 0
+3 -2
View File
@@ -2,14 +2,15 @@
## explicit; go 1.16
github.com/checkpoint-restore/go-criu/v6
github.com/checkpoint-restore/go-criu/v6/rpc
# github.com/cilium/ebpf v0.11.0
## explicit; go 1.19
# github.com/cilium/ebpf v0.12.0
## explicit; go 1.20
github.com/cilium/ebpf
github.com/cilium/ebpf/asm
github.com/cilium/ebpf/btf
github.com/cilium/ebpf/internal
github.com/cilium/ebpf/internal/kconfig
github.com/cilium/ebpf/internal/sys
github.com/cilium/ebpf/internal/sysenc
github.com/cilium/ebpf/internal/tracefs
github.com/cilium/ebpf/internal/unix
github.com/cilium/ebpf/link