diff --git a/libbpf-tools/readahead.bpf.c b/libbpf-tools/readahead.bpf.c index 89863e67d60d..44d4121fe8e2 100644 --- a/libbpf-tools/readahead.bpf.c +++ b/libbpf-tools/readahead.bpf.c @@ -34,8 +34,8 @@ int BPF_PROG(do_page_cache_ra) return 0; } -SEC("fexit/__page_cache_alloc") -int BPF_PROG(page_cache_alloc_ret, gfp_t gfp, struct page *ret) +static __always_inline +int alloc_done(struct page *page) { u32 pid = bpf_get_current_pid_tgid(); u64 ts; @@ -44,13 +44,33 @@ int BPF_PROG(page_cache_alloc_ret, gfp_t gfp, struct page *ret) return 0; ts = bpf_ktime_get_ns(); - bpf_map_update_elem(&birth, &ret, &ts, 0); + bpf_map_update_elem(&birth, &page, &ts, 0); __sync_fetch_and_add(&hist.unused, 1); __sync_fetch_and_add(&hist.total, 1); return 0; } +SEC("fexit/__page_cache_alloc") +int BPF_PROG(page_cache_alloc_ret, gfp_t gfp, struct page *ret) +{ + return alloc_done(ret); +} + +SEC("fexit/filemap_alloc_folio") +int BPF_PROG(filemap_alloc_folio_ret, gfp_t gfp, unsigned int order, + struct folio *ret) +{ + return alloc_done(&ret->page); +} + +SEC("fexit/filemap_alloc_folio_noprof") +int BPF_PROG(filemap_alloc_folio_noprof_ret, gfp_t gfp, unsigned int order, + struct folio *ret) +{ + return alloc_done(&ret->page); +} + SEC("fexit/do_page_cache_ra") int BPF_PROG(do_page_cache_ra_ret) { diff --git a/libbpf-tools/readahead.c b/libbpf-tools/readahead.c index df1854afffa2..37501582c7c5 100644 --- a/libbpf-tools/readahead.c +++ b/libbpf-tools/readahead.c @@ -80,6 +80,18 @@ static int readahead__set_attach_target(struct bpf_program *prog) { int err; + /* + * 56a4d67c264e ("mm/readahead: Switch to page_cache_ra_order") in v5.18 + * renamed do_page_cache_ra to page_cache_ra_order + */ + err = bpf_program__set_attach_target(prog, 0, "page_cache_ra_order"); + if (!err) + return 0; + + /* + * 8238287eadb2 ("mm/readahead: make do_page_cache_ra take a readahead_control") + * in v5.10 renamed __do_page_cache_readahead to do_page_cache_ra + */ err = bpf_program__set_attach_target(prog, 0, "do_page_cache_ra"); if (!err) return 0; @@ -94,6 +106,33 @@ static int readahead__set_attach_target(struct bpf_program *prog) return err; } +static int attach_alloc_ret(struct readahead_bpf *obj) +{ + bpf_program__set_autoload(obj->progs.page_cache_alloc_ret, false); + bpf_program__set_autoload(obj->progs.filemap_alloc_folio_ret, false); + bpf_program__set_autoload(obj->progs.filemap_alloc_folio_noprof_ret, false); + + /* + * b951aaff5035 ("mm: enable page allocation tagging") in v6.10 + * renamed filemap_alloc_folio to filemap_alloc_folio_noprof + */ + if (fentry_can_attach("filemap_alloc_folio_noprof", NULL)) + return bpf_program__set_autoload(obj->progs.filemap_alloc_folio_noprof_ret, true); + + /* + * bb3c579e25e5 ("mm/filemap: Add filemap_alloc_folio") in v5.16 + * changed __page_cache_alloc to be a wrapper of filemap_alloc_folio + */ + if (fentry_can_attach("filemap_alloc_folio", NULL)) + return bpf_program__set_autoload(obj->progs.filemap_alloc_folio_ret, true); + + if (fentry_can_attach("__page_cache_alloc", NULL)) + return bpf_program__set_autoload(obj->progs.page_cache_alloc_ret, true); + + fprintf(stderr, "failed to attach to alloc functions\n"); + return -1; +} + int main(int argc, char **argv) { static const struct argp argp = { @@ -117,10 +156,9 @@ int main(int argc, char **argv) return 1; } - /* - * Starting from v5.10-rc1 (8238287), __do_page_cache_readahead has - * renamed to do_page_cache_ra. So we specify the function dynamically. - */ + err = attach_alloc_ret(obj); + if (err) + goto cleanup; err = readahead__set_attach_target(obj->progs.do_page_cache_ra); if (err) goto cleanup;