bpf: Use kvmalloc for map keys in syscalls

Same as previous patch but for the keys. memdup_bpfptr is renamed
to kvmemdup_bpfptr (and converted to kvmalloc).

Signed-off-by: Stanislav Fomichev <sdf@google.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Song Liu <songliubraving@fb.com>
Link: https://lore.kernel.org/bpf/20210818235216.1159202-2-sdf@google.com

diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h
index 5cdeab497cb3b140bb3dfe55bb14579d7a996094..546e27fc6d462409450c195f22722826cf1a7988 100644 (file)
--- a/include/linux/bpfptr.h
+++ b/include/linux/bpfptr.h
@@ -62,9 +62,17 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, size_t offset,
        return copy_to_sockptr_offset((sockptr_t) dst, offset, src, size);
 }
 
-static inline void *memdup_bpfptr(bpfptr_t src, size_t len)
+static inline void *kvmemdup_bpfptr(bpfptr_t src, size_t len)
 {
-       return memdup_sockptr((sockptr_t) src, len);
+       void *p = kvmalloc(len, GFP_USER | __GFP_NOWARN);
+
+       if (!p)
+               return ERR_PTR(-ENOMEM);
+       if (copy_from_bpfptr(p, src, len)) {
+               kvfree(p);
+               return ERR_PTR(-EFAULT);
+       }
+       return p;
 }
 
 static inline long strncpy_from_bpfptr(char *dst, bpfptr_t src, size_t count)

diff --git a/kernel/bpf/syscall.c b/kernel/bpf/syscall.c
index 075f650d297ad247d428ff0e426b38ec61ff2d14..4e50c0bfdb7d38e1c929a499c0688f59b9caf618 100644 (file)
--- a/kernel/bpf/syscall.c
+++ b/kernel/bpf/syscall.c
@@ -1013,7 +1013,7 @@ int __weak bpf_stackmap_copy(struct bpf_map *map, void *key, void *value)
 static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 {
        if (key_size)
-               return memdup_user(ukey, key_size);
+               return vmemdup_user(ukey, key_size);
 
        if (ukey)
                return ERR_PTR(-EINVAL);
@@ -1024,7 +1024,7 @@ static void *__bpf_copy_key(void __user *ukey, u64 key_size)
 static void *___bpf_copy_key(bpfptr_t ukey, u64 key_size)
 {
        if (key_size)
-               return memdup_bpfptr(ukey, key_size);
+               return kvmemdup_bpfptr(ukey, key_size);
 
        if (!bpfptr_is_null(ukey))
                return ERR_PTR(-EINVAL);
@@ -1093,7 +1093,7 @@ static int map_lookup_elem(union bpf_attr *attr)
 free_value:
        kvfree(value);
 free_key:
-       kfree(key);
+       kvfree(key);
 err_put:
        fdput(f);
        return err;
@@ -1153,7 +1153,7 @@ static int map_update_elem(union bpf_attr *attr, bpfptr_t uattr)
 free_value:
        kvfree(value);
 free_key:
-       kfree(key);
+       kvfree(key);
 err_put:
        fdput(f);
        return err;
@@ -1205,7 +1205,7 @@ static int map_delete_elem(union bpf_attr *attr)
        bpf_enable_instrumentation();
        maybe_wait_bpf_programs(map);
 out:
-       kfree(key);
+       kvfree(key);
 err_put:
        fdput(f);
        return err;
@@ -1247,7 +1247,7 @@ static int map_get_next_key(union bpf_attr *attr)
        }
 
        err = -ENOMEM;
-       next_key = kmalloc(map->key_size, GFP_USER);
+       next_key = kvmalloc(map->key_size, GFP_USER);
        if (!next_key)
                goto free_key;
 
@@ -1270,9 +1270,9 @@ out:
        err = 0;
 
 free_next_key:
-       kfree(next_key);
+       kvfree(next_key);
 free_key:
-       kfree(key);
+       kvfree(key);
 err_put:
        fdput(f);
        return err;
@@ -1299,7 +1299,7 @@ int generic_map_delete_batch(struct bpf_map *map,
        if (!max_count)
                return 0;
 
-       key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
+       key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
        if (!key)
                return -ENOMEM;
 
@@ -1326,7 +1326,7 @@ int generic_map_delete_batch(struct bpf_map *map,
        if (copy_to_user(&uattr->batch.count, &cp, sizeof(cp)))
                err = -EFAULT;
 
-       kfree(key);
+       kvfree(key);
        return err;
 }
 
@@ -1357,13 +1357,13 @@ int generic_map_update_batch(struct bpf_map *map,
        if (!max_count)
                return 0;
 
-       key = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
+       key = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
        if (!key)
                return -ENOMEM;
 
        value = kvmalloc(value_size, GFP_USER | __GFP_NOWARN);
        if (!value) {
-               kfree(key);
+               kvfree(key);
                return -ENOMEM;
        }
 
@@ -1385,7 +1385,7 @@ int generic_map_update_batch(struct bpf_map *map,
                err = -EFAULT;
 
        kvfree(value);
-       kfree(key);
+       kvfree(key);
        return err;
 }
 
@@ -1419,13 +1419,13 @@ int generic_map_lookup_batch(struct bpf_map *map,
        if (put_user(0, &uattr->batch.count))
                return -EFAULT;
 
-       buf_prevkey = kmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
+       buf_prevkey = kvmalloc(map->key_size, GFP_USER | __GFP_NOWARN);
        if (!buf_prevkey)
                return -ENOMEM;
 
        buf = kvmalloc(map->key_size + value_size, GFP_USER | __GFP_NOWARN);
        if (!buf) {
-               kfree(buf_prevkey);
+               kvfree(buf_prevkey);
                return -ENOMEM;
        }
 
@@ -1485,7 +1485,7 @@ int generic_map_lookup_batch(struct bpf_map *map,
                err = -EFAULT;
 
 free_buf:
-       kfree(buf_prevkey);
+       kvfree(buf_prevkey);
        kvfree(buf);
        return err;
 }
@@ -1575,7 +1575,7 @@ static int map_lookup_and_delete_elem(union bpf_attr *attr)
 free_value:
        kvfree(value);
 free_key:
-       kfree(key);
+       kvfree(key);
 err_put:
        fdput(f);
        return err;