Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Narrow bounds on kernel malloc allocations. #2261

Open
wants to merge 9 commits into
base: dev
Choose a base branch
from
138 changes: 115 additions & 23 deletions sys/kern/kern_malloc.c
Original file line number Diff line number Diff line change
@@ -110,7 +110,7 @@
#define MALLOC_DEBUG 1
#endif

#if defined(KASAN) || defined(DEBUG_REDZONE)
#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__)

Check warning on line 113 in sys/kern/kern_malloc.c

GitHub Actions / Style Checker

line over 80 characters
#define DEBUG_REDZONE_ARG_DEF , unsigned long osize
#define DEBUG_REDZONE_ARG , osize
#else
@@ -633,14 +633,24 @@
size = roundup(size, PAGE_SIZE);
va = kmem_malloc_domainset(policy, size, flags);
if (va != NULL) {
/* Use low bits unused for slab pointers. */
vsetzoneslab((uintptr_t)va, NULL, MALLOC_LARGE_SLAB(size));
/*
* Use low bits unused for slab pointers.
* XXX-AM: Abuse the zone pointer to stash the original pointer.
* On CHERI systems, this is necessary to recover the bounds of
* the original allocation.
*/
vsetzoneslab((uintptr_t)va, va, MALLOC_LARGE_SLAB(size));
uma_total_inc(size);
#ifdef __CHERI_PURE_CAPABILITY__
KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size),
va = cheri_setbounds(va, osize);
KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize),
("Invalid bounds: expected %zx found %zx",
(size_t)CHERI_REPRESENTABLE_LENGTH(size),
(size_t)CHERI_REPRESENTABLE_LENGTH(osize),
(size_t)cheri_getlen(va)));
if ((flags & M_ZERO) == 0 && osize < cheri_getlen(va)) {
bzero((void *)((uintptr_t)va + osize),
cheri_getlen(va) - osize);
}
#endif
}
malloc_type_allocated(mtp, va, va == NULL ? 0 : size);
@@ -659,10 +669,32 @@
static void
free_large(void *addr, size_t size)
{

kmem_free(addr, size);
uma_total_dec(size);
}

#ifdef __CHERI_PURE_CAPABILITY__
/*
* Recover original bounds for a malloc_large allocation.
*
* Error conditions:
* - Clear the capability tag if the given capability is not a subset of
* the saved object capability.
*/
static void *
malloc_large_grow_bounds(void *saved_ptr, void *addr)
{
KASSERT(cheri_is_subset(saved_ptr, addr),
("Unexpected malloc_large grow bounds: pointer %#p is "
"not derived from %#p", addr, saved_ptr));

addr = cheri_setaddress(saved_ptr, (vm_offset_t)addr);
addr = cheri_setboundsexact(addr, cheri_getlen(saved_ptr));
KASSERT(cheri_gettag(addr),
("Failed to recover malloc_large bounds for %#p", addr));
return (addr);
}
#endif
#undef IS_MALLOC_LARGE
#undef MALLOC_LARGE_SLAB

@@ -680,7 +712,7 @@
int indx;
caddr_t va;
uma_zone_t zone;
#if defined(DEBUG_REDZONE) || defined(KASAN)
#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__)

Check warning on line 715 in sys/kern/kern_malloc.c

GitHub Actions / Style Checker

line over 80 characters
unsigned long osize = size;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I admit a temptation to slap a __diagused on at least the local variables and make them unconditional.

#endif

@@ -722,10 +754,17 @@
kasan_mark((void *)va, osize, size, KASAN_MALLOC_REDZONE);
#endif
#ifdef __CHERI_PURE_CAPABILITY__
KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(size),
("Invalid bounds: expected %zx found %zx",
(size_t)CHERI_REPRESENTABLE_LENGTH(size),
(size_t)cheri_getlen(va)));
/* Intentionally inexect bounds allow for non-representable sizes */
va = cheri_setbounds(va, osize);
KASSERT(cheri_gettag(va),
("Invalid malloc: %#p requested size %zx", va, osize));
KASSERT(cheri_getlen(va) <= CHERI_REPRESENTABLE_LENGTH(osize),
("Invalid malloc: %#p expected length %zx", va,
(size_t)CHERI_REPRESENTABLE_LENGTH(osize)));
if (va != NULL && (flags & M_ZERO) == 0 && osize < cheri_getlen(va)) {
bzero((void *)((uintptr_t)va + osize),
cheri_getlen(va) - osize);
}
#endif
return ((void *) va);
}
@@ -765,7 +804,7 @@
caddr_t va;
int domain;
int indx;
#if defined(KASAN) || defined(DEBUG_REDZONE)
#if defined(KASAN) || defined(DEBUG_REDZONE) || defined(__CHERI_PURE_CAPABILITY__)

Check warning on line 807 in sys/kern/kern_malloc.c

GitHub Actions / Style Checker

line over 80 characters
unsigned long osize = size;
#endif

@@ -781,6 +820,11 @@
return (malloc_large(size, mtp, DOMAINSET_RR(), flags
DEBUG_REDZONE_ARG));

/* XXX-AM: see malloc() */
if (size != CHERI_REPRESENTABLE_LENGTH(size)) {
flags |= M_ZERO;
}

vm_domainset_iter_policy_init(&di, ds, &domain, &flags);
do {
va = malloc_domain(&size, &indx, mtp, domain, flags);
@@ -803,6 +847,15 @@
kmsan_mark(va, size, KMSAN_STATE_UNINIT);
kmsan_orig(va, size, KMSAN_TYPE_MALLOC, KMSAN_RET_ADDR);
}
#endif
#ifdef __CHERI_PURE_CAPABILITY__
/* Intentionally inexect bounds allow for non-representable sizes */
va = cheri_setbounds(va, osize);
KASSERT(cheri_gettag(va),
("Invalid malloc: %#p requested size %zx", va, osize));
KASSERT(cheri_getlen(va) == CHERI_REPRESENTABLE_LENGTH(osize),
("Invalid malloc: %#p expected length %zx", va,
(size_t)CHERI_REPRESENTABLE_LENGTH(osize)));
#endif
return (va);
}
@@ -821,7 +874,7 @@
malloc_domainset_exec(size_t size, struct malloc_type *mtp, struct domainset *ds,
int flags)
{
#if defined(DEBUG_REDZONE) || defined(KASAN)
#if defined(DEBUG_REDZONE) || defined(KASAN) || defined(__CHERI_PURE_CAPABILITY__)

Check warning on line 877 in sys/kern/kern_malloc.c

GitHub Actions / Style Checker

line over 80 characters
unsigned long osize = size;
#endif
#ifdef MALLOC_DEBUG
@@ -996,11 +1049,10 @@
case __predict_true(SLAB_COOKIE_SLAB_PTR):
size = zone->uz_size;
#ifdef __CHERI_PURE_CAPABILITY__
if (__predict_false(cheri_getlen(addr) !=
CHERI_REPRESENTABLE_LENGTH(size)))
panic("Invalid bounds: expected %zx found %zx",
(size_t)CHERI_REPRESENTABLE_LENGTH(size),
cheri_getlen(addr));
addr = uma_zgrow_bounds(zone, addr);
KASSERT(cheri_getlen(addr) == size,
("vtozoneslab disagrees with uma_zgrow_bounds: %zx != %zx",
cheri_getlen(addr), size));
#endif
#if defined(INVARIANTS) && !defined(KASAN)
free_save_type(addr, mtp, size);
@@ -1014,11 +1066,10 @@
case SLAB_COOKIE_MALLOC_LARGE:
size = malloc_large_size(slab);
#ifdef __CHERI_PURE_CAPABILITY__
if (__predict_false(cheri_getlen(addr) !=
CHERI_REPRESENTABLE_LENGTH(size)))
panic("Invalid bounds: expected %zx found %zx",
(size_t)CHERI_REPRESENTABLE_LENGTH(size),
cheri_getlen(addr));
addr = malloc_large_grow_bounds(zone, addr);
KASSERT(cheri_getlen(addr) == size,
("malloc_large object bounds mismatch: %zx != %zx",
cheri_getlen(addr), size));
#endif
if (dozero) {
kasan_mark(addr, size, size, 0);
@@ -1074,6 +1125,9 @@
#endif
unsigned long alloc;
void *newaddr;
#ifdef __CHERI_PURE_CAPABILITY__
size_t olength;
#endif

KASSERT(mtp->ks_version == M_VERSION,
("realloc: bad malloc type version"));
@@ -1088,6 +1142,7 @@
panic("Expect valid capability");
if (__predict_false(cheri_getsealed(addr)))
panic("Expect unsealed capability");
olength = cheri_getlen(addr);
#endif

/*
@@ -1113,9 +1168,25 @@
switch (GET_SLAB_COOKIE(slab)) {
case __predict_true(SLAB_COOKIE_SLAB_PTR):
alloc = zone->uz_size;
#ifdef __CHERI_PURE_CAPABILITY__
if (size > olength) {
addr = uma_zgrow_bounds(zone, addr);
KASSERT(cheri_getlen(addr) == alloc,
("realloc mismatch uma_zgrow_bounds: %zx != %zx",
cheri_getlen(addr), alloc));
}
#endif
break;
case SLAB_COOKIE_MALLOC_LARGE:
alloc = malloc_large_size(slab);
#ifdef __CHERI_PURE_CAPABILITY__
if (size > olength) {
addr = malloc_large_grow_bounds(zone, addr);
KASSERT(cheri_getlen(addr) == size,
("realloc large object bounds mismatch: %zx != %zx",
cheri_getlen(addr), size));
}
#endif
break;
default:
#ifdef INVARIANTS
@@ -1129,7 +1200,19 @@
if (size <= alloc &&
(size > (alloc >> REALLOC_FRACTION) || alloc == MINALLOCSIZE)) {
kasan_mark((void *)addr, size, alloc, KASAN_MALLOC_REDZONE);
#ifdef __CHERI_PURE_CAPABILITY__
addr = cheri_setbounds(addr, size);
/*
* Zero only the non-representable portion of allocation
* that was not reachable from the original capability.
*/
if (size > olength) {
bzero((void *)((uintptr_t)addr + olength),
cheri_getlen(addr) - olength);
}
#else
return (addr);
#endif
}
#endif /* !DEBUG_REDZONE */

@@ -1142,7 +1225,16 @@
* valid before performing the copy.
*/
kasan_mark(addr, alloc, alloc, 0);
#ifdef __CHERI_PURE_CAPABILITY__
/*
* We have unbounded addr here, need to avoid copying
* past the original length.
* XXX-AM: is it worth it re-setting bounds on addr?
*/
bcopy(addr, newaddr, min(size, olength));
#else
bcopy(addr, newaddr, min(size, alloc));
#endif
free(addr, mtp);
return (newaddr);
}
25 changes: 25 additions & 0 deletions sys/vm/uma.h
Original file line number Diff line number Diff line change
@@ -414,6 +414,31 @@ uma_zfree_pcpu(uma_zone_t zone, void *item)
uma_zfree_pcpu_arg(zone, item, NULL);
}

#ifdef __CHERI_PURE_CAPABILITY__
/*
* Reset bounds on an item to the full item size.
*
* This is intended for nested allocators to recover items capabilities before
* free-ing them.
* Note that UMA maintains the invariant that a full item capability must be
* passed to uma_zfree functions.
* XXX-AM: Should we instead relax that invariant (optionally via a zone flag),
* or add a different set of uma_zfree_narrow functions that allow for partial
* item capabilities?
* Given that this is a privileged operation, I would like to maintain the
* intentionality of the operation here.
*
* Error conditions:
* - Clear the item capability tag if the item does not belong
* to the given zone.
* - Clear the item capability tag if the item does not belong
* to the slab from vtoslab.
*/
void *uma_zgrow_bounds(uma_zone_t zone, void *item);
#else
#define uma_zgrow_bounds(zone, item) (item)
#endif

/*
* Wait until the specified zone can allocate an item.
*/
34 changes: 34 additions & 0 deletions sys/vm/uma_core.c
Original file line number Diff line number Diff line change
@@ -5036,6 +5036,40 @@ zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip)
zone_free_limit(zone, 1);
}

#ifdef __CHERI_PURE_CAPABILITY__
/* See uma.h */
void *
uma_zgrow_bounds(uma_zone_t zone, void *item)
{
uma_keg_t keg = zone->uz_keg;
uma_slab_t slab;
int index;

KASSERT((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0,
("Purecap kernel UMA zone missing UMA_ZFLAG_VTOSLAB"));
KASSERT((zone->uz_flags & (UMA_ZONE_PCPU | UMA_ZONE_SMR)) == 0,
("Can not grow bounds on PCPU and SMR zones"));

/*
* XXX-AM: It should be safe to only check the index range
* with INVARIANTS. If an item does not belong to the slab the computed
* index will be garbage but it will fail setbounds with the slab_data()
* capability.
*/
slab = vtoslab((vm_offset_t)item);
index = slab_item_index(slab, keg, item);
KASSERT(index >= 0 && index < keg->uk_ipers,
("Invalid item index %d for slab %#p from item %#p for zone %s",
index, slab, item, zone->uz_name));
item = cheri_setboundsexact(slab_item(slab, keg, index),
keg->uk_size);
KASSERT(cheri_gettag(item),
("Failed to recover item %#p bounds for zone %s",
item, zone->uz_name));
return (item);
}
#endif

/* See uma.h */
int
uma_zone_set_max(uma_zone_t zone, int nitems)
2 changes: 2 additions & 0 deletions sys/vm/uma_int.h
Original file line number Diff line number Diff line change
@@ -403,6 +403,7 @@

typedef struct uma_hash_slab * uma_hash_slab_t;

#ifdef _KERNEL
static inline uma_hash_slab_t
slab_tohashslab(uma_slab_t slab)
{
@@ -437,9 +438,10 @@
data = (uintptr_t)slab_data(slab, keg);
return (((ptraddr_t)item - (ptraddr_t)data) / keg->uk_rsize);
}
#endif /* _KERNEL */

STAILQ_HEAD(uma_bucketlist, uma_bucket);

Check warning on line 444 in sys/vm/uma_int.h

GitHub Actions / Style Checker

Missing Signed-off-by: line
struct uma_zone_domain {
struct uma_bucketlist uzd_buckets; /* full buckets */
uma_bucket_t uzd_cross; /* Fills from cross buckets. */