#include <linux/pfn.h>
#include <linux/rbtree.h>
#include <linux/slab.h>
+#include <linux/spinlock.h>
#include <linux/vmalloc.h>
#include <linux/workqueue.h>
static int pcpu_reserved_chunk_limit;
/*
- * One mutex to rule them all.
- *
- * The following mutex is grabbed in the outermost public alloc/free
- * interface functions and released only when the operation is
- * complete. As such, every function in this file other than the
- * outermost functions are called under pcpu_mutex.
- *
- * It can easily be switched to use spinlock such that only the area
- * allocation and page population commit are protected with it doing
- * actual [de]allocation without holding any lock. However, given
- * what this allocator does, I think it's better to let them run
- * sequentially.
+ * Synchronization rules.
+ *
+ * There are two locks - pcpu_alloc_mutex and pcpu_lock. The former
+ * protects allocation/reclaim paths, chunks and chunk->page arrays.
+ * The latter is a spinlock and protects the index data structures -
+ * chunk slots, rbtree, chunks and area maps in chunks.
+ *
+ * During allocation, pcpu_alloc_mutex is kept locked all the time and
+ * pcpu_lock is grabbed and released as necessary. All actual memory
+ * allocations are done using GFP_KERNEL with pcpu_lock released.
+ *
+ * Free path accesses and alters only the index data structures, so it
+ * can be safely called from atomic context. When memory needs to be
+ * returned to the system, free path schedules reclaim_work which
+ * grabs both pcpu_alloc_mutex and pcpu_lock, unlinks chunks to be
+ * reclaimed, release both locks and frees the chunks. Note that it's
+ * necessary to grab both locks to remove a chunk from circulation as
+ * allocation path might be referencing the chunk with only
+ * pcpu_alloc_mutex locked.
*/
-static DEFINE_MUTEX(pcpu_mutex);
+static DEFINE_MUTEX(pcpu_alloc_mutex); /* protects whole alloc and reclaim */
+static DEFINE_SPINLOCK(pcpu_lock); /* protects index data structures */
static struct list_head *pcpu_slot __read_mostly; /* chunk list slots */
static struct rb_root pcpu_addr_root = RB_ROOT; /* chunks by address */
* kzalloc() is used; otherwise, vmalloc() is used. The returned
* memory is always zeroed.
*
+ * CONTEXT:
+ * Does GFP_KERNEL allocation.
+ *
* RETURNS:
* Pointer to the allocated area on success, NULL on failure.
*/
* New slot according to the changed state is determined and @chunk is
* moved to the slot. Note that the reserved chunk is never put on
* chunk slots.
+ *
+ * CONTEXT:
+ * pcpu_lock.
*/
static void pcpu_chunk_relocate(struct pcpu_chunk *chunk, int oslot)
{
* searchs for the chunk with the highest start address which isn't
* beyond @addr.
*
+ * CONTEXT:
+ * pcpu_lock.
+ *
* RETURNS:
* The address of the found chunk.
*/
* @new: chunk to insert
*
* Insert @new into address rb tree.
+ *
+ * CONTEXT:
+ * pcpu_lock.
*/
static void pcpu_chunk_addr_insert(struct pcpu_chunk *new)
{
* A single allocation can split an area into three areas, so this
* function makes sure that @chunk->map has at least two extra slots.
*
+ * CONTEXT:
+ * pcpu_alloc_mutex, pcpu_lock. pcpu_lock is released and reacquired
+ * if area map is extended.
+ *
* RETURNS:
* 0 if noop, 1 if successfully extended, -errno on failure.
*/
if (chunk->map_alloc >= chunk->map_used + 2)
return 0;
+ spin_unlock_irq(&pcpu_lock);
+
new_alloc = PCPU_DFL_MAP_ALLOC;
while (new_alloc < chunk->map_used + 2)
new_alloc *= 2;
new = pcpu_mem_alloc(new_alloc * sizeof(new[0]));
- if (!new)
+ if (!new) {
+ spin_lock_irq(&pcpu_lock);
return -ENOMEM;
+ }
+
+ /*
+ * Acquire pcpu_lock and switch to new area map. Only free
+ * could have happened inbetween, so map_used couldn't have
+ * grown.
+ */
+ spin_lock_irq(&pcpu_lock);
+ BUG_ON(new_alloc < chunk->map_used + 2);
size = chunk->map_alloc * sizeof(chunk->map[0]);
memcpy(new, chunk->map, size);
* is inserted after the target block.
*
* @chunk->map must have enough free slots to accomodate the split.
+ *
+ * CONTEXT:
+ * pcpu_lock.
*/
static void pcpu_split_block(struct pcpu_chunk *chunk, int i,
int head, int tail)
*
* @chunk->map must have at least two free slots.
*
+ * CONTEXT:
+ * pcpu_lock.
+ *
* RETURNS:
* Allocated offset in @chunk on success, -1 if no matching area is
* found.
* Free area starting from @freeme to @chunk. Note that this function
* only modifies the allocation map. It doesn't depopulate or unmap
* the area.
+ *
+ * CONTEXT:
+ * pcpu_lock.
*/
static void pcpu_free_area(struct pcpu_chunk *chunk, int freeme)
{
* For each cpu, depopulate and unmap pages [@page_start,@page_end)
* from @chunk. If @flush is true, vcache is flushed before unmapping
* and tlb after.
+ *
+ * CONTEXT:
+ * pcpu_alloc_mutex.
*/
static void pcpu_depopulate_chunk(struct pcpu_chunk *chunk, int off, int size,
bool flush)
*
* For each cpu, populate and map pages [@page_start,@page_end) into
* @chunk. The area is cleared on return.
+ *
+ * CONTEXT:
+ * pcpu_alloc_mutex, does GFP_KERNEL allocation.
*/
static int pcpu_populate_chunk(struct pcpu_chunk *chunk, int off, int size)
{
* @align: alignment of area (max PAGE_SIZE)
* @reserved: allocate from the reserved chunk if available
*
- * Allocate percpu area of @size bytes aligned at @align. Might
- * sleep. Might trigger writeouts.
+ * Allocate percpu area of @size bytes aligned at @align.
+ *
+ * CONTEXT:
+ * Does GFP_KERNEL allocation.
*
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
static void *pcpu_alloc(size_t size, size_t align, bool reserved)
{
- void *ptr = NULL;
struct pcpu_chunk *chunk;
int slot, off;
return NULL;
}
- mutex_lock(&pcpu_mutex);
+ mutex_lock(&pcpu_alloc_mutex);
+ spin_lock_irq(&pcpu_lock);
/* serve reserved allocations from the reserved chunk if available */
if (reserved && pcpu_reserved_chunk) {
chunk = pcpu_reserved_chunk;
if (size > chunk->contig_hint ||
pcpu_extend_area_map(chunk) < 0)
- goto out_unlock;
+ goto fail_unlock;
off = pcpu_alloc_area(chunk, size, align);
if (off >= 0)
goto area_found;
- goto out_unlock;
+ goto fail_unlock;
}
+restart:
/* search through normal chunks */
for (slot = pcpu_size_to_slot(size); slot < pcpu_nr_slots; slot++) {
list_for_each_entry(chunk, &pcpu_slot[slot], list) {
if (size > chunk->contig_hint)
continue;
- if (pcpu_extend_area_map(chunk) < 0)
- goto out_unlock;
+
+ switch (pcpu_extend_area_map(chunk)) {
+ case 0:
+ break;
+ case 1:
+ goto restart; /* pcpu_lock dropped, restart */
+ default:
+ goto fail_unlock;
+ }
+
off = pcpu_alloc_area(chunk, size, align);
if (off >= 0)
goto area_found;
}
/* hmmm... no space left, create a new chunk */
+ spin_unlock_irq(&pcpu_lock);
+
chunk = alloc_pcpu_chunk();
if (!chunk)
- goto out_unlock;
+ goto fail_unlock_mutex;
+
+ spin_lock_irq(&pcpu_lock);
pcpu_chunk_relocate(chunk, -1);
pcpu_chunk_addr_insert(chunk);
-
- off = pcpu_alloc_area(chunk, size, align);
- if (off < 0)
- goto out_unlock;
+ goto restart;
area_found:
+ spin_unlock_irq(&pcpu_lock);
+
/* populate, map and clear the area */
if (pcpu_populate_chunk(chunk, off, size)) {
+ spin_lock_irq(&pcpu_lock);
pcpu_free_area(chunk, off);
- goto out_unlock;
+ goto fail_unlock;
}
- ptr = __addr_to_pcpu_ptr(chunk->vm->addr + off);
-out_unlock:
- mutex_unlock(&pcpu_mutex);
- return ptr;
+ mutex_unlock(&pcpu_alloc_mutex);
+
+ return __addr_to_pcpu_ptr(chunk->vm->addr + off);
+
+fail_unlock:
+ spin_unlock_irq(&pcpu_lock);
+fail_unlock_mutex:
+ mutex_unlock(&pcpu_alloc_mutex);
+ return NULL;
}
/**
* Allocate percpu area of @size bytes aligned at @align. Might
* sleep. Might trigger writeouts.
*
+ * CONTEXT:
+ * Does GFP_KERNEL allocation.
+ *
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
* percpu area if arch has set it up; otherwise, allocation is served
* from the same dynamic area. Might sleep. Might trigger writeouts.
*
+ * CONTEXT:
+ * Does GFP_KERNEL allocation.
+ *
* RETURNS:
* Percpu pointer to the allocated area on success, NULL on failure.
*/
* @work: unused
*
* Reclaim all fully free chunks except for the first one.
+ *
+ * CONTEXT:
+ * workqueue context.
*/
static void pcpu_reclaim(struct work_struct *work)
{
struct list_head *head = &pcpu_slot[pcpu_nr_slots - 1];
struct pcpu_chunk *chunk, *next;
- mutex_lock(&pcpu_mutex);
+ mutex_lock(&pcpu_alloc_mutex);
+ spin_lock_irq(&pcpu_lock);
list_for_each_entry_safe(chunk, next, head, list) {
WARN_ON(chunk->immutable);
list_move(&chunk->list, &todo);
}
- mutex_unlock(&pcpu_mutex);
+ spin_unlock_irq(&pcpu_lock);
+ mutex_unlock(&pcpu_alloc_mutex);
list_for_each_entry_safe(chunk, next, &todo, list) {
pcpu_depopulate_chunk(chunk, 0, pcpu_unit_size, false);
* free_percpu - free percpu area
* @ptr: pointer to area to free
*
- * Free percpu area @ptr. Might sleep.
+ * Free percpu area @ptr.
+ *
+ * CONTEXT:
+ * Can be called from atomic context.
*/
void free_percpu(void *ptr)
{
void *addr = __pcpu_ptr_to_addr(ptr);
struct pcpu_chunk *chunk;
+ unsigned long flags;
int off;
if (!ptr)
return;
- mutex_lock(&pcpu_mutex);
+ spin_lock_irqsave(&pcpu_lock, flags);
chunk = pcpu_chunk_addr_search(addr);
off = addr - chunk->vm->addr;
}
}
- mutex_unlock(&pcpu_mutex);
+ spin_unlock_irqrestore(&pcpu_lock, flags);
}
EXPORT_SYMBOL_GPL(free_percpu);
projects / openwrt / staging / blogic.git / commitdiff