#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/rbtree.h>
-#include <linux/rtmutex.h>
-#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/seq_file.h>
#include <linux/uaccess.h>
static int ion_buffer_alloc_dirty(struct ion_buffer *buffer);
-static bool ion_heap_drain_freelist(struct ion_heap *heap);
/* this function should only be called while dev->lock is held */
static struct ion_buffer *ion_buffer_create(struct ion_heap *heap,
struct ion_device *dev,
if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE))
goto err2;
- ion_heap_drain_freelist(heap);
+ ion_heap_freelist_drain(heap, 0);
ret = heap->ops->allocate(heap, buffer, len, align,
flags);
if (ret)
return ERR_PTR(ret);
}
-static void _ion_buffer_destroy(struct ion_buffer *buffer)
+void ion_buffer_destroy(struct ion_buffer *buffer)
{
if (WARN_ON(buffer->kmap_cnt > 0))
buffer->heap->ops->unmap_kernel(buffer->heap, buffer);
kfree(buffer);
}
-static void ion_buffer_destroy(struct kref *kref)
+static void _ion_buffer_destroy(struct kref *kref)
{
struct ion_buffer *buffer = container_of(kref, struct ion_buffer, ref);
struct ion_heap *heap = buffer->heap;
rb_erase(&buffer->node, &dev->buffers);
mutex_unlock(&dev->buffer_lock);
- if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) {
- rt_mutex_lock(&heap->lock);
- list_add(&buffer->list, &heap->free_list);
- rt_mutex_unlock(&heap->lock);
- wake_up(&heap->waitqueue);
- return;
- }
- _ion_buffer_destroy(buffer);
+ if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
+ ion_heap_freelist_add(heap, buffer);
+ else
+ ion_buffer_destroy(buffer);
}
static void ion_buffer_get(struct ion_buffer *buffer)
static int ion_buffer_put(struct ion_buffer *buffer)
{
- return kref_put(&buffer->ref, ion_buffer_destroy);
+ return kref_put(&buffer->ref, _ion_buffer_destroy);
}
static void ion_buffer_add_to_handle(struct ion_buffer *buffer)
.release = single_release,
};
-static size_t ion_heap_free_list_is_empty(struct ion_heap *heap)
+#ifdef DEBUG_HEAP_SHRINKER
+static int debug_shrink_set(void *data, u64 val)
{
- bool is_empty;
+ struct ion_heap *heap = data;
+ struct shrink_control sc;
+ int objs;
- rt_mutex_lock(&heap->lock);
- is_empty = list_empty(&heap->free_list);
- rt_mutex_unlock(&heap->lock);
+ sc.gfp_mask = -1;
+ sc.nr_to_scan = 0;
- return is_empty;
-}
-
-static int ion_heap_deferred_free(void *data)
-{
- struct ion_heap *heap = data;
+ if (!val)
+ return 0;
- while (true) {
- struct ion_buffer *buffer;
+ objs = heap->shrinker.shrink(&heap->shrinker, &sc);
+ sc.nr_to_scan = objs;
- wait_event_freezable(heap->waitqueue,
- !ion_heap_free_list_is_empty(heap));
-
- rt_mutex_lock(&heap->lock);
- if (list_empty(&heap->free_list)) {
- rt_mutex_unlock(&heap->lock);
- continue;
- }
- buffer = list_first_entry(&heap->free_list, struct ion_buffer,
- list);
- list_del(&buffer->list);
- rt_mutex_unlock(&heap->lock);
- _ion_buffer_destroy(buffer);
- }
-
- return 0;
+ heap->shrinker.shrink(&heap->shrinker, &sc);
+ return 0;
}
-static bool ion_heap_drain_freelist(struct ion_heap *heap)
+static int debug_shrink_get(void *data, u64 *val)
{
- struct ion_buffer *buffer, *tmp;
-
- if (ion_heap_free_list_is_empty(heap))
- return false;
- rt_mutex_lock(&heap->lock);
- list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) {
- list_del(&buffer->list);
- _ion_buffer_destroy(buffer);
- }
- BUG_ON(!list_empty(&heap->free_list));
- rt_mutex_unlock(&heap->lock);
+ struct ion_heap *heap = data;
+ struct shrink_control sc;
+ int objs;
+ sc.gfp_mask = -1;
+ sc.nr_to_scan = 0;
- return true;
+ objs = heap->shrinker.shrink(&heap->shrinker, &sc);
+ *val = objs;
+ return 0;
}
+DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get,
+ debug_shrink_set, "%llu\n");
+#endif
+
void ion_device_add_heap(struct ion_device *dev, struct ion_heap *heap)
{
- struct sched_param param = { .sched_priority = 0 };
-
if (!heap->ops->allocate || !heap->ops->free || !heap->ops->map_dma ||
!heap->ops->unmap_dma)
pr_err("%s: can not add heap with invalid ops struct.\n",
__func__);
- if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) {
- INIT_LIST_HEAD(&heap->free_list);
- rt_mutex_init(&heap->lock);
- init_waitqueue_head(&heap->waitqueue);
- heap->task = kthread_run(ion_heap_deferred_free, heap,
- "%s", heap->name);
- sched_setscheduler(heap->task, SCHED_IDLE, ¶m);
- if (IS_ERR(heap->task))
- pr_err("%s: creating thread for deferred free failed\n",
- __func__);
- }
+ if (heap->flags & ION_HEAP_FLAG_DEFER_FREE)
+ ion_heap_init_deferred_free(heap);
heap->dev = dev;
down_write(&dev->lock);
plist_add(&heap->node, &dev->heaps);
debugfs_create_file(heap->name, 0664, dev->debug_root, heap,
&debug_heap_fops);
+#ifdef DEBUG_HEAP_SHRINKER
+ if (heap->shrinker.shrink) {
+ char debug_name[64];
+
+ snprintf(debug_name, 64, "%s_shrink", heap->name);
+ debugfs_create_file(debug_name, 0644, dev->debug_root, heap,
+ &debug_shrink_fops);
+ }
+#endif
up_write(&dev->lock);
}
*/
#include <linux/err.h>
+#include <linux/freezer.h>
+#include <linux/kthread.h>
#include <linux/mm.h>
+#include <linux/rtmutex.h>
+#include <linux/sched.h>
#include <linux/scatterlist.h>
#include <linux/vmalloc.h>
#include "ion.h"
return ret;
}
+void ion_heap_free_page(struct ion_buffer *buffer, struct page *page,
+ unsigned int order)
+{
+ int i;
+
+ if (!ion_buffer_fault_user_mappings(buffer)) {
+ __free_pages(page, order);
+ return;
+ }
+ for (i = 0; i < (1 << order); i++)
+ __free_page(page + i);
+}
+
+void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer * buffer)
+{
+ rt_mutex_lock(&heap->lock);
+ list_add(&buffer->list, &heap->free_list);
+ heap->free_list_size += buffer->size;
+ rt_mutex_unlock(&heap->lock);
+ wake_up(&heap->waitqueue);
+}
+
+size_t ion_heap_freelist_size(struct ion_heap *heap)
+{
+ size_t size;
+
+ rt_mutex_lock(&heap->lock);
+ size = heap->free_list_size;
+ rt_mutex_unlock(&heap->lock);
+
+ return size;
+}
+
+size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size)
+{
+ struct ion_buffer *buffer, *tmp;
+ size_t total_drained = 0;
+
+ if (ion_heap_freelist_size(heap) == 0)
+ return 0;
+
+ rt_mutex_lock(&heap->lock);
+ if (size == 0)
+ size = heap->free_list_size;
+
+ list_for_each_entry_safe(buffer, tmp, &heap->free_list, list) {
+ if (total_drained >= size)
+ break;
+ list_del(&buffer->list);
+ ion_buffer_destroy(buffer);
+ heap->free_list_size -= buffer->size;
+ total_drained += buffer->size;
+ }
+ rt_mutex_unlock(&heap->lock);
+
+ return total_drained;
+}
+
+int ion_heap_deferred_free(void *data)
+{
+ struct ion_heap *heap = data;
+
+ while (true) {
+ struct ion_buffer *buffer;
+
+ wait_event_freezable(heap->waitqueue,
+ ion_heap_freelist_size(heap) > 0);
+
+ rt_mutex_lock(&heap->lock);
+ if (list_empty(&heap->free_list)) {
+ rt_mutex_unlock(&heap->lock);
+ continue;
+ }
+ buffer = list_first_entry(&heap->free_list, struct ion_buffer,
+ list);
+ list_del(&buffer->list);
+ heap->free_list_size -= buffer->size;
+ rt_mutex_unlock(&heap->lock);
+ ion_buffer_destroy(buffer);
+ }
+
+ return 0;
+}
+
+int ion_heap_init_deferred_free(struct ion_heap *heap)
+{
+ struct sched_param param = { .sched_priority = 0 };
+
+ INIT_LIST_HEAD(&heap->free_list);
+ heap->free_list_size = 0;
+ rt_mutex_init(&heap->lock);
+ init_waitqueue_head(&heap->waitqueue);
+ heap->task = kthread_run(ion_heap_deferred_free, heap,
+ "%s", heap->name);
+ sched_setscheduler(heap->task, SCHED_IDLE, ¶m);
+ if (IS_ERR(heap->task)) {
+ pr_err("%s: creating thread for deferred free failed\n",
+ __func__);
+ return PTR_RET(heap->task);
+ }
+ return 0;
+}
+
struct ion_heap *ion_heap_create(struct ion_platform_heap *heap_data)
{
struct ion_heap *heap = NULL;
#include <linux/list.h>
#include <linux/module.h>
#include <linux/slab.h>
-#include <linux/shrinker.h>
#include "ion_priv.h"
-/* #define DEBUG_PAGE_POOL_SHRINKER */
-
-static struct plist_head pools = PLIST_HEAD_INIT(pools);
-static struct shrinker shrinker;
-
struct ion_page_pool_item {
struct page *page;
struct list_head list;
ion_page_pool_free_pages(pool, page);
}
-#ifdef DEBUG_PAGE_POOL_SHRINKER
-static int debug_drop_pools_set(void *data, u64 val)
-{
- struct shrink_control sc;
- int objs;
-
- sc.gfp_mask = -1;
- sc.nr_to_scan = 0;
-
- if (!val)
- return 0;
-
- objs = shrinker.shrink(&shrinker, &sc);
- sc.nr_to_scan = objs;
-
- shrinker.shrink(&shrinker, &sc);
- return 0;
-}
-
-static int debug_drop_pools_get(void *data, u64 *val)
+static int ion_page_pool_total(struct ion_page_pool *pool, bool high)
{
- struct shrink_control sc;
- int objs;
-
- sc.gfp_mask = -1;
- sc.nr_to_scan = 0;
-
- objs = shrinker.shrink(&shrinker, &sc);
- *val = objs;
- return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(debug_drop_pools_fops, debug_drop_pools_get,
- debug_drop_pools_set, "%llu\n");
-
-static int debug_grow_pools_set(void *data, u64 val)
-{
- struct ion_page_pool *pool;
- struct page *page;
-
- plist_for_each_entry(pool, &pools, list) {
- if (val != pool->list.prio)
- continue;
- page = ion_page_pool_alloc_pages(pool);
- if (page)
- ion_page_pool_add(pool, page);
- }
-
- return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(debug_grow_pools_fops, debug_drop_pools_get,
- debug_grow_pools_set, "%llu\n");
-#endif
-
-static int ion_page_pool_total(bool high)
-{
- struct ion_page_pool *pool;
int total = 0;
- plist_for_each_entry(pool, &pools, list) {
- total += high ? (pool->high_count + pool->low_count) *
- (1 << pool->order) :
+ total += high ? (pool->high_count + pool->low_count) *
+ (1 << pool->order) :
pool->low_count * (1 << pool->order);
- }
return total;
}
-static int ion_page_pool_shrink(struct shrinker *shrinker,
- struct shrink_control *sc)
+int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
+ int nr_to_scan)
{
- struct ion_page_pool *pool;
int nr_freed = 0;
int i;
bool high;
- int nr_to_scan = sc->nr_to_scan;
- high = sc->gfp_mask & __GFP_HIGHMEM;
+ high = gfp_mask & __GFP_HIGHMEM;
if (nr_to_scan == 0)
- return ion_page_pool_total(high);
-
- plist_for_each_entry(pool, &pools, list) {
- for (i = 0; i < nr_to_scan; i++) {
- struct page *page;
-
- mutex_lock(&pool->mutex);
- if (high && pool->high_count) {
- page = ion_page_pool_remove(pool, true);
- } else if (pool->low_count) {
- page = ion_page_pool_remove(pool, false);
- } else {
- mutex_unlock(&pool->mutex);
- break;
- }
+ return ion_page_pool_total(pool, high);
+
+ for (i = 0; i < nr_to_scan; i++) {
+ struct page *page;
+
+ mutex_lock(&pool->mutex);
+ if (high && pool->high_count) {
+ page = ion_page_pool_remove(pool, true);
+ } else if (pool->low_count) {
+ page = ion_page_pool_remove(pool, false);
+ } else {
mutex_unlock(&pool->mutex);
- ion_page_pool_free_pages(pool, page);
- nr_freed += (1 << pool->order);
+ break;
}
- nr_to_scan -= i;
+ mutex_unlock(&pool->mutex);
+ ion_page_pool_free_pages(pool, page);
+ nr_freed += (1 << pool->order);
}
- return ion_page_pool_total(high);
+ return nr_freed;
}
struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order)
pool->order = order;
mutex_init(&pool->mutex);
plist_node_init(&pool->list, order);
- plist_add(&pool->list, &pools);
return pool;
}
void ion_page_pool_destroy(struct ion_page_pool *pool)
{
- plist_del(&pool->list, &pools);
kfree(pool);
}
static int __init ion_page_pool_init(void)
{
- shrinker.shrink = ion_page_pool_shrink;
- shrinker.seeks = DEFAULT_SEEKS;
- shrinker.batch = 0;
- register_shrinker(&shrinker);
-#ifdef DEBUG_PAGE_POOL_SHRINKER
- debugfs_create_file("ion_pools_shrink", 0644, NULL, NULL,
- &debug_drop_pools_fops);
- debugfs_create_file("ion_pools_grow", 0644, NULL, NULL,
- &debug_grow_pools_fops);
-#endif
return 0;
}
static void __exit ion_page_pool_exit(void)
{
- unregister_shrinker(&shrinker);
}
module_init(ion_page_pool_init);
char task_comm[TASK_COMM_LEN];
pid_t pid;
};
+void ion_buffer_destroy(struct ion_buffer *buffer);
/**
* struct ion_heap_ops - ops to operate on a given heap
* allocating. These are specified by platform data and
* MUST be unique
* @name: used for debugging
+ * @shrinker: a shrinker for the heap, if the heap caches system
+ * memory, it must define a shrinker to return it on low
+ * memory conditions, this includes system memory cached
+ * in the deferred free lists for heaps that support it
* @free_list: free list head if deferred free is used
+ * @free_list_size size of the deferred free list in bytes
* @lock: protects the free list
* @waitqueue: queue to wait on from deferred free thread
* @task: task struct of deferred free thread
unsigned long flags;
unsigned int id;
const char *name;
+ struct shrinker shrinker;
struct list_head free_list;
+ size_t free_list_size;
struct rt_mutex lock;
wait_queue_head_t waitqueue;
struct task_struct *task;
struct vm_area_struct *);
int ion_heap_buffer_zero(struct ion_buffer *buffer);
+/**
+ * ion_heap_init_deferred_free -- initialize deferred free functionality
+ * @heap: the heap
+ *
+ * If a heap sets the ION_HEAP_FLAG_DEFER_FREE flag this function will
+ * be called to setup deferred frees. Calls to free the buffer will
+ * return immediately and the actual free will occur some time later
+ */
+int ion_heap_init_deferred_free(struct ion_heap *heap);
+
+/**
+ * ion_heap_freelist_add - add a buffer to the deferred free list
+ * @heap: the heap
+ * @buffer: the buffer
+ *
+ * Adds an item to the deferred freelist.
+ */
+void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer);
+
+/**
+ * ion_heap_freelist_drain - drain the deferred free list
+ * @heap: the heap
+ * @size: ammount of memory to drain in bytes
+ *
+ * Drains the indicated amount of memory from the deferred freelist immediately.
+ * Returns the total amount freed. The total freed may be higher depending
+ * on the size of the items in the list, or lower if there is insufficient
+ * total memory on the freelist.
+ */
+size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size);
+
+/**
+ * ion_heap_freelist_size - returns the size of the freelist in bytes
+ * @heap: the heap
+ */
+size_t ion_heap_freelist_size(struct ion_heap *heap);
+
/**
* functions for creating and destroying the built in ion heaps.
struct list_head high_items;
struct list_head low_items;
struct mutex mutex;
- void *(*alloc)(struct ion_page_pool *pool);
- void (*free)(struct ion_page_pool *pool, struct page *page);
gfp_t gfp_mask;
unsigned int order;
struct plist_node list;
void *ion_page_pool_alloc(struct ion_page_pool *);
void ion_page_pool_free(struct ion_page_pool *, struct page *);
+/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool
+ * @pool: the pool
+ * @gfp_mask: the memory type to reclaim
+ * @nr_to_scan: number of items to shrink in pages
+ *
+ * returns the number of items freed in pages
+ */
+int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask,
+ int nr_to_scan);
+
#endif /* _ION_PRIV_H */
.map_user = ion_heap_map_user,
};
+static int ion_system_heap_shrink(struct shrinker *shrinker,
+ struct shrink_control *sc) {
+
+ struct ion_heap *heap = container_of(shrinker, struct ion_heap,
+ shrinker);
+ struct ion_system_heap *sys_heap = container_of(heap,
+ struct ion_system_heap,
+ heap);
+ int nr_total = 0;
+ int nr_freed = 0;
+ int i;
+
+ if (sc->nr_to_scan == 0)
+ goto end;
+
+ /* shrink the free list first, no point in zeroing the memory if
+ we're just going to reclaim it */
+ nr_freed += ion_heap_freelist_drain(heap, sc->nr_to_scan * PAGE_SIZE) /
+ PAGE_SIZE;
+
+ if (nr_freed >= sc->nr_to_scan)
+ goto end;
+
+ for (i = 0; i < num_orders; i++) {
+ struct ion_page_pool *pool = sys_heap->pools[i];
+
+ nr_freed += ion_page_pool_shrink(pool, sc->gfp_mask,
+ sc->nr_to_scan);
+ if (nr_freed >= sc->nr_to_scan)
+ break;
+ }
+
+end:
+ /* total number of items is whatever the page pools are holding
+ plus whatever's in the freelist */
+ for (i = 0; i < num_orders; i++) {
+ struct ion_page_pool *pool = sys_heap->pools[i];
+ nr_total += ion_page_pool_shrink(pool, sc->gfp_mask, 0);
+ }
+ nr_total += ion_heap_freelist_size(heap) / PAGE_SIZE;
+ return nr_total;
+
+}
+
static int ion_system_heap_debug_show(struct ion_heap *heap, struct seq_file *s,
void *unused)
{
goto err_create_pool;
heap->pools[i] = pool;
}
+
+ heap->heap.shrinker.shrink = ion_system_heap_shrink;
+ heap->heap.shrinker.seeks = DEFAULT_SEEKS;
+ heap->heap.shrinker.batch = 0;
+ register_shrinker(&heap->heap.shrinker);
heap->heap.debug_show = ion_system_heap_debug_show;
return &heap->heap;
err_create_pool:
projects / openwrt / staging / blogic.git / commitdiff