perf: Support overwrite mode for the AUX area

This adds support for overwrite mode in the AUX area, which means "keep
collecting data till you're stopped", turning AUX area into a circular
buffer, where new data overwrites old data. It does not depend on data
buffer's overwrite mode, so that it doesn't lose sideband data that is
instrumental for processing AUX data.

Overwrite mode is enabled at mapping AUX area read only. Even though
aux_tail in the buffer's user page might be user writable, it will be
ignored in this mode.

A PERF_RECORD_AUX with PERF_AUX_FLAG_OVERWRITE set is written to the perf
data stream every time an event writes new data to the AUX area. The pmu
driver might not be able to infer the exact beginning of the new data in
each snapshot, some drivers will only provide the tail, which is
aux_offset + aux_size in the AUX record. Consumer has to be able to tell
the new data from the old one, for example, by means of time stamps if
such are provided in the trace.

Consumer is also responsible for disabling any events that might write
to the AUX area (thus potentially racing with the consumer) before
collecting the data.

Signed-off-by: Alexander Shishkin <alexander.shishkin@linux.intel.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kaixu Xia <kaixu.xia@linaro.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Paul Mackerras <paulus@samba.org>
Cc: Robert Richter <rric@kernel.org>
Cc: Stephane Eranian <eranian@google.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: acme@infradead.org
Cc: adrian.hunter@intel.com
Cc: kan.liang@intel.com
Cc: markus.t.metzger@intel.com
Cc: mathieu.poirier@linaro.org
Link: http://lkml.kernel.org/r/1421237903-181015-9-git-send-email-alexander.shishkin@linux.intel.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 8904ad3a850b1368e52a8502a8b59d1a9e08791f..29ef2f73bb4a4fb0566347179871938e773ace79 100644 (file)
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -803,6 +803,7 @@ enum perf_callchain_context {
  * PERF_RECORD_AUX::flags bits
  */
 #define PERF_AUX_FLAG_TRUNCATED                0x01    /* record was truncated to fit */
+#define PERF_AUX_FLAG_OVERWRITE                0x02    /* snapshot from overwrite mode */
 
 #define PERF_FLAG_FD_NO_GROUP          (1UL << 0)
 #define PERF_FLAG_FD_OUTPUT            (1UL << 1)

diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index b701ebc32570ab41dbb8254d13186fd0cb452ff4..ffd51d9f59454e77b369209c55c20a42f328be2b 100644 (file)
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -40,6 +40,7 @@ struct ring_buffer {
        local_t                         aux_nest;
        unsigned long                   aux_pgoff;
        int                             aux_nr_pages;
+       int                             aux_overwrite;
        atomic_t                        aux_mmap_count;
        unsigned long                   aux_mmap_locked;
        void                            (*free_aux)(void *);

diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 0cc7b0f390585130ef0608f3da43800e13001d69..67b328337a416360ffd116e30cab360da108b65a 100644 (file)
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -283,26 +283,33 @@ void *perf_aux_output_begin(struct perf_output_handle *handle,
                goto err_put;
 
        aux_head = local_read(&rb->aux_head);
-       aux_tail = ACCESS_ONCE(rb->user_page->aux_tail);
 
        handle->rb = rb;
        handle->event = event;
        handle->head = aux_head;
-       if (aux_head - aux_tail < perf_aux_size(rb))
-               handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
-       else
-               handle->size = 0;
+       handle->size = 0;
 
        /*
-        * handle->size computation depends on aux_tail load; this forms a
-        * control dependency barrier separating aux_tail load from aux data
-        * store that will be enabled on successful return
+        * In overwrite mode, AUX data stores do not depend on aux_tail,
+        * therefore (A) control dependency barrier does not exist. The
+        * (B) <-> (C) ordering is still observed by the pmu driver.
         */
-       if (!handle->size) { /* A, matches D */
-               event->pending_disable = 1;
-               perf_output_wakeup(handle);
-               local_set(&rb->aux_nest, 0);
-               goto err_put;
+       if (!rb->aux_overwrite) {
+               aux_tail = ACCESS_ONCE(rb->user_page->aux_tail);
+               if (aux_head - aux_tail < perf_aux_size(rb))
+                       handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
+
+               /*
+                * handle->size computation depends on aux_tail load; this forms a
+                * control dependency barrier separating aux_tail load from aux data
+                * store that will be enabled on successful return
+                */
+               if (!handle->size) { /* A, matches D */
+                       event->pending_disable = 1;
+                       perf_output_wakeup(handle);
+                       local_set(&rb->aux_nest, 0);
+                       goto err_put;
+               }
        }
 
        return handle->rb->aux_priv;
@@ -327,13 +334,22 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size,
                         bool truncated)
 {
        struct ring_buffer *rb = handle->rb;
-       unsigned long aux_head = local_read(&rb->aux_head);
+       unsigned long aux_head;
        u64 flags = 0;
 
        if (truncated)
                flags |= PERF_AUX_FLAG_TRUNCATED;
 
-       local_add(size, &rb->aux_head);
+       /* in overwrite mode, driver provides aux_head via handle */
+       if (rb->aux_overwrite) {
+               flags |= PERF_AUX_FLAG_OVERWRITE;
+
+               aux_head = handle->head;
+               local_set(&rb->aux_head, aux_head);
+       } else {
+               aux_head = local_read(&rb->aux_head);
+               local_add(size, &rb->aux_head);
+       }
 
        if (size || flags) {
                /*
@@ -480,6 +496,8 @@ int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
         */
        atomic_set(&rb->aux_refcount, 1);
 
+       rb->aux_overwrite = overwrite;
+
 out:
        if (!ret)
                rb->aux_pgoff = pgoff;