summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--include/uapi/linux/perf_event.h1
-rw-r--r--kernel/events/internal.h1
-rw-r--r--kernel/events/ring_buffer.c48
3 files changed, 35 insertions, 15 deletions
diff --git a/include/uapi/linux/perf_event.h b/include/uapi/linux/perf_event.h
index 8904ad3a850b..29ef2f73bb4a 100644
--- a/include/uapi/linux/perf_event.h
+++ b/include/uapi/linux/perf_event.h
@@ -803,6 +803,7 @@ enum perf_callchain_context {
* PERF_RECORD_AUX::flags bits
*/
#define PERF_AUX_FLAG_TRUNCATED 0x01 /* record was truncated to fit */
+#define PERF_AUX_FLAG_OVERWRITE 0x02 /* snapshot from overwrite mode */
#define PERF_FLAG_FD_NO_GROUP (1UL << 0)
#define PERF_FLAG_FD_OUTPUT (1UL << 1)
diff --git a/kernel/events/internal.h b/kernel/events/internal.h
index b701ebc32570..ffd51d9f5945 100644
--- a/kernel/events/internal.h
+++ b/kernel/events/internal.h
@@ -40,6 +40,7 @@ struct ring_buffer {
local_t aux_nest;
unsigned long aux_pgoff;
int aux_nr_pages;
+ int aux_overwrite;
atomic_t aux_mmap_count;
unsigned long aux_mmap_locked;
void (*free_aux)(void *);
diff --git a/kernel/events/ring_buffer.c b/kernel/events/ring_buffer.c
index 0cc7b0f39058..67b328337a41 100644
--- a/kernel/events/ring_buffer.c
+++ b/kernel/events/ring_buffer.c
@@ -283,26 +283,33 @@ void *perf_aux_output_begin(struct perf_output_handle *handle,
goto err_put;
aux_head = local_read(&rb->aux_head);
- aux_tail = ACCESS_ONCE(rb->user_page->aux_tail);
handle->rb = rb;
handle->event = event;
handle->head = aux_head;
- if (aux_head - aux_tail < perf_aux_size(rb))
- handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
- else
- handle->size = 0;
+ handle->size = 0;
/*
- * handle->size computation depends on aux_tail load; this forms a
- * control dependency barrier separating aux_tail load from aux data
- * store that will be enabled on successful return
+ * In overwrite mode, AUX data stores do not depend on aux_tail,
+ * therefore (A) control dependency barrier does not exist. The
+ * (B) <-> (C) ordering is still observed by the pmu driver.
*/
- if (!handle->size) { /* A, matches D */
- event->pending_disable = 1;
- perf_output_wakeup(handle);
- local_set(&rb->aux_nest, 0);
- goto err_put;
+ if (!rb->aux_overwrite) {
+ aux_tail = ACCESS_ONCE(rb->user_page->aux_tail);
+ if (aux_head - aux_tail < perf_aux_size(rb))
+ handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));
+
+ /*
+ * handle->size computation depends on aux_tail load; this forms a
+ * control dependency barrier separating aux_tail load from aux data
+ * store that will be enabled on successful return
+ */
+ if (!handle->size) { /* A, matches D */
+ event->pending_disable = 1;
+ perf_output_wakeup(handle);
+ local_set(&rb->aux_nest, 0);
+ goto err_put;
+ }
}
return handle->rb->aux_priv;
@@ -327,13 +334,22 @@ void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size,
bool truncated)
{
struct ring_buffer *rb = handle->rb;
- unsigned long aux_head = local_read(&rb->aux_head);
+ unsigned long aux_head;
u64 flags = 0;
if (truncated)
flags |= PERF_AUX_FLAG_TRUNCATED;
- local_add(size, &rb->aux_head);
+ /* in overwrite mode, driver provides aux_head via handle */
+ if (rb->aux_overwrite) {
+ flags |= PERF_AUX_FLAG_OVERWRITE;
+
+ aux_head = handle->head;
+ local_set(&rb->aux_head, aux_head);
+ } else {
+ aux_head = local_read(&rb->aux_head);
+ local_add(size, &rb->aux_head);
+ }
if (size || flags) {
/*
@@ -480,6 +496,8 @@ int rb_alloc_aux(struct ring_buffer *rb, struct perf_event *event,
*/
atomic_set(&rb->aux_refcount, 1);
+ rb->aux_overwrite = overwrite;
+
out:
if (!ret)
rb->aux_pgoff = pgoff;