DPDK logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright(C) 2020 Marvell International Ltd.
 */

#ifndef _RTE_TRACE_POINT_H_
#define _RTE_TRACE_POINT_H_

/**
 * @file
 *
 * RTE Tracepoint API
 *
 * This file provides the tracepoint API to RTE applications.
 *
 * @warning
 * @b EXPERIMENTAL: this API may change without prior notice
 */

#ifdef __cplusplus
extern "C" {
#endif

#include <stdbool.h>
#include <stdio.h>

#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_compat.h>
#include <rte_cycles.h>
#include <rte_per_lcore.h>
#include <rte_string_fns.h>
#include <rte_uuid.h>

/** The tracepoint object. */
typedef uint64_t rte_trace_point_t;

/**
 * Macro to define the tracepoint arguments in RTE_TRACE_POINT macro.

 * @see RTE_TRACE_POINT, RTE_TRACE_POINT_FP
 */
#define RTE_TRACE_POINT_ARGS

/** @internal Helper macro to support RTE_TRACE_POINT and RTE_TRACE_POINT_FP */
#define __RTE_TRACE_POINT(_mode, _tp, _args, ...) \
extern rte_trace_point_t __##_tp; \
static __rte_always_inline void \
_tp _args \
{ \
	__rte_trace_point_emit_header_##_mode(&__##_tp); \
	__VA_ARGS__ \
}

/**
 * Create a tracepoint.
 *
 * A tracepoint is defined by specifying:
 * - its input arguments: they are the C function style parameters to define
 *   the arguments of tracepoint function. These input arguments are embedded
 *   using the RTE_TRACE_POINT_ARGS macro.
 * - its output event fields: they are the sources of event fields that form
 *   the payload of any event that the execution of the tracepoint macro emits
 *   for this particular tracepoint. The application uses
 *   rte_trace_point_emit_* macros to emit the output event fields.
 *
 * @param tp
 *   Tracepoint object. Before using the tracepoint, an application needs to
 *   define the tracepoint using RTE_TRACE_POINT_REGISTER macro.
 * @param args
 *   C function style input arguments to define the arguments to tracepoint
 *   function.
 * @param ...
 *   Define the payload of trace function. The payload will be formed using
 *   rte_trace_point_emit_* macros. Use ";" delimiter between two payloads.
 *
 * @see RTE_TRACE_POINT_ARGS, RTE_TRACE_POINT_REGISTER, rte_trace_point_emit_*
 */
#define RTE_TRACE_POINT(tp, args, ...) \
	__RTE_TRACE_POINT(generic, tp, args, __VA_ARGS__)

/**
 * Create a tracepoint for fast path.
 *
 * Similar to RTE_TRACE_POINT, except that it is removed at compilation time
 * unless the RTE_ENABLE_TRACE_FP configuration parameter is set.
 *
 * @param tp
 *   Tracepoint object. Before using the tracepoint, an application needs to
 *   define the tracepoint using RTE_TRACE_POINT_REGISTER macro.
 * @param args
 *   C function style input arguments to define the arguments to tracepoint.
 *   function.
 * @param ...
 *   Define the payload of trace function. The payload will be formed using
 *   rte_trace_point_emit_* macros, Use ";" delimiter between two payloads.
 *
 * @see RTE_TRACE_POINT
 */
#define RTE_TRACE_POINT_FP(tp, args, ...) \
	__RTE_TRACE_POINT(fp, tp, args, __VA_ARGS__)

#ifdef __DOXYGEN__

/**
 * Register a tracepoint.
 *
 * @param trace
 *   The tracepoint object created using RTE_TRACE_POINT_REGISTER.
 * @param name
 *   The name of the tracepoint object.
 * @return
 *   - 0: Successfully registered the tracepoint.
 *   - <0: Failure to register the tracepoint.
 */
#define RTE_TRACE_POINT_REGISTER(trace, name)

/** Tracepoint function payload for uint64_t datatype */
#define rte_trace_point_emit_u64(val)
/** Tracepoint function payload for int64_t datatype */
#define rte_trace_point_emit_i64(val)
/** Tracepoint function payload for uint32_t datatype */
#define rte_trace_point_emit_u32(val)
/** Tracepoint function payload for int32_t datatype */
#define rte_trace_point_emit_i32(val)
/** Tracepoint function payload for uint16_t datatype */
#define rte_trace_point_emit_u16(val)
/** Tracepoint function payload for int16_t datatype */
#define rte_trace_point_emit_i16(val)
/** Tracepoint function payload for uint8_t datatype */
#define rte_trace_point_emit_u8(val)
/** Tracepoint function payload for int8_t datatype */
#define rte_trace_point_emit_i8(val)
/** Tracepoint function payload for int datatype */
#define rte_trace_point_emit_int(val)
/** Tracepoint function payload for long datatype */
#define rte_trace_point_emit_long(val)
/** Tracepoint function payload for size_t datatype */
#define rte_trace_point_emit_size_t(val)
/** Tracepoint function payload for float datatype */
#define rte_trace_point_emit_float(val)
/** Tracepoint function payload for double datatype */
#define rte_trace_point_emit_double(val)
/** Tracepoint function payload for pointer datatype */
#define rte_trace_point_emit_ptr(val)
/** Tracepoint function payload for string datatype */
#define rte_trace_point_emit_string(val)

#endif /* __DOXYGEN__ */

/** @internal Macro to define maximum emit length of string datatype. */
#define __RTE_TRACE_EMIT_STRING_LEN_MAX 32
/** @internal Macro to define event header size. */
#define __RTE_TRACE_EVENT_HEADER_SZ sizeof(uint64_t)

/**
 * Enable recording events of the given tracepoint in the trace buffer.
 *
 * @param tp
 *   The tracepoint object to enable.
 * @return
 *   - 0: Success.
 *   - (-ERANGE): Trace object is not registered.
 */
__rte_experimental
int rte_trace_point_enable(rte_trace_point_t *tp);

/**
 * Disable recording events of the given tracepoint in the trace buffer.
 *
 * @param tp
 *   The tracepoint object to disable.
 * @return
 *   - 0: Success.
 *   - (-ERANGE): Trace object is not registered.
 */
__rte_experimental
int rte_trace_point_disable(rte_trace_point_t *tp);

/**
 * Test if recording events from the given tracepoint is enabled.
 *
 * @param tp
 *    The tracepoint object.
 * @return
 *    true if tracepoint is enabled, false otherwise.
 */
__rte_experimental
bool rte_trace_point_is_enabled(rte_trace_point_t *tp);

/**
 * Lookup a tracepoint object from its name.
 *
 * @param name
 *   The name of the tracepoint.
 * @return
 *   The tracepoint object or NULL if not found.
 */
__rte_experimental
rte_trace_point_t *rte_trace_point_lookup(const char *name);

/**
 * @internal
 *
 * Test if the tracepoint fast path compile-time option is enabled.
 *
 * @return
 *   true if tracepoint fast path enabled, false otherwise.
 */
__rte_experimental
static __rte_always_inline bool
__rte_trace_point_fp_is_enabled(void)
{
#ifdef RTE_ENABLE_TRACE_FP
	return true;
#else
	return false;
#endif
}

/**
 * @internal
 *
 * Allocate trace memory buffer per thread.
 *
 */
__rte_experimental
void __rte_trace_mem_per_thread_alloc(void);

/**
 * @internal
 *
 * Helper function to emit field.
 *
 * @param sz
 *   The tracepoint size.
 * @param field
 *   The name of the trace event.
 * @param type
 *   The datatype of the trace event as string.
 * @return
 *   - 0: Success.
 *   - <0: Failure.
 */
__rte_experimental
void __rte_trace_point_emit_field(size_t sz, const char *field,
	const char *type);

/**
 * @internal
 *
 * Helper function to register a dynamic tracepoint.
 * Use RTE_TRACE_POINT_REGISTER macro for tracepoint registration.
 *
 * @param trace
 *   The tracepoint object created using RTE_TRACE_POINT_REGISTER.
 * @param name
 *   The name of the tracepoint object.
 * @param register_fn
 *   Trace registration function.
 * @return
 *   - 0: Successfully registered the tracepoint.
 *   - <0: Failure to register the tracepoint.
 */
__rte_experimental
int __rte_trace_point_register(rte_trace_point_t *trace, const char *name,
	void (*register_fn)(void));

#ifndef __DOXYGEN__

#ifndef _RTE_TRACE_POINT_REGISTER_H_
#ifdef ALLOW_EXPERIMENTAL_API

#define __RTE_TRACE_EVENT_HEADER_ID_SHIFT (48)

#define __RTE_TRACE_FIELD_SIZE_SHIFT 0
#define __RTE_TRACE_FIELD_SIZE_MASK (0xffffULL << __RTE_TRACE_FIELD_SIZE_SHIFT)
#define __RTE_TRACE_FIELD_ID_SHIFT (16)
#define __RTE_TRACE_FIELD_ID_MASK (0xffffULL << __RTE_TRACE_FIELD_ID_SHIFT)
#define __RTE_TRACE_FIELD_ENABLE_MASK (1ULL << 63)
#define __RTE_TRACE_FIELD_ENABLE_DISCARD (1ULL << 62)

struct __rte_trace_stream_header {
	uint32_t magic;
	rte_uuid_t uuid;
	uint32_t lcore_id;
	char thread_name[__RTE_TRACE_EMIT_STRING_LEN_MAX];
} __rte_packed;

struct __rte_trace_header {
	uint32_t offset;
	uint32_t len;
	struct __rte_trace_stream_header stream_header;
	uint8_t mem[];
};

RTE_DECLARE_PER_LCORE(void *, trace_mem);

static __rte_always_inline void *
__rte_trace_mem_get(uint64_t in)
{
	struct __rte_trace_header *trace =
		(struct __rte_trace_header *)(RTE_PER_LCORE(trace_mem));
	const uint16_t sz = in & __RTE_TRACE_FIELD_SIZE_MASK;

	/* Trace memory is not initialized for this thread */
	if (unlikely(trace == NULL)) {
		__rte_trace_mem_per_thread_alloc();
		trace = (struct __rte_trace_header *)(RTE_PER_LCORE(trace_mem));
		if (unlikely(trace == NULL))
			return NULL;
	}
	/* Check the wrap around case */
	uint32_t offset = trace->offset;
	if (unlikely((offset + sz) >= trace->len)) {
		/* Disable the trace event if it in DISCARD mode */
		if (unlikely(in & __RTE_TRACE_FIELD_ENABLE_DISCARD))
			return NULL;

		offset = 0;
	}
	/* Align to event header size */
	offset = RTE_ALIGN_CEIL(offset, __RTE_TRACE_EVENT_HEADER_SZ);
	void *mem = RTE_PTR_ADD(&trace->mem[0], offset);
	offset += sz;
	trace->offset = offset;

	return mem;
}

static __rte_always_inline void *
__rte_trace_point_emit_ev_header(void *mem, uint64_t in)
{
	uint64_t val;

	/* Event header [63:0] = id [63:48] | timestamp [47:0] */
	val = rte_get_tsc_cycles() &
		~(0xffffULL << __RTE_TRACE_EVENT_HEADER_ID_SHIFT);
	val |= ((in & __RTE_TRACE_FIELD_ID_MASK) <<
		(__RTE_TRACE_EVENT_HEADER_ID_SHIFT -
		 __RTE_TRACE_FIELD_ID_SHIFT));

	*(uint64_t *)mem = val;
	return RTE_PTR_ADD(mem, __RTE_TRACE_EVENT_HEADER_SZ);
}

#define __rte_trace_point_emit_header_generic(t) \
void *mem; \
do { \
	const uint64_t val = __atomic_load_n(t, __ATOMIC_ACQUIRE); \
	if (likely(!(val & __RTE_TRACE_FIELD_ENABLE_MASK))) \
		return; \
	mem = __rte_trace_mem_get(val); \
	if (unlikely(mem == NULL)) \
		return; \
	mem = __rte_trace_point_emit_ev_header(mem, val); \
} while (0)

#define __rte_trace_point_emit_header_fp(t) \
	if (!__rte_trace_point_fp_is_enabled()) \
		return; \
	__rte_trace_point_emit_header_generic(t)

#define __rte_trace_point_emit(in, type) \
do { \
	memcpy(mem, &(in), sizeof(in)); \
	mem = RTE_PTR_ADD(mem, sizeof(in)); \
} while (0)

#define rte_trace_point_emit_string(in) \
do { \
	if (unlikely(in == NULL)) \
		return; \
	rte_strscpy(mem, in, __RTE_TRACE_EMIT_STRING_LEN_MAX); \
	mem = RTE_PTR_ADD(mem, __RTE_TRACE_EMIT_STRING_LEN_MAX); \
} while (0)

#else

#define __rte_trace_point_emit_header_generic(t) RTE_SET_USED(t)
#define __rte_trace_point_emit_header_fp(t) RTE_SET_USED(t)
#define __rte_trace_point_emit(in, type) RTE_SET_USED(in)
#define rte_trace_point_emit_string(in) RTE_SET_USED(in)

#endif /* ALLOW_EXPERIMENTAL_API */
#endif /* _RTE_TRACE_POINT_REGISTER_H_ */

#define rte_trace_point_emit_u64(in) __rte_trace_point_emit(in, uint64_t)
#define rte_trace_point_emit_i64(in) __rte_trace_point_emit(in, int64_t)
#define rte_trace_point_emit_u32(in) __rte_trace_point_emit(in, uint32_t)
#define rte_trace_point_emit_i32(in) __rte_trace_point_emit(in, int32_t)
#define rte_trace_point_emit_u16(in) __rte_trace_point_emit(in, uint16_t)
#define rte_trace_point_emit_i16(in) __rte_trace_point_emit(in, int16_t)
#define rte_trace_point_emit_u8(in) __rte_trace_point_emit(in, uint8_t)
#define rte_trace_point_emit_i8(in) __rte_trace_point_emit(in, int8_t)
#define rte_trace_point_emit_int(in) __rte_trace_point_emit(in, int32_t)
#define rte_trace_point_emit_long(in) __rte_trace_point_emit(in, long)
#define rte_trace_point_emit_size_t(in) __rte_trace_point_emit(in, size_t)
#define rte_trace_point_emit_float(in) __rte_trace_point_emit(in, float)
#define rte_trace_point_emit_double(in) __rte_trace_point_emit(in, double)
#define rte_trace_point_emit_ptr(in) __rte_trace_point_emit(in, uintptr_t)

#endif /* __DOXYGEN__ */

#ifdef __cplusplus
}
#endif

#endif /* _RTE_TRACE_POINT_H_ */