DPDK logo

Elixir Cross Referencer

  1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
/*   SPDX-License-Identifier: BSD-3-Clause
 *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
 */

#ifndef _CCP_DEV_H_
#define _CCP_DEV_H_

#include <limits.h>
#include <stdbool.h>
#include <stdint.h>
#include <string.h>

#include <rte_bus_pci.h>
#include <rte_atomic.h>
#include <rte_byteorder.h>
#include <rte_io.h>
#include <rte_pci.h>
#include <rte_spinlock.h>
#include <rte_crypto_sym.h>
#include <rte_cryptodev.h>

/**< CCP sspecific */
#define MAX_HW_QUEUES                   5
#define CCP_MAX_TRNG_RETRIES		10
#define CCP_ALIGN(x, y) ((((x) + (y - 1)) / y) * y)

/**< CCP Register Mappings */
#define Q_MASK_REG                      0x000
#define TRNG_OUT_REG                    0x00c

/* CCP Version 5 Specifics */
#define CMD_QUEUE_MASK_OFFSET		0x00
#define	CMD_QUEUE_PRIO_OFFSET		0x04
#define CMD_REQID_CONFIG_OFFSET		0x08
#define	CMD_CMD_TIMEOUT_OFFSET		0x10
#define LSB_PUBLIC_MASK_LO_OFFSET	0x18
#define LSB_PUBLIC_MASK_HI_OFFSET	0x1C
#define LSB_PRIVATE_MASK_LO_OFFSET	0x20
#define LSB_PRIVATE_MASK_HI_OFFSET	0x24

#define CMD_Q_CONTROL_BASE		0x0000
#define CMD_Q_TAIL_LO_BASE		0x0004
#define CMD_Q_HEAD_LO_BASE		0x0008
#define CMD_Q_INT_ENABLE_BASE		0x000C
#define CMD_Q_INTERRUPT_STATUS_BASE	0x0010

#define CMD_Q_STATUS_BASE		0x0100
#define CMD_Q_INT_STATUS_BASE		0x0104

#define	CMD_CONFIG_0_OFFSET		0x6000
#define	CMD_TRNG_CTL_OFFSET		0x6008
#define	CMD_AES_MASK_OFFSET		0x6010
#define	CMD_CLK_GATE_CTL_OFFSET		0x603C

/* Address offset between two virtual queue registers */
#define CMD_Q_STATUS_INCR		0x1000

/* Bit masks */
#define CMD_Q_RUN			0x1
#define CMD_Q_SIZE			0x1F
#define CMD_Q_SHIFT			3
#define COMMANDS_PER_QUEUE		8192

#define QUEUE_SIZE_VAL                  ((ffs(COMMANDS_PER_QUEUE) - 2) & \
					 CMD_Q_SIZE)
#define Q_DESC_SIZE                     sizeof(struct ccp_desc)
#define Q_SIZE(n)                       (COMMANDS_PER_QUEUE*(n))

#define INT_COMPLETION                  0x1
#define INT_ERROR                       0x2
#define INT_QUEUE_STOPPED               0x4
#define ALL_INTERRUPTS                  (INT_COMPLETION| \
					 INT_ERROR| \
					 INT_QUEUE_STOPPED)

#define LSB_REGION_WIDTH                5
#define MAX_LSB_CNT                     8

#define LSB_SIZE                        16
#define LSB_ITEM_SIZE                   32
#define SLSB_MAP_SIZE                   (MAX_LSB_CNT * LSB_SIZE)
#define LSB_ENTRY_NUMBER(LSB_ADDR)      (LSB_ADDR / LSB_ITEM_SIZE)

/* General CCP Defines */

#define CCP_SB_BYTES                    32
/* Word 0 */
#define CCP_CMD_DW0(p)		((p)->dw0)
#define CCP_CMD_SOC(p)		(CCP_CMD_DW0(p).soc)
#define CCP_CMD_IOC(p)		(CCP_CMD_DW0(p).ioc)
#define CCP_CMD_INIT(p)	        (CCP_CMD_DW0(p).init)
#define CCP_CMD_EOM(p)		(CCP_CMD_DW0(p).eom)
#define CCP_CMD_FUNCTION(p)	(CCP_CMD_DW0(p).function)
#define CCP_CMD_ENGINE(p)	(CCP_CMD_DW0(p).engine)
#define CCP_CMD_PROT(p)	        (CCP_CMD_DW0(p).prot)

/* Word 1 */
#define CCP_CMD_DW1(p)		((p)->length)
#define CCP_CMD_LEN(p)		(CCP_CMD_DW1(p))

/* Word 2 */
#define CCP_CMD_DW2(p)		((p)->src_lo)
#define CCP_CMD_SRC_LO(p)	(CCP_CMD_DW2(p))

/* Word 3 */
#define CCP_CMD_DW3(p)		((p)->dw3)
#define CCP_CMD_SRC_MEM(p)	((p)->dw3.src_mem)
#define CCP_CMD_SRC_HI(p)	((p)->dw3.src_hi)
#define CCP_CMD_LSB_ID(p)	((p)->dw3.lsb_cxt_id)
#define CCP_CMD_FIX_SRC(p)	((p)->dw3.fixed)

/* Words 4/5 */
#define CCP_CMD_DW4(p)		((p)->dw4)
#define CCP_CMD_DST_LO(p)	(CCP_CMD_DW4(p).dst_lo)
#define CCP_CMD_DW5(p)		((p)->dw5.fields.dst_hi)
#define CCP_CMD_DST_HI(p)	(CCP_CMD_DW5(p))
#define CCP_CMD_DST_MEM(p)	((p)->dw5.fields.dst_mem)
#define CCP_CMD_FIX_DST(p)	((p)->dw5.fields.fixed)
#define CCP_CMD_SHA_LO(p)	((p)->dw4.sha_len_lo)
#define CCP_CMD_SHA_HI(p)	((p)->dw5.sha_len_hi)

/* Word 6/7 */
#define CCP_CMD_DW6(p)		((p)->key_lo)
#define CCP_CMD_KEY_LO(p)	(CCP_CMD_DW6(p))
#define CCP_CMD_DW7(p)		((p)->dw7)
#define CCP_CMD_KEY_HI(p)	((p)->dw7.key_hi)
#define CCP_CMD_KEY_MEM(p)	((p)->dw7.key_mem)

/* bitmap */
enum {
	BITS_PER_WORD = sizeof(unsigned long) * CHAR_BIT
};

#define WORD_OFFSET(b) ((b) / BITS_PER_WORD)
#define BIT_OFFSET(b)  ((b) % BITS_PER_WORD)

#define CCP_DIV_ROUND_UP(n, d)  (((n) + (d) - 1) / (d))
#define CCP_BITMAP_SIZE(nr) \
	CCP_DIV_ROUND_UP(nr, CHAR_BIT * sizeof(unsigned long))

#define CCP_BITMAP_FIRST_WORD_MASK(start) \
	(~0UL << ((start) & (BITS_PER_WORD - 1)))
#define CCP_BITMAP_LAST_WORD_MASK(nbits) \
	(~0UL >> (-(nbits) & (BITS_PER_WORD - 1)))

#define __ccp_round_mask(x, y) ((typeof(x))((y)-1))
#define ccp_round_down(x, y) ((x) & ~__ccp_round_mask(x, y))

/** CCP registers Write/Read */

static inline void ccp_pci_reg_write(void *base, int offset,
				     uint32_t value)
{
	volatile void *reg_addr = ((uint8_t *)base + offset);

	rte_write32((rte_cpu_to_le_32(value)), reg_addr);
}

static inline uint32_t ccp_pci_reg_read(void *base, int offset)
{
	volatile void *reg_addr = ((uint8_t *)base + offset);

	return rte_le_to_cpu_32(rte_read32(reg_addr));
}

#define CCP_READ_REG(hw_addr, reg_offset) \
	ccp_pci_reg_read(hw_addr, reg_offset)

#define CCP_WRITE_REG(hw_addr, reg_offset, value) \
	ccp_pci_reg_write(hw_addr, reg_offset, value)

TAILQ_HEAD(ccp_list, ccp_device);

extern struct ccp_list ccp_list;

/**
 * CCP device version
 */
enum ccp_device_version {
	CCP_VERSION_5A = 0,
	CCP_VERSION_5B,
};

/**
 * A structure describing a CCP command queue.
 */
struct ccp_queue {
	struct ccp_device *dev;
	char memz_name[RTE_MEMZONE_NAMESIZE];

	rte_atomic64_t free_slots;
	/**< available free slots updated from enq/deq calls */

	/* Queue identifier */
	uint64_t id;	/**< queue id */
	uint64_t qidx;	/**< queue index */
	uint64_t qsize;	/**< queue size */

	/* Queue address */
	struct ccp_desc *qbase_desc;
	void *qbase_addr;
	phys_addr_t qbase_phys_addr;
	/**< queue-page registers addr */
	void *reg_base;

	uint32_t qcontrol;
	/**< queue ctrl reg */

	int lsb;
	/**< lsb region assigned to queue */
	unsigned long lsbmask;
	/**< lsb regions queue can access */
	unsigned long lsbmap[CCP_BITMAP_SIZE(LSB_SIZE)];
	/**< all lsb resources which queue is using */
	uint32_t sb_key;
	/**< lsb assigned for queue */
	uint32_t sb_iv;
	/**< lsb assigned for iv */
	uint32_t sb_sha;
	/**< lsb assigned for sha ctx */
	uint32_t sb_hmac;
	/**< lsb assigned for hmac ctx */
} __rte_cache_aligned;

/**
 * A structure describing a CCP device.
 */
struct ccp_device {
	TAILQ_ENTRY(ccp_device) next;
	int id;
	/**< ccp dev id on platform */
	struct ccp_queue cmd_q[MAX_HW_QUEUES];
	/**< ccp queue */
	int cmd_q_count;
	/**< no. of ccp Queues */
	struct rte_pci_device pci;
	/**< ccp pci identifier */
	unsigned long lsbmap[CCP_BITMAP_SIZE(SLSB_MAP_SIZE)];
	/**< shared lsb mask of ccp */
	rte_spinlock_t lsb_lock;
	/**< protection for shared lsb region allocation */
	int qidx;
	/**< current queue index */
	int hwrng_retries;
	/**< retry counter for CCP TRNG */
} __rte_cache_aligned;

/**< CCP H/W engine related */
/**
 * ccp_engine - CCP operation identifiers
 *
 * @CCP_ENGINE_AES: AES operation
 * @CCP_ENGINE_XTS_AES: 128-bit XTS AES operation
 * @CCP_ENGINE_3DES: DES/3DES operation
 * @CCP_ENGINE_SHA: SHA operation
 * @CCP_ENGINE_RSA: RSA operation
 * @CCP_ENGINE_PASSTHRU: pass-through operation
 * @CCP_ENGINE_ZLIB_DECOMPRESS: unused
 * @CCP_ENGINE_ECC: ECC operation
 */
enum ccp_engine {
	CCP_ENGINE_AES = 0,
	CCP_ENGINE_XTS_AES_128,
	CCP_ENGINE_3DES,
	CCP_ENGINE_SHA,
	CCP_ENGINE_RSA,
	CCP_ENGINE_PASSTHRU,
	CCP_ENGINE_ZLIB_DECOMPRESS,
	CCP_ENGINE_ECC,
	CCP_ENGINE__LAST,
};

/* Passthru engine */
/**
 * ccp_passthru_bitwise - type of bitwise passthru operation
 *
 * @CCP_PASSTHRU_BITWISE_NOOP: no bitwise operation performed
 * @CCP_PASSTHRU_BITWISE_AND: perform bitwise AND of src with mask
 * @CCP_PASSTHRU_BITWISE_OR: perform bitwise OR of src with mask
 * @CCP_PASSTHRU_BITWISE_XOR: perform bitwise XOR of src with mask
 * @CCP_PASSTHRU_BITWISE_MASK: overwrite with mask
 */
enum ccp_passthru_bitwise {
	CCP_PASSTHRU_BITWISE_NOOP = 0,
	CCP_PASSTHRU_BITWISE_AND,
	CCP_PASSTHRU_BITWISE_OR,
	CCP_PASSTHRU_BITWISE_XOR,
	CCP_PASSTHRU_BITWISE_MASK,
	CCP_PASSTHRU_BITWISE__LAST,
};

/**
 * ccp_passthru_byteswap - type of byteswap passthru operation
 *
 * @CCP_PASSTHRU_BYTESWAP_NOOP: no byte swapping performed
 * @CCP_PASSTHRU_BYTESWAP_32BIT: swap bytes within 32-bit words
 * @CCP_PASSTHRU_BYTESWAP_256BIT: swap bytes within 256-bit words
 */
enum ccp_passthru_byteswap {
	CCP_PASSTHRU_BYTESWAP_NOOP = 0,
	CCP_PASSTHRU_BYTESWAP_32BIT,
	CCP_PASSTHRU_BYTESWAP_256BIT,
	CCP_PASSTHRU_BYTESWAP__LAST,
};

/**
 * CCP passthru
 */
struct ccp_passthru {
	phys_addr_t src_addr;
	phys_addr_t dest_addr;
	enum ccp_passthru_bitwise bit_mod;
	enum ccp_passthru_byteswap byte_swap;
	int len;
	int dir;
};

/* CCP version 5: Union to define the function field (cmd_reg1/dword0) */
union ccp_function {
	struct {
		uint16_t size:7;
		uint16_t encrypt:1;
		uint16_t mode:5;
		uint16_t type:2;
	} aes;
	struct {
		uint16_t size:7;
		uint16_t encrypt:1;
		uint16_t mode:5;
		uint16_t type:2;
	} des;
	struct {
		uint16_t size:7;
		uint16_t encrypt:1;
		uint16_t rsvd:5;
		uint16_t type:2;
	} aes_xts;
	struct {
		uint16_t rsvd1:10;
		uint16_t type:4;
		uint16_t rsvd2:1;
	} sha;
	struct {
		uint16_t mode:3;
		uint16_t size:12;
	} rsa;
	struct {
		uint16_t byteswap:2;
		uint16_t bitwise:3;
		uint16_t reflect:2;
		uint16_t rsvd:8;
	} pt;
	struct  {
		uint16_t rsvd:13;
	} zlib;
	struct {
		uint16_t size:10;
		uint16_t type:2;
		uint16_t mode:3;
	} ecc;
	uint16_t raw;
};


/**
 * descriptor for version 5 CPP commands
 * 8 32-bit words:
 * word 0: function; engine; control bits
 * word 1: length of source data
 * word 2: low 32 bits of source pointer
 * word 3: upper 16 bits of source pointer; source memory type
 * word 4: low 32 bits of destination pointer
 * word 5: upper 16 bits of destination pointer; destination memory
 * type
 * word 6: low 32 bits of key pointer
 * word 7: upper 16 bits of key pointer; key memory type
 */
struct dword0 {
	uint32_t soc:1;
	uint32_t ioc:1;
	uint32_t rsvd1:1;
	uint32_t init:1;
	uint32_t eom:1;
	uint32_t function:15;
	uint32_t engine:4;
	uint32_t prot:1;
	uint32_t rsvd2:7;
};

struct dword3 {
	uint32_t src_hi:16;
	uint32_t src_mem:2;
	uint32_t lsb_cxt_id:8;
	uint32_t rsvd1:5;
	uint32_t fixed:1;
};

union dword4 {
	uint32_t dst_lo;	/* NON-SHA */
	uint32_t sha_len_lo;	/* SHA */
};

union dword5 {
	struct {
		uint32_t dst_hi:16;
		uint32_t dst_mem:2;
		uint32_t rsvd1:13;
		uint32_t fixed:1;
	}
	fields;
	uint32_t sha_len_hi;
};

struct dword7 {
	uint32_t key_hi:16;
	uint32_t key_mem:2;
	uint32_t rsvd1:14;
};

struct ccp_desc {
	struct dword0 dw0;
	uint32_t length;
	uint32_t src_lo;
	struct dword3 dw3;
	union dword4 dw4;
	union dword5 dw5;
	uint32_t key_lo;
	struct dword7 dw7;
};

/**
 * ccp memory type
 */
enum ccp_memtype {
	CCP_MEMTYPE_SYSTEM = 0,
	CCP_MEMTYPE_SB,
	CCP_MEMTYPE_LOCAL,
	CCP_MEMTYPE_LAST,
};

/**
 * cmd id to follow order
 */
enum ccp_cmd_order {
	CCP_CMD_CIPHER = 0,
	CCP_CMD_AUTH,
	CCP_CMD_CIPHER_HASH,
	CCP_CMD_HASH_CIPHER,
	CCP_CMD_COMBINED,
	CCP_CMD_NOT_SUPPORTED,
};

static inline uint32_t
low32_value(unsigned long addr)
{
	return ((uint64_t)addr) & 0x0ffffffff;
}

static inline uint32_t
high32_value(unsigned long addr)
{
	return ((uint64_t)addr >> 32) & 0x00000ffff;
}

/*
 * Start CCP device
 */
int ccp_dev_start(struct rte_cryptodev *dev);

/**
 * Detect ccp platform and initialize all ccp devices
 *
 * @param ccp_id rte_pci_id list for supported CCP devices
 * @return no. of successfully initialized CCP devices
 */
int ccp_probe_devices(struct rte_pci_device *pci_dev,
		const struct rte_pci_id *ccp_id);

/**
 * allocate a ccp command queue
 *
 * @dev rte crypto device
 * @param slot_req number of required
 * @return allotted CCP queue on success otherwise NULL
 */
struct ccp_queue *ccp_allot_queue(struct rte_cryptodev *dev, int slot_req);

/**
 * read hwrng value
 *
 * @param trng_value data pointer to write RNG value
 * @return 0 on success otherwise -1
 */
int ccp_read_hwrng(uint32_t *trng_value);

#endif /* _CCP_DEV_H_ */