X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=drivers%2Fcrypto%2Fhifn_795x.c;h=0c79fe7f15673563257fe6a3a01a0c451f6dd997;hb=84d9077b3391e6966813e380e63b57f53e58e120;hp=fec32aa1ec529323bece6475f68d34d6590d7fae;hpb=102d49d3d0f0f471b338b6805001fc3ca7bf663b;p=safe%2Fjmp%2Flinux-2.6 diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index fec32aa..0c79fe7 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -28,17 +29,15 @@ #include #include #include -#include #include +#include +#include #include #include #include -#undef dprintk - -#define HIFN_TEST //#define HIFN_DEBUG #ifdef HIFN_DEBUG @@ -47,6 +46,11 @@ #define dprintk(f, a...) do {} while (0) #endif +static char hifn_pll_ref[sizeof("extNNN")] = "ext"; +module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); +MODULE_PARM_DESC(hifn_pll_ref, + "PLL reference clock (pci[freq] or ext[freq], default ext)"); + static atomic_t hifn_dev_number; #define ACRYPTO_OP_DECRYPT 0 @@ -286,7 +290,26 @@ static atomic_t hifn_dev_number; #define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */ #define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */ -#define HIFN_PLL_7956 0x00001d18 /* 7956 PLL config value */ +/* PLL configuration register */ +#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */ +#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */ +#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */ +#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */ +#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */ +#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */ +#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */ +#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */ +#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */ +#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */ +#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */ +#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */ +#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */ +#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */ +#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */ +#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */ +#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */ + +#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */ /* Public key reset register (HIFN_1_PUB_RESET) */ #define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */ @@ -337,12 +360,14 @@ static atomic_t hifn_dev_number; #define HIFN_NAMESIZE 32 #define HIFN_MAX_RESULT_ORDER 5 -#define HIFN_D_CMD_RSIZE 24*4 -#define HIFN_D_SRC_RSIZE 80*4 -#define HIFN_D_DST_RSIZE 80*4 -#define HIFN_D_RES_RSIZE 24*4 +#define HIFN_D_CMD_RSIZE 24*1 +#define HIFN_D_SRC_RSIZE 80*1 +#define HIFN_D_DST_RSIZE 80*1 +#define HIFN_D_RES_RSIZE 24*1 -#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5 +#define HIFN_D_DST_DALIGN 4 + +#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1) #define AES_MIN_KEY_SIZE 16 #define AES_MAX_KEY_SIZE 32 @@ -365,8 +390,8 @@ static atomic_t hifn_dev_number; struct hifn_desc { - volatile u32 l; - volatile u32 p; + volatile __le32 l; + volatile __le32 p; }; struct hifn_dma { @@ -378,8 +403,6 @@ struct hifn_dma { u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; - u64 test_src, test_dst; - /* * Our current positions for insertion and removal from the descriptor * rings. @@ -406,9 +429,6 @@ struct hifn_device struct pci_dev *pdev; void __iomem *bar[3]; - unsigned long result_mem; - dma_addr_t dst; - void *desc_virt; dma_addr_t desc_dma; @@ -418,8 +438,6 @@ struct hifn_device spinlock_t lock; - void *priv; - u32 flags; int active, started; struct delayed_work work; @@ -433,6 +451,14 @@ struct hifn_device struct crypto_queue queue; struct list_head alg_list; + + unsigned int pk_clk_freq; + +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG + unsigned int rng_wait_time; + ktime_t rngtime; + struct hwrng rng; +#endif }; #define HIFN_D_LENGTH 0x0000ffff @@ -446,10 +472,10 @@ struct hifn_device struct hifn_base_command { - volatile u16 masks; - volatile u16 session_num; - volatile u16 total_source_count; - volatile u16 total_dest_count; + volatile __le16 masks; + volatile __le16 session_num; + volatile __le16 total_source_count; + volatile __le16 total_dest_count; }; #define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */ @@ -469,10 +495,10 @@ struct hifn_base_command */ struct hifn_crypt_command { - volatile u16 masks; - volatile u16 header_skip; - volatile u16 source_count; - volatile u16 reserved; + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ @@ -500,10 +526,10 @@ struct hifn_crypt_command */ struct hifn_mac_command { - volatile u16 masks; - volatile u16 header_skip; - volatile u16 source_count; - volatile u16 reserved; + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_MAC_CMD_ALG_MASK 0x0001 @@ -529,10 +555,10 @@ struct hifn_mac_command struct hifn_comp_command { - volatile u16 masks; - volatile u16 header_skip; - volatile u16 source_count; - volatile u16 reserved; + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_COMP_CMD_SRCLEN_M 0xc000 @@ -548,10 +574,10 @@ struct hifn_comp_command struct hifn_base_result { - volatile u16 flags; - volatile u16 session; - volatile u16 src_cnt; /* 15:0 of source count */ - volatile u16 dst_cnt; /* 15:0 of dest count */ + volatile __le16 flags; + volatile __le16 session; + volatile __le16 src_cnt; /* 15:0 of source count */ + volatile __le16 dst_cnt; /* 15:0 of dest count */ }; #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ @@ -562,8 +588,8 @@ struct hifn_base_result struct hifn_comp_result { - volatile u16 flags; - volatile u16 crc; + volatile __le16 flags; + volatile __le16 crc; }; #define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */ @@ -574,8 +600,8 @@ struct hifn_comp_result struct hifn_mac_result { - volatile u16 flags; - volatile u16 reserved; + volatile __le16 flags; + volatile __le16 reserved; /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ }; @@ -584,8 +610,8 @@ struct hifn_mac_result struct hifn_crypt_result { - volatile u16 flags; - volatile u16 reserved; + volatile __le16 flags; + volatile __le16 reserved; }; #define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */ @@ -621,21 +647,26 @@ struct ablkcipher_walk struct hifn_context { - u8 key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv; + u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; struct hifn_device *dev; - unsigned int keysize, ivsize; + unsigned int keysize; +}; + +struct hifn_request_context +{ + u8 *iv; + unsigned int ivsize; u8 op, type, mode, unused; struct ablkcipher_walk walk; - atomic_t sg_num; }; -#define crypto_alg_to_hifn(alg) container_of(alg, struct hifn_crypto_alg, alg) +#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg) static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg) { u32 ret; - ret = readl((char *)(dev->bar[0]) + reg); + ret = readl(dev->bar[0] + reg); return ret; } @@ -644,19 +675,19 @@ static inline u32 hifn_read_1(struct hifn_device *dev, u32 reg) { u32 ret; - ret = readl((char *)(dev->bar[1]) + reg); + ret = readl(dev->bar[1] + reg); return ret; } static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val) { - writel(val, (char *)(dev->bar[0]) + reg); + writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg); } static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val) { - writel(val, (char *)(dev->bar[1]) + reg); + writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg); } static void hifn_wait_puc(struct hifn_device *dev) @@ -760,6 +791,56 @@ static struct pci2id { } }; +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG +static int hifn_rng_data_present(struct hwrng *rng, int wait) +{ + struct hifn_device *dev = (struct hifn_device *)rng->priv; + s64 nsec; + + nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime)); + nsec -= dev->rng_wait_time; + if (nsec <= 0) + return 1; + if (!wait) + return 0; + ndelay(nsec); + return 1; +} + +static int hifn_rng_data_read(struct hwrng *rng, u32 *data) +{ + struct hifn_device *dev = (struct hifn_device *)rng->priv; + + *data = hifn_read_1(dev, HIFN_1_RNG_DATA); + dev->rngtime = ktime_get(); + return 4; +} + +static int hifn_register_rng(struct hifn_device *dev) +{ + /* + * We must wait at least 256 Pk_clk cycles between two reads of the rng. + */ + dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) * + 256; + + dev->rng.name = dev->name; + dev->rng.data_present = hifn_rng_data_present, + dev->rng.data_read = hifn_rng_data_read, + dev->rng.priv = (unsigned long)dev; + + return hwrng_register(&dev->rng); +} + +static void hifn_unregister_rng(struct hifn_device *dev) +{ + hwrng_unregister(&dev->rng); +} +#else +#define hifn_register_rng(dev) 0 +#define hifn_unregister_rng(dev) +#endif + static int hifn_init_pubrng(struct hifn_device *dev) { int i; @@ -795,6 +876,11 @@ static int hifn_init_pubrng(struct hifn_device *dev) dprintk("Chip %s: RNG engine has been successfully initialised.\n", dev->name); +#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG + /* First value must be discarded */ + hifn_read_1(dev, HIFN_1_RNG_DATA); + dev->rngtime = ktime_get(); +#endif return 0; } @@ -804,7 +890,7 @@ static int hifn_enable_crypto(struct hifn_device *dev) char *offtbl = NULL; int i; - for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) { + for (i = 0; i < ARRAY_SIZE(pci2id); i++) { if (pci2id[i].pci_vendor == dev->pdev->vendor && pci2id[i].pci_prod == dev->pdev->device) { offtbl = pci2id[i].card_id; @@ -871,6 +957,72 @@ static void hifn_init_dma(struct hifn_device *dev) dma->cmdk = dma->srck = dma->dstk = dma->resk = 0; } +/* + * Initialize the PLL. We need to know the frequency of the reference clock + * to calculate the optimal multiplier. For PCI we assume 66MHz, since that + * allows us to operate without the risk of overclocking the chip. If it + * actually uses 33MHz, the chip will operate at half the speed, this can be + * overriden by specifying the frequency as module parameter (pci33). + * + * Unfortunately the PCI clock is not very suitable since the HIFN needs a + * stable clock and the PCI clock frequency may vary, so the default is the + * external clock. There is no way to find out its frequency, we default to + * 66MHz since according to Mike Ham of HiFn, almost every board in existence + * has an external crystal populated at 66MHz. + */ +static void hifn_init_pll(struct hifn_device *dev) +{ + unsigned int freq, m; + u32 pllcfg; + + pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1; + + if (strncmp(hifn_pll_ref, "ext", 3) == 0) + pllcfg |= HIFN_PLL_REF_CLK_PLL; + else + pllcfg |= HIFN_PLL_REF_CLK_HBI; + + if (hifn_pll_ref[3] != '\0') + freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); + else { + freq = 66; + printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, " + "override with hifn_pll_ref=%.3s\n", + freq, hifn_pll_ref); + } + + m = HIFN_PLL_FCK_MAX / freq; + + pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT; + if (m <= 8) + pllcfg |= HIFN_PLL_IS_1_8; + else + pllcfg |= HIFN_PLL_IS_9_12; + + /* Select clock source and enable clock bypass */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP); + + /* Let the chip lock to the input clock */ + mdelay(10); + + /* Disable clock bypass */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI); + + /* Switch the engines to the PLL */ + hifn_write_1(dev, HIFN_1_PLL, pllcfg | + HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL); + + /* + * The Fpk_clk runs at half the total speed. Its frequency is needed to + * calculate the minimum time between two reads of the rng. Since 33MHz + * is actually 33.333... we overestimate the frequency here, resulting + * in slightly larger intervals. + */ + dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2; +} + static void hifn_init_registers(struct hifn_device *dev) { u32 dptr = dev->desc_dma; @@ -881,14 +1033,14 @@ static void hifn_init_registers(struct hifn_device *dev) hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER); /* write all 4 ring address registers */ - hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, cmdr[0]))); - hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, srcr[0]))); - hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, dstr[0]))); - hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr + - offsetof(struct hifn_dma, resr[0]))); + hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr + + offsetof(struct hifn_dma, cmdr[0])); + hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr + + offsetof(struct hifn_dma, srcr[0])); + hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr + + offsetof(struct hifn_dma, dstr[0])); + hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr + + offsetof(struct hifn_dma, resr[0])); mdelay(2); #if 0 @@ -938,7 +1090,7 @@ static void hifn_init_registers(struct hifn_device *dev) #else hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342); #endif - hifn_write_1(dev, HIFN_1_PLL, HIFN_PLL_7956); + hifn_init_pll(dev); hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET | @@ -1010,113 +1162,20 @@ static int hifn_setup_crypto_command(struct hifn_device *dev, return cmd_len; } -static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, - unsigned int offset, unsigned int size) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - int idx; - dma_addr_t addr; - - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); - - idx = dma->srci; - - dma->srcr[idx].p = __cpu_to_le32(addr); - dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST; - - if (++idx == HIFN_D_SRC_RSIZE) { - dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | - HIFN_D_MASKDONEIRQ | HIFN_D_LAST); - idx = 0; - } - - dma->srci = idx; - dma->srcu++; - - if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); - dev->flags |= HIFN_FLAG_SRC_BUSY; - } - - return size; -} - -static void hifn_setup_res_desc(struct hifn_device *dev) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - - dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | - HIFN_D_VALID | HIFN_D_LAST); - /* - * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | - * HIFN_D_LAST | HIFN_D_NOINVALID); - */ - - if (++dma->resi == HIFN_D_RES_RSIZE) { - dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); - dma->resi = 0; - } - - dma->resu++; - - if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); - dev->flags |= HIFN_FLAG_RES_BUSY; - } -} - -static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, - unsigned offset, unsigned size) -{ - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - int idx; - dma_addr_t addr; - - addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); - - idx = dma->dsti; - dma->dstr[idx].p = __cpu_to_le32(addr); - dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | - HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST); - - if (++idx == HIFN_D_DST_RSIZE) { - dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | - HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | - HIFN_D_LAST | HIFN_D_NOINVALID); - idx = 0; - } - dma->dsti = idx; - dma->dstu++; - - if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { - hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); - dev->flags |= HIFN_FLAG_DST_BUSY; - } -} - -static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff, - struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv, - struct hifn_context *ctx) +static int hifn_setup_cmd_desc(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + void *priv, unsigned int nbytes) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int cmd_len, sa_idx; u8 *buf, *buf_pos; u16 mask; - dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n", - dev->name, spage, soff, dpage, doff, nbytes, priv, ctx); - - sa_idx = dma->resi; - - hifn_setup_src_desc(dev, spage, soff, nbytes); - + sa_idx = dma->cmdi; buf_pos = buf = dma->command_bufs[dma->cmdi]; mask = 0; - switch (ctx->op) { + switch (rctx->op) { case ACRYPTO_OP_DECRYPT: mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; break; @@ -1133,15 +1192,15 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, nbytes, mask, dev->snum); - if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) { + if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) { u16 md = 0; if (ctx->keysize) md |= HIFN_CRYPT_CMD_NEW_KEY; - if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB) md |= HIFN_CRYPT_CMD_NEW_IV; - switch (ctx->mode) { + switch (rctx->mode) { case ACRYPTO_MODE_ECB: md |= HIFN_CRYPT_CMD_MODE_ECB; break; @@ -1158,7 +1217,7 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned goto err_out; } - switch (ctx->type) { + switch (rctx->type) { case ACRYPTO_TYPE_AES_128: if (ctx->keysize != 16) goto err_out; @@ -1193,17 +1252,18 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned buf_pos += hifn_setup_crypto_command(dev, buf_pos, nbytes, nbytes, ctx->key, ctx->keysize, - ctx->iv, ctx->ivsize, md); + rctx->iv, rctx->ivsize, md); } dev->sa[sa_idx] = priv; + dev->started++; cmd_len = buf_pos - buf; dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ); if (++dma->cmdi == HIFN_D_CMD_RSIZE) { - dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND | + dma->cmdr[dma->cmdi].l = __cpu_to_le32( HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); dma->cmdi = 0; @@ -1214,16 +1274,149 @@ static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); dev->flags |= HIFN_FLAG_CMD_BUSY; } - - hifn_setup_dst_desc(dev, dpage, doff, nbytes); - hifn_setup_res_desc(dev); - return 0; err_out: return -EINVAL; } +static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page, + unsigned int offset, unsigned int size, int last) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE); + + idx = dma->srci; + + dma->srcr[idx].p = __cpu_to_le32(addr); + dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_SRC_RSIZE) { + dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + + dma->srci = idx; + dma->srcu++; + + if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA); + dev->flags |= HIFN_FLAG_SRC_BUSY; + } + + return size; +} + +static void hifn_setup_res_desc(struct hifn_device *dev) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + + dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT | + HIFN_D_VALID | HIFN_D_LAST); + /* + * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID | + * HIFN_D_LAST); + */ + + if (++dma->resi == HIFN_D_RES_RSIZE) { + dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST); + dma->resi = 0; + } + + dma->resu++; + + if (!(dev->flags & HIFN_FLAG_RES_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA); + dev->flags |= HIFN_FLAG_RES_BUSY; + } +} + +static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page, + unsigned offset, unsigned size, int last) +{ + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + int idx; + dma_addr_t addr; + + addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE); + + idx = dma->dsti; + dma->dstr[idx].p = __cpu_to_le32(addr); + dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID | + HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0)); + + if (++idx == HIFN_D_DST_RSIZE) { + dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID | + HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | + (last ? HIFN_D_LAST : 0)); + idx = 0; + } + dma->dsti = idx; + dma->dstu++; + + if (!(dev->flags & HIFN_FLAG_DST_BUSY)) { + hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA); + dev->flags |= HIFN_FLAG_DST_BUSY; + } +} + +static int hifn_setup_dma(struct hifn_device *dev, + struct hifn_context *ctx, struct hifn_request_context *rctx, + struct scatterlist *src, struct scatterlist *dst, + unsigned int nbytes, void *priv) +{ + struct scatterlist *t; + struct page *spage, *dpage; + unsigned int soff, doff; + unsigned int n, len; + + n = nbytes; + while (n) { + spage = sg_page(src); + soff = src->offset; + len = min(src->length, n); + + hifn_setup_src_desc(dev, spage, soff, len, n - len == 0); + + src++; + n -= len; + } + + t = &rctx->walk.cache[0]; + n = nbytes; + while (n) { + if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + BUG_ON(!sg_page(t)); + dpage = sg_page(t); + doff = 0; + len = t->length; + } else { + BUG_ON(!sg_page(dst)); + dpage = sg_page(dst); + doff = dst->offset; + len = dst->length; + } + len = min(len, n); + + hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0); + + dst++; + t++; + n -= len; + } + + hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes); + hifn_setup_res_desc(dev); + return 0; +} + static int ablkcipher_walk_init(struct ablkcipher_walk *w, int num, gfp_t gfp_flags) { @@ -1264,32 +1457,26 @@ static void ablkcipher_walk_exit(struct ablkcipher_walk *w) w->num = 0; } -static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src, +static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst, unsigned int size, unsigned int *nbytesp) { unsigned int copy, drest = *drestp, nbytes = *nbytesp; int idx = 0; - void *saddr; if (drest < size || size > nbytes) return -EINVAL; while (size) { - copy = min(drest, src->length); - - saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1); - memcpy(daddr, saddr + src->offset, copy); - kunmap_atomic(saddr, KM_SOFTIRQ1); + copy = min(drest, min(size, dst->length)); size -= copy; drest -= copy; nbytes -= copy; - daddr += copy; dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", __func__, copy, size, drest, nbytes); - src++; + dst++; idx++; } @@ -1302,12 +1489,7 @@ static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist static int ablkcipher_walk(struct ablkcipher_request *req, struct ablkcipher_walk *w) { - unsigned blocksize = - crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req)); - unsigned alignmask = - crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req)); - struct scatterlist *src, *dst, *t; - void *daddr; + struct scatterlist *dst, *t; unsigned int nbytes = req->nbytes, offset, copy, diff; int idx, tidx, err; @@ -1317,33 +1499,27 @@ static int ablkcipher_walk(struct ablkcipher_request *req, if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED)) return -EINVAL; - src = &req->src[idx]; dst = &req->dst[idx]; - dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, " - "blocksize: %u, nbytes: %u.\n", - __func__, src->length, dst->length, src->offset, - dst->offset, offset, blocksize, nbytes); - - if (src->length & (blocksize - 1) || - src->offset & (alignmask - 1) || - dst->length & (blocksize - 1) || - dst->offset & (alignmask - 1) || - offset) { - unsigned slen = src->length - offset; + dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", + __func__, dst->length, dst->offset, offset, nbytes); + + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || + offset) { + unsigned slen = min(dst->length - offset, nbytes); unsigned dlen = PAGE_SIZE; t = &w->cache[idx]; - daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0); - err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes); + err = ablkcipher_add(&dlen, dst, slen, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; - copy = slen & ~(blocksize - 1); - diff = slen & (blocksize - 1); + copy = slen & ~(HIFN_D_DST_DALIGN - 1); + diff = slen & (HIFN_D_DST_DALIGN - 1); if (dlen < nbytes) { /* @@ -1351,7 +1527,7 @@ static int ablkcipher_walk(struct ablkcipher_request *req, * to put there additional blocksized chunk, * so we mark that page as containing only * blocksize aligned chunks: - * t->length = (slen & ~(blocksize - 1)); + * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); * and increase number of bytes to be processed * in next chunk: * nbytes += diff; @@ -1374,21 +1550,19 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } else { copy += diff + nbytes; - src = &req->src[idx]; + dst = &req->dst[idx]; - err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes); + err = ablkcipher_add(&dlen, dst, nbytes, &nbytes); if (err < 0) - goto err_out_unmap; + return err; idx += err; } t->length = copy; t->offset = offset; - - kunmap_atomic(daddr, KM_SOFTIRQ0); } else { - nbytes -= src->length; + nbytes -= min(dst->length, nbytes); idx++; } @@ -1396,58 +1570,46 @@ static int ablkcipher_walk(struct ablkcipher_request *req, } return tidx; - -err_out_unmap: - kunmap_atomic(daddr, KM_SOFTIRQ0); - return err; } static int hifn_setup_session(struct ablkcipher_request *req) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); struct hifn_device *dev = ctx->dev; - struct page *spage, *dpage; - unsigned long soff, doff, flags; - unsigned int nbytes = req->nbytes, idx = 0, len; + unsigned long dlen, flags; + unsigned int nbytes = req->nbytes, idx = 0; int err = -EINVAL, sg_num; - struct scatterlist *src, *dst, *t; - unsigned blocksize = - crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req)); - unsigned alignmask = - crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req)); + struct scatterlist *dst; - if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB) + if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB) goto err_out_exit; - ctx->walk.flags = 0; + rctx->walk.flags = 0; while (nbytes) { - src = &req->src[idx]; dst = &req->dst[idx]; + dlen = min(dst->length, nbytes); - if (src->length & (blocksize - 1) || - src->offset & (alignmask - 1) || - dst->length & (blocksize - 1) || - dst->offset & (alignmask - 1)) { - ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; - } + if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || + !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN)) + rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED; - nbytes -= src->length; + nbytes -= dlen; idx++; } - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC); + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { + err = ablkcipher_walk_init(&rctx->walk, idx, GFP_ATOMIC); if (err < 0) return err; } - nbytes = req->nbytes; - idx = 0; - - sg_num = ablkcipher_walk(req, &ctx->walk); - - atomic_set(&ctx->sg_num, sg_num); + sg_num = ablkcipher_walk(req, &rctx->walk); + if (sg_num < 0) { + err = sg_num; + goto err_out_exit; + } spin_lock_irqsave(&dev->lock, flags); if (dev->started + sg_num > HIFN_QUEUE_LENGTH) { @@ -1455,37 +1617,11 @@ static int hifn_setup_session(struct ablkcipher_request *req) goto err_out; } - dev->snum++; - dev->started += sg_num; - - while (nbytes) { - src = &req->src[idx]; - dst = &req->dst[idx]; - t = &ctx->walk.cache[idx]; - - if (t->length) { - spage = dpage = sg_page(t); - soff = doff = 0; - len = t->length; - } else { - spage = sg_page(src); - soff = src->offset; - - dpage = sg_page(dst); - doff = dst->offset; - - len = dst->length; - } - - idx++; - - err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes, - req, ctx); - if (err) - goto err_out; + err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req); + if (err) + goto err_out; - nbytes -= len; - } + dev->snum++; dev->active = HIFN_DEFAULT_ACTIVE_NUM; spin_unlock_irqrestore(&dev->lock, flags); @@ -1495,12 +1631,13 @@ static int hifn_setup_session(struct ablkcipher_request *req) err_out: spin_unlock_irqrestore(&dev->lock, flags); err_out_exit: - if (err && printk_ratelimit()) - dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " + if (err) { + printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " "type: %u, err: %d.\n", - dev->name, ctx->iv, ctx->ivsize, + dev->name, rctx->iv, rctx->ivsize, ctx->key, ctx->keysize, - ctx->mode, ctx->op, ctx->type, err); + rctx->mode, rctx->op, rctx->type, err); + } return err; } @@ -1510,31 +1647,33 @@ static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) int n, err; u8 src[16]; struct hifn_context ctx; + struct hifn_request_context rctx; u8 fips_aes_ecb_from_zero[16] = { 0x66, 0xE9, 0x4B, 0xD4, 0xEF, 0x8A, 0x2C, 0x3B, 0x88, 0x4C, 0xFA, 0x59, 0xCA, 0x34, 0x2B, 0x2E}; + struct scatterlist sg; memset(src, 0, sizeof(src)); memset(ctx.key, 0, sizeof(ctx.key)); ctx.dev = dev; ctx.keysize = 16; - ctx.ivsize = 0; - ctx.iv = NULL; - ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; - ctx.mode = ACRYPTO_MODE_ECB; - ctx.type = ACRYPTO_TYPE_AES_128; - atomic_set(&ctx.sg_num, 1); - - err = hifn_setup_dma(dev, - virt_to_page(src), offset_in_page(src), - virt_to_page(src), offset_in_page(src), - sizeof(src), NULL, &ctx); + rctx.ivsize = 0; + rctx.iv = NULL; + rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; + rctx.mode = ACRYPTO_MODE_ECB; + rctx.type = ACRYPTO_TYPE_AES_128; + rctx.walk.cache[0].length = 0; + + sg_init_one(&sg, &src, sizeof(src)); + + err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL); if (err) goto err_out; + dev->started = 0; msleep(200); dprintk("%s: decoded: ", dev->name); @@ -1561,6 +1700,7 @@ static int hifn_start_device(struct hifn_device *dev) { int err; + dev->started = dev->active = 0; hifn_reset_dma(dev, 1); err = hifn_enable_crypto(dev); @@ -1589,8 +1729,7 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset return -EINVAL; while (size) { - - copy = min(dst->length, srest); + copy = min(srest, min(dst->length, size)); daddr = kmap_atomic(sg_page(dst), KM_IRQ0); memcpy(daddr + dst->offset + offset, saddr, copy); @@ -1615,90 +1754,65 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset return idx; } -static void hifn_process_ready(struct ablkcipher_request *req, int error) +static inline void hifn_complete_sa(struct hifn_device *dev, int i) { - struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); - struct hifn_device *dev; - - dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx); + unsigned long flags; - dev = ctx->dev; - dprintk("%s: req: %p, started: %d, sg_num: %d.\n", - __func__, req, dev->started, atomic_read(&ctx->sg_num)); + spin_lock_irqsave(&dev->lock, flags); + dev->sa[i] = NULL; + dev->started--; + if (dev->started < 0) + printk("%s: started: %d.\n", __func__, dev->started); + spin_unlock_irqrestore(&dev->lock, flags); + BUG_ON(dev->started < 0); +} - if (--dev->started < 0) - BUG(); +static void hifn_process_ready(struct ablkcipher_request *req, int error) +{ + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); - if (atomic_dec_and_test(&ctx->sg_num)) { + if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { unsigned int nbytes = req->nbytes; int idx = 0, err; struct scatterlist *dst, *t; void *saddr; - if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) { - while (nbytes) { - t = &ctx->walk.cache[idx]; - dst = &req->dst[idx]; + while (nbytes) { + t = &rctx->walk.cache[idx]; + dst = &req->dst[idx]; - dprintk("\n%s: sg_page(t): %p, t->length: %u, " - "sg_page(dst): %p, dst->length: %u, " - "nbytes: %u.\n", - __func__, sg_page(t), t->length, - sg_page(dst), dst->length, nbytes); - - if (!t->length) { - nbytes -= dst->length; - idx++; - continue; - } + dprintk("\n%s: sg_page(t): %p, t->length: %u, " + "sg_page(dst): %p, dst->length: %u, " + "nbytes: %u.\n", + __func__, sg_page(t), t->length, + sg_page(dst), dst->length, nbytes); - saddr = kmap_atomic(sg_page(t), KM_IRQ1); + if (!t->length) { + nbytes -= min(dst->length, nbytes); + idx++; + continue; + } - err = ablkcipher_get(saddr, &t->length, t->offset, - dst, nbytes, &nbytes); - if (err < 0) { - kunmap_atomic(saddr, KM_IRQ1); - break; - } + saddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0); - idx += err; - kunmap_atomic(saddr, KM_IRQ1); + err = ablkcipher_get(saddr, &t->length, t->offset, + dst, nbytes, &nbytes); + if (err < 0) { + kunmap_atomic(saddr, KM_SOFTIRQ0); + break; } - ablkcipher_walk_exit(&ctx->walk); + idx += err; + kunmap_atomic(saddr, KM_SOFTIRQ0); } - req->base.complete(&req->base, error); + ablkcipher_walk_exit(&rctx->walk); } -} - -static void hifn_check_for_completion(struct hifn_device *dev, int error) -{ - int i; - struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - - for (i=0; iresr[i]; - if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) { - dev->success++; - dev->reset = 0; - hifn_process_ready(dev->sa[i], error); - dev->sa[i] = NULL; - } - - if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER)) - if (printk_ratelimit()) - printk("%s: overflow detected [d: %u, o: %u] " - "at %d resr: l: %08x, p: %08x.\n", - dev->name, - !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)), - !!(d->l & __cpu_to_le32(HIFN_D_OVER)), - i, d->l, d->p); - } + req->base.complete(&req->base, error); } -static void hifn_clear_rings(struct hifn_device *dev) +static void hifn_clear_rings(struct hifn_device *dev, int error) { struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i, u; @@ -1715,21 +1829,26 @@ static void hifn_clear_rings(struct hifn_device *dev) if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_RES_RSIZE) - u--; + if (dev->sa[i]) { + dev->success++; + dev->reset = 0; + hifn_process_ready(dev->sa[i], error); + hifn_complete_sa(dev, i); + } - if (++i == (HIFN_D_RES_RSIZE + 1)) + if (++i == HIFN_D_RES_RSIZE) i = 0; + u--; } dma->resk = i; dma->resu = u; i = dma->srck; u = dma->srcu; while (u != 0) { - if (i == HIFN_D_SRC_RSIZE) - i = 0; if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_SRC_RSIZE) + i = 0; + u--; } dma->srck = i; dma->srcu = u; @@ -1737,20 +1856,19 @@ static void hifn_clear_rings(struct hifn_device *dev) while (u != 0) { if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - if (i != HIFN_D_CMD_RSIZE) - u--; - if (++i == (HIFN_D_CMD_RSIZE + 1)) + if (++i == HIFN_D_CMD_RSIZE) i = 0; + u--; } dma->cmdk = i; dma->cmdu = u; i = dma->dstk; u = dma->dstu; while (u != 0) { - if (i == HIFN_D_DST_RSIZE) - i = 0; if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID)) break; - i++, u--; + if (++i == HIFN_D_DST_RSIZE) + i = 0; + u--; } dma->dstk = i; dma->dstu = u; @@ -1795,30 +1913,39 @@ static void hifn_work(struct work_struct *work) } else dev->active--; - if (dev->prev_success == dev->success && dev->started) + if ((dev->prev_success == dev->success) && dev->started) reset = 1; dev->prev_success = dev->success; spin_unlock_irqrestore(&dev->lock, flags); if (reset) { - dprintk("%s: r: %08x, active: %d, started: %d, " - "success: %lu: reset: %d.\n", - dev->name, r, dev->active, dev->started, - dev->success, reset); - if (++dev->reset >= 5) { - dprintk("%s: really hard reset.\n", dev->name); + int i; + struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; + + printk("%s: r: %08x, active: %d, started: %d, " + "success: %lu: qlen: %u/%u, reset: %d.\n", + dev->name, r, dev->active, dev->started, + dev->success, dev->queue.qlen, dev->queue.max_qlen, + reset); + + printk("%s: res: ", __func__); + for (i=0; iresr[i].l, dev->sa[i]); + if (dev->sa[i]) { + hifn_process_ready(dev->sa[i], -ENODEV); + hifn_complete_sa(dev, i); + } + } + printk("\n"); + hifn_reset_dma(dev, 1); hifn_stop_device(dev); hifn_start_device(dev); dev->reset = 0; } - spin_lock_irqsave(&dev->lock, flags); - hifn_check_for_completion(dev, -EBUSY); - hifn_clear_rings(dev); - dev->started = 0; - spin_unlock_irqrestore(&dev->lock, flags); + tasklet_schedule(&dev->tasklet); } schedule_delayed_work(&dev->work, HZ); @@ -1835,8 +1962,8 @@ static irqreturn_t hifn_interrupt(int irq, void *data) dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, - dma->cmdu, dma->srcu, dma->dstu, dma->resu, - dma->cmdi, dma->srci, dma->dsti, dma->resi); + dma->cmdi, dma->srci, dma->dsti, dma->resi, + dma->cmdu, dma->srcu, dma->dstu, dma->resu); if ((dmacsr & dev->dmareg) == 0) return IRQ_NONE; @@ -1853,11 +1980,10 @@ static irqreturn_t hifn_interrupt(int irq, void *data) if (restart) { u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); - if (printk_ratelimit()) - printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), - !!(dmacsr & HIFN_DMACSR_D_OVER), - puisr, !!(puisr & HIFN_PUISR_DSTOVER)); + printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), + !!(dmacsr & HIFN_DMACSR_D_OVER), + puisr, !!(puisr & HIFN_PUISR_DSTOVER)); if (!!(puisr & HIFN_PUISR_DSTOVER)) hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER | @@ -1867,12 +1993,11 @@ static irqreturn_t hifn_interrupt(int irq, void *data) restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); if (restart) { - if (printk_ratelimit()) - printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), - !!(dmacsr & HIFN_DMACSR_S_ABORT), - !!(dmacsr & HIFN_DMACSR_D_ABORT), - !!(dmacsr & HIFN_DMACSR_R_ABORT)); + printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n", + dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), + !!(dmacsr & HIFN_DMACSR_S_ABORT), + !!(dmacsr & HIFN_DMACSR_D_ABORT), + !!(dmacsr & HIFN_DMACSR_R_ABORT)); hifn_reset_dma(dev, 1); hifn_init_dma(dev); hifn_init_registers(dev); @@ -1885,7 +2010,6 @@ static irqreturn_t hifn_interrupt(int irq, void *data) } tasklet_schedule(&dev->tasklet); - hifn_clear_rings(dev); return IRQ_HANDLED; } @@ -1899,21 +2023,25 @@ static void hifn_flush(struct hifn_device *dev) struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i; - spin_lock_irqsave(&dev->lock, flags); for (i=0; iresr[i]; if (dev->sa[i]) { hifn_process_ready(dev->sa[i], (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0); + hifn_complete_sa(dev, i); } } + spin_lock_irqsave(&dev->lock, flags); while ((async_req = crypto_dequeue_request(&dev->queue))) { ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); + spin_unlock_irqrestore(&dev->lock, flags); hifn_process_ready(req, -ENODEV); + + spin_lock_irqsave(&dev->lock, flags); } spin_unlock_irqrestore(&dev->lock, flags); } @@ -1972,6 +2100,7 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, u8 type, u8 mode) { struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm); + struct hifn_request_context *rctx = ablkcipher_request_ctx(req); unsigned ivsize; ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req)); @@ -1992,11 +2121,11 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, type = ACRYPTO_TYPE_AES_256; } - ctx->op = op; - ctx->mode = mode; - ctx->type = type; - ctx->iv = req->info; - ctx->ivsize = ivsize; + rctx->op = op; + rctx->mode = mode; + rctx->type = type; + rctx->iv = req->info; + rctx->ivsize = ivsize; /* * HEAVY TODO: needs to kick Herbert XU to write documentation. @@ -2009,7 +2138,7 @@ static int hifn_setup_crypto_req(struct ablkcipher_request *req, u8 op, static int hifn_process_queue(struct hifn_device *dev) { - struct crypto_async_request *async_req; + struct crypto_async_request *async_req, *backlog; struct hifn_context *ctx; struct ablkcipher_request *req; unsigned long flags; @@ -2017,12 +2146,16 @@ static int hifn_process_queue(struct hifn_device *dev) while (dev->started < HIFN_QUEUE_LENGTH) { spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); async_req = crypto_dequeue_request(&dev->queue); spin_unlock_irqrestore(&dev->lock, flags); if (!async_req) break; + if (backlog) + backlog->complete(backlog, -EINPROGRESS); + ctx = crypto_tfm_ctx(async_req->tfm); req = container_of(async_req, struct ablkcipher_request, base); @@ -2046,9 +2179,9 @@ static int hifn_setup_crypto(struct ablkcipher_request *req, u8 op, return err; if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) - err = hifn_process_queue(dev); + hifn_process_queue(dev); - return err; + return -EINPROGRESS; } /* @@ -2208,7 +2341,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * 3DES ECB, CBC, CFB and OFB modes. */ { - .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2218,7 +2351,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2228,8 +2361,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8, .ablkcipher = { + .ivsize = HIFN_IV_LENGTH, .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, .setkey = hifn_setkey, @@ -2238,7 +2372,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8, + .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_3DES_KEY_LENGTH, .max_keysize = HIFN_3DES_KEY_LENGTH, @@ -2252,7 +2386,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * DES ECB, CBC, CFB and OFB modes. */ { - .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2262,7 +2396,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2272,8 +2406,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8, .ablkcipher = { + .ivsize = HIFN_IV_LENGTH, .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, .setkey = hifn_setkey, @@ -2282,7 +2417,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8, + .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8, .ablkcipher = { .min_keysize = HIFN_DES_KEY_LENGTH, .max_keysize = HIFN_DES_KEY_LENGTH, @@ -2296,7 +2431,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { * AES ECB, CBC, CFB and OFB modes. */ { - .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2306,8 +2441,9 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16, .ablkcipher = { + .ivsize = HIFN_AES_IV_LENGTH, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = hifn_setkey, @@ -2316,7 +2452,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2326,7 +2462,7 @@ static struct hifn_alg_template hifn_alg_templates[] = { }, }, { - .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16, + .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16, .ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, @@ -2344,7 +2480,7 @@ static int hifn_cra_init(struct crypto_tfm *tfm) struct hifn_context *ctx = crypto_tfm_ctx(tfm); ctx->dev = ha->dev; - + tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context); return 0; } @@ -2358,15 +2494,14 @@ static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t) return -ENOMEM; snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name); - snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name); + snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s", + t->drv_name, dev->name); alg->alg.cra_priority = 300; - alg->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_ASYNC; + alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; alg->alg.cra_blocksize = t->bsize; alg->alg.cra_ctxsize = sizeof(struct hifn_context); - alg->alg.cra_alignmask = 15; - if (t->bsize == 8) - alg->alg.cra_alignmask = 3; + alg->alg.cra_alignmask = 0; alg->alg.cra_type = &crypto_ablkcipher_type; alg->alg.cra_module = THIS_MODULE; alg->alg.cra_u.ablkcipher = t->ablkcipher; @@ -2423,7 +2558,10 @@ static void hifn_tasklet_callback(unsigned long data) * (like dev->success), but they are used in process * context or update is atomic (like setting dev->sa[i] to NULL). */ - hifn_check_for_completion(dev, 0); + hifn_clear_rings(dev, 0); + + if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen) + hifn_process_queue(dev); } static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -2480,22 +2618,11 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_out_unmap_bars; } - dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER); - if (!dev->result_mem) { - dprintk("Failed to allocate %d pages for result_mem.\n", - HIFN_MAX_RESULT_ORDER); - goto err_out_unmap_bars; - } - memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<dst = pci_map_single(pdev, (void *)dev->result_mem, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE); - dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma), &dev->desc_dma); if (!dev->desc_virt) { dprintk("Failed to allocate descriptor rings.\n"); - goto err_out_free_result_pages; + goto err_out_unmap_bars; } memset(dev->desc_virt, 0, sizeof(struct hifn_dma)); @@ -2526,10 +2653,14 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_out_stop_device; - err = hifn_register_alg(dev); + err = hifn_register_rng(dev); if (err) goto err_out_stop_device; + err = hifn_register_alg(dev); + if (err) + goto err_out_unregister_rng; + INIT_DELAYED_WORK(&dev->work, hifn_work); schedule_delayed_work(&dev->work, HZ); @@ -2539,6 +2670,8 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) return 0; +err_out_unregister_rng: + hifn_unregister_rng(dev); err_out_stop_device: hifn_reset_dma(dev, 1); hifn_stop_device(dev); @@ -2549,11 +2682,6 @@ err_out_free_desc: pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); -err_out_free_result_pages: - pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); - err_out_unmap_bars: for (i=0; i<3; ++i) if (dev->bar[i]) @@ -2579,6 +2707,7 @@ static void hifn_remove(struct pci_dev *pdev) cancel_delayed_work(&dev->work); flush_scheduled_work(); + hifn_unregister_rng(dev); hifn_unregister_alg(dev); hifn_reset_dma(dev, 1); hifn_stop_device(dev); @@ -2590,10 +2719,6 @@ static void hifn_remove(struct pci_dev *pdev) pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); - pci_unmap_single(pdev, dev->dst, - PAGE_SIZE << HIFN_MAX_RESULT_ORDER, - PCI_DMA_FROMDEVICE); - free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER); for (i=0; i<3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2621,8 +2746,36 @@ static struct pci_driver hifn_pci_driver = { static int __devinit hifn_init(void) { + unsigned int freq; int err; + if (sizeof(dma_addr_t) > 4) { + printk(KERN_INFO "HIFN supports only 32-bit addresses.\n"); + return -EINVAL; + } + + if (strncmp(hifn_pll_ref, "ext", 3) && + strncmp(hifn_pll_ref, "pci", 3)) { + printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, " + "must be pci or ext"); + return -EINVAL; + } + + /* + * For the 7955/7956 the reference clock frequency must be in the + * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz, + * but this chip is currently not supported. + */ + if (hifn_pll_ref[3] != '\0') { + freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); + if (freq < 20 || freq > 100) { + printk(KERN_ERR "hifn795x: invalid hifn_pll_ref " + "frequency, must be in the range " + "of 20-100"); + return -EINVAL; + } + } + err = pci_register_driver(&hifn_pci_driver); if (err < 0) { dprintk("Failed to register PCI driver for %s device.\n",