#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/moduleparam.h>
#include <linux/mod_devicetable.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
#include <linux/highmem.h>
-#include <linux/interrupt.h>
#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/ktime.h>
#include <crypto/algapi.h>
#include <crypto/des.h>
#include <asm/kmap_types.h>
-#undef dprintk
-
-#define HIFN_TEST
//#define HIFN_DEBUG
#ifdef HIFN_DEBUG
#define dprintk(f, a...) do {} while (0)
#endif
+static char hifn_pll_ref[sizeof("extNNN")] = "ext";
+module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
+MODULE_PARM_DESC(hifn_pll_ref,
+ "PLL reference clock (pci[freq] or ext[freq], default ext)");
+
static atomic_t hifn_dev_number;
#define ACRYPTO_OP_DECRYPT 0
#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
-#define HIFN_PLL_7956 0x00001d18 /* 7956 PLL config value */
+/* PLL configuration register */
+#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
+#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
+#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
+#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
+#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
+#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
+#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
+#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
+#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
+#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
+#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
+#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
+#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
+#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
+#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
+#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
+#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
+
+#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
/* Public key reset register (HIFN_1_PUB_RESET) */
#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
#define HIFN_NAMESIZE 32
#define HIFN_MAX_RESULT_ORDER 5
-#define HIFN_D_CMD_RSIZE 24*4
-#define HIFN_D_SRC_RSIZE 80*4
-#define HIFN_D_DST_RSIZE 80*4
-#define HIFN_D_RES_RSIZE 24*4
+#define HIFN_D_CMD_RSIZE 24*1
+#define HIFN_D_SRC_RSIZE 80*1
+#define HIFN_D_DST_RSIZE 80*1
+#define HIFN_D_RES_RSIZE 24*1
-#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5
+#define HIFN_D_DST_DALIGN 4
+
+#define HIFN_QUEUE_LENGTH (HIFN_D_CMD_RSIZE - 1)
#define AES_MIN_KEY_SIZE 16
#define AES_MAX_KEY_SIZE 32
struct hifn_desc
{
- volatile u32 l;
- volatile u32 p;
+ volatile __le32 l;
+ volatile __le32 p;
};
struct hifn_dma {
u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND];
u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT];
- u64 test_src, test_dst;
-
/*
* Our current positions for insertion and removal from the descriptor
* rings.
struct pci_dev *pdev;
void __iomem *bar[3];
- unsigned long result_mem;
- dma_addr_t dst;
-
void *desc_virt;
dma_addr_t desc_dma;
spinlock_t lock;
- void *priv;
-
u32 flags;
int active, started;
struct delayed_work work;
struct crypto_queue queue;
struct list_head alg_list;
+
+ unsigned int pk_clk_freq;
+
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ unsigned int rng_wait_time;
+ ktime_t rngtime;
+ struct hwrng rng;
+#endif
};
#define HIFN_D_LENGTH 0x0000ffff
struct hifn_base_command
{
- volatile u16 masks;
- volatile u16 session_num;
- volatile u16 total_source_count;
- volatile u16 total_dest_count;
+ volatile __le16 masks;
+ volatile __le16 session_num;
+ volatile __le16 total_source_count;
+ volatile __le16 total_dest_count;
};
#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
*/
struct hifn_crypt_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
*/
struct hifn_mac_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_MAC_CMD_ALG_MASK 0x0001
struct hifn_comp_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_COMP_CMD_SRCLEN_M 0xc000
struct hifn_base_result
{
- volatile u16 flags;
- volatile u16 session;
- volatile u16 src_cnt; /* 15:0 of source count */
- volatile u16 dst_cnt; /* 15:0 of dest count */
+ volatile __le16 flags;
+ volatile __le16 session;
+ volatile __le16 src_cnt; /* 15:0 of source count */
+ volatile __le16 dst_cnt; /* 15:0 of dest count */
};
#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
struct hifn_comp_result
{
- volatile u16 flags;
- volatile u16 crc;
+ volatile __le16 flags;
+ volatile __le16 crc;
};
#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
struct hifn_mac_result
{
- volatile u16 flags;
- volatile u16 reserved;
+ volatile __le16 flags;
+ volatile __le16 reserved;
/* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
};
struct hifn_crypt_result
{
- volatile u16 flags;
- volatile u16 reserved;
+ volatile __le16 flags;
+ volatile __le16 reserved;
};
#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
struct hifn_context
{
- u8 key[HIFN_MAX_CRYPT_KEY_LENGTH], *iv;
+ u8 key[HIFN_MAX_CRYPT_KEY_LENGTH];
struct hifn_device *dev;
- unsigned int keysize, ivsize;
+ unsigned int keysize;
+};
+
+struct hifn_request_context
+{
+ u8 *iv;
+ unsigned int ivsize;
u8 op, type, mode, unused;
struct ablkcipher_walk walk;
- atomic_t sg_num;
};
-#define crypto_alg_to_hifn(alg) container_of(alg, struct hifn_crypto_alg, alg)
+#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
{
u32 ret;
- ret = readl((char *)(dev->bar[0]) + reg);
+ ret = readl(dev->bar[0] + reg);
return ret;
}
{
u32 ret;
- ret = readl((char *)(dev->bar[1]) + reg);
+ ret = readl(dev->bar[1] + reg);
return ret;
}
static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
{
- writel(val, (char *)(dev->bar[0]) + reg);
+ writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
}
static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
{
- writel(val, (char *)(dev->bar[1]) + reg);
+ writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
}
static void hifn_wait_puc(struct hifn_device *dev)
}
};
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+static int hifn_rng_data_present(struct hwrng *rng, int wait)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+ s64 nsec;
+
+ nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
+ nsec -= dev->rng_wait_time;
+ if (nsec <= 0)
+ return 1;
+ if (!wait)
+ return 0;
+ ndelay(nsec);
+ return 1;
+}
+
+static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+
+ *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+ return 4;
+}
+
+static int hifn_register_rng(struct hifn_device *dev)
+{
+ /*
+ * We must wait at least 256 Pk_clk cycles between two reads of the rng.
+ */
+ dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) *
+ 256;
+
+ dev->rng.name = dev->name;
+ dev->rng.data_present = hifn_rng_data_present,
+ dev->rng.data_read = hifn_rng_data_read,
+ dev->rng.priv = (unsigned long)dev;
+
+ return hwrng_register(&dev->rng);
+}
+
+static void hifn_unregister_rng(struct hifn_device *dev)
+{
+ hwrng_unregister(&dev->rng);
+}
+#else
+#define hifn_register_rng(dev) 0
+#define hifn_unregister_rng(dev)
+#endif
+
static int hifn_init_pubrng(struct hifn_device *dev)
{
int i;
dev->dmareg |= HIFN_DMAIER_PUBDONE;
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
- dprintk("Chip %s: Public key engine has been sucessfully "
+ dprintk("Chip %s: Public key engine has been successfully "
"initialised.\n", dev->name);
}
dprintk("Chip %s: RNG engine has been successfully initialised.\n",
dev->name);
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ /* First value must be discarded */
+ hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+#endif
return 0;
}
char *offtbl = NULL;
int i;
- for (i = 0; i < sizeof(pci2id)/sizeof(pci2id[0]); i++) {
+ for (i = 0; i < ARRAY_SIZE(pci2id); i++) {
if (pci2id[i].pci_vendor == dev->pdev->vendor &&
pci2id[i].pci_prod == dev->pdev->device) {
offtbl = pci2id[i].card_id;
dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
}
+/*
+ * Initialize the PLL. We need to know the frequency of the reference clock
+ * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
+ * allows us to operate without the risk of overclocking the chip. If it
+ * actually uses 33MHz, the chip will operate at half the speed, this can be
+ * overriden by specifying the frequency as module parameter (pci33).
+ *
+ * Unfortunately the PCI clock is not very suitable since the HIFN needs a
+ * stable clock and the PCI clock frequency may vary, so the default is the
+ * external clock. There is no way to find out its frequency, we default to
+ * 66MHz since according to Mike Ham of HiFn, almost every board in existence
+ * has an external crystal populated at 66MHz.
+ */
+static void hifn_init_pll(struct hifn_device *dev)
+{
+ unsigned int freq, m;
+ u32 pllcfg;
+
+ pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
+
+ if (strncmp(hifn_pll_ref, "ext", 3) == 0)
+ pllcfg |= HIFN_PLL_REF_CLK_PLL;
+ else
+ pllcfg |= HIFN_PLL_REF_CLK_HBI;
+
+ if (hifn_pll_ref[3] != '\0')
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ else {
+ freq = 66;
+ printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, "
+ "override with hifn_pll_ref=%.3s<frequency>\n",
+ freq, hifn_pll_ref);
+ }
+
+ m = HIFN_PLL_FCK_MAX / freq;
+
+ pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
+ if (m <= 8)
+ pllcfg |= HIFN_PLL_IS_1_8;
+ else
+ pllcfg |= HIFN_PLL_IS_9_12;
+
+ /* Select clock source and enable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
+
+ /* Let the chip lock to the input clock */
+ mdelay(10);
+
+ /* Disable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
+
+ /* Switch the engines to the PLL */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
+
+ /*
+ * The Fpk_clk runs at half the total speed. Its frequency is needed to
+ * calculate the minimum time between two reads of the rng. Since 33MHz
+ * is actually 33.333... we overestimate the frequency here, resulting
+ * in slightly larger intervals.
+ */
+ dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
+}
+
static void hifn_init_registers(struct hifn_device *dev)
{
u32 dptr = dev->desc_dma;
hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
/* write all 4 ring address registers */
- hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, cmdr[0])));
- hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, srcr[0])));
- hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, dstr[0])));
- hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, resr[0])));
+ hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
+ offsetof(struct hifn_dma, cmdr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
+ offsetof(struct hifn_dma, srcr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
+ offsetof(struct hifn_dma, dstr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
+ offsetof(struct hifn_dma, resr[0]));
mdelay(2);
#if 0
#else
hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
#endif
- hifn_write_1(dev, HIFN_1_PLL, HIFN_PLL_7956);
+ hifn_init_pll(dev);
hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
return cmd_len;
}
-static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
- unsigned int offset, unsigned int size)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
- int idx;
- dma_addr_t addr;
-
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
-
- idx = dma->srci;
-
- dma->srcr[idx].p = __cpu_to_le32(addr);
- dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID |
- HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST;
-
- if (++idx == HIFN_D_SRC_RSIZE) {
- dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP |
- HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
- idx = 0;
- }
-
- dma->srci = idx;
- dma->srcu++;
-
- if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
- dev->flags |= HIFN_FLAG_SRC_BUSY;
- }
-
- return size;
-}
-
-static void hifn_setup_res_desc(struct hifn_device *dev)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
-
- dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
- HIFN_D_VALID | HIFN_D_LAST);
- /*
- * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
- * HIFN_D_LAST | HIFN_D_NOINVALID);
- */
-
- if (++dma->resi == HIFN_D_RES_RSIZE) {
- dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
- dma->resi = 0;
- }
-
- dma->resu++;
-
- if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
- dev->flags |= HIFN_FLAG_RES_BUSY;
- }
-}
-
-static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
- unsigned offset, unsigned size)
-{
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
- int idx;
- dma_addr_t addr;
-
- addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
-
- idx = dma->dsti;
- dma->dstr[idx].p = __cpu_to_le32(addr);
- dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
- HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
-
- if (++idx == HIFN_D_DST_RSIZE) {
- dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
- HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
- HIFN_D_LAST | HIFN_D_NOINVALID);
- idx = 0;
- }
- dma->dsti = idx;
- dma->dstu++;
-
- if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
- hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
- dev->flags |= HIFN_FLAG_DST_BUSY;
- }
-}
-
-static int hifn_setup_dma(struct hifn_device *dev, struct page *spage, unsigned int soff,
- struct page *dpage, unsigned int doff, unsigned int nbytes, void *priv,
- struct hifn_context *ctx)
+static int hifn_setup_cmd_desc(struct hifn_device *dev,
+ struct hifn_context *ctx, struct hifn_request_context *rctx,
+ void *priv, unsigned int nbytes)
{
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
int cmd_len, sa_idx;
u8 *buf, *buf_pos;
u16 mask;
- dprintk("%s: spage: %p, soffset: %u, dpage: %p, doffset: %u, nbytes: %u, priv: %p, ctx: %p.\n",
- dev->name, spage, soff, dpage, doff, nbytes, priv, ctx);
-
- sa_idx = dma->resi;
-
- hifn_setup_src_desc(dev, spage, soff, nbytes);
-
+ sa_idx = dma->cmdi;
buf_pos = buf = dma->command_bufs[dma->cmdi];
mask = 0;
- switch (ctx->op) {
+ switch (rctx->op) {
case ACRYPTO_OP_DECRYPT:
mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE;
break;
buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes,
nbytes, mask, dev->snum);
- if (ctx->op == ACRYPTO_OP_ENCRYPT || ctx->op == ACRYPTO_OP_DECRYPT) {
+ if (rctx->op == ACRYPTO_OP_ENCRYPT || rctx->op == ACRYPTO_OP_DECRYPT) {
u16 md = 0;
if (ctx->keysize)
md |= HIFN_CRYPT_CMD_NEW_KEY;
- if (ctx->iv && ctx->mode != ACRYPTO_MODE_ECB)
+ if (rctx->iv && rctx->mode != ACRYPTO_MODE_ECB)
md |= HIFN_CRYPT_CMD_NEW_IV;
- switch (ctx->mode) {
+ switch (rctx->mode) {
case ACRYPTO_MODE_ECB:
md |= HIFN_CRYPT_CMD_MODE_ECB;
break;
goto err_out;
}
- switch (ctx->type) {
+ switch (rctx->type) {
case ACRYPTO_TYPE_AES_128:
if (ctx->keysize != 16)
goto err_out;
buf_pos += hifn_setup_crypto_command(dev, buf_pos,
nbytes, nbytes, ctx->key, ctx->keysize,
- ctx->iv, ctx->ivsize, md);
+ rctx->iv, rctx->ivsize, md);
}
dev->sa[sa_idx] = priv;
+ dev->started++;
cmd_len = buf_pos - buf;
dma->cmdr[dma->cmdi].l = __cpu_to_le32(cmd_len | HIFN_D_VALID |
HIFN_D_LAST | HIFN_D_MASKDONEIRQ);
if (++dma->cmdi == HIFN_D_CMD_RSIZE) {
- dma->cmdr[dma->cmdi].l = __cpu_to_le32(HIFN_MAX_COMMAND |
+ dma->cmdr[dma->cmdi].l = __cpu_to_le32(
HIFN_D_VALID | HIFN_D_LAST |
HIFN_D_MASKDONEIRQ | HIFN_D_JUMP);
dma->cmdi = 0;
hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA);
dev->flags |= HIFN_FLAG_CMD_BUSY;
}
-
- hifn_setup_dst_desc(dev, dpage, doff, nbytes);
- hifn_setup_res_desc(dev);
-
return 0;
err_out:
return -EINVAL;
}
+static int hifn_setup_src_desc(struct hifn_device *dev, struct page *page,
+ unsigned int offset, unsigned int size, int last)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_TODEVICE);
+
+ idx = dma->srci;
+
+ dma->srcr[idx].p = __cpu_to_le32(addr);
+ dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
+
+ if (++idx == HIFN_D_SRC_RSIZE) {
+ dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
+ (last ? HIFN_D_LAST : 0));
+ idx = 0;
+ }
+
+ dma->srci = idx;
+ dma->srcu++;
+
+ if (!(dev->flags & HIFN_FLAG_SRC_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_S_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_SRC_BUSY;
+ }
+
+ return size;
+}
+
+static void hifn_setup_res_desc(struct hifn_device *dev)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ dma->resr[dma->resi].l = __cpu_to_le32(HIFN_USED_RESULT |
+ HIFN_D_VALID | HIFN_D_LAST);
+ /*
+ * dma->resr[dma->resi].l = __cpu_to_le32(HIFN_MAX_RESULT | HIFN_D_VALID |
+ * HIFN_D_LAST);
+ */
+
+ if (++dma->resi == HIFN_D_RES_RSIZE) {
+ dma->resr[HIFN_D_RES_RSIZE].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ | HIFN_D_LAST);
+ dma->resi = 0;
+ }
+
+ dma->resu++;
+
+ if (!(dev->flags & HIFN_FLAG_RES_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_R_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_RES_BUSY;
+ }
+}
+
+static void hifn_setup_dst_desc(struct hifn_device *dev, struct page *page,
+ unsigned offset, unsigned size, int last)
+{
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+ int idx;
+ dma_addr_t addr;
+
+ addr = pci_map_page(dev->pdev, page, offset, size, PCI_DMA_FROMDEVICE);
+
+ idx = dma->dsti;
+ dma->dstr[idx].p = __cpu_to_le32(addr);
+ dma->dstr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | (last ? HIFN_D_LAST : 0));
+
+ if (++idx == HIFN_D_DST_RSIZE) {
+ dma->dstr[idx].l = __cpu_to_le32(HIFN_D_VALID |
+ HIFN_D_JUMP | HIFN_D_MASKDONEIRQ |
+ (last ? HIFN_D_LAST : 0));
+ idx = 0;
+ }
+ dma->dsti = idx;
+ dma->dstu++;
+
+ if (!(dev->flags & HIFN_FLAG_DST_BUSY)) {
+ hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_D_CTRL_ENA);
+ dev->flags |= HIFN_FLAG_DST_BUSY;
+ }
+}
+
+static int hifn_setup_dma(struct hifn_device *dev,
+ struct hifn_context *ctx, struct hifn_request_context *rctx,
+ struct scatterlist *src, struct scatterlist *dst,
+ unsigned int nbytes, void *priv)
+{
+ struct scatterlist *t;
+ struct page *spage, *dpage;
+ unsigned int soff, doff;
+ unsigned int n, len;
+
+ n = nbytes;
+ while (n) {
+ spage = sg_page(src);
+ soff = src->offset;
+ len = min(src->length, n);
+
+ hifn_setup_src_desc(dev, spage, soff, len, n - len == 0);
+
+ src++;
+ n -= len;
+ }
+
+ t = &rctx->walk.cache[0];
+ n = nbytes;
+ while (n) {
+ if (t->length && rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
+ BUG_ON(!sg_page(t));
+ dpage = sg_page(t);
+ doff = 0;
+ len = t->length;
+ } else {
+ BUG_ON(!sg_page(dst));
+ dpage = sg_page(dst);
+ doff = dst->offset;
+ len = dst->length;
+ }
+ len = min(len, n);
+
+ hifn_setup_dst_desc(dev, dpage, doff, len, n - len == 0);
+
+ dst++;
+ t++;
+ n -= len;
+ }
+
+ hifn_setup_cmd_desc(dev, ctx, rctx, priv, nbytes);
+ hifn_setup_res_desc(dev);
+ return 0;
+}
+
static int ablkcipher_walk_init(struct ablkcipher_walk *w,
int num, gfp_t gfp_flags)
{
w->num = 0;
}
-static int ablkcipher_add(void *daddr, unsigned int *drestp, struct scatterlist *src,
+static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst,
unsigned int size, unsigned int *nbytesp)
{
unsigned int copy, drest = *drestp, nbytes = *nbytesp;
int idx = 0;
- void *saddr;
if (drest < size || size > nbytes)
return -EINVAL;
while (size) {
- copy = min(drest, src->length);
-
- saddr = kmap_atomic(sg_page(src), KM_SOFTIRQ1);
- memcpy(daddr, saddr + src->offset, copy);
- kunmap_atomic(saddr, KM_SOFTIRQ1);
+ copy = min(drest, min(size, dst->length));
size -= copy;
drest -= copy;
nbytes -= copy;
- daddr += copy;
dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n",
__func__, copy, size, drest, nbytes);
- src++;
+ dst++;
idx++;
}
static int ablkcipher_walk(struct ablkcipher_request *req,
struct ablkcipher_walk *w)
{
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
- struct scatterlist *src, *dst, *t;
- void *daddr;
+ struct scatterlist *dst, *t;
unsigned int nbytes = req->nbytes, offset, copy, diff;
int idx, tidx, err;
if (idx >= w->num && (w->flags & ASYNC_FLAGS_MISALIGNED))
return -EINVAL;
- src = &req->src[idx];
dst = &req->dst[idx];
- dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
- "blocksize: %u, nbytes: %u.\n",
- __func__, src->length, dst->length, src->offset,
- dst->offset, offset, blocksize, nbytes);
-
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1) ||
- offset) {
- unsigned slen = src->length - offset;
+ dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n",
+ __func__, dst->length, dst->offset, offset, nbytes);
+
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
+ offset) {
+ unsigned slen = min(dst->length - offset, nbytes);
unsigned dlen = PAGE_SIZE;
t = &w->cache[idx];
- daddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0);
- err = ablkcipher_add(daddr, &dlen, src, slen, &nbytes);
+ err = ablkcipher_add(&dlen, dst, slen, &nbytes);
if (err < 0)
- goto err_out_unmap;
+ return err;
idx += err;
- copy = slen & ~(blocksize - 1);
- diff = slen & (blocksize - 1);
+ copy = slen & ~(HIFN_D_DST_DALIGN - 1);
+ diff = slen & (HIFN_D_DST_DALIGN - 1);
if (dlen < nbytes) {
/*
* to put there additional blocksized chunk,
* so we mark that page as containing only
* blocksize aligned chunks:
- * t->length = (slen & ~(blocksize - 1));
+ * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
* and increase number of bytes to be processed
* in next chunk:
* nbytes += diff;
} else {
copy += diff + nbytes;
- src = &req->src[idx];
+ dst = &req->dst[idx];
- err = ablkcipher_add(daddr + slen, &dlen, src, nbytes, &nbytes);
+ err = ablkcipher_add(&dlen, dst, nbytes, &nbytes);
if (err < 0)
- goto err_out_unmap;
+ return err;
idx += err;
}
t->length = copy;
t->offset = offset;
-
- kunmap_atomic(daddr, KM_SOFTIRQ0);
} else {
- nbytes -= src->length;
+ nbytes -= min(dst->length, nbytes);
idx++;
}
}
return tidx;
-
-err_out_unmap:
- kunmap_atomic(daddr, KM_SOFTIRQ0);
- return err;
}
static int hifn_setup_session(struct ablkcipher_request *req)
{
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
struct hifn_device *dev = ctx->dev;
- struct page *spage, *dpage;
- unsigned long soff, doff, flags;
- unsigned int nbytes = req->nbytes, idx = 0, len;
+ unsigned long dlen, flags;
+ unsigned int nbytes = req->nbytes, idx = 0;
int err = -EINVAL, sg_num;
- struct scatterlist *src, *dst, *t;
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
+ struct scatterlist *dst;
- if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
+ if (rctx->iv && !rctx->ivsize && rctx->mode != ACRYPTO_MODE_ECB)
goto err_out_exit;
- ctx->walk.flags = 0;
+ rctx->walk.flags = 0;
while (nbytes) {
- src = &req->src[idx];
dst = &req->dst[idx];
+ dlen = min(dst->length, nbytes);
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1)) {
- ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
- }
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dlen, HIFN_D_DST_DALIGN))
+ rctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
- nbytes -= src->length;
+ nbytes -= dlen;
idx++;
}
- if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
- err = ablkcipher_walk_init(&ctx->walk, idx, GFP_ATOMIC);
+ if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
+ err = ablkcipher_walk_init(&rctx->walk, idx, GFP_ATOMIC);
if (err < 0)
return err;
}
- nbytes = req->nbytes;
- idx = 0;
-
- sg_num = ablkcipher_walk(req, &ctx->walk);
-
- atomic_set(&ctx->sg_num, sg_num);
+ sg_num = ablkcipher_walk(req, &rctx->walk);
+ if (sg_num < 0) {
+ err = sg_num;
+ goto err_out_exit;
+ }
spin_lock_irqsave(&dev->lock, flags);
if (dev->started + sg_num > HIFN_QUEUE_LENGTH) {
goto err_out;
}
- dev->snum++;
- dev->started += sg_num;
-
- while (nbytes) {
- src = &req->src[idx];
- dst = &req->dst[idx];
- t = &ctx->walk.cache[idx];
-
- if (t->length) {
- spage = dpage = sg_page(t);
- soff = doff = 0;
- len = t->length;
- } else {
- spage = sg_page(src);
- soff = src->offset;
-
- dpage = sg_page(dst);
- doff = dst->offset;
-
- len = dst->length;
- }
-
- idx++;
-
- err = hifn_setup_dma(dev, spage, soff, dpage, doff, nbytes,
- req, ctx);
- if (err)
- goto err_out;
+ err = hifn_setup_dma(dev, ctx, rctx, req->src, req->dst, req->nbytes, req);
+ if (err)
+ goto err_out;
- nbytes -= len;
- }
+ dev->snum++;
dev->active = HIFN_DEFAULT_ACTIVE_NUM;
spin_unlock_irqrestore(&dev->lock, flags);
err_out:
spin_unlock_irqrestore(&dev->lock, flags);
err_out_exit:
- if (err && printk_ratelimit())
- dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
+ if (err) {
+ printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
"type: %u, err: %d.\n",
- dev->name, ctx->iv, ctx->ivsize,
+ dev->name, rctx->iv, rctx->ivsize,
ctx->key, ctx->keysize,
- ctx->mode, ctx->op, ctx->type, err);
+ rctx->mode, rctx->op, rctx->type, err);
+ }
return err;
}
int n, err;
u8 src[16];
struct hifn_context ctx;
+ struct hifn_request_context rctx;
u8 fips_aes_ecb_from_zero[16] = {
0x66, 0xE9, 0x4B, 0xD4,
0xEF, 0x8A, 0x2C, 0x3B,
0x88, 0x4C, 0xFA, 0x59,
0xCA, 0x34, 0x2B, 0x2E};
+ struct scatterlist sg;
memset(src, 0, sizeof(src));
memset(ctx.key, 0, sizeof(ctx.key));
ctx.dev = dev;
ctx.keysize = 16;
- ctx.ivsize = 0;
- ctx.iv = NULL;
- ctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
- ctx.mode = ACRYPTO_MODE_ECB;
- ctx.type = ACRYPTO_TYPE_AES_128;
- atomic_set(&ctx.sg_num, 1);
-
- err = hifn_setup_dma(dev,
- virt_to_page(src), offset_in_page(src),
- virt_to_page(src), offset_in_page(src),
- sizeof(src), NULL, &ctx);
+ rctx.ivsize = 0;
+ rctx.iv = NULL;
+ rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT;
+ rctx.mode = ACRYPTO_MODE_ECB;
+ rctx.type = ACRYPTO_TYPE_AES_128;
+ rctx.walk.cache[0].length = 0;
+
+ sg_init_one(&sg, &src, sizeof(src));
+
+ err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL);
if (err)
goto err_out;
+ dev->started = 0;
msleep(200);
dprintk("%s: decoded: ", dev->name);
{
int err;
+ dev->started = dev->active = 0;
hifn_reset_dma(dev, 1);
err = hifn_enable_crypto(dev);
return -EINVAL;
while (size) {
-
- copy = min(dst->length, srest);
+ copy = min(srest, min(dst->length, size));
daddr = kmap_atomic(sg_page(dst), KM_IRQ0);
memcpy(daddr + dst->offset + offset, saddr, copy);
return idx;
}
-static void hifn_process_ready(struct ablkcipher_request *req, int error)
+static inline void hifn_complete_sa(struct hifn_device *dev, int i)
{
- struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
- struct hifn_device *dev;
-
- dprintk("%s: req: %p, ctx: %p.\n", __func__, req, ctx);
+ unsigned long flags;
- dev = ctx->dev;
- dprintk("%s: req: %p, started: %d, sg_num: %d.\n",
- __func__, req, dev->started, atomic_read(&ctx->sg_num));
+ spin_lock_irqsave(&dev->lock, flags);
+ dev->sa[i] = NULL;
+ dev->started--;
+ if (dev->started < 0)
+ printk("%s: started: %d.\n", __func__, dev->started);
+ spin_unlock_irqrestore(&dev->lock, flags);
+ BUG_ON(dev->started < 0);
+}
- if (--dev->started < 0)
- BUG();
+static void hifn_process_ready(struct ablkcipher_request *req, int error)
+{
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
- if (atomic_dec_and_test(&ctx->sg_num)) {
+ if (rctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
unsigned int nbytes = req->nbytes;
int idx = 0, err;
struct scatterlist *dst, *t;
void *saddr;
- if (ctx->walk.flags & ASYNC_FLAGS_MISALIGNED) {
- while (nbytes) {
- t = &ctx->walk.cache[idx];
- dst = &req->dst[idx];
+ while (nbytes) {
+ t = &rctx->walk.cache[idx];
+ dst = &req->dst[idx];
- dprintk("\n%s: sg_page(t): %p, t->length: %u, "
- "sg_page(dst): %p, dst->length: %u, "
- "nbytes: %u.\n",
- __func__, sg_page(t), t->length,
- sg_page(dst), dst->length, nbytes);
-
- if (!t->length) {
- nbytes -= dst->length;
- idx++;
- continue;
- }
+ dprintk("\n%s: sg_page(t): %p, t->length: %u, "
+ "sg_page(dst): %p, dst->length: %u, "
+ "nbytes: %u.\n",
+ __func__, sg_page(t), t->length,
+ sg_page(dst), dst->length, nbytes);
- saddr = kmap_atomic(sg_page(t), KM_IRQ1);
+ if (!t->length) {
+ nbytes -= min(dst->length, nbytes);
+ idx++;
+ continue;
+ }
- err = ablkcipher_get(saddr, &t->length, t->offset,
- dst, nbytes, &nbytes);
- if (err < 0) {
- kunmap_atomic(saddr, KM_IRQ1);
- break;
- }
+ saddr = kmap_atomic(sg_page(t), KM_SOFTIRQ0);
- idx += err;
- kunmap_atomic(saddr, KM_IRQ1);
+ err = ablkcipher_get(saddr, &t->length, t->offset,
+ dst, nbytes, &nbytes);
+ if (err < 0) {
+ kunmap_atomic(saddr, KM_SOFTIRQ0);
+ break;
}
- ablkcipher_walk_exit(&ctx->walk);
+ idx += err;
+ kunmap_atomic(saddr, KM_SOFTIRQ0);
}
- req->base.complete(&req->base, error);
+ ablkcipher_walk_exit(&rctx->walk);
}
-}
-
-static void hifn_check_for_completion(struct hifn_device *dev, int error)
-{
- int i;
- struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
-
- for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
- struct hifn_desc *d = &dma->resr[i];
- if (!(d->l & __cpu_to_le32(HIFN_D_VALID)) && dev->sa[i]) {
- dev->success++;
- dev->reset = 0;
- hifn_process_ready(dev->sa[i], error);
- dev->sa[i] = NULL;
- }
-
- if (d->l & __cpu_to_le32(HIFN_D_DESTOVER | HIFN_D_OVER))
- if (printk_ratelimit())
- printk("%s: overflow detected [d: %u, o: %u] "
- "at %d resr: l: %08x, p: %08x.\n",
- dev->name,
- !!(d->l & __cpu_to_le32(HIFN_D_DESTOVER)),
- !!(d->l & __cpu_to_le32(HIFN_D_OVER)),
- i, d->l, d->p);
- }
+ req->base.complete(&req->base, error);
}
-static void hifn_clear_rings(struct hifn_device *dev)
+static void hifn_clear_rings(struct hifn_device *dev, int error)
{
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
int i, u;
if (dma->resr[i].l & __cpu_to_le32(HIFN_D_VALID))
break;
- if (i != HIFN_D_RES_RSIZE)
- u--;
+ if (dev->sa[i]) {
+ dev->success++;
+ dev->reset = 0;
+ hifn_process_ready(dev->sa[i], error);
+ hifn_complete_sa(dev, i);
+ }
- if (++i == (HIFN_D_RES_RSIZE + 1))
+ if (++i == HIFN_D_RES_RSIZE)
i = 0;
+ u--;
}
dma->resk = i; dma->resu = u;
i = dma->srck; u = dma->srcu;
while (u != 0) {
- if (i == HIFN_D_SRC_RSIZE)
- i = 0;
if (dma->srcr[i].l & __cpu_to_le32(HIFN_D_VALID))
break;
- i++, u--;
+ if (++i == HIFN_D_SRC_RSIZE)
+ i = 0;
+ u--;
}
dma->srck = i; dma->srcu = u;
while (u != 0) {
if (dma->cmdr[i].l & __cpu_to_le32(HIFN_D_VALID))
break;
- if (i != HIFN_D_CMD_RSIZE)
- u--;
- if (++i == (HIFN_D_CMD_RSIZE + 1))
+ if (++i == HIFN_D_CMD_RSIZE)
i = 0;
+ u--;
}
dma->cmdk = i; dma->cmdu = u;
i = dma->dstk; u = dma->dstu;
while (u != 0) {
- if (i == HIFN_D_DST_RSIZE)
- i = 0;
if (dma->dstr[i].l & __cpu_to_le32(HIFN_D_VALID))
break;
- i++, u--;
+ if (++i == HIFN_D_DST_RSIZE)
+ i = 0;
+ u--;
}
dma->dstk = i; dma->dstu = u;
static void hifn_work(struct work_struct *work)
{
- struct delayed_work *dw = container_of(work, struct delayed_work, work);
+ struct delayed_work *dw = to_delayed_work(work);
struct hifn_device *dev = container_of(dw, struct hifn_device, work);
unsigned long flags;
int reset = 0;
} else
dev->active--;
- if (dev->prev_success == dev->success && dev->started)
+ if ((dev->prev_success == dev->success) && dev->started)
reset = 1;
dev->prev_success = dev->success;
spin_unlock_irqrestore(&dev->lock, flags);
if (reset) {
- dprintk("%s: r: %08x, active: %d, started: %d, "
- "success: %lu: reset: %d.\n",
- dev->name, r, dev->active, dev->started,
- dev->success, reset);
-
if (++dev->reset >= 5) {
- dprintk("%s: really hard reset.\n", dev->name);
+ int i;
+ struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
+
+ printk("%s: r: %08x, active: %d, started: %d, "
+ "success: %lu: qlen: %u/%u, reset: %d.\n",
+ dev->name, r, dev->active, dev->started,
+ dev->success, dev->queue.qlen, dev->queue.max_qlen,
+ reset);
+
+ printk("%s: res: ", __func__);
+ for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
+ printk("%x.%p ", dma->resr[i].l, dev->sa[i]);
+ if (dev->sa[i]) {
+ hifn_process_ready(dev->sa[i], -ENODEV);
+ hifn_complete_sa(dev, i);
+ }
+ }
+ printk("\n");
+
hifn_reset_dma(dev, 1);
hifn_stop_device(dev);
hifn_start_device(dev);
dev->reset = 0;
}
- spin_lock_irqsave(&dev->lock, flags);
- hifn_check_for_completion(dev, -EBUSY);
- hifn_clear_rings(dev);
- dev->started = 0;
- spin_unlock_irqrestore(&dev->lock, flags);
+ tasklet_schedule(&dev->tasklet);
}
schedule_delayed_work(&dev->work, HZ);
dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], "
"i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n",
dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi,
- dma->cmdu, dma->srcu, dma->dstu, dma->resu,
- dma->cmdi, dma->srci, dma->dsti, dma->resi);
+ dma->cmdi, dma->srci, dma->dsti, dma->resi,
+ dma->cmdu, dma->srcu, dma->dstu, dma->resu);
if ((dmacsr & dev->dmareg) == 0)
return IRQ_NONE;
if (restart) {
u32 puisr = hifn_read_0(dev, HIFN_0_PUISR);
- if (printk_ratelimit())
- printk("%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
- dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
- !!(dmacsr & HIFN_DMACSR_D_OVER),
- puisr, !!(puisr & HIFN_PUISR_DSTOVER));
+ printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n",
+ dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER),
+ !!(dmacsr & HIFN_DMACSR_D_OVER),
+ puisr, !!(puisr & HIFN_PUISR_DSTOVER));
if (!!(puisr & HIFN_PUISR_DSTOVER))
hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
hifn_write_1(dev, HIFN_1_DMA_CSR, dmacsr & (HIFN_DMACSR_R_OVER |
restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT |
HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT);
if (restart) {
- if (printk_ratelimit())
- printk("%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
- dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
- !!(dmacsr & HIFN_DMACSR_S_ABORT),
- !!(dmacsr & HIFN_DMACSR_D_ABORT),
- !!(dmacsr & HIFN_DMACSR_R_ABORT));
+ printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n",
+ dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT),
+ !!(dmacsr & HIFN_DMACSR_S_ABORT),
+ !!(dmacsr & HIFN_DMACSR_D_ABORT),
+ !!(dmacsr & HIFN_DMACSR_R_ABORT));
hifn_reset_dma(dev, 1);
hifn_init_dma(dev);
hifn_init_registers(dev);
}
tasklet_schedule(&dev->tasklet);
- hifn_clear_rings(dev);
return IRQ_HANDLED;
}
struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt;
int i;
- spin_lock_irqsave(&dev->lock, flags);
for (i=0; i<HIFN_D_RES_RSIZE; ++i) {
struct hifn_desc *d = &dma->resr[i];
if (dev->sa[i]) {
hifn_process_ready(dev->sa[i],
(d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0);
+ hifn_complete_sa(dev, i);
}
}
+ spin_lock_irqsave(&dev->lock, flags);
while ((async_req = crypto_dequeue_request(&dev->queue))) {
ctx = crypto_tfm_ctx(async_req->tfm);
req = container_of(async_req, struct ablkcipher_request, base);
+ spin_unlock_irqrestore(&dev->lock, flags);
hifn_process_ready(req, -ENODEV);
+
+ spin_lock_irqsave(&dev->lock, flags);
}
spin_unlock_irqrestore(&dev->lock, flags);
}
u8 type, u8 mode)
{
struct hifn_context *ctx = crypto_tfm_ctx(req->base.tfm);
+ struct hifn_request_context *rctx = ablkcipher_request_ctx(req);
unsigned ivsize;
ivsize = crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(req));
type = ACRYPTO_TYPE_AES_256;
}
- ctx->op = op;
- ctx->mode = mode;
- ctx->type = type;
- ctx->iv = req->info;
- ctx->ivsize = ivsize;
+ rctx->op = op;
+ rctx->mode = mode;
+ rctx->type = type;
+ rctx->iv = req->info;
+ rctx->ivsize = ivsize;
/*
* HEAVY TODO: needs to kick Herbert XU to write documentation.
static int hifn_process_queue(struct hifn_device *dev)
{
- struct crypto_async_request *async_req;
+ struct crypto_async_request *async_req, *backlog;
struct hifn_context *ctx;
struct ablkcipher_request *req;
unsigned long flags;
while (dev->started < HIFN_QUEUE_LENGTH) {
spin_lock_irqsave(&dev->lock, flags);
+ backlog = crypto_get_backlog(&dev->queue);
async_req = crypto_dequeue_request(&dev->queue);
spin_unlock_irqrestore(&dev->lock, flags);
if (!async_req)
break;
+ if (backlog)
+ backlog->complete(backlog, -EINPROGRESS);
+
ctx = crypto_tfm_ctx(async_req->tfm);
req = container_of(async_req, struct ablkcipher_request, base);
return err;
if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
- err = hifn_process_queue(dev);
+ hifn_process_queue(dev);
- return err;
+ return -EINPROGRESS;
}
/*
* 3DES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "cfb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "cfb(des3_ede)", .drv_name = "cfb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
},
},
{
- .name = "ofb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "ofb(des3_ede)", .drv_name = "ofb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
},
},
{
- .name = "cbc(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "cbc(des3_ede)", .drv_name = "cbc-3des", .bsize = 8,
.ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
.setkey = hifn_setkey,
},
},
{
- .name = "ecb(des3_ede)", .drv_name = "hifn-3des", .bsize = 8,
+ .name = "ecb(des3_ede)", .drv_name = "ecb-3des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_3DES_KEY_LENGTH,
.max_keysize = HIFN_3DES_KEY_LENGTH,
* DES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "cfb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "cfb(des)", .drv_name = "cfb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
},
},
{
- .name = "ofb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "ofb(des)", .drv_name = "ofb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
},
},
{
- .name = "cbc(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "cbc(des)", .drv_name = "cbc-des", .bsize = 8,
.ablkcipher = {
+ .ivsize = HIFN_IV_LENGTH,
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
.setkey = hifn_setkey,
},
},
{
- .name = "ecb(des)", .drv_name = "hifn-des", .bsize = 8,
+ .name = "ecb(des)", .drv_name = "ecb-des", .bsize = 8,
.ablkcipher = {
.min_keysize = HIFN_DES_KEY_LENGTH,
.max_keysize = HIFN_DES_KEY_LENGTH,
* AES ECB, CBC, CFB and OFB modes.
*/
{
- .name = "ecb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "ecb(aes)", .drv_name = "ecb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
},
},
{
- .name = "cbc(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "cbc(aes)", .drv_name = "cbc-aes", .bsize = 16,
.ablkcipher = {
+ .ivsize = HIFN_AES_IV_LENGTH,
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
.setkey = hifn_setkey,
},
},
{
- .name = "cfb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "cfb(aes)", .drv_name = "cfb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
},
},
{
- .name = "ofb(aes)", .drv_name = "hifn-aes", .bsize = 16,
+ .name = "ofb(aes)", .drv_name = "ofb-aes", .bsize = 16,
.ablkcipher = {
.min_keysize = AES_MIN_KEY_SIZE,
.max_keysize = AES_MAX_KEY_SIZE,
struct hifn_context *ctx = crypto_tfm_ctx(tfm);
ctx->dev = ha->dev;
-
+ tfm->crt_ablkcipher.reqsize = sizeof(struct hifn_request_context);
return 0;
}
return -ENOMEM;
snprintf(alg->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s", t->name);
- snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name);
+ snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s-%s",
+ t->drv_name, dev->name);
alg->alg.cra_priority = 300;
- alg->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_ASYNC;
+ alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
alg->alg.cra_blocksize = t->bsize;
alg->alg.cra_ctxsize = sizeof(struct hifn_context);
- alg->alg.cra_alignmask = 15;
- if (t->bsize == 8)
- alg->alg.cra_alignmask = 3;
+ alg->alg.cra_alignmask = 0;
alg->alg.cra_type = &crypto_ablkcipher_type;
alg->alg.cra_module = THIS_MODULE;
alg->alg.cra_u.ablkcipher = t->ablkcipher;
* (like dev->success), but they are used in process
* context or update is atomic (like setting dev->sa[i] to NULL).
*/
- hifn_check_for_completion(dev, 0);
+ hifn_clear_rings(dev, 0);
+
+ if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
+ hifn_process_queue(dev);
}
-static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static int __devinit hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
int err, i;
struct hifn_device *dev;
return err;
pci_set_master(pdev);
- err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
+ err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
if (err)
goto err_out_disable_pci_device;
goto err_out_unmap_bars;
}
- dev->result_mem = __get_free_pages(GFP_KERNEL, HIFN_MAX_RESULT_ORDER);
- if (!dev->result_mem) {
- dprintk("Failed to allocate %d pages for result_mem.\n",
- HIFN_MAX_RESULT_ORDER);
- goto err_out_unmap_bars;
- }
- memset((void *)dev->result_mem, 0, PAGE_SIZE*(1<<HIFN_MAX_RESULT_ORDER));
-
- dev->dst = pci_map_single(pdev, (void *)dev->result_mem,
- PAGE_SIZE << HIFN_MAX_RESULT_ORDER, PCI_DMA_FROMDEVICE);
-
dev->desc_virt = pci_alloc_consistent(pdev, sizeof(struct hifn_dma),
&dev->desc_dma);
if (!dev->desc_virt) {
dprintk("Failed to allocate descriptor rings.\n");
- goto err_out_free_result_pages;
+ goto err_out_unmap_bars;
}
memset(dev->desc_virt, 0, sizeof(struct hifn_dma));
if (err)
goto err_out_stop_device;
- err = hifn_register_alg(dev);
+ err = hifn_register_rng(dev);
if (err)
goto err_out_stop_device;
+ err = hifn_register_alg(dev);
+ if (err)
+ goto err_out_unregister_rng;
+
INIT_DELAYED_WORK(&dev->work, hifn_work);
schedule_delayed_work(&dev->work, HZ);
return 0;
+err_out_unregister_rng:
+ hifn_unregister_rng(dev);
err_out_stop_device:
hifn_reset_dma(dev, 1);
hifn_stop_device(dev);
pci_free_consistent(pdev, sizeof(struct hifn_dma),
dev->desc_virt, dev->desc_dma);
-err_out_free_result_pages:
- pci_unmap_single(pdev, dev->dst, PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
- PCI_DMA_FROMDEVICE);
- free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
-
err_out_unmap_bars:
for (i=0; i<3; ++i)
if (dev->bar[i])
return err;
}
-static void hifn_remove(struct pci_dev *pdev)
+static void __devexit hifn_remove(struct pci_dev *pdev)
{
int i;
struct hifn_device *dev;
cancel_delayed_work(&dev->work);
flush_scheduled_work();
+ hifn_unregister_rng(dev);
hifn_unregister_alg(dev);
hifn_reset_dma(dev, 1);
hifn_stop_device(dev);
pci_free_consistent(pdev, sizeof(struct hifn_dma),
dev->desc_virt, dev->desc_dma);
- pci_unmap_single(pdev, dev->dst,
- PAGE_SIZE << HIFN_MAX_RESULT_ORDER,
- PCI_DMA_FROMDEVICE);
- free_pages(dev->result_mem, HIFN_MAX_RESULT_ORDER);
for (i=0; i<3; ++i)
if (dev->bar[i])
iounmap(dev->bar[i]);
.remove = __devexit_p(hifn_remove),
};
-static int __devinit hifn_init(void)
+static int __init hifn_init(void)
{
+ unsigned int freq;
int err;
+ if (sizeof(dma_addr_t) > 4) {
+ printk(KERN_INFO "HIFN supports only 32-bit addresses.\n");
+ return -EINVAL;
+ }
+
+ if (strncmp(hifn_pll_ref, "ext", 3) &&
+ strncmp(hifn_pll_ref, "pci", 3)) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, "
+ "must be pci or ext");
+ return -EINVAL;
+ }
+
+ /*
+ * For the 7955/7956 the reference clock frequency must be in the
+ * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
+ * but this chip is currently not supported.
+ */
+ if (hifn_pll_ref[3] != '\0') {
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ if (freq < 20 || freq > 100) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref "
+ "frequency, must be in the range "
+ "of 20-100");
+ return -EINVAL;
+ }
+ }
+
err = pci_register_driver(&hifn_pci_driver);
if (err < 0) {
dprintk("Failed to register PCI driver for %s device.\n",
return 0;
}
-static void __devexit hifn_fini(void)
+static void __exit hifn_fini(void)
{
pci_unregister_driver(&hifn_pci_driver);
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff