#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/moduleparam.h>
#include <linux/mod_devicetable.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/slab.h>
#include <linux/delay.h>
#include <linux/mm.h>
+#include <linux/dma-mapping.h>
+#include <linux/scatterlist.h>
#include <linux/highmem.h>
+#include <linux/interrupt.h>
#include <linux/crypto.h>
+#include <linux/hw_random.h>
+#include <linux/ktime.h>
#include <crypto/algapi.h>
+#include <crypto/des.h>
#include <asm/kmap_types.h>
#define dprintk(f, a...) do {} while (0)
#endif
+static char hifn_pll_ref[sizeof("extNNN")] = "ext";
+module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444);
+MODULE_PARM_DESC(hifn_pll_ref,
+ "PLL reference clock (pci[freq] or ext[freq], default ext)");
+
static atomic_t hifn_dev_number;
#define ACRYPTO_OP_DECRYPT 0
#define HIFN_DMACNFG_DMARESET 0x00000002 /* DMA Reset # */
#define HIFN_DMACNFG_MSTRESET 0x00000001 /* Master Reset # */
-#define HIFN_PLL_7956 0x00001d18 /* 7956 PLL config value */
+/* PLL configuration register */
+#define HIFN_PLL_REF_CLK_HBI 0x00000000 /* HBI reference clock */
+#define HIFN_PLL_REF_CLK_PLL 0x00000001 /* PLL reference clock */
+#define HIFN_PLL_BP 0x00000002 /* Reference clock bypass */
+#define HIFN_PLL_PK_CLK_HBI 0x00000000 /* PK engine HBI clock */
+#define HIFN_PLL_PK_CLK_PLL 0x00000008 /* PK engine PLL clock */
+#define HIFN_PLL_PE_CLK_HBI 0x00000000 /* PE engine HBI clock */
+#define HIFN_PLL_PE_CLK_PLL 0x00000010 /* PE engine PLL clock */
+#define HIFN_PLL_RESERVED_1 0x00000400 /* Reserved bit, must be 1 */
+#define HIFN_PLL_ND_SHIFT 11 /* Clock multiplier shift */
+#define HIFN_PLL_ND_MULT_2 0x00000000 /* PLL clock multiplier 2 */
+#define HIFN_PLL_ND_MULT_4 0x00000800 /* PLL clock multiplier 4 */
+#define HIFN_PLL_ND_MULT_6 0x00001000 /* PLL clock multiplier 6 */
+#define HIFN_PLL_ND_MULT_8 0x00001800 /* PLL clock multiplier 8 */
+#define HIFN_PLL_ND_MULT_10 0x00002000 /* PLL clock multiplier 10 */
+#define HIFN_PLL_ND_MULT_12 0x00002800 /* PLL clock multiplier 12 */
+#define HIFN_PLL_IS_1_8 0x00000000 /* charge pump (mult. 1-8) */
+#define HIFN_PLL_IS_9_12 0x00010000 /* charge pump (mult. 9-12) */
+
+#define HIFN_PLL_FCK_MAX 266 /* Maximum PLL frequency */
/* Public key reset register (HIFN_1_PUB_RESET) */
#define HIFN_PUBRST_RESET 0x00000001 /* reset public/rng unit */
#define HIFN_D_DST_RSIZE 80*4
#define HIFN_D_RES_RSIZE 24*4
+#define HIFN_D_DST_DALIGN 4
+
#define HIFN_QUEUE_LENGTH HIFN_D_CMD_RSIZE-5
#define AES_MIN_KEY_SIZE 16
struct hifn_desc
{
- volatile u32 l;
- volatile u32 p;
+ volatile __le32 l;
+ volatile __le32 p;
};
struct hifn_dma {
u8 snum;
+ struct tasklet_struct tasklet;
+
struct crypto_queue queue;
struct list_head alg_list;
+
+ unsigned int pk_clk_freq;
+
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ unsigned int rng_wait_time;
+ ktime_t rngtime;
+ struct hwrng rng;
+#endif
};
#define HIFN_D_LENGTH 0x0000ffff
struct hifn_base_command
{
- volatile u16 masks;
- volatile u16 session_num;
- volatile u16 total_source_count;
- volatile u16 total_dest_count;
+ volatile __le16 masks;
+ volatile __le16 session_num;
+ volatile __le16 total_source_count;
+ volatile __le16 total_dest_count;
};
#define HIFN_BASE_CMD_COMP 0x0100 /* enable compression engine */
*/
struct hifn_crypt_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */
*/
struct hifn_mac_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_MAC_CMD_ALG_MASK 0x0001
struct hifn_comp_command
{
- volatile u16 masks;
- volatile u16 header_skip;
- volatile u16 source_count;
- volatile u16 reserved;
+ volatile __le16 masks;
+ volatile __le16 header_skip;
+ volatile __le16 source_count;
+ volatile __le16 reserved;
};
#define HIFN_COMP_CMD_SRCLEN_M 0xc000
struct hifn_base_result
{
- volatile u16 flags;
- volatile u16 session;
- volatile u16 src_cnt; /* 15:0 of source count */
- volatile u16 dst_cnt; /* 15:0 of dest count */
+ volatile __le16 flags;
+ volatile __le16 session;
+ volatile __le16 src_cnt; /* 15:0 of source count */
+ volatile __le16 dst_cnt; /* 15:0 of dest count */
};
#define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */
struct hifn_comp_result
{
- volatile u16 flags;
- volatile u16 crc;
+ volatile __le16 flags;
+ volatile __le16 crc;
};
#define HIFN_COMP_RES_LCB_M 0xff00 /* longitudinal check byte */
struct hifn_mac_result
{
- volatile u16 flags;
- volatile u16 reserved;
+ volatile __le16 flags;
+ volatile __le16 reserved;
/* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */
};
struct hifn_crypt_result
{
- volatile u16 flags;
- volatile u16 reserved;
+ volatile __le16 flags;
+ volatile __le16 reserved;
};
#define HIFN_CRYPT_RES_SRC_NOTZERO 0x0001 /* source expired */
atomic_t sg_num;
};
-#define crypto_alg_to_hifn(alg) container_of(alg, struct hifn_crypto_alg, alg)
+#define crypto_alg_to_hifn(a) container_of(a, struct hifn_crypto_alg, alg)
static inline u32 hifn_read_0(struct hifn_device *dev, u32 reg)
{
u32 ret;
- ret = readl((char *)(dev->bar[0]) + reg);
+ ret = readl(dev->bar[0] + reg);
return ret;
}
{
u32 ret;
- ret = readl((char *)(dev->bar[1]) + reg);
+ ret = readl(dev->bar[1] + reg);
return ret;
}
static inline void hifn_write_0(struct hifn_device *dev, u32 reg, u32 val)
{
- writel(val, (char *)(dev->bar[0]) + reg);
+ writel((__force u32)cpu_to_le32(val), dev->bar[0] + reg);
}
static inline void hifn_write_1(struct hifn_device *dev, u32 reg, u32 val)
{
- writel(val, (char *)(dev->bar[1]) + reg);
+ writel((__force u32)cpu_to_le32(val), dev->bar[1] + reg);
}
static void hifn_wait_puc(struct hifn_device *dev)
}
};
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+static int hifn_rng_data_present(struct hwrng *rng, int wait)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+ s64 nsec;
+
+ nsec = ktime_to_ns(ktime_sub(ktime_get(), dev->rngtime));
+ nsec -= dev->rng_wait_time;
+ if (nsec <= 0)
+ return 1;
+ if (!wait)
+ return 0;
+ ndelay(nsec);
+ return 1;
+}
+
+static int hifn_rng_data_read(struct hwrng *rng, u32 *data)
+{
+ struct hifn_device *dev = (struct hifn_device *)rng->priv;
+
+ *data = hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+ return 4;
+}
+
+static int hifn_register_rng(struct hifn_device *dev)
+{
+ /*
+ * We must wait at least 256 Pk_clk cycles between two reads of the rng.
+ */
+ dev->rng_wait_time = DIV_ROUND_UP(NSEC_PER_SEC, dev->pk_clk_freq) *
+ 256;
+
+ dev->rng.name = dev->name;
+ dev->rng.data_present = hifn_rng_data_present,
+ dev->rng.data_read = hifn_rng_data_read,
+ dev->rng.priv = (unsigned long)dev;
+
+ return hwrng_register(&dev->rng);
+}
+
+static void hifn_unregister_rng(struct hifn_device *dev)
+{
+ hwrng_unregister(&dev->rng);
+}
+#else
+#define hifn_register_rng(dev) 0
+#define hifn_unregister_rng(dev)
+#endif
+
static int hifn_init_pubrng(struct hifn_device *dev)
{
int i;
dprintk("Chip %s: RNG engine has been successfully initialised.\n",
dev->name);
+#ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG
+ /* First value must be discarded */
+ hifn_read_1(dev, HIFN_1_RNG_DATA);
+ dev->rngtime = ktime_get();
+#endif
return 0;
}
dma->cmdk = dma->srck = dma->dstk = dma->resk = 0;
}
+/*
+ * Initialize the PLL. We need to know the frequency of the reference clock
+ * to calculate the optimal multiplier. For PCI we assume 66MHz, since that
+ * allows us to operate without the risk of overclocking the chip. If it
+ * actually uses 33MHz, the chip will operate at half the speed, this can be
+ * overriden by specifying the frequency as module parameter (pci33).
+ *
+ * Unfortunately the PCI clock is not very suitable since the HIFN needs a
+ * stable clock and the PCI clock frequency may vary, so the default is the
+ * external clock. There is no way to find out its frequency, we default to
+ * 66MHz since according to Mike Ham of HiFn, almost every board in existence
+ * has an external crystal populated at 66MHz.
+ */
+static void hifn_init_pll(struct hifn_device *dev)
+{
+ unsigned int freq, m;
+ u32 pllcfg;
+
+ pllcfg = HIFN_1_PLL | HIFN_PLL_RESERVED_1;
+
+ if (strncmp(hifn_pll_ref, "ext", 3) == 0)
+ pllcfg |= HIFN_PLL_REF_CLK_PLL;
+ else
+ pllcfg |= HIFN_PLL_REF_CLK_HBI;
+
+ if (hifn_pll_ref[3] != '\0')
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ else {
+ freq = 66;
+ printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, "
+ "override with hifn_pll_ref=%.3s<frequency>\n",
+ freq, hifn_pll_ref);
+ }
+
+ m = HIFN_PLL_FCK_MAX / freq;
+
+ pllcfg |= (m / 2 - 1) << HIFN_PLL_ND_SHIFT;
+ if (m <= 8)
+ pllcfg |= HIFN_PLL_IS_1_8;
+ else
+ pllcfg |= HIFN_PLL_IS_9_12;
+
+ /* Select clock source and enable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI | HIFN_PLL_BP);
+
+ /* Let the chip lock to the input clock */
+ mdelay(10);
+
+ /* Disable clock bypass */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_HBI | HIFN_PLL_PE_CLK_HBI);
+
+ /* Switch the engines to the PLL */
+ hifn_write_1(dev, HIFN_1_PLL, pllcfg |
+ HIFN_PLL_PK_CLK_PLL | HIFN_PLL_PE_CLK_PLL);
+
+ /*
+ * The Fpk_clk runs at half the total speed. Its frequency is needed to
+ * calculate the minimum time between two reads of the rng. Since 33MHz
+ * is actually 33.333... we overestimate the frequency here, resulting
+ * in slightly larger intervals.
+ */
+ dev->pk_clk_freq = 1000000 * (freq + 1) * m / 2;
+}
+
static void hifn_init_registers(struct hifn_device *dev)
{
u32 dptr = dev->desc_dma;
hifn_write_0(dev, HIFN_0_PUIER, HIFN_PUIER_DSTOVER);
/* write all 4 ring address registers */
- hifn_write_1(dev, HIFN_1_DMA_CRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, cmdr[0])));
- hifn_write_1(dev, HIFN_1_DMA_SRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, srcr[0])));
- hifn_write_1(dev, HIFN_1_DMA_DRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, dstr[0])));
- hifn_write_1(dev, HIFN_1_DMA_RRAR, __cpu_to_le32(dptr +
- offsetof(struct hifn_dma, resr[0])));
+ hifn_write_1(dev, HIFN_1_DMA_CRAR, dptr +
+ offsetof(struct hifn_dma, cmdr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_SRAR, dptr +
+ offsetof(struct hifn_dma, srcr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_DRAR, dptr +
+ offsetof(struct hifn_dma, dstr[0]));
+ hifn_write_1(dev, HIFN_1_DMA_RRAR, dptr +
+ offsetof(struct hifn_dma, resr[0]));
mdelay(2);
#if 0
#else
hifn_write_0(dev, HIFN_0_PUCNFG, 0x10342);
#endif
- hifn_write_1(dev, HIFN_1_PLL, HIFN_PLL_7956);
+ hifn_init_pll(dev);
hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER);
hifn_write_1(dev, HIFN_1_DMA_CNFG, HIFN_DMACNFG_MSTRESET |
idx = dma->srci;
dma->srcr[idx].p = __cpu_to_le32(addr);
- dma->srcr[idx].l = __cpu_to_le32(size) | HIFN_D_VALID |
- HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST;
+ dma->srcr[idx].l = __cpu_to_le32(size | HIFN_D_VALID |
+ HIFN_D_MASKDONEIRQ | HIFN_D_NOINVALID | HIFN_D_LAST);
if (++idx == HIFN_D_SRC_RSIZE) {
dma->srcr[idx].l = __cpu_to_le32(HIFN_D_VALID |
static int ablkcipher_walk(struct ablkcipher_request *req,
struct ablkcipher_walk *w)
{
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
struct scatterlist *src, *dst, *t;
void *daddr;
unsigned int nbytes = req->nbytes, offset, copy, diff;
dst = &req->dst[idx];
dprintk("\n%s: slen: %u, dlen: %u, soff: %u, doff: %u, offset: %u, "
- "blocksize: %u, nbytes: %u.\n",
+ "nbytes: %u.\n",
__func__, src->length, dst->length, src->offset,
- dst->offset, offset, blocksize, nbytes);
+ dst->offset, offset, nbytes);
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1) ||
- offset) {
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) ||
+ offset) {
unsigned slen = src->length - offset;
unsigned dlen = PAGE_SIZE;
idx += err;
- copy = slen & ~(blocksize - 1);
- diff = slen & (blocksize - 1);
+ copy = slen & ~(HIFN_D_DST_DALIGN - 1);
+ diff = slen & (HIFN_D_DST_DALIGN - 1);
if (dlen < nbytes) {
/*
* to put there additional blocksized chunk,
* so we mark that page as containing only
* blocksize aligned chunks:
- * t->length = (slen & ~(blocksize - 1));
+ * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1));
* and increase number of bytes to be processed
* in next chunk:
* nbytes += diff;
unsigned int nbytes = req->nbytes, idx = 0, len;
int err = -EINVAL, sg_num;
struct scatterlist *src, *dst, *t;
- unsigned blocksize =
- crypto_ablkcipher_blocksize(crypto_ablkcipher_reqtfm(req));
- unsigned alignmask =
- crypto_ablkcipher_alignmask(crypto_ablkcipher_reqtfm(req));
if (ctx->iv && !ctx->ivsize && ctx->mode != ACRYPTO_MODE_ECB)
goto err_out_exit;
ctx->walk.flags = 0;
while (nbytes) {
- src = &req->src[idx];
dst = &req->dst[idx];
- if (src->length & (blocksize - 1) ||
- src->offset & (alignmask - 1) ||
- dst->length & (blocksize - 1) ||
- dst->offset & (alignmask - 1)) {
+ if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) ||
+ !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN))
ctx->walk.flags |= ASYNC_FLAGS_MISALIGNED;
- }
- nbytes -= src->length;
+ nbytes -= dst->length;
idx++;
}
idx = 0;
sg_num = ablkcipher_walk(req, &ctx->walk);
-
+ if (sg_num < 0) {
+ err = sg_num;
+ goto err_out_exit;
+ }
atomic_set(&ctx->sg_num, sg_num);
spin_lock_irqsave(&dev->lock, flags);
err_out:
spin_unlock_irqrestore(&dev->lock, flags);
err_out_exit:
- if (err && printk_ratelimit())
+ if (err)
dprintk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, "
"type: %u, err: %d.\n",
dev->name, ctx->iv, ctx->ivsize,
hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg);
}
- hifn_check_for_completion(dev, 0);
+ tasklet_schedule(&dev->tasklet);
hifn_clear_rings(dev);
return IRQ_HANDLED;
return -1;
}
+ if (len == HIFN_DES_KEY_LENGTH) {
+ u32 tmp[DES_EXPKEY_WORDS];
+ int ret = des_ekey(tmp, key);
+
+ if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) {
+ tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY;
+ return -EINVAL;
+ }
+ }
+
dev->flags &= ~HIFN_FLAG_OLD_KEY;
memcpy(ctx->key, key, len);
return err;
if (dev->started < HIFN_QUEUE_LENGTH && dev->queue.qlen)
- err = hifn_process_queue(dev);
+ hifn_process_queue(dev);
- return err;
+ return -EINPROGRESS;
}
/*
snprintf(alg->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", t->drv_name);
alg->alg.cra_priority = 300;
- alg->alg.cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_ASYNC;
+ alg->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC;
alg->alg.cra_blocksize = t->bsize;
alg->alg.cra_ctxsize = sizeof(struct hifn_context);
- alg->alg.cra_alignmask = 15;
- if (t->bsize == 8)
- alg->alg.cra_alignmask = 3;
+ alg->alg.cra_alignmask = 0;
alg->alg.cra_type = &crypto_ablkcipher_type;
alg->alg.cra_module = THIS_MODULE;
alg->alg.cra_u.ablkcipher = t->ablkcipher;
return err;
}
+static void hifn_tasklet_callback(unsigned long data)
+{
+ struct hifn_device *dev = (struct hifn_device *)data;
+
+ /*
+ * This is ok to call this without lock being held,
+ * althogh it modifies some parameters used in parallel,
+ * (like dev->success), but they are used in process
+ * context or update is atomic (like setting dev->sa[i] to NULL).
+ */
+ hifn_check_for_completion(dev, 0);
+}
+
static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
int err, i;
pci_set_drvdata(pdev, dev);
+ tasklet_init(&dev->tasklet, hifn_tasklet_callback, (unsigned long)dev);
+
crypto_init_queue(&dev->queue, 1);
err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev);
if (err)
goto err_out_stop_device;
- err = hifn_register_alg(dev);
+ err = hifn_register_rng(dev);
if (err)
goto err_out_stop_device;
+ err = hifn_register_alg(dev);
+ if (err)
+ goto err_out_unregister_rng;
+
INIT_DELAYED_WORK(&dev->work, hifn_work);
schedule_delayed_work(&dev->work, HZ);
return 0;
+err_out_unregister_rng:
+ hifn_unregister_rng(dev);
err_out_stop_device:
hifn_reset_dma(dev, 1);
hifn_stop_device(dev);
err_out_free_irq:
free_irq(dev->irq, dev->name);
+ tasklet_kill(&dev->tasklet);
err_out_free_desc:
pci_free_consistent(pdev, sizeof(struct hifn_dma),
dev->desc_virt, dev->desc_dma);
cancel_delayed_work(&dev->work);
flush_scheduled_work();
+ hifn_unregister_rng(dev);
hifn_unregister_alg(dev);
hifn_reset_dma(dev, 1);
hifn_stop_device(dev);
free_irq(dev->irq, dev->name);
+ tasklet_kill(&dev->tasklet);
hifn_flush(dev);
static int __devinit hifn_init(void)
{
+ unsigned int freq;
int err;
+ if (strncmp(hifn_pll_ref, "ext", 3) &&
+ strncmp(hifn_pll_ref, "pci", 3)) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, "
+ "must be pci or ext");
+ return -EINVAL;
+ }
+
+ /*
+ * For the 7955/7956 the reference clock frequency must be in the
+ * range of 20MHz-100MHz. For the 7954 the upper bound is 66.67MHz,
+ * but this chip is currently not supported.
+ */
+ if (hifn_pll_ref[3] != '\0') {
+ freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10);
+ if (freq < 20 || freq > 100) {
+ printk(KERN_ERR "hifn795x: invalid hifn_pll_ref "
+ "frequency, must be in the range "
+ "of 20-100");
+ return -EINVAL;
+ }
+ }
+
err = pci_register_driver(&hifn_pci_driver);
if (err < 0) {
dprintk("Failed to register PCI driver for %s device.\n",
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff