*
* Copyright (c) 2004 Michal Ludvig <michal@logix.cz>
*
- * Key expansion routine taken from crypto/aes.c
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * ---------------------------------------------------------------------------
- * Copyright (c) 2002, Dr Brian Gladman <brg@gladman.me.uk>, Worcester, UK.
- * All rights reserved.
- *
- * LICENSE TERMS
- *
- * The free distribution and use of this software in both source and binary
- * form is allowed (with or without changes) provided that:
- *
- * 1. distributions of this source code include the above copyright
- * notice, this list of conditions and the following disclaimer;
- *
- * 2. distributions in binary form include the above copyright
- * notice, this list of conditions and the following disclaimer
- * in the documentation and/or other associated materials;
- *
- * 3. the copyright holder's name is not used to endorse products
- * built using this software without specific written permission.
- *
- * ALTERNATIVELY, provided that this notice is retained in full, this product
- * may be distributed under the terms of the GNU General Public License (GPL),
- * in which case the provisions of the GPL apply INSTEAD OF those given above.
- *
- * DISCLAIMER
- *
- * This software is provided 'as is' with no explicit or implied warranties
- * in respect of its properties, including, but not limited to, correctness
- * and/or fitness for purpose.
- * ---------------------------------------------------------------------------
*/
+#include <crypto/algapi.h>
+#include <crypto/aes.h>
#include <linux/module.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/errno.h>
-#include <linux/crypto.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
+#include <linux/percpu.h>
+#include <linux/smp.h>
+#include <linux/slab.h>
#include <asm/byteorder.h>
+#include <asm/processor.h>
+#include <asm/i387.h>
#include "padlock.h"
-#define AES_MIN_KEY_SIZE 16 /* in uint8_t units */
-#define AES_MAX_KEY_SIZE 32 /* ditto */
-#define AES_BLOCK_SIZE 16 /* ditto */
-#define AES_EXTENDED_KEY_SIZE 64 /* in uint32_t units */
-#define AES_EXTENDED_KEY_SIZE_B (AES_EXTENDED_KEY_SIZE * sizeof(uint32_t))
-
+/*
+ * Number of data blocks actually fetched for each xcrypt insn.
+ * Processors with prefetch errata will fetch extra blocks.
+ */
+static unsigned int ecb_fetch_blocks = 2;
+#define MAX_ECB_FETCH_BLOCKS (8)
+#define ecb_fetch_bytes (ecb_fetch_blocks * AES_BLOCK_SIZE)
+
+static unsigned int cbc_fetch_blocks = 1;
+#define MAX_CBC_FETCH_BLOCKS (4)
+#define cbc_fetch_bytes (cbc_fetch_blocks * AES_BLOCK_SIZE)
+
+/* Control word. */
+struct cword {
+ unsigned int __attribute__ ((__packed__))
+ rounds:4,
+ algo:3,
+ keygen:1,
+ interm:1,
+ encdec:1,
+ ksize:2;
+} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+
+/* Whenever making any changes to the following
+ * structure *make sure* you keep E, d_data
+ * and cword aligned on 16 Bytes boundaries and
+ * the Hardware can access 16 * 16 bytes of E and d_data
+ * (only the first 15 * 16 bytes matter but the HW reads
+ * more).
+ */
struct aes_ctx {
- uint32_t e_data[AES_EXTENDED_KEY_SIZE];
- uint32_t d_data[AES_EXTENDED_KEY_SIZE];
+ u32 E[AES_MAX_KEYLENGTH_U32]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+ u32 d_data[AES_MAX_KEYLENGTH_U32]
+ __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
struct {
struct cword encrypt;
struct cword decrypt;
} cword;
- uint32_t *E;
- uint32_t *D;
- int key_length;
+ u32 *D;
};
-/* ====== Key management routines ====== */
-
-static inline uint32_t
-generic_rotr32 (const uint32_t x, const unsigned bits)
-{
- const unsigned n = bits % 32;
- return (x >> n) | (x << (32 - n));
-}
-
-static inline uint32_t
-generic_rotl32 (const uint32_t x, const unsigned bits)
-{
- const unsigned n = bits % 32;
- return (x << n) | (x >> (32 - n));
-}
-
-#define rotl generic_rotl32
-#define rotr generic_rotr32
-
-/*
- * #define byte(x, nr) ((unsigned char)((x) >> (nr*8)))
- */
-static inline uint8_t
-byte(const uint32_t x, const unsigned n)
-{
- return x >> (n << 3);
-}
-
-#define E_KEY ctx->E
-#define D_KEY ctx->D
-
-static uint8_t pow_tab[256];
-static uint8_t log_tab[256];
-static uint8_t sbx_tab[256];
-static uint8_t isb_tab[256];
-static uint32_t rco_tab[10];
-static uint32_t ft_tab[4][256];
-static uint32_t it_tab[4][256];
-
-static uint32_t fl_tab[4][256];
-static uint32_t il_tab[4][256];
-
-static inline uint8_t
-f_mult (uint8_t a, uint8_t b)
-{
- uint8_t aa = log_tab[a], cc = aa + log_tab[b];
-
- return pow_tab[cc + (cc < aa ? 1 : 0)];
-}
-
-#define ff_mult(a,b) (a && b ? f_mult(a, b) : 0)
-
-#define f_rn(bo, bi, n, k) \
- bo[n] = ft_tab[0][byte(bi[n],0)] ^ \
- ft_tab[1][byte(bi[(n + 1) & 3],1)] ^ \
- ft_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
- ft_tab[3][byte(bi[(n + 3) & 3],3)] ^ *(k + n)
-
-#define i_rn(bo, bi, n, k) \
- bo[n] = it_tab[0][byte(bi[n],0)] ^ \
- it_tab[1][byte(bi[(n + 3) & 3],1)] ^ \
- it_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
- it_tab[3][byte(bi[(n + 1) & 3],3)] ^ *(k + n)
-
-#define ls_box(x) \
- ( fl_tab[0][byte(x, 0)] ^ \
- fl_tab[1][byte(x, 1)] ^ \
- fl_tab[2][byte(x, 2)] ^ \
- fl_tab[3][byte(x, 3)] )
-
-#define f_rl(bo, bi, n, k) \
- bo[n] = fl_tab[0][byte(bi[n],0)] ^ \
- fl_tab[1][byte(bi[(n + 1) & 3],1)] ^ \
- fl_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
- fl_tab[3][byte(bi[(n + 3) & 3],3)] ^ *(k + n)
-
-#define i_rl(bo, bi, n, k) \
- bo[n] = il_tab[0][byte(bi[n],0)] ^ \
- il_tab[1][byte(bi[(n + 3) & 3],1)] ^ \
- il_tab[2][byte(bi[(n + 2) & 3],2)] ^ \
- il_tab[3][byte(bi[(n + 1) & 3],3)] ^ *(k + n)
-
-static void
-gen_tabs (void)
-{
- uint32_t i, t;
- uint8_t p, q;
-
- /* log and power tables for GF(2**8) finite field with
- 0x011b as modular polynomial - the simplest prmitive
- root is 0x03, used here to generate the tables */
-
- for (i = 0, p = 1; i < 256; ++i) {
- pow_tab[i] = (uint8_t) p;
- log_tab[p] = (uint8_t) i;
-
- p ^= (p << 1) ^ (p & 0x80 ? 0x01b : 0);
- }
-
- log_tab[1] = 0;
-
- for (i = 0, p = 1; i < 10; ++i) {
- rco_tab[i] = p;
-
- p = (p << 1) ^ (p & 0x80 ? 0x01b : 0);
- }
-
- for (i = 0; i < 256; ++i) {
- p = (i ? pow_tab[255 - log_tab[i]] : 0);
- q = ((p >> 7) | (p << 1)) ^ ((p >> 6) | (p << 2));
- p ^= 0x63 ^ q ^ ((q >> 6) | (q << 2));
- sbx_tab[i] = p;
- isb_tab[p] = (uint8_t) i;
- }
-
- for (i = 0; i < 256; ++i) {
- p = sbx_tab[i];
-
- t = p;
- fl_tab[0][i] = t;
- fl_tab[1][i] = rotl (t, 8);
- fl_tab[2][i] = rotl (t, 16);
- fl_tab[3][i] = rotl (t, 24);
-
- t = ((uint32_t) ff_mult (2, p)) |
- ((uint32_t) p << 8) |
- ((uint32_t) p << 16) | ((uint32_t) ff_mult (3, p) << 24);
-
- ft_tab[0][i] = t;
- ft_tab[1][i] = rotl (t, 8);
- ft_tab[2][i] = rotl (t, 16);
- ft_tab[3][i] = rotl (t, 24);
-
- p = isb_tab[i];
-
- t = p;
- il_tab[0][i] = t;
- il_tab[1][i] = rotl (t, 8);
- il_tab[2][i] = rotl (t, 16);
- il_tab[3][i] = rotl (t, 24);
-
- t = ((uint32_t) ff_mult (14, p)) |
- ((uint32_t) ff_mult (9, p) << 8) |
- ((uint32_t) ff_mult (13, p) << 16) |
- ((uint32_t) ff_mult (11, p) << 24);
-
- it_tab[0][i] = t;
- it_tab[1][i] = rotl (t, 8);
- it_tab[2][i] = rotl (t, 16);
- it_tab[3][i] = rotl (t, 24);
- }
-}
-
-#define star_x(x) (((x) & 0x7f7f7f7f) << 1) ^ ((((x) & 0x80808080) >> 7) * 0x1b)
-
-#define imix_col(y,x) \
- u = star_x(x); \
- v = star_x(u); \
- w = star_x(v); \
- t = w ^ (x); \
- (y) = u ^ v ^ w; \
- (y) ^= rotr(u ^ t, 8) ^ \
- rotr(v ^ t, 16) ^ \
- rotr(t,24)
-
-/* initialise the key schedule from the user supplied key */
-
-#define loop4(i) \
-{ t = rotr(t, 8); t = ls_box(t) ^ rco_tab[i]; \
- t ^= E_KEY[4 * i]; E_KEY[4 * i + 4] = t; \
- t ^= E_KEY[4 * i + 1]; E_KEY[4 * i + 5] = t; \
- t ^= E_KEY[4 * i + 2]; E_KEY[4 * i + 6] = t; \
- t ^= E_KEY[4 * i + 3]; E_KEY[4 * i + 7] = t; \
-}
-
-#define loop6(i) \
-{ t = rotr(t, 8); t = ls_box(t) ^ rco_tab[i]; \
- t ^= E_KEY[6 * i]; E_KEY[6 * i + 6] = t; \
- t ^= E_KEY[6 * i + 1]; E_KEY[6 * i + 7] = t; \
- t ^= E_KEY[6 * i + 2]; E_KEY[6 * i + 8] = t; \
- t ^= E_KEY[6 * i + 3]; E_KEY[6 * i + 9] = t; \
- t ^= E_KEY[6 * i + 4]; E_KEY[6 * i + 10] = t; \
- t ^= E_KEY[6 * i + 5]; E_KEY[6 * i + 11] = t; \
-}
-
-#define loop8(i) \
-{ t = rotr(t, 8); ; t = ls_box(t) ^ rco_tab[i]; \
- t ^= E_KEY[8 * i]; E_KEY[8 * i + 8] = t; \
- t ^= E_KEY[8 * i + 1]; E_KEY[8 * i + 9] = t; \
- t ^= E_KEY[8 * i + 2]; E_KEY[8 * i + 10] = t; \
- t ^= E_KEY[8 * i + 3]; E_KEY[8 * i + 11] = t; \
- t = E_KEY[8 * i + 4] ^ ls_box(t); \
- E_KEY[8 * i + 12] = t; \
- t ^= E_KEY[8 * i + 5]; E_KEY[8 * i + 13] = t; \
- t ^= E_KEY[8 * i + 6]; E_KEY[8 * i + 14] = t; \
- t ^= E_KEY[8 * i + 7]; E_KEY[8 * i + 15] = t; \
-}
+static DEFINE_PER_CPU(struct cword *, paes_last_cword);
/* Tells whether the ACE is capable to generate
the extended key for a given key_len. */
return 0;
}
-static inline struct aes_ctx *aes_ctx(void *ctx)
+static inline struct aes_ctx *aes_ctx_common(void *ctx)
{
- return (struct aes_ctx *)ALIGN((unsigned long)ctx, PADLOCK_ALIGNMENT);
+ unsigned long addr = (unsigned long)ctx;
+ unsigned long align = PADLOCK_ALIGNMENT;
+
+ if (align <= crypto_tfm_ctx_alignment())
+ align = 1;
+ return (struct aes_ctx *)ALIGN(addr, align);
+}
+
+static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
+{
+ return aes_ctx_common(crypto_tfm_ctx(tfm));
+}
+
+static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
+{
+ return aes_ctx_common(crypto_blkcipher_ctx(tfm));
}
-static int
-aes_set_key(void *ctx_arg, const uint8_t *in_key, unsigned int key_len, uint32_t *flags)
+static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
+ unsigned int key_len)
{
- struct aes_ctx *ctx = aes_ctx(ctx_arg);
+ struct aes_ctx *ctx = aes_ctx(tfm);
const __le32 *key = (const __le32 *)in_key;
- uint32_t i, t, u, v, w;
- uint32_t P[AES_EXTENDED_KEY_SIZE];
- uint32_t rounds;
+ u32 *flags = &tfm->crt_flags;
+ struct crypto_aes_ctx gen_aes;
+ int cpu;
- if (key_len != 16 && key_len != 24 && key_len != 32) {
+ if (key_len % 8) {
*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
return -EINVAL;
}
- ctx->key_length = key_len;
-
/*
* If the hardware is capable of generating the extended key
* itself we must supply the plain key for both encryption
* and decryption.
*/
- ctx->E = ctx->e_data;
- ctx->D = ctx->e_data;
+ ctx->D = ctx->E;
- E_KEY[0] = le32_to_cpu(key[0]);
- E_KEY[1] = le32_to_cpu(key[1]);
- E_KEY[2] = le32_to_cpu(key[2]);
- E_KEY[3] = le32_to_cpu(key[3]);
+ ctx->E[0] = le32_to_cpu(key[0]);
+ ctx->E[1] = le32_to_cpu(key[1]);
+ ctx->E[2] = le32_to_cpu(key[2]);
+ ctx->E[3] = le32_to_cpu(key[3]);
/* Prepare control words. */
memset(&ctx->cword, 0, sizeof(ctx->cword));
/* Don't generate extended keys if the hardware can do it. */
if (aes_hw_extkey_available(key_len))
- return 0;
+ goto ok;
ctx->D = ctx->d_data;
ctx->cword.encrypt.keygen = 1;
ctx->cword.decrypt.keygen = 1;
- switch (key_len) {
- case 16:
- t = E_KEY[3];
- for (i = 0; i < 10; ++i)
- loop4 (i);
- break;
-
- case 24:
- E_KEY[4] = le32_to_cpu(key[4]);
- t = E_KEY[5] = le32_to_cpu(key[5]);
- for (i = 0; i < 8; ++i)
- loop6 (i);
- break;
-
- case 32:
- E_KEY[4] = le32_to_cpu(in_key[4]);
- E_KEY[5] = le32_to_cpu(in_key[5]);
- E_KEY[6] = le32_to_cpu(in_key[6]);
- t = E_KEY[7] = le32_to_cpu(in_key[7]);
- for (i = 0; i < 7; ++i)
- loop8 (i);
- break;
- }
-
- D_KEY[0] = E_KEY[0];
- D_KEY[1] = E_KEY[1];
- D_KEY[2] = E_KEY[2];
- D_KEY[3] = E_KEY[3];
-
- for (i = 4; i < key_len + 24; ++i) {
- imix_col (D_KEY[i], E_KEY[i]);
- }
-
- /* PadLock needs a different format of the decryption key. */
- rounds = 10 + (key_len - 16) / 4;
-
- for (i = 0; i < rounds; i++) {
- P[((i + 1) * 4) + 0] = D_KEY[((rounds - i - 1) * 4) + 0];
- P[((i + 1) * 4) + 1] = D_KEY[((rounds - i - 1) * 4) + 1];
- P[((i + 1) * 4) + 2] = D_KEY[((rounds - i - 1) * 4) + 2];
- P[((i + 1) * 4) + 3] = D_KEY[((rounds - i - 1) * 4) + 3];
+ if (crypto_aes_expand_key(&gen_aes, in_key, key_len)) {
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
}
- P[0] = E_KEY[(rounds * 4) + 0];
- P[1] = E_KEY[(rounds * 4) + 1];
- P[2] = E_KEY[(rounds * 4) + 2];
- P[3] = E_KEY[(rounds * 4) + 3];
+ memcpy(ctx->E, gen_aes.key_enc, AES_MAX_KEYLENGTH);
+ memcpy(ctx->D, gen_aes.key_dec, AES_MAX_KEYLENGTH);
- memcpy(D_KEY, P, AES_EXTENDED_KEY_SIZE_B);
+ok:
+ for_each_online_cpu(cpu)
+ if (&ctx->cword.encrypt == per_cpu(paes_last_cword, cpu) ||
+ &ctx->cword.decrypt == per_cpu(paes_last_cword, cpu))
+ per_cpu(paes_last_cword, cpu) = NULL;
return 0;
}
/* ====== Encryption/decryption routines ====== */
/* These are the real call to PadLock. */
-static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
- void *control_word, u32 count)
+static inline void padlock_reset_key(struct cword *cword)
+{
+ int cpu = raw_smp_processor_id();
+
+ if (cword != per_cpu(paes_last_cword, cpu))
+#ifndef CONFIG_X86_64
+ asm volatile ("pushfl; popfl");
+#else
+ asm volatile ("pushfq; popfq");
+#endif
+}
+
+static inline void padlock_store_cword(struct cword *cword)
+{
+ per_cpu(paes_last_cword, raw_smp_processor_id()) = cword;
+}
+
+/*
+ * While the padlock instructions don't use FP/SSE registers, they
+ * generate a spurious DNA fault when cr0.ts is '1'. These instructions
+ * should be used only inside the irq_ts_save/restore() context
+ */
+
+static inline void rep_xcrypt_ecb(const u8 *input, u8 *output, void *key,
+ struct cword *control_word, int count)
{
- asm volatile ("pushfl; popfl"); /* enforce key reload. */
asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
: "+S"(input), "+D"(output)
: "d"(control_word), "b"(key), "c"(count));
}
-static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
- u8 *iv, void *control_word, u32 count)
+static inline u8 *rep_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, struct cword *control_word, int count)
{
- /* Enforce key reload. */
- asm volatile ("pushfl; popfl");
- /* rep xcryptcbc */
- asm volatile (".byte 0xf3,0x0f,0xa7,0xd0"
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
: "+S" (input), "+D" (output), "+a" (iv)
: "d" (control_word), "b" (key), "c" (count));
return iv;
}
-static void
-aes_encrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
+static void ecb_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ struct cword *cword, int count)
+{
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * (MAX_ECB_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
+ u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ rep_xcrypt_ecb(tmp, out, key, cword, count);
+}
+
+static u8 *cbc_crypt_copy(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
{
- struct aes_ctx *ctx = aes_ctx(ctx_arg);
- padlock_xcrypt_ecb(in, out, ctx->E, &ctx->cword.encrypt, 1);
+ /*
+ * Padlock prefetches extra data so we must provide mapped input buffers.
+ * Assume there are at least 16 bytes of stack already in use.
+ */
+ u8 buf[AES_BLOCK_SIZE * (MAX_CBC_FETCH_BLOCKS - 1) + PADLOCK_ALIGNMENT - 1];
+ u8 *tmp = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
+
+ memcpy(tmp, in, count * AES_BLOCK_SIZE);
+ return rep_xcrypt_cbc(tmp, out, key, iv, cword, count);
}
-static void
-aes_decrypt(void *ctx_arg, uint8_t *out, const uint8_t *in)
+static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key,
+ struct cword *cword, int count)
{
- struct aes_ctx *ctx = aes_ctx(ctx_arg);
- padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt, 1);
+ /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data.
+ * We could avoid some copying here but it's probably not worth it.
+ */
+ if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) {
+ ecb_crypt_copy(in, out, key, cword, count);
+ return;
+ }
+
+ rep_xcrypt_ecb(in, out, key, cword, count);
}
-static unsigned int aes_encrypt_ecb(const struct cipher_desc *desc, u8 *out,
- const u8 *in, unsigned int nbytes)
+static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key,
+ u8 *iv, struct cword *cword, int count)
{
- struct aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(desc->tfm));
- padlock_xcrypt_ecb(in, out, ctx->E, &ctx->cword.encrypt,
- nbytes / AES_BLOCK_SIZE);
- return nbytes & ~(AES_BLOCK_SIZE - 1);
+ /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */
+ if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE))
+ return cbc_crypt_copy(in, out, key, iv, cword, count);
+
+ return rep_xcrypt_cbc(in, out, key, iv, cword, count);
}
-static unsigned int aes_decrypt_ecb(const struct cipher_desc *desc, u8 *out,
- const u8 *in, unsigned int nbytes)
+static inline void padlock_xcrypt_ecb(const u8 *input, u8 *output, void *key,
+ void *control_word, u32 count)
{
- struct aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(desc->tfm));
- padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt,
- nbytes / AES_BLOCK_SIZE);
- return nbytes & ~(AES_BLOCK_SIZE - 1);
+ u32 initial = count & (ecb_fetch_blocks - 1);
+
+ if (count < ecb_fetch_blocks) {
+ ecb_crypt(input, output, key, control_word, count);
+ return;
+ }
+
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(initial));
+
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xc8" /* rep xcryptecb */
+ : "+S"(input), "+D"(output)
+ : "d"(control_word), "b"(key), "c"(count - initial));
}
-static unsigned int aes_encrypt_cbc(const struct cipher_desc *desc, u8 *out,
- const u8 *in, unsigned int nbytes)
+static inline u8 *padlock_xcrypt_cbc(const u8 *input, u8 *output, void *key,
+ u8 *iv, void *control_word, u32 count)
{
- struct aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(desc->tfm));
- u8 *iv;
+ u32 initial = count & (cbc_fetch_blocks - 1);
+
+ if (count < cbc_fetch_blocks)
+ return cbc_crypt(input, output, key, iv, control_word, count);
- iv = padlock_xcrypt_cbc(in, out, ctx->E, desc->info,
- &ctx->cword.encrypt, nbytes / AES_BLOCK_SIZE);
- memcpy(desc->info, iv, AES_BLOCK_SIZE);
+ if (initial)
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count));
- return nbytes & ~(AES_BLOCK_SIZE - 1);
+ asm volatile (".byte 0xf3,0x0f,0xa7,0xd0" /* rep xcryptcbc */
+ : "+S" (input), "+D" (output), "+a" (iv)
+ : "d" (control_word), "b" (key), "c" (count-initial));
+ return iv;
+}
+
+static void aes_encrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
+{
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+ ts_state = irq_ts_save();
+ ecb_crypt(in, out, ctx->E, &ctx->cword.encrypt, 1);
+ irq_ts_restore(ts_state);
+ padlock_store_cword(&ctx->cword.encrypt);
}
-static unsigned int aes_decrypt_cbc(const struct cipher_desc *desc, u8 *out,
- const u8 *in, unsigned int nbytes)
+static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
{
- struct aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(desc->tfm));
- padlock_xcrypt_cbc(in, out, ctx->D, desc->info, &ctx->cword.decrypt,
- nbytes / AES_BLOCK_SIZE);
- return nbytes & ~(AES_BLOCK_SIZE - 1);
+ struct aes_ctx *ctx = aes_ctx(tfm);
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+ ts_state = irq_ts_save();
+ ecb_crypt(in, out, ctx->D, &ctx->cword.decrypt, 1);
+ irq_ts_restore(ts_state);
+ padlock_store_cword(&ctx->cword.encrypt);
}
static struct crypto_alg aes_alg = {
.cra_name = "aes",
.cra_driver_name = "aes-padlock",
- .cra_priority = 300,
+ .cra_priority = PADLOCK_CRA_PRIORITY,
.cra_flags = CRYPTO_ALG_TYPE_CIPHER,
.cra_blocksize = AES_BLOCK_SIZE,
.cra_ctxsize = sizeof(struct aes_ctx),
.cia_setkey = aes_set_key,
.cia_encrypt = aes_encrypt,
.cia_decrypt = aes_decrypt,
- .cia_encrypt_ecb = aes_encrypt_ecb,
- .cia_decrypt_ecb = aes_decrypt_ecb,
- .cia_encrypt_cbc = aes_encrypt_cbc,
- .cia_decrypt_cbc = aes_decrypt_cbc,
}
}
};
-int __init padlock_init_aes(void)
+static int ecb_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->E, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static int ecb_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.decrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static struct crypto_alg ecb_aes_alg = {
+ .cra_name = "ecb(aes)",
+ .cra_driver_name = "ecb-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(ecb_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = ecb_aes_encrypt,
+ .decrypt = ecb_aes_decrypt,
+ }
+ }
+};
+
+static int cbc_aes_encrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
+ walk.dst.virt.addr, ctx->E,
+ walk.iv, &ctx->cword.encrypt,
+ nbytes / AES_BLOCK_SIZE);
+ memcpy(walk.iv, iv, AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.decrypt);
+
+ return err;
+}
+
+static int cbc_aes_decrypt(struct blkcipher_desc *desc,
+ struct scatterlist *dst, struct scatterlist *src,
+ unsigned int nbytes)
+{
+ struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+ struct blkcipher_walk walk;
+ int err;
+ int ts_state;
+
+ padlock_reset_key(&ctx->cword.encrypt);
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ ts_state = irq_ts_save();
+ while ((nbytes = walk.nbytes)) {
+ padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
+ ctx->D, walk.iv, &ctx->cword.decrypt,
+ nbytes / AES_BLOCK_SIZE);
+ nbytes &= AES_BLOCK_SIZE - 1;
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ irq_ts_restore(ts_state);
+
+ padlock_store_cword(&ctx->cword.encrypt);
+
+ return err;
+}
+
+static struct crypto_alg cbc_aes_alg = {
+ .cra_name = "cbc(aes)",
+ .cra_driver_name = "cbc-aes-padlock",
+ .cra_priority = PADLOCK_COMPOSITE_PRIORITY,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = AES_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct aes_ctx),
+ .cra_alignmask = PADLOCK_ALIGNMENT - 1,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(cbc_aes_alg.cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = AES_MIN_KEY_SIZE,
+ .max_keysize = AES_MAX_KEY_SIZE,
+ .ivsize = AES_BLOCK_SIZE,
+ .setkey = aes_set_key,
+ .encrypt = cbc_aes_encrypt,
+ .decrypt = cbc_aes_decrypt,
+ }
+ }
+};
+
+static int __init padlock_init(void)
+{
+ int ret;
+ struct cpuinfo_x86 *c = &cpu_data(0);
+
+ if (!cpu_has_xcrypt) {
+ printk(KERN_NOTICE PFX "VIA PadLock not detected.\n");
+ return -ENODEV;
+ }
+
+ if (!cpu_has_xcrypt_enabled) {
+ printk(KERN_NOTICE PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
+ return -ENODEV;
+ }
+
+ if ((ret = crypto_register_alg(&aes_alg)))
+ goto aes_err;
+
+ if ((ret = crypto_register_alg(&ecb_aes_alg)))
+ goto ecb_aes_err;
+
+ if ((ret = crypto_register_alg(&cbc_aes_alg)))
+ goto cbc_aes_err;
+
printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
- gen_tabs();
- return crypto_register_alg(&aes_alg);
+ if (c->x86 == 6 && c->x86_model == 15 && c->x86_mask == 2) {
+ ecb_fetch_blocks = MAX_ECB_FETCH_BLOCKS;
+ cbc_fetch_blocks = MAX_CBC_FETCH_BLOCKS;
+ printk(KERN_NOTICE PFX "VIA Nano stepping 2 detected: enabling workaround.\n");
+ }
+
+out:
+ return ret;
+
+cbc_aes_err:
+ crypto_unregister_alg(&ecb_aes_alg);
+ecb_aes_err:
+ crypto_unregister_alg(&aes_alg);
+aes_err:
+ printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
+ goto out;
}
-void __exit padlock_fini_aes(void)
+static void __exit padlock_fini(void)
{
+ crypto_unregister_alg(&cbc_aes_alg);
+ crypto_unregister_alg(&ecb_aes_alg);
crypto_unregister_alg(&aes_alg);
}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("aes");
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff