2 * GCM: Galois/Counter Mode.
4 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation.
11 #include <crypto/gf128mul.h>
12 #include <crypto/internal/skcipher.h>
13 #include <crypto/scatterwalk.h>
14 #include <linux/completion.h>
15 #include <linux/err.h>
16 #include <linux/init.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/slab.h>
21 struct gcm_instance_ctx {
22 struct crypto_skcipher_spawn ctr;
25 struct crypto_gcm_ctx {
26 struct crypto_ablkcipher *ctr;
27 struct gf128mul_4k *gf128;
30 struct crypto_gcm_ghash_ctx {
33 struct gf128mul_4k *gf128;
37 struct crypto_gcm_req_priv_ctx {
40 struct scatterlist src[2];
41 struct scatterlist dst[2];
42 struct crypto_gcm_ghash_ctx ghash;
43 struct ablkcipher_request abreq;
46 struct crypto_gcm_setkey_result {
48 struct completion completion;
51 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
52 struct aead_request *req)
54 unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
56 return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
59 static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags,
60 struct gf128mul_4k *gf128)
65 memset(ctx->buffer, 0, 16);
68 static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx,
69 const u8 *src, unsigned int srclen)
71 u8 *dst = ctx->buffer;
74 int n = min(srclen, ctx->bytes);
75 u8 *pos = dst + (16 - ctx->bytes);
84 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
87 while (srclen >= 16) {
88 crypto_xor(dst, src, 16);
89 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
95 ctx->bytes = 16 - srclen;
101 static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx,
102 struct scatterlist *sg, int len)
104 struct scatter_walk walk;
111 scatterwalk_start(&walk, sg);
114 n = scatterwalk_clamp(&walk, len);
117 scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg));
118 n = scatterwalk_clamp(&walk, len);
121 src = scatterwalk_map(&walk, 0);
123 crypto_gcm_ghash_update(ctx, src, n);
126 scatterwalk_unmap(src, 0);
127 scatterwalk_advance(&walk, n);
128 scatterwalk_done(&walk, 0, len);
130 crypto_yield(ctx->flags);
134 static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx)
136 u8 *dst = ctx->buffer;
139 u8 *tmp = dst + (16 - ctx->bytes);
144 gf128mul_4k_lle((be128 *)dst, ctx->gf128);
150 static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx,
151 unsigned int authlen,
152 unsigned int cryptlen, u8 *dst)
154 u8 *buf = ctx->buffer;
157 lengths.a = cpu_to_be64(authlen * 8);
158 lengths.b = cpu_to_be64(cryptlen * 8);
160 crypto_gcm_ghash_flush(ctx);
161 crypto_xor(buf, (u8 *)&lengths, 16);
162 gf128mul_4k_lle((be128 *)buf, ctx->gf128);
163 crypto_xor(dst, buf, 16);
166 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
168 struct crypto_gcm_setkey_result *result = req->data;
170 if (err == -EINPROGRESS)
174 complete(&result->completion);
177 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
180 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
181 struct crypto_ablkcipher *ctr = ctx->ctr;
186 struct crypto_gcm_setkey_result result;
188 struct scatterlist sg[1];
189 struct ablkcipher_request req;
193 crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
194 crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
195 CRYPTO_TFM_REQ_MASK);
197 err = crypto_ablkcipher_setkey(ctr, key, keylen);
201 crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
202 CRYPTO_TFM_RES_MASK);
204 data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
209 init_completion(&data->result.completion);
210 sg_init_one(data->sg, &data->hash, sizeof(data->hash));
211 ablkcipher_request_set_tfm(&data->req, ctr);
212 ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
213 CRYPTO_TFM_REQ_MAY_BACKLOG,
214 crypto_gcm_setkey_done,
216 ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
217 sizeof(data->hash), data->iv);
219 err = crypto_ablkcipher_encrypt(&data->req);
220 if (err == -EINPROGRESS || err == -EBUSY) {
221 err = wait_for_completion_interruptible(
222 &data->result.completion);
224 err = data->result.err;
230 if (ctx->gf128 != NULL)
231 gf128mul_free_4k(ctx->gf128);
233 ctx->gf128 = gf128mul_init_4k_lle(&data->hash);
235 if (ctx->gf128 == NULL)
243 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
244 struct aead_request *req,
245 unsigned int cryptlen)
247 struct crypto_aead *aead = crypto_aead_reqtfm(req);
248 struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
249 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
250 u32 flags = req->base.tfm->crt_flags;
251 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
252 struct scatterlist *dst;
253 __be32 counter = cpu_to_be32(1);
255 memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
256 memcpy(req->iv + 12, &counter, 4);
258 sg_init_table(pctx->src, 2);
259 sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
260 scatterwalk_sg_chain(pctx->src, 2, req->src);
263 if (req->src != req->dst) {
264 sg_init_table(pctx->dst, 2);
265 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
266 scatterwalk_sg_chain(pctx->dst, 2, req->dst);
270 ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
271 ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
272 cryptlen + sizeof(pctx->auth_tag),
275 crypto_gcm_ghash_init(ghash, flags, ctx->gf128);
277 crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen);
278 crypto_gcm_ghash_flush(ghash);
281 static int crypto_gcm_hash(struct aead_request *req)
283 struct crypto_aead *aead = crypto_aead_reqtfm(req);
284 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
285 u8 *auth_tag = pctx->auth_tag;
286 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
288 crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen);
289 crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen,
292 scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
293 crypto_aead_authsize(aead), 1);
297 static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err)
299 struct aead_request *req = areq->data;
302 err = crypto_gcm_hash(req);
304 aead_request_complete(req, err);
307 static int crypto_gcm_encrypt(struct aead_request *req)
309 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
310 struct ablkcipher_request *abreq = &pctx->abreq;
313 crypto_gcm_init_crypt(abreq, req, req->cryptlen);
314 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
315 crypto_gcm_encrypt_done, req);
317 err = crypto_ablkcipher_encrypt(abreq);
321 return crypto_gcm_hash(req);
324 static int crypto_gcm_verify(struct aead_request *req)
326 struct crypto_aead *aead = crypto_aead_reqtfm(req);
327 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
328 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
329 u8 *auth_tag = pctx->auth_tag;
330 u8 *iauth_tag = pctx->iauth_tag;
331 unsigned int authsize = crypto_aead_authsize(aead);
332 unsigned int cryptlen = req->cryptlen - authsize;
334 crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag);
336 authsize = crypto_aead_authsize(aead);
337 scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
338 return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
341 static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err)
343 struct aead_request *req = areq->data;
346 err = crypto_gcm_verify(req);
348 aead_request_complete(req, err);
351 static int crypto_gcm_decrypt(struct aead_request *req)
353 struct crypto_aead *aead = crypto_aead_reqtfm(req);
354 struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
355 struct ablkcipher_request *abreq = &pctx->abreq;
356 struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash;
357 unsigned int cryptlen = req->cryptlen;
358 unsigned int authsize = crypto_aead_authsize(aead);
361 if (cryptlen < authsize)
363 cryptlen -= authsize;
365 crypto_gcm_init_crypt(abreq, req, cryptlen);
366 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
367 crypto_gcm_decrypt_done, req);
369 crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen);
371 err = crypto_ablkcipher_decrypt(abreq);
375 return crypto_gcm_verify(req);
378 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
380 struct crypto_instance *inst = (void *)tfm->__crt_alg;
381 struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
382 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
383 struct crypto_ablkcipher *ctr;
387 ctr = crypto_spawn_skcipher(&ictx->ctr);
395 align = crypto_tfm_alg_alignmask(tfm);
396 align &= ~(crypto_tfm_ctx_alignment() - 1);
397 tfm->crt_aead.reqsize = align +
398 sizeof(struct crypto_gcm_req_priv_ctx) +
399 crypto_ablkcipher_reqsize(ctr);
404 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
406 struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
408 if (ctx->gf128 != NULL)
409 gf128mul_free_4k(ctx->gf128);
411 crypto_free_ablkcipher(ctx->ctr);
414 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
415 const char *full_name,
416 const char *ctr_name)
418 struct crypto_attr_type *algt;
419 struct crypto_instance *inst;
420 struct crypto_alg *ctr;
421 struct gcm_instance_ctx *ctx;
424 algt = crypto_get_attr_type(tb);
429 if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
430 return ERR_PTR(-EINVAL);
432 inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
434 return ERR_PTR(-ENOMEM);
436 ctx = crypto_instance_ctx(inst);
437 crypto_set_skcipher_spawn(&ctx->ctr, inst);
438 err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
439 crypto_requires_sync(algt->type,
444 ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
446 /* We only support 16-byte blocks. */
447 if (ctr->cra_ablkcipher.ivsize != 16)
450 /* Not a stream cipher? */
452 if (ctr->cra_blocksize != 1)
456 if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
457 "gcm_base(%s)", ctr->cra_driver_name) >=
461 memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
463 inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
464 inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
465 inst->alg.cra_priority = ctr->cra_priority;
466 inst->alg.cra_blocksize = 1;
467 inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
468 inst->alg.cra_type = &crypto_aead_type;
469 inst->alg.cra_aead.ivsize = 16;
470 inst->alg.cra_aead.maxauthsize = 16;
471 inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
472 inst->alg.cra_init = crypto_gcm_init_tfm;
473 inst->alg.cra_exit = crypto_gcm_exit_tfm;
474 inst->alg.cra_aead.setkey = crypto_gcm_setkey;
475 inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
476 inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
482 crypto_drop_skcipher(&ctx->ctr);
489 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
492 const char *cipher_name;
493 char ctr_name[CRYPTO_MAX_ALG_NAME];
494 char full_name[CRYPTO_MAX_ALG_NAME];
496 cipher_name = crypto_attr_alg_name(tb[1]);
497 err = PTR_ERR(cipher_name);
498 if (IS_ERR(cipher_name))
501 if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
503 return ERR_PTR(-ENAMETOOLONG);
505 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
507 return ERR_PTR(-ENAMETOOLONG);
509 return crypto_gcm_alloc_common(tb, full_name, ctr_name);
512 static void crypto_gcm_free(struct crypto_instance *inst)
514 struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
516 crypto_drop_skcipher(&ctx->ctr);
520 static struct crypto_template crypto_gcm_tmpl = {
522 .alloc = crypto_gcm_alloc,
523 .free = crypto_gcm_free,
524 .module = THIS_MODULE,
527 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
530 const char *ctr_name;
531 char full_name[CRYPTO_MAX_ALG_NAME];
533 ctr_name = crypto_attr_alg_name(tb[1]);
534 err = PTR_ERR(ctr_name);
535 if (IS_ERR(ctr_name))
538 if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)",
539 ctr_name) >= CRYPTO_MAX_ALG_NAME)
540 return ERR_PTR(-ENAMETOOLONG);
542 return crypto_gcm_alloc_common(tb, full_name, ctr_name);
545 static struct crypto_template crypto_gcm_base_tmpl = {
547 .alloc = crypto_gcm_base_alloc,
548 .free = crypto_gcm_free,
549 .module = THIS_MODULE,
552 static int __init crypto_gcm_module_init(void)
556 err = crypto_register_template(&crypto_gcm_base_tmpl);
560 err = crypto_register_template(&crypto_gcm_tmpl);
568 crypto_unregister_template(&crypto_gcm_base_tmpl);
572 static void __exit crypto_gcm_module_exit(void)
574 crypto_unregister_template(&crypto_gcm_tmpl);
575 crypto_unregister_template(&crypto_gcm_base_tmpl);
578 module_init(crypto_gcm_module_init);
579 module_exit(crypto_gcm_module_exit);
581 MODULE_LICENSE("GPL");
582 MODULE_DESCRIPTION("Galois/Counter Mode");
583 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
584 MODULE_ALIAS("gcm_base");