SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/crypto/talitos.c

   1 /*
   2  * talitos - Freescale Integrated Security Engine (SEC) device driver
   3  *
   4  * Copyright (c) 2008 Freescale Semiconductor, Inc.
   5  *
   6  * Scatterlist Crypto API glue code copied from files with the following:
   7  * Copyright (c) 2006-2007 Herbert Xu <herbert@gondor.apana.org.au>
   8  *
   9  * Crypto algorithm registration code copied from hifn driver:
  10  * 2007+ Copyright (c) Evgeniy Polyakov <johnpol@2ka.mipt.ru>
  11  * All rights reserved.
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  26  */
  27
  28 #include <linux/kernel.h>
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/device.h>
  32 #include <linux/interrupt.h>
  33 #include <linux/crypto.h>
  34 #include <linux/hw_random.h>
  35 #include <linux/of_platform.h>
  36 #include <linux/dma-mapping.h>
  37 #include <linux/io.h>
  38 #include <linux/spinlock.h>
  39 #include <linux/rtnetlink.h>
  40
  41 #include <crypto/algapi.h>
  42 #include <crypto/aes.h>
  43 #include <crypto/des.h>
  44 #include <crypto/sha.h>
  45 #include <crypto/aead.h>
  46 #include <crypto/authenc.h>
  47 #include <crypto/skcipher.h>
  48 #include <crypto/scatterwalk.h>
  49
  50 #include "talitos.h"
  51
  52 #define TALITOS_TIMEOUT 100000
  53 #define TALITOS_MAX_DATA_LEN 65535
  54
  55 #define DESC_TYPE(desc_hdr) ((be32_to_cpu(desc_hdr) >> 3) & 0x1f)
  56 #define PRIMARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 28) & 0xf)
  57 #define SECONDARY_EU(desc_hdr) ((be32_to_cpu(desc_hdr) >> 16) & 0xf)
  58
  59 /* descriptor pointer entry */
  60 struct talitos_ptr {
  61         __be16 len;     /* length */
  62         u8 j_extent;    /* jump to sg link table and/or extent */
  63         u8 eptr;        /* extended address */
  64         __be32 ptr;     /* address */
  65 };
  66
  67 /* descriptor */
  68 struct talitos_desc {
  69         __be32 hdr;                     /* header high bits */
  70         __be32 hdr_lo;                  /* header low bits */
  71         struct talitos_ptr ptr[7];      /* ptr/len pair array */
  72 };
  73
  74 /**
  75  * talitos_request - descriptor submission request
  76  * @desc: descriptor pointer (kernel virtual)
  77  * @dma_desc: descriptor's physical bus address
  78  * @callback: whom to call when descriptor processing is done
  79  * @context: caller context (optional)
  80  */
  81 struct talitos_request {
  82         struct talitos_desc *desc;
  83         dma_addr_t dma_desc;
  84         void (*callback) (struct device *dev, struct talitos_desc *desc,
  85                           void *context, int error);
  86         void *context;
  87 };
  88
  89 /* per-channel fifo management */
  90 struct talitos_channel {
  91         /* request fifo */
  92         struct talitos_request *fifo;
  93
  94         /* number of requests pending in channel h/w fifo */
  95         atomic_t submit_count ____cacheline_aligned;
  96
  97         /* request submission (head) lock */
  98         spinlock_t head_lock ____cacheline_aligned;
  99         /* index to next free descriptor request */
 100         int head;
 101
 102         /* request release (tail) lock */
 103         spinlock_t tail_lock ____cacheline_aligned;
 104         /* index to next in-progress/done descriptor request */
 105         int tail;
 106 };
 107
 108 struct talitos_private {
 109         struct device *dev;
 110         struct of_device *ofdev;
 111         void __iomem *reg;
 112         int irq;
 113
 114         /* SEC version geometry (from device tree node) */
 115         unsigned int num_channels;
 116         unsigned int chfifo_len;
 117         unsigned int exec_units;
 118         unsigned int desc_types;
 119
 120         /* SEC Compatibility info */
 121         unsigned long features;
 122
 123         /*
 124          * length of the request fifo
 125          * fifo_len is chfifo_len rounded up to next power of 2
 126          * so we can use bitwise ops to wrap
 127          */
 128         unsigned int fifo_len;
 129
 130         struct talitos_channel *chan;
 131
 132         /* next channel to be assigned next incoming descriptor */
 133         atomic_t last_chan ____cacheline_aligned;
 134
 135         /* request callback tasklet */
 136         struct tasklet_struct done_task;
 137
 138         /* list of registered algorithms */
 139         struct list_head alg_list;
 140
 141         /* hwrng device */
 142         struct hwrng rng;
 143 };
 144
 145 /* .features flag */
 146 #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001
 147 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002
 148
 149 /*
 150  * map virtual single (contiguous) pointer to h/w descriptor pointer
 151  */
 152 static void map_single_talitos_ptr(struct device *dev,
 153                                    struct talitos_ptr *talitos_ptr,
 154                                    unsigned short len, void *data,
 155                                    unsigned char extent,
 156                                    enum dma_data_direction dir)
 157 {
 158         talitos_ptr->len = cpu_to_be16(len);
 159         talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir));
 160         talitos_ptr->j_extent = extent;
 161 }
 162
 163 /*
 164  * unmap bus single (contiguous) h/w descriptor pointer
 165  */
 166 static void unmap_single_talitos_ptr(struct device *dev,
 167                                      struct talitos_ptr *talitos_ptr,
 168                                      enum dma_data_direction dir)
 169 {
 170         dma_unmap_single(dev, be32_to_cpu(talitos_ptr->ptr),
 171                          be16_to_cpu(talitos_ptr->len), dir);
 172 }
 173
 174 static int reset_channel(struct device *dev, int ch)
 175 {
 176         struct talitos_private *priv = dev_get_drvdata(dev);
 177         unsigned int timeout = TALITOS_TIMEOUT;
 178
 179         setbits32(priv->reg + TALITOS_CCCR(ch), TALITOS_CCCR_RESET);
 180
 181         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) & TALITOS_CCCR_RESET)
 182                && --timeout)
 183                 cpu_relax();
 184
 185         if (timeout == 0) {
 186                 dev_err(dev, "failed to reset channel %d\n", ch);
 187                 return -EIO;
 188         }
 189
 190         /* set done writeback and IRQ */
 191         setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE |
 192                   TALITOS_CCCR_LO_CDIE);
 193
 194         /* and ICCR writeback, if available */
 195         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 196                 setbits32(priv->reg + TALITOS_CCCR_LO(ch),
 197                           TALITOS_CCCR_LO_IWSE);
 198
 199         return 0;
 200 }
 201
 202 static int reset_device(struct device *dev)
 203 {
 204         struct talitos_private *priv = dev_get_drvdata(dev);
 205         unsigned int timeout = TALITOS_TIMEOUT;
 206
 207         setbits32(priv->reg + TALITOS_MCR, TALITOS_MCR_SWR);
 208
 209         while ((in_be32(priv->reg + TALITOS_MCR) & TALITOS_MCR_SWR)
 210                && --timeout)
 211                 cpu_relax();
 212
 213         if (timeout == 0) {
 214                 dev_err(dev, "failed to reset device\n");
 215                 return -EIO;
 216         }
 217
 218         return 0;
 219 }
 220
 221 /*
 222  * Reset and initialize the device
 223  */
 224 static int init_device(struct device *dev)
 225 {
 226         struct talitos_private *priv = dev_get_drvdata(dev);
 227         int ch, err;
 228
 229         /*
 230          * Master reset
 231          * errata documentation: warning: certain SEC interrupts
 232          * are not fully cleared by writing the MCR:SWR bit,
 233          * set bit twice to completely reset
 234          */
 235         err = reset_device(dev);
 236         if (err)
 237                 return err;
 238
 239         err = reset_device(dev);
 240         if (err)
 241                 return err;
 242
 243         /* reset channels */
 244         for (ch = 0; ch < priv->num_channels; ch++) {
 245                 err = reset_channel(dev, ch);
 246                 if (err)
 247                         return err;
 248         }
 249
 250         /* enable channel done and error interrupts */
 251         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
 252         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
 253
 254         /* disable integrity check error interrupts (use writeback instead) */
 255         if (priv->features & TALITOS_FTR_HW_AUTH_CHECK)
 256                 setbits32(priv->reg + TALITOS_MDEUICR_LO,
 257                           TALITOS_MDEUICR_LO_ICE);
 258
 259         return 0;
 260 }
 261
 262 /**
 263  * talitos_submit - submits a descriptor to the device for processing
 264  * @dev:        the SEC device to be used
 265  * @desc:       the descriptor to be processed by the device
 266  * @callback:   whom to call when processing is complete
 267  * @context:    a handle for use by caller (optional)
 268  *
 269  * desc must contain valid dma-mapped (bus physical) address pointers.
 270  * callback must check err and feedback in descriptor header
 271  * for device processing status.
 272  */
 273 static int talitos_submit(struct device *dev, struct talitos_desc *desc,
 274                           void (*callback)(struct device *dev,
 275                                            struct talitos_desc *desc,
 276                                            void *context, int error),
 277                           void *context)
 278 {
 279         struct talitos_private *priv = dev_get_drvdata(dev);
 280         struct talitos_request *request;
 281         unsigned long flags, ch;
 282         int head;
 283
 284         /* select done notification */
 285         desc->hdr |= DESC_HDR_DONE_NOTIFY;
 286
 287         /* emulate SEC's round-robin channel fifo polling scheme */
 288         ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1);
 289
 290         spin_lock_irqsave(&priv->chan[ch].head_lock, flags);
 291
 292         if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) {
 293                 /* h/w fifo is full */
 294                 spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 295                 return -EAGAIN;
 296         }
 297
 298         head = priv->chan[ch].head;
 299         request = &priv->chan[ch].fifo[head];
 300
 301         /* map descriptor and save caller data */
 302         request->dma_desc = dma_map_single(dev, desc, sizeof(*desc),
 303                                            DMA_BIDIRECTIONAL);
 304         request->callback = callback;
 305         request->context = context;
 306
 307         /* increment fifo head */
 308         priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1);
 309
 310         smp_wmb();
 311         request->desc = desc;
 312
 313         /* GO! */
 314         wmb();
 315         out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc);
 316
 317         spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags);
 318
 319         return -EINPROGRESS;
 320 }
 321
 322 /*
 323  * process what was done, notify callback of error if not
 324  */
 325 static void flush_channel(struct device *dev, int ch, int error, int reset_ch)
 326 {
 327         struct talitos_private *priv = dev_get_drvdata(dev);
 328         struct talitos_request *request, saved_req;
 329         unsigned long flags;
 330         int tail, status;
 331
 332         spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 333
 334         tail = priv->chan[ch].tail;
 335         while (priv->chan[ch].fifo[tail].desc) {
 336                 request = &priv->chan[ch].fifo[tail];
 337
 338                 /* descriptors with their done bits set don't get the error */
 339                 rmb();
 340                 if ((request->desc->hdr & DESC_HDR_DONE) == DESC_HDR_DONE)
 341                         status = 0;
 342                 else
 343                         if (!error)
 344                                 break;
 345                         else
 346                                 status = error;
 347
 348                 dma_unmap_single(dev, request->dma_desc,
 349                                  sizeof(struct talitos_desc),
 350                                  DMA_BIDIRECTIONAL);
 351
 352                 /* copy entries so we can call callback outside lock */
 353                 saved_req.desc = request->desc;
 354                 saved_req.callback = request->callback;
 355                 saved_req.context = request->context;
 356
 357                 /* release request entry in fifo */
 358                 smp_wmb();
 359                 request->desc = NULL;
 360
 361                 /* increment fifo tail */
 362                 priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1);
 363
 364                 spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 365
 366                 atomic_dec(&priv->chan[ch].submit_count);
 367
 368                 saved_req.callback(dev, saved_req.desc, saved_req.context,
 369                                    status);
 370                 /* channel may resume processing in single desc error case */
 371                 if (error && !reset_ch && status == error)
 372                         return;
 373                 spin_lock_irqsave(&priv->chan[ch].tail_lock, flags);
 374                 tail = priv->chan[ch].tail;
 375         }
 376
 377         spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags);
 378 }
 379
 380 /*
 381  * process completed requests for channels that have done status
 382  */
 383 static void talitos_done(unsigned long data)
 384 {
 385         struct device *dev = (struct device *)data;
 386         struct talitos_private *priv = dev_get_drvdata(dev);
 387         int ch;
 388
 389         for (ch = 0; ch < priv->num_channels; ch++)
 390                 flush_channel(dev, ch, 0, 0);
 391
 392         /* At this point, all completed channels have been processed.
 393          * Unmask done interrupts for channels completed later on.
 394          */
 395         setbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_INIT);
 396         setbits32(priv->reg + TALITOS_IMR_LO, TALITOS_IMR_LO_INIT);
 397 }
 398
 399 /*
 400  * locate current (offending) descriptor
 401  */
 402 static struct talitos_desc *current_desc(struct device *dev, int ch)
 403 {
 404         struct talitos_private *priv = dev_get_drvdata(dev);
 405         int tail = priv->chan[ch].tail;
 406         dma_addr_t cur_desc;
 407
 408         cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch));
 409
 410         while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) {
 411                 tail = (tail + 1) & (priv->fifo_len - 1);
 412                 if (tail == priv->chan[ch].tail) {
 413                         dev_err(dev, "couldn't locate current descriptor\n");
 414                         return NULL;
 415                 }
 416         }
 417
 418         return priv->chan[ch].fifo[tail].desc;
 419 }
 420
 421 /*
 422  * user diagnostics; report root cause of error based on execution unit status
 423  */
 424 static void report_eu_error(struct device *dev, int ch,
 425                             struct talitos_desc *desc)
 426 {
 427         struct talitos_private *priv = dev_get_drvdata(dev);
 428         int i;
 429
 430         switch (desc->hdr & DESC_HDR_SEL0_MASK) {
 431         case DESC_HDR_SEL0_AFEU:
 432                 dev_err(dev, "AFEUISR 0x%08x_%08x\n",
 433                         in_be32(priv->reg + TALITOS_AFEUISR),
 434                         in_be32(priv->reg + TALITOS_AFEUISR_LO));
 435                 break;
 436         case DESC_HDR_SEL0_DEU:
 437                 dev_err(dev, "DEUISR 0x%08x_%08x\n",
 438                         in_be32(priv->reg + TALITOS_DEUISR),
 439                         in_be32(priv->reg + TALITOS_DEUISR_LO));
 440                 break;
 441         case DESC_HDR_SEL0_MDEUA:
 442         case DESC_HDR_SEL0_MDEUB:
 443                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 444                         in_be32(priv->reg + TALITOS_MDEUISR),
 445                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 446                 break;
 447         case DESC_HDR_SEL0_RNG:
 448                 dev_err(dev, "RNGUISR 0x%08x_%08x\n",
 449                         in_be32(priv->reg + TALITOS_RNGUISR),
 450                         in_be32(priv->reg + TALITOS_RNGUISR_LO));
 451                 break;
 452         case DESC_HDR_SEL0_PKEU:
 453                 dev_err(dev, "PKEUISR 0x%08x_%08x\n",
 454                         in_be32(priv->reg + TALITOS_PKEUISR),
 455                         in_be32(priv->reg + TALITOS_PKEUISR_LO));
 456                 break;
 457         case DESC_HDR_SEL0_AESU:
 458                 dev_err(dev, "AESUISR 0x%08x_%08x\n",
 459                         in_be32(priv->reg + TALITOS_AESUISR),
 460                         in_be32(priv->reg + TALITOS_AESUISR_LO));
 461                 break;
 462         case DESC_HDR_SEL0_CRCU:
 463                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 464                         in_be32(priv->reg + TALITOS_CRCUISR),
 465                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 466                 break;
 467         case DESC_HDR_SEL0_KEU:
 468                 dev_err(dev, "KEUISR 0x%08x_%08x\n",
 469                         in_be32(priv->reg + TALITOS_KEUISR),
 470                         in_be32(priv->reg + TALITOS_KEUISR_LO));
 471                 break;
 472         }
 473
 474         switch (desc->hdr & DESC_HDR_SEL1_MASK) {
 475         case DESC_HDR_SEL1_MDEUA:
 476         case DESC_HDR_SEL1_MDEUB:
 477                 dev_err(dev, "MDEUISR 0x%08x_%08x\n",
 478                         in_be32(priv->reg + TALITOS_MDEUISR),
 479                         in_be32(priv->reg + TALITOS_MDEUISR_LO));
 480                 break;
 481         case DESC_HDR_SEL1_CRCU:
 482                 dev_err(dev, "CRCUISR 0x%08x_%08x\n",
 483                         in_be32(priv->reg + TALITOS_CRCUISR),
 484                         in_be32(priv->reg + TALITOS_CRCUISR_LO));
 485                 break;
 486         }
 487
 488         for (i = 0; i < 8; i++)
 489                 dev_err(dev, "DESCBUF 0x%08x_%08x\n",
 490                         in_be32(priv->reg + TALITOS_DESCBUF(ch) + 8*i),
 491                         in_be32(priv->reg + TALITOS_DESCBUF_LO(ch) + 8*i));
 492 }
 493
 494 /*
 495  * recover from error interrupts
 496  */
 497 static void talitos_error(unsigned long data, u32 isr, u32 isr_lo)
 498 {
 499         struct device *dev = (struct device *)data;
 500         struct talitos_private *priv = dev_get_drvdata(dev);
 501         unsigned int timeout = TALITOS_TIMEOUT;
 502         int ch, error, reset_dev = 0, reset_ch = 0;
 503         u32 v, v_lo;
 504
 505         for (ch = 0; ch < priv->num_channels; ch++) {
 506                 /* skip channels without errors */
 507                 if (!(isr & (1 << (ch * 2 + 1))))
 508                         continue;
 509
 510                 error = -EINVAL;
 511
 512                 v = in_be32(priv->reg + TALITOS_CCPSR(ch));
 513                 v_lo = in_be32(priv->reg + TALITOS_CCPSR_LO(ch));
 514
 515                 if (v_lo & TALITOS_CCPSR_LO_DOF) {
 516                         dev_err(dev, "double fetch fifo overflow error\n");
 517                         error = -EAGAIN;
 518                         reset_ch = 1;
 519                 }
 520                 if (v_lo & TALITOS_CCPSR_LO_SOF) {
 521                         /* h/w dropped descriptor */
 522                         dev_err(dev, "single fetch fifo overflow error\n");
 523                         error = -EAGAIN;
 524                 }
 525                 if (v_lo & TALITOS_CCPSR_LO_MDTE)
 526                         dev_err(dev, "master data transfer error\n");
 527                 if (v_lo & TALITOS_CCPSR_LO_SGDLZ)
 528                         dev_err(dev, "s/g data length zero error\n");
 529                 if (v_lo & TALITOS_CCPSR_LO_FPZ)
 530                         dev_err(dev, "fetch pointer zero error\n");
 531                 if (v_lo & TALITOS_CCPSR_LO_IDH)
 532                         dev_err(dev, "illegal descriptor header error\n");
 533                 if (v_lo & TALITOS_CCPSR_LO_IEU)
 534                         dev_err(dev, "invalid execution unit error\n");
 535                 if (v_lo & TALITOS_CCPSR_LO_EU)
 536                         report_eu_error(dev, ch, current_desc(dev, ch));
 537                 if (v_lo & TALITOS_CCPSR_LO_GB)
 538                         dev_err(dev, "gather boundary error\n");
 539                 if (v_lo & TALITOS_CCPSR_LO_GRL)
 540                         dev_err(dev, "gather return/length error\n");
 541                 if (v_lo & TALITOS_CCPSR_LO_SB)
 542                         dev_err(dev, "scatter boundary error\n");
 543                 if (v_lo & TALITOS_CCPSR_LO_SRL)
 544                         dev_err(dev, "scatter return/length error\n");
 545
 546                 flush_channel(dev, ch, error, reset_ch);
 547
 548                 if (reset_ch) {
 549                         reset_channel(dev, ch);
 550                 } else {
 551                         setbits32(priv->reg + TALITOS_CCCR(ch),
 552                                   TALITOS_CCCR_CONT);
 553                         setbits32(priv->reg + TALITOS_CCCR_LO(ch), 0);
 554                         while ((in_be32(priv->reg + TALITOS_CCCR(ch)) &
 555                                TALITOS_CCCR_CONT) && --timeout)
 556                                 cpu_relax();
 557                         if (timeout == 0) {
 558                                 dev_err(dev, "failed to restart channel %d\n",
 559                                         ch);
 560                                 reset_dev = 1;
 561                         }
 562                 }
 563         }
 564         if (reset_dev || isr & ~TALITOS_ISR_CHERR || isr_lo) {
 565                 dev_err(dev, "done overflow, internal time out, or rngu error: "
 566                         "ISR 0x%08x_%08x\n", isr, isr_lo);
 567
 568                 /* purge request queues */
 569                 for (ch = 0; ch < priv->num_channels; ch++)
 570                         flush_channel(dev, ch, -EIO, 1);
 571
 572                 /* reset and reinitialize the device */
 573                 init_device(dev);
 574         }
 575 }
 576
 577 static irqreturn_t talitos_interrupt(int irq, void *data)
 578 {
 579         struct device *dev = data;
 580         struct talitos_private *priv = dev_get_drvdata(dev);
 581         u32 isr, isr_lo;
 582
 583         isr = in_be32(priv->reg + TALITOS_ISR);
 584         isr_lo = in_be32(priv->reg + TALITOS_ISR_LO);
 585         /* Acknowledge interrupt */
 586         out_be32(priv->reg + TALITOS_ICR, isr);
 587         out_be32(priv->reg + TALITOS_ICR_LO, isr_lo);
 588
 589         if (unlikely((isr & ~TALITOS_ISR_CHDONE) || isr_lo))
 590                 talitos_error((unsigned long)data, isr, isr_lo);
 591         else
 592                 if (likely(isr & TALITOS_ISR_CHDONE)) {
 593                         /* mask further done interrupts. */
 594                         clrbits32(priv->reg + TALITOS_IMR, TALITOS_IMR_DONE);
 595                         /* done_task will unmask done interrupts at exit */
 596                         tasklet_schedule(&priv->done_task);
 597                 }
 598
 599         return (isr || isr_lo) ? IRQ_HANDLED : IRQ_NONE;
 600 }
 601
 602 /*
 603  * hwrng
 604  */
 605 static int talitos_rng_data_present(struct hwrng *rng, int wait)
 606 {
 607         struct device *dev = (struct device *)rng->priv;
 608         struct talitos_private *priv = dev_get_drvdata(dev);
 609         u32 ofl;
 610         int i;
 611
 612         for (i = 0; i < 20; i++) {
 613                 ofl = in_be32(priv->reg + TALITOS_RNGUSR_LO) &
 614                       TALITOS_RNGUSR_LO_OFL;
 615                 if (ofl || !wait)
 616                         break;
 617                 udelay(10);
 618         }
 619
 620         return !!ofl;
 621 }
 622
 623 static int talitos_rng_data_read(struct hwrng *rng, u32 *data)
 624 {
 625         struct device *dev = (struct device *)rng->priv;
 626         struct talitos_private *priv = dev_get_drvdata(dev);
 627
 628         /* rng fifo requires 64-bit accesses */
 629         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO);
 630         *data = in_be32(priv->reg + TALITOS_RNGU_FIFO_LO);
 631
 632         return sizeof(u32);
 633 }
 634
 635 static int talitos_rng_init(struct hwrng *rng)
 636 {
 637         struct device *dev = (struct device *)rng->priv;
 638         struct talitos_private *priv = dev_get_drvdata(dev);
 639         unsigned int timeout = TALITOS_TIMEOUT;
 640
 641         setbits32(priv->reg + TALITOS_RNGURCR_LO, TALITOS_RNGURCR_LO_SR);
 642         while (!(in_be32(priv->reg + TALITOS_RNGUSR_LO) & TALITOS_RNGUSR_LO_RD)
 643                && --timeout)
 644                 cpu_relax();
 645         if (timeout == 0) {
 646                 dev_err(dev, "failed to reset rng hw\n");
 647                 return -ENODEV;
 648         }
 649
 650         /* start generating */
 651         setbits32(priv->reg + TALITOS_RNGUDSR_LO, 0);
 652
 653         return 0;
 654 }
 655
 656 static int talitos_register_rng(struct device *dev)
 657 {
 658         struct talitos_private *priv = dev_get_drvdata(dev);
 659
 660         priv->rng.name          = dev_driver_string(dev),
 661         priv->rng.init          = talitos_rng_init,
 662         priv->rng.data_present  = talitos_rng_data_present,
 663         priv->rng.data_read     = talitos_rng_data_read,
 664         priv->rng.priv          = (unsigned long)dev;
 665
 666         return hwrng_register(&priv->rng);
 667 }
 668
 669 static void talitos_unregister_rng(struct device *dev)
 670 {
 671         struct talitos_private *priv = dev_get_drvdata(dev);
 672
 673         hwrng_unregister(&priv->rng);
 674 }
 675
 676 /*
 677  * crypto alg
 678  */
 679 #define TALITOS_CRA_PRIORITY            3000
 680 #define TALITOS_MAX_KEY_SIZE            64
 681 #define TALITOS_MAX_IV_LENGTH           16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */
 682
 683 #define MD5_DIGEST_SIZE   16
 684
 685 struct talitos_ctx {
 686         struct device *dev;
 687         __be32 desc_hdr_template;
 688         u8 key[TALITOS_MAX_KEY_SIZE];
 689         u8 iv[TALITOS_MAX_IV_LENGTH];
 690         unsigned int keylen;
 691         unsigned int enckeylen;
 692         unsigned int authkeylen;
 693         unsigned int authsize;
 694 };
 695
 696 static int aead_setauthsize(struct crypto_aead *authenc,
 697                             unsigned int authsize)
 698 {
 699         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 700
 701         ctx->authsize = authsize;
 702
 703         return 0;
 704 }
 705
 706 static int aead_setkey(struct crypto_aead *authenc,
 707                        const u8 *key, unsigned int keylen)
 708 {
 709         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 710         struct rtattr *rta = (void *)key;
 711         struct crypto_authenc_key_param *param;
 712         unsigned int authkeylen;
 713         unsigned int enckeylen;
 714
 715         if (!RTA_OK(rta, keylen))
 716                 goto badkey;
 717
 718         if (rta->rta_type != CRYPTO_AUTHENC_KEYA_PARAM)
 719                 goto badkey;
 720
 721         if (RTA_PAYLOAD(rta) < sizeof(*param))
 722                 goto badkey;
 723
 724         param = RTA_DATA(rta);
 725         enckeylen = be32_to_cpu(param->enckeylen);
 726
 727         key += RTA_ALIGN(rta->rta_len);
 728         keylen -= RTA_ALIGN(rta->rta_len);
 729
 730         if (keylen < enckeylen)
 731                 goto badkey;
 732
 733         authkeylen = keylen - enckeylen;
 734
 735         if (keylen > TALITOS_MAX_KEY_SIZE)
 736                 goto badkey;
 737
 738         memcpy(&ctx->key, key, keylen);
 739
 740         ctx->keylen = keylen;
 741         ctx->enckeylen = enckeylen;
 742         ctx->authkeylen = authkeylen;
 743
 744         return 0;
 745
 746 badkey:
 747         crypto_aead_set_flags(authenc, CRYPTO_TFM_RES_BAD_KEY_LEN);
 748         return -EINVAL;
 749 }
 750
 751 /*
 752  * talitos_edesc - s/w-extended descriptor
 753  * @src_nents: number of segments in input scatterlist
 754  * @dst_nents: number of segments in output scatterlist
 755  * @dma_len: length of dma mapped link_tbl space
 756  * @dma_link_tbl: bus physical address of link_tbl
 757  * @desc: h/w descriptor
 758  * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1)
 759  *
 760  * if decrypting (with authcheck), or either one of src_nents or dst_nents
 761  * is greater than 1, an integrity check value is concatenated to the end
 762  * of link_tbl data
 763  */
 764 struct talitos_edesc {
 765         int src_nents;
 766         int dst_nents;
 767         int src_is_chained;
 768         int dst_is_chained;
 769         int dma_len;
 770         dma_addr_t dma_link_tbl;
 771         struct talitos_desc desc;
 772         struct talitos_ptr link_tbl[0];
 773 };
 774
 775 static int talitos_map_sg(struct device *dev, struct scatterlist *sg,
 776                           unsigned int nents, enum dma_data_direction dir,
 777                           int chained)
 778 {
 779         if (unlikely(chained))
 780                 while (sg) {
 781                         dma_map_sg(dev, sg, 1, dir);
 782                         sg = scatterwalk_sg_next(sg);
 783                 }
 784         else
 785                 dma_map_sg(dev, sg, nents, dir);
 786         return nents;
 787 }
 788
 789 static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg,
 790                                    enum dma_data_direction dir)
 791 {
 792         while (sg) {
 793                 dma_unmap_sg(dev, sg, 1, dir);
 794                 sg = scatterwalk_sg_next(sg);
 795         }
 796 }
 797
 798 static void talitos_sg_unmap(struct device *dev,
 799                              struct talitos_edesc *edesc,
 800                              struct scatterlist *src,
 801                              struct scatterlist *dst)
 802 {
 803         unsigned int src_nents = edesc->src_nents ? : 1;
 804         unsigned int dst_nents = edesc->dst_nents ? : 1;
 805
 806         if (src != dst) {
 807                 if (edesc->src_is_chained)
 808                         talitos_unmap_sg_chain(dev, src, DMA_TO_DEVICE);
 809                 else
 810                         dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE);
 811
 812                 if (edesc->dst_is_chained)
 813                         talitos_unmap_sg_chain(dev, dst, DMA_FROM_DEVICE);
 814                 else
 815                         dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE);
 816         } else
 817                 if (edesc->src_is_chained)
 818                         talitos_unmap_sg_chain(dev, src, DMA_BIDIRECTIONAL);
 819                 else
 820                         dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL);
 821 }
 822
 823 static void ipsec_esp_unmap(struct device *dev,
 824                             struct talitos_edesc *edesc,
 825                             struct aead_request *areq)
 826 {
 827         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[6], DMA_FROM_DEVICE);
 828         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[3], DMA_TO_DEVICE);
 829         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
 830         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[0], DMA_TO_DEVICE);
 831
 832         dma_unmap_sg(dev, areq->assoc, 1, DMA_TO_DEVICE);
 833
 834         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
 835
 836         if (edesc->dma_len)
 837                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
 838                                  DMA_BIDIRECTIONAL);
 839 }
 840
 841 /*
 842  * ipsec_esp descriptor callbacks
 843  */
 844 static void ipsec_esp_encrypt_done(struct device *dev,
 845                                    struct talitos_desc *desc, void *context,
 846                                    int err)
 847 {
 848         struct aead_request *areq = context;
 849         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
 850         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 851         struct talitos_edesc *edesc;
 852         struct scatterlist *sg;
 853         void *icvdata;
 854
 855         edesc = container_of(desc, struct talitos_edesc, desc);
 856
 857         ipsec_esp_unmap(dev, edesc, areq);
 858
 859         /* copy the generated ICV to dst */
 860         if (edesc->dma_len) {
 861                 icvdata = &edesc->link_tbl[edesc->src_nents +
 862                                            edesc->dst_nents + 2];
 863                 sg = sg_last(areq->dst, edesc->dst_nents);
 864                 memcpy((char *)sg_virt(sg) + sg->length - ctx->authsize,
 865                        icvdata, ctx->authsize);
 866         }
 867
 868         kfree(edesc);
 869
 870         aead_request_complete(areq, err);
 871 }
 872
 873 static void ipsec_esp_decrypt_swauth_done(struct device *dev,
 874                                           struct talitos_desc *desc,
 875                                           void *context, int err)
 876 {
 877         struct aead_request *req = context;
 878         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
 879         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
 880         struct talitos_edesc *edesc;
 881         struct scatterlist *sg;
 882         void *icvdata;
 883
 884         edesc = container_of(desc, struct talitos_edesc, desc);
 885
 886         ipsec_esp_unmap(dev, edesc, req);
 887
 888         if (!err) {
 889                 /* auth check */
 890                 if (edesc->dma_len)
 891                         icvdata = &edesc->link_tbl[edesc->src_nents +
 892                                                    edesc->dst_nents + 2];
 893                 else
 894                         icvdata = &edesc->link_tbl[0];
 895
 896                 sg = sg_last(req->dst, edesc->dst_nents ? : 1);
 897                 err = memcmp(icvdata, (char *)sg_virt(sg) + sg->length -
 898                              ctx->authsize, ctx->authsize) ? -EBADMSG : 0;
 899         }
 900
 901         kfree(edesc);
 902
 903         aead_request_complete(req, err);
 904 }
 905
 906 static void ipsec_esp_decrypt_hwauth_done(struct device *dev,
 907                                           struct talitos_desc *desc,
 908                                           void *context, int err)
 909 {
 910         struct aead_request *req = context;
 911         struct talitos_edesc *edesc;
 912
 913         edesc = container_of(desc, struct talitos_edesc, desc);
 914
 915         ipsec_esp_unmap(dev, edesc, req);
 916
 917         /* check ICV auth status */
 918         if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) !=
 919                      DESC_HDR_LO_ICCR1_PASS))
 920                 err = -EBADMSG;
 921
 922         kfree(edesc);
 923
 924         aead_request_complete(req, err);
 925 }
 926
 927 /*
 928  * convert scatterlist to SEC h/w link table format
 929  * stop at cryptlen bytes
 930  */
 931 static int sg_to_link_tbl(struct scatterlist *sg, int sg_count,
 932                            int cryptlen, struct talitos_ptr *link_tbl_ptr)
 933 {
 934         int n_sg = sg_count;
 935
 936         while (n_sg--) {
 937                 link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg));
 938                 link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg));
 939                 link_tbl_ptr->j_extent = 0;
 940                 link_tbl_ptr++;
 941                 cryptlen -= sg_dma_len(sg);
 942                 sg = scatterwalk_sg_next(sg);
 943         }
 944
 945         /* adjust (decrease) last one (or two) entry's len to cryptlen */
 946         link_tbl_ptr--;
 947         while (be16_to_cpu(link_tbl_ptr->len) <= (-cryptlen)) {
 948                 /* Empty this entry, and move to previous one */
 949                 cryptlen += be16_to_cpu(link_tbl_ptr->len);
 950                 link_tbl_ptr->len = 0;
 951                 sg_count--;
 952                 link_tbl_ptr--;
 953         }
 954         link_tbl_ptr->len = cpu_to_be16(be16_to_cpu(link_tbl_ptr->len)
 955                                         + cryptlen);
 956
 957         /* tag end of link table */
 958         link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
 959
 960         return sg_count;
 961 }
 962
 963 /*
 964  * fill in and submit ipsec_esp descriptor
 965  */
 966 static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq,
 967                      u8 *giv, u64 seq,
 968                      void (*callback) (struct device *dev,
 969                                        struct talitos_desc *desc,
 970                                        void *context, int error))
 971 {
 972         struct crypto_aead *aead = crypto_aead_reqtfm(areq);
 973         struct talitos_ctx *ctx = crypto_aead_ctx(aead);
 974         struct device *dev = ctx->dev;
 975         struct talitos_desc *desc = &edesc->desc;
 976         unsigned int cryptlen = areq->cryptlen;
 977         unsigned int authsize = ctx->authsize;
 978         unsigned int ivsize = crypto_aead_ivsize(aead);
 979         int sg_count, ret;
 980         int sg_link_tbl_len;
 981
 982         /* hmac key */
 983         map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key,
 984                                0, DMA_TO_DEVICE);
 985         /* hmac data */
 986         map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize,
 987                                sg_virt(areq->assoc), 0, DMA_TO_DEVICE);
 988         /* cipher iv */
 989         map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0,
 990                                DMA_TO_DEVICE);
 991
 992         /* cipher key */
 993         map_single_talitos_ptr(dev, &desc->ptr[3], ctx->enckeylen,
 994                                (char *)&ctx->key + ctx->authkeylen, 0,
 995                                DMA_TO_DEVICE);
 996
 997         /*
 998          * cipher in
 999          * map and adjust cipher len to aead request cryptlen.
1000          * extent is bytes of HMAC postpended to ciphertext,
1001          * typically 12 for ipsec
1002          */
1003         desc->ptr[4].len = cpu_to_be16(cryptlen);
1004         desc->ptr[4].j_extent = authsize;
1005
1006         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1007                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1008                                                            : DMA_TO_DEVICE,
1009                                   edesc->src_is_chained);
1010
1011         if (sg_count == 1) {
1012                 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src));
1013         } else {
1014                 sg_link_tbl_len = cryptlen;
1015
1016                 if (edesc->desc.hdr & DESC_HDR_MODE1_MDEU_CICV)
1017                         sg_link_tbl_len = cryptlen + authsize;
1018
1019                 sg_count = sg_to_link_tbl(areq->src, sg_count, sg_link_tbl_len,
1020                                           &edesc->link_tbl[0]);
1021                 if (sg_count > 1) {
1022                         desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1023                         desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl);
1024                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1025                                                    edesc->dma_len,
1026                                                    DMA_BIDIRECTIONAL);
1027                 } else {
1028                         /* Only one segment now, so no link tbl needed */
1029                         desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->
1030                                                                       src));
1031                 }
1032         }
1033
1034         /* cipher out */
1035         desc->ptr[5].len = cpu_to_be16(cryptlen);
1036         desc->ptr[5].j_extent = authsize;
1037
1038         if (areq->src != areq->dst)
1039                 sg_count = talitos_map_sg(dev, areq->dst,
1040                                           edesc->dst_nents ? : 1,
1041                                           DMA_FROM_DEVICE,
1042                                           edesc->dst_is_chained);
1043
1044         if (sg_count == 1) {
1045                 desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst));
1046         } else {
1047                 struct talitos_ptr *link_tbl_ptr =
1048                         &edesc->link_tbl[edesc->src_nents + 1];
1049
1050                 desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *)
1051                                                edesc->dma_link_tbl +
1052                                                edesc->src_nents + 1);
1053                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1054                                           link_tbl_ptr);
1055
1056                 /* Add an entry to the link table for ICV data */
1057                 link_tbl_ptr += sg_count - 1;
1058                 link_tbl_ptr->j_extent = 0;
1059                 sg_count++;
1060                 link_tbl_ptr++;
1061                 link_tbl_ptr->j_extent = DESC_PTR_LNKTBL_RETURN;
1062                 link_tbl_ptr->len = cpu_to_be16(authsize);
1063
1064                 /* icv data follows link tables */
1065                 link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *)
1066                                                 edesc->dma_link_tbl +
1067                                                 edesc->src_nents +
1068                                                 edesc->dst_nents + 2);
1069
1070                 desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP;
1071                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1072                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1073         }
1074
1075         /* iv out */
1076         map_single_talitos_ptr(dev, &desc->ptr[6], ivsize, ctx->iv, 0,
1077                                DMA_FROM_DEVICE);
1078
1079         ret = talitos_submit(dev, desc, callback, areq);
1080         if (ret != -EINPROGRESS) {
1081                 ipsec_esp_unmap(dev, edesc, areq);
1082                 kfree(edesc);
1083         }
1084         return ret;
1085 }
1086
1087 /*
1088  * derive number of elements in scatterlist
1089  */
1090 static int sg_count(struct scatterlist *sg_list, int nbytes, int *chained)
1091 {
1092         struct scatterlist *sg = sg_list;
1093         int sg_nents = 0;
1094
1095         *chained = 0;
1096         while (nbytes > 0) {
1097                 sg_nents++;
1098                 nbytes -= sg->length;
1099                 if (!sg_is_last(sg) && (sg + 1)->length == 0)
1100                         *chained = 1;
1101                 sg = scatterwalk_sg_next(sg);
1102         }
1103
1104         return sg_nents;
1105 }
1106
1107 /*
1108  * allocate and map the extended descriptor
1109  */
1110 static struct talitos_edesc *talitos_edesc_alloc(struct device *dev,
1111                                                  struct scatterlist *src,
1112                                                  struct scatterlist *dst,
1113                                                  unsigned int cryptlen,
1114                                                  unsigned int authsize,
1115                                                  int icv_stashing,
1116                                                  u32 cryptoflags)
1117 {
1118         struct talitos_edesc *edesc;
1119         int src_nents, dst_nents, alloc_len, dma_len;
1120         int src_chained, dst_chained = 0;
1121         gfp_t flags = cryptoflags & CRYPTO_TFM_REQ_MAY_SLEEP ? GFP_KERNEL :
1122                       GFP_ATOMIC;
1123
1124         if (cryptlen + authsize > TALITOS_MAX_DATA_LEN) {
1125                 dev_err(dev, "length exceeds h/w max limit\n");
1126                 return ERR_PTR(-EINVAL);
1127         }
1128
1129         src_nents = sg_count(src, cryptlen + authsize, &src_chained);
1130         src_nents = (src_nents == 1) ? 0 : src_nents;
1131
1132         if (dst == src) {
1133                 dst_nents = src_nents;
1134         } else {
1135                 dst_nents = sg_count(dst, cryptlen + authsize, &dst_chained);
1136                 dst_nents = (dst_nents == 1) ? 0 : dst_nents;
1137         }
1138
1139         /*
1140          * allocate space for base edesc plus the link tables,
1141          * allowing for two separate entries for ICV and generated ICV (+ 2),
1142          * and the ICV data itself
1143          */
1144         alloc_len = sizeof(struct talitos_edesc);
1145         if (src_nents || dst_nents) {
1146                 dma_len = (src_nents + dst_nents + 2) *
1147                                  sizeof(struct talitos_ptr) + authsize;
1148                 alloc_len += dma_len;
1149         } else {
1150                 dma_len = 0;
1151                 alloc_len += icv_stashing ? authsize : 0;
1152         }
1153
1154         edesc = kmalloc(alloc_len, GFP_DMA | flags);
1155         if (!edesc) {
1156                 dev_err(dev, "could not allocate edescriptor\n");
1157                 return ERR_PTR(-ENOMEM);
1158         }
1159
1160         edesc->src_nents = src_nents;
1161         edesc->dst_nents = dst_nents;
1162         edesc->src_is_chained = src_chained;
1163         edesc->dst_is_chained = dst_chained;
1164         edesc->dma_len = dma_len;
1165         edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0],
1166                                              edesc->dma_len, DMA_BIDIRECTIONAL);
1167
1168         return edesc;
1169 }
1170
1171 static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq,
1172                                               int icv_stashing)
1173 {
1174         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1175         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1176
1177         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst,
1178                                    areq->cryptlen, ctx->authsize, icv_stashing,
1179                                    areq->base.flags);
1180 }
1181
1182 static int aead_encrypt(struct aead_request *req)
1183 {
1184         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1185         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1186         struct talitos_edesc *edesc;
1187
1188         /* allocate extended descriptor */
1189         edesc = aead_edesc_alloc(req, 0);
1190         if (IS_ERR(edesc))
1191                 return PTR_ERR(edesc);
1192
1193         /* set encrypt */
1194         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1195
1196         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_encrypt_done);
1197 }
1198
1199 static int aead_decrypt(struct aead_request *req)
1200 {
1201         struct crypto_aead *authenc = crypto_aead_reqtfm(req);
1202         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1203         unsigned int authsize = ctx->authsize;
1204         struct talitos_private *priv = dev_get_drvdata(ctx->dev);
1205         struct talitos_edesc *edesc;
1206         struct scatterlist *sg;
1207         void *icvdata;
1208
1209         req->cryptlen -= authsize;
1210
1211         /* allocate extended descriptor */
1212         edesc = aead_edesc_alloc(req, 1);
1213         if (IS_ERR(edesc))
1214                 return PTR_ERR(edesc);
1215
1216         if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) &&
1217             ((!edesc->src_nents && !edesc->dst_nents) ||
1218              priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) {
1219
1220                 /* decrypt and check the ICV */
1221                 edesc->desc.hdr = ctx->desc_hdr_template |
1222                                   DESC_HDR_DIR_INBOUND |
1223                                   DESC_HDR_MODE1_MDEU_CICV;
1224
1225                 /* reset integrity check result bits */
1226                 edesc->desc.hdr_lo = 0;
1227
1228                 return ipsec_esp(edesc, req, NULL, 0,
1229                                  ipsec_esp_decrypt_hwauth_done);
1230
1231         }
1232
1233         /* Have to check the ICV with software */
1234         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1235
1236         /* stash incoming ICV for later cmp with ICV generated by the h/w */
1237         if (edesc->dma_len)
1238                 icvdata = &edesc->link_tbl[edesc->src_nents +
1239                                            edesc->dst_nents + 2];
1240         else
1241                 icvdata = &edesc->link_tbl[0];
1242
1243         sg = sg_last(req->src, edesc->src_nents ? : 1);
1244
1245         memcpy(icvdata, (char *)sg_virt(sg) + sg->length - ctx->authsize,
1246                ctx->authsize);
1247
1248         return ipsec_esp(edesc, req, NULL, 0, ipsec_esp_decrypt_swauth_done);
1249 }
1250
1251 static int aead_givencrypt(struct aead_givcrypt_request *req)
1252 {
1253         struct aead_request *areq = &req->areq;
1254         struct crypto_aead *authenc = crypto_aead_reqtfm(areq);
1255         struct talitos_ctx *ctx = crypto_aead_ctx(authenc);
1256         struct talitos_edesc *edesc;
1257
1258         /* allocate extended descriptor */
1259         edesc = aead_edesc_alloc(areq, 0);
1260         if (IS_ERR(edesc))
1261                 return PTR_ERR(edesc);
1262
1263         /* set encrypt */
1264         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1265
1266         memcpy(req->giv, ctx->iv, crypto_aead_ivsize(authenc));
1267         /* avoid consecutive packets going out with same IV */
1268         *(__be64 *)req->giv ^= cpu_to_be64(req->seq);
1269
1270         return ipsec_esp(edesc, areq, req->giv, req->seq,
1271                          ipsec_esp_encrypt_done);
1272 }
1273
1274 static int ablkcipher_setkey(struct crypto_ablkcipher *cipher,
1275                              const u8 *key, unsigned int keylen)
1276 {
1277         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1278         struct ablkcipher_alg *alg = crypto_ablkcipher_alg(cipher);
1279
1280         if (keylen > TALITOS_MAX_KEY_SIZE)
1281                 goto badkey;
1282
1283         if (keylen < alg->min_keysize || keylen > alg->max_keysize)
1284                 goto badkey;
1285
1286         memcpy(&ctx->key, key, keylen);
1287         ctx->keylen = keylen;
1288
1289         return 0;
1290
1291 badkey:
1292         crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN);
1293         return -EINVAL;
1294 }
1295
1296 static void common_nonsnoop_unmap(struct device *dev,
1297                                   struct talitos_edesc *edesc,
1298                                   struct ablkcipher_request *areq)
1299 {
1300         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE);
1301         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[2], DMA_TO_DEVICE);
1302         unmap_single_talitos_ptr(dev, &edesc->desc.ptr[1], DMA_TO_DEVICE);
1303
1304         talitos_sg_unmap(dev, edesc, areq->src, areq->dst);
1305
1306         if (edesc->dma_len)
1307                 dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len,
1308                                  DMA_BIDIRECTIONAL);
1309 }
1310
1311 static void ablkcipher_done(struct device *dev,
1312                             struct talitos_desc *desc, void *context,
1313                             int err)
1314 {
1315         struct ablkcipher_request *areq = context;
1316         struct talitos_edesc *edesc;
1317
1318         edesc = container_of(desc, struct talitos_edesc, desc);
1319
1320         common_nonsnoop_unmap(dev, edesc, areq);
1321
1322         kfree(edesc);
1323
1324         areq->base.complete(&areq->base, err);
1325 }
1326
1327 static int common_nonsnoop(struct talitos_edesc *edesc,
1328                            struct ablkcipher_request *areq,
1329                            u8 *giv,
1330                            void (*callback) (struct device *dev,
1331                                              struct talitos_desc *desc,
1332                                              void *context, int error))
1333 {
1334         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1335         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1336         struct device *dev = ctx->dev;
1337         struct talitos_desc *desc = &edesc->desc;
1338         unsigned int cryptlen = areq->nbytes;
1339         unsigned int ivsize;
1340         int sg_count, ret;
1341
1342         /* first DWORD empty */
1343         desc->ptr[0].len = 0;
1344         desc->ptr[0].ptr = 0;
1345         desc->ptr[0].j_extent = 0;
1346
1347         /* cipher iv */
1348         ivsize = crypto_ablkcipher_ivsize(cipher);
1349         map_single_talitos_ptr(dev, &desc->ptr[1], ivsize, giv ?: areq->info, 0,
1350                                DMA_TO_DEVICE);
1351
1352         /* cipher key */
1353         map_single_talitos_ptr(dev, &desc->ptr[2], ctx->keylen,
1354                                (char *)&ctx->key, 0, DMA_TO_DEVICE);
1355
1356         /*
1357          * cipher in
1358          */
1359         desc->ptr[3].len = cpu_to_be16(cryptlen);
1360         desc->ptr[3].j_extent = 0;
1361
1362         sg_count = talitos_map_sg(dev, areq->src, edesc->src_nents ? : 1,
1363                                   (areq->src == areq->dst) ? DMA_BIDIRECTIONAL
1364                                                            : DMA_TO_DEVICE,
1365                                   edesc->src_is_chained);
1366
1367         if (sg_count == 1) {
1368                 desc->ptr[3].ptr = cpu_to_be32(sg_dma_address(areq->src));
1369         } else {
1370                 sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen,
1371                                           &edesc->link_tbl[0]);
1372                 if (sg_count > 1) {
1373                         desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP;
1374                         desc->ptr[3].ptr = cpu_to_be32(edesc->dma_link_tbl);
1375                         dma_sync_single_for_device(dev, edesc->dma_link_tbl,
1376                                                    edesc->dma_len,
1377                                                    DMA_BIDIRECTIONAL);
1378                 } else {
1379                         /* Only one segment now, so no link tbl needed */
1380                         desc->ptr[3].ptr = cpu_to_be32(sg_dma_address(areq->
1381                                                                       src));
1382                 }
1383         }
1384
1385         /* cipher out */
1386         desc->ptr[4].len = cpu_to_be16(cryptlen);
1387         desc->ptr[4].j_extent = 0;
1388
1389         if (areq->src != areq->dst)
1390                 sg_count = talitos_map_sg(dev, areq->dst,
1391                                           edesc->dst_nents ? : 1,
1392                                           DMA_FROM_DEVICE,
1393                                           edesc->dst_is_chained);
1394
1395         if (sg_count == 1) {
1396                 desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->dst));
1397         } else {
1398                 struct talitos_ptr *link_tbl_ptr =
1399                         &edesc->link_tbl[edesc->src_nents + 1];
1400
1401                 desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP;
1402                 desc->ptr[4].ptr = cpu_to_be32((struct talitos_ptr *)
1403                                                edesc->dma_link_tbl +
1404                                                edesc->src_nents + 1);
1405                 sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen,
1406                                           link_tbl_ptr);
1407                 dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl,
1408                                            edesc->dma_len, DMA_BIDIRECTIONAL);
1409         }
1410
1411         /* iv out */
1412         map_single_talitos_ptr(dev, &desc->ptr[5], ivsize, ctx->iv, 0,
1413                                DMA_FROM_DEVICE);
1414
1415         /* last DWORD empty */
1416         desc->ptr[6].len = 0;
1417         desc->ptr[6].ptr = 0;
1418         desc->ptr[6].j_extent = 0;
1419
1420         ret = talitos_submit(dev, desc, callback, areq);
1421         if (ret != -EINPROGRESS) {
1422                 common_nonsnoop_unmap(dev, edesc, areq);
1423                 kfree(edesc);
1424         }
1425         return ret;
1426 }
1427
1428 static struct talitos_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request *
1429                                                     areq)
1430 {
1431         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1432         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1433
1434         return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, areq->nbytes,
1435                                    0, 0, areq->base.flags);
1436 }
1437
1438 static int ablkcipher_encrypt(struct ablkcipher_request *areq)
1439 {
1440         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1441         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1442         struct talitos_edesc *edesc;
1443
1444         /* allocate extended descriptor */
1445         edesc = ablkcipher_edesc_alloc(areq);
1446         if (IS_ERR(edesc))
1447                 return PTR_ERR(edesc);
1448
1449         /* set encrypt */
1450         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT;
1451
1452         return common_nonsnoop(edesc, areq, NULL, ablkcipher_done);
1453 }
1454
1455 static int ablkcipher_decrypt(struct ablkcipher_request *areq)
1456 {
1457         struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq);
1458         struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher);
1459         struct talitos_edesc *edesc;
1460
1461         /* allocate extended descriptor */
1462         edesc = ablkcipher_edesc_alloc(areq);
1463         if (IS_ERR(edesc))
1464                 return PTR_ERR(edesc);
1465
1466         edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND;
1467
1468         return common_nonsnoop(edesc, areq, NULL, ablkcipher_done);
1469 }
1470
1471 struct talitos_alg_template {
1472         struct crypto_alg alg;
1473         __be32 desc_hdr_template;
1474 };
1475
1476 static struct talitos_alg_template driver_algs[] = {
1477         /* AEAD algorithms.  These use a single-pass ipsec_esp descriptor */
1478         {
1479                 .alg = {
1480                         .cra_name = "authenc(hmac(sha1),cbc(aes))",
1481                         .cra_driver_name = "authenc-hmac-sha1-cbc-aes-talitos",
1482                         .cra_blocksize = AES_BLOCK_SIZE,
1483                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1484                         .cra_type = &crypto_aead_type,
1485                         .cra_aead = {
1486                                 .setkey = aead_setkey,
1487                                 .setauthsize = aead_setauthsize,
1488                                 .encrypt = aead_encrypt,
1489                                 .decrypt = aead_decrypt,
1490                                 .givencrypt = aead_givencrypt,
1491                                 .geniv = "<built-in>",
1492                                 .ivsize = AES_BLOCK_SIZE,
1493                                 .maxauthsize = SHA1_DIGEST_SIZE,
1494                         }
1495                 },
1496                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1497                                      DESC_HDR_SEL0_AESU |
1498                                      DESC_HDR_MODE0_AESU_CBC |
1499                                      DESC_HDR_SEL1_MDEUA |
1500                                      DESC_HDR_MODE1_MDEU_INIT |
1501                                      DESC_HDR_MODE1_MDEU_PAD |
1502                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1503         },
1504         {
1505                 .alg = {
1506                         .cra_name = "authenc(hmac(sha1),cbc(des3_ede))",
1507                         .cra_driver_name = "authenc-hmac-sha1-cbc-3des-talitos",
1508                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1509                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1510                         .cra_type = &crypto_aead_type,
1511                         .cra_aead = {
1512                                 .setkey = aead_setkey,
1513                                 .setauthsize = aead_setauthsize,
1514                                 .encrypt = aead_encrypt,
1515                                 .decrypt = aead_decrypt,
1516                                 .givencrypt = aead_givencrypt,
1517                                 .geniv = "<built-in>",
1518                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1519                                 .maxauthsize = SHA1_DIGEST_SIZE,
1520                         }
1521                 },
1522                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1523                                      DESC_HDR_SEL0_DEU |
1524                                      DESC_HDR_MODE0_DEU_CBC |
1525                                      DESC_HDR_MODE0_DEU_3DES |
1526                                      DESC_HDR_SEL1_MDEUA |
1527                                      DESC_HDR_MODE1_MDEU_INIT |
1528                                      DESC_HDR_MODE1_MDEU_PAD |
1529                                      DESC_HDR_MODE1_MDEU_SHA1_HMAC,
1530         },
1531         {
1532                 .alg = {
1533                         .cra_name = "authenc(hmac(sha256),cbc(aes))",
1534                         .cra_driver_name = "authenc-hmac-sha256-cbc-aes-talitos",
1535                         .cra_blocksize = AES_BLOCK_SIZE,
1536                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1537                         .cra_type = &crypto_aead_type,
1538                         .cra_aead = {
1539                                 .setkey = aead_setkey,
1540                                 .setauthsize = aead_setauthsize,
1541                                 .encrypt = aead_encrypt,
1542                                 .decrypt = aead_decrypt,
1543                                 .givencrypt = aead_givencrypt,
1544                                 .geniv = "<built-in>",
1545                                 .ivsize = AES_BLOCK_SIZE,
1546                                 .maxauthsize = SHA256_DIGEST_SIZE,
1547                         }
1548                 },
1549                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1550                                      DESC_HDR_SEL0_AESU |
1551                                      DESC_HDR_MODE0_AESU_CBC |
1552                                      DESC_HDR_SEL1_MDEUA |
1553                                      DESC_HDR_MODE1_MDEU_INIT |
1554                                      DESC_HDR_MODE1_MDEU_PAD |
1555                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1556         },
1557         {
1558                 .alg = {
1559                         .cra_name = "authenc(hmac(sha256),cbc(des3_ede))",
1560                         .cra_driver_name = "authenc-hmac-sha256-cbc-3des-talitos",
1561                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1562                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1563                         .cra_type = &crypto_aead_type,
1564                         .cra_aead = {
1565                                 .setkey = aead_setkey,
1566                                 .setauthsize = aead_setauthsize,
1567                                 .encrypt = aead_encrypt,
1568                                 .decrypt = aead_decrypt,
1569                                 .givencrypt = aead_givencrypt,
1570                                 .geniv = "<built-in>",
1571                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1572                                 .maxauthsize = SHA256_DIGEST_SIZE,
1573                         }
1574                 },
1575                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1576                                      DESC_HDR_SEL0_DEU |
1577                                      DESC_HDR_MODE0_DEU_CBC |
1578                                      DESC_HDR_MODE0_DEU_3DES |
1579                                      DESC_HDR_SEL1_MDEUA |
1580                                      DESC_HDR_MODE1_MDEU_INIT |
1581                                      DESC_HDR_MODE1_MDEU_PAD |
1582                                      DESC_HDR_MODE1_MDEU_SHA256_HMAC,
1583         },
1584         {
1585                 .alg = {
1586                         .cra_name = "authenc(hmac(md5),cbc(aes))",
1587                         .cra_driver_name = "authenc-hmac-md5-cbc-aes-talitos",
1588                         .cra_blocksize = AES_BLOCK_SIZE,
1589                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1590                         .cra_type = &crypto_aead_type,
1591                         .cra_aead = {
1592                                 .setkey = aead_setkey,
1593                                 .setauthsize = aead_setauthsize,
1594                                 .encrypt = aead_encrypt,
1595                                 .decrypt = aead_decrypt,
1596                                 .givencrypt = aead_givencrypt,
1597                                 .geniv = "<built-in>",
1598                                 .ivsize = AES_BLOCK_SIZE,
1599                                 .maxauthsize = MD5_DIGEST_SIZE,
1600                         }
1601                 },
1602                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1603                                      DESC_HDR_SEL0_AESU |
1604                                      DESC_HDR_MODE0_AESU_CBC |
1605                                      DESC_HDR_SEL1_MDEUA |
1606                                      DESC_HDR_MODE1_MDEU_INIT |
1607                                      DESC_HDR_MODE1_MDEU_PAD |
1608                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1609         },
1610         {
1611                 .alg = {
1612                         .cra_name = "authenc(hmac(md5),cbc(des3_ede))",
1613                         .cra_driver_name = "authenc-hmac-md5-cbc-3des-talitos",
1614                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1615                         .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC,
1616                         .cra_type = &crypto_aead_type,
1617                         .cra_aead = {
1618                                 .setkey = aead_setkey,
1619                                 .setauthsize = aead_setauthsize,
1620                                 .encrypt = aead_encrypt,
1621                                 .decrypt = aead_decrypt,
1622                                 .givencrypt = aead_givencrypt,
1623                                 .geniv = "<built-in>",
1624                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1625                                 .maxauthsize = MD5_DIGEST_SIZE,
1626                         }
1627                 },
1628                 .desc_hdr_template = DESC_HDR_TYPE_IPSEC_ESP |
1629                                      DESC_HDR_SEL0_DEU |
1630                                      DESC_HDR_MODE0_DEU_CBC |
1631                                      DESC_HDR_MODE0_DEU_3DES |
1632                                      DESC_HDR_SEL1_MDEUA |
1633                                      DESC_HDR_MODE1_MDEU_INIT |
1634                                      DESC_HDR_MODE1_MDEU_PAD |
1635                                      DESC_HDR_MODE1_MDEU_MD5_HMAC,
1636         },
1637         /* ABLKCIPHER algorithms. */
1638         {
1639                 .alg = {
1640                         .cra_name = "cbc(aes)",
1641                         .cra_driver_name = "cbc-aes-talitos",
1642                         .cra_blocksize = AES_BLOCK_SIZE,
1643                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1644                                      CRYPTO_ALG_ASYNC,
1645                         .cra_type = &crypto_ablkcipher_type,
1646                         .cra_ablkcipher = {
1647                                 .setkey = ablkcipher_setkey,
1648                                 .encrypt = ablkcipher_encrypt,
1649                                 .decrypt = ablkcipher_decrypt,
1650                                 .geniv = "eseqiv",
1651                                 .min_keysize = AES_MIN_KEY_SIZE,
1652                                 .max_keysize = AES_MAX_KEY_SIZE,
1653                                 .ivsize = AES_BLOCK_SIZE,
1654                         }
1655                 },
1656                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
1657                                      DESC_HDR_SEL0_AESU |
1658                                      DESC_HDR_MODE0_AESU_CBC,
1659         },
1660         {
1661                 .alg = {
1662                         .cra_name = "cbc(des3_ede)",
1663                         .cra_driver_name = "cbc-3des-talitos",
1664                         .cra_blocksize = DES3_EDE_BLOCK_SIZE,
1665                         .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER |
1666                                      CRYPTO_ALG_ASYNC,
1667                         .cra_type = &crypto_ablkcipher_type,
1668                         .cra_ablkcipher = {
1669                                 .setkey = ablkcipher_setkey,
1670                                 .encrypt = ablkcipher_encrypt,
1671                                 .decrypt = ablkcipher_decrypt,
1672                                 .geniv = "eseqiv",
1673                                 .min_keysize = DES3_EDE_KEY_SIZE,
1674                                 .max_keysize = DES3_EDE_KEY_SIZE,
1675                                 .ivsize = DES3_EDE_BLOCK_SIZE,
1676                         }
1677                 },
1678                 .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU |
1679                                      DESC_HDR_SEL0_DEU |
1680                                      DESC_HDR_MODE0_DEU_CBC |
1681                                      DESC_HDR_MODE0_DEU_3DES,
1682         }
1683 };
1684
1685 struct talitos_crypto_alg {
1686         struct list_head entry;
1687         struct device *dev;
1688         __be32 desc_hdr_template;
1689         struct crypto_alg crypto_alg;
1690 };
1691
1692 static int talitos_cra_init(struct crypto_tfm *tfm)
1693 {
1694         struct crypto_alg *alg = tfm->__crt_alg;
1695         struct talitos_crypto_alg *talitos_alg;
1696         struct talitos_ctx *ctx = crypto_tfm_ctx(tfm);
1697
1698         talitos_alg =  container_of(alg, struct talitos_crypto_alg, crypto_alg);
1699
1700         /* update context with ptr to dev */
1701         ctx->dev = talitos_alg->dev;
1702
1703         /* copy descriptor header template value */
1704         ctx->desc_hdr_template = talitos_alg->desc_hdr_template;
1705
1706         /* random first IV */
1707         get_random_bytes(ctx->iv, TALITOS_MAX_IV_LENGTH);
1708
1709         return 0;
1710 }
1711
1712 /*
1713  * given the alg's descriptor header template, determine whether descriptor
1714  * type and primary/secondary execution units required match the hw
1715  * capabilities description provided in the device tree node.
1716  */
1717 static int hw_supports(struct device *dev, __be32 desc_hdr_template)
1718 {
1719         struct talitos_private *priv = dev_get_drvdata(dev);
1720         int ret;
1721
1722         ret = (1 << DESC_TYPE(desc_hdr_template) & priv->desc_types) &&
1723               (1 << PRIMARY_EU(desc_hdr_template) & priv->exec_units);
1724
1725         if (SECONDARY_EU(desc_hdr_template))
1726                 ret = ret && (1 << SECONDARY_EU(desc_hdr_template)
1727                               & priv->exec_units);
1728
1729         return ret;
1730 }
1731
1732 static int talitos_remove(struct of_device *ofdev)
1733 {
1734         struct device *dev = &ofdev->dev;
1735         struct talitos_private *priv = dev_get_drvdata(dev);
1736         struct talitos_crypto_alg *t_alg, *n;
1737         int i;
1738
1739         list_for_each_entry_safe(t_alg, n, &priv->alg_list, entry) {
1740                 crypto_unregister_alg(&t_alg->crypto_alg);
1741                 list_del(&t_alg->entry);
1742                 kfree(t_alg);
1743         }
1744
1745         if (hw_supports(dev, DESC_HDR_SEL0_RNG))
1746                 talitos_unregister_rng(dev);
1747
1748         for (i = 0; i < priv->num_channels; i++)
1749                 if (priv->chan[i].fifo)
1750                         kfree(priv->chan[i].fifo);
1751
1752         kfree(priv->chan);
1753
1754         if (priv->irq != NO_IRQ) {
1755                 free_irq(priv->irq, dev);
1756                 irq_dispose_mapping(priv->irq);
1757         }
1758
1759         tasklet_kill(&priv->done_task);
1760
1761         iounmap(priv->reg);
1762
1763         dev_set_drvdata(dev, NULL);
1764
1765         kfree(priv);
1766
1767         return 0;
1768 }
1769
1770 static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev,
1771                                                     struct talitos_alg_template
1772                                                            *template)
1773 {
1774         struct talitos_crypto_alg *t_alg;
1775         struct crypto_alg *alg;
1776
1777         t_alg = kzalloc(sizeof(struct talitos_crypto_alg), GFP_KERNEL);
1778         if (!t_alg)
1779                 return ERR_PTR(-ENOMEM);
1780
1781         alg = &t_alg->crypto_alg;
1782         *alg = template->alg;
1783
1784         alg->cra_module = THIS_MODULE;
1785         alg->cra_init = talitos_cra_init;
1786         alg->cra_priority = TALITOS_CRA_PRIORITY;
1787         alg->cra_alignmask = 0;
1788         alg->cra_ctxsize = sizeof(struct talitos_ctx);
1789
1790         t_alg->desc_hdr_template = template->desc_hdr_template;
1791         t_alg->dev = dev;
1792
1793         return t_alg;
1794 }
1795
1796 static int talitos_probe(struct of_device *ofdev,
1797                          const struct of_device_id *match)
1798 {
1799         struct device *dev = &ofdev->dev;
1800         struct device_node *np = ofdev->node;
1801         struct talitos_private *priv;
1802         const unsigned int *prop;
1803         int i, err;
1804
1805         priv = kzalloc(sizeof(struct talitos_private), GFP_KERNEL);
1806         if (!priv)
1807                 return -ENOMEM;
1808
1809         dev_set_drvdata(dev, priv);
1810
1811         priv->ofdev = ofdev;
1812
1813         tasklet_init(&priv->done_task, talitos_done, (unsigned long)dev);
1814
1815         INIT_LIST_HEAD(&priv->alg_list);
1816
1817         priv->irq = irq_of_parse_and_map(np, 0);
1818
1819         if (priv->irq == NO_IRQ) {
1820                 dev_err(dev, "failed to map irq\n");
1821                 err = -EINVAL;
1822                 goto err_out;
1823         }
1824
1825         /* get the irq line */
1826         err = request_irq(priv->irq, talitos_interrupt, 0,
1827                           dev_driver_string(dev), dev);
1828         if (err) {
1829                 dev_err(dev, "failed to request irq %d\n", priv->irq);
1830                 irq_dispose_mapping(priv->irq);
1831                 priv->irq = NO_IRQ;
1832                 goto err_out;
1833         }
1834
1835         priv->reg = of_iomap(np, 0);
1836         if (!priv->reg) {
1837                 dev_err(dev, "failed to of_iomap\n");
1838                 err = -ENOMEM;
1839                 goto err_out;
1840         }
1841
1842         /* get SEC version capabilities from device tree */
1843         prop = of_get_property(np, "fsl,num-channels", NULL);
1844         if (prop)
1845                 priv->num_channels = *prop;
1846
1847         prop = of_get_property(np, "fsl,channel-fifo-len", NULL);
1848         if (prop)
1849                 priv->chfifo_len = *prop;
1850
1851         prop = of_get_property(np, "fsl,exec-units-mask", NULL);
1852         if (prop)
1853                 priv->exec_units = *prop;
1854
1855         prop = of_get_property(np, "fsl,descriptor-types-mask", NULL);
1856         if (prop)
1857                 priv->desc_types = *prop;
1858
1859         if (!is_power_of_2(priv->num_channels) || !priv->chfifo_len ||
1860             !priv->exec_units || !priv->desc_types) {
1861                 dev_err(dev, "invalid property data in device tree node\n");
1862                 err = -EINVAL;
1863                 goto err_out;
1864         }
1865
1866         if (of_device_is_compatible(np, "fsl,sec3.0"))
1867                 priv->features |= TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT;
1868
1869         if (of_device_is_compatible(np, "fsl,sec2.1"))
1870                 priv->features |= TALITOS_FTR_HW_AUTH_CHECK;
1871
1872         priv->chan = kzalloc(sizeof(struct talitos_channel) *
1873                              priv->num_channels, GFP_KERNEL);
1874         if (!priv->chan) {
1875                 dev_err(dev, "failed to allocate channel management space\n");
1876                 err = -ENOMEM;
1877                 goto err_out;
1878         }
1879
1880         for (i = 0; i < priv->num_channels; i++) {
1881                 spin_lock_init(&priv->chan[i].head_lock);
1882                 spin_lock_init(&priv->chan[i].tail_lock);
1883         }
1884
1885         priv->fifo_len = roundup_pow_of_two(priv->chfifo_len);
1886
1887         for (i = 0; i < priv->num_channels; i++) {
1888                 priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) *
1889                                              priv->fifo_len, GFP_KERNEL);
1890                 if (!priv->chan[i].fifo) {
1891                         dev_err(dev, "failed to allocate request fifo %d\n", i);
1892                         err = -ENOMEM;
1893                         goto err_out;
1894                 }
1895         }
1896
1897         for (i = 0; i < priv->num_channels; i++)
1898                 atomic_set(&priv->chan[i].submit_count,
1899                            -(priv->chfifo_len - 1));
1900
1901         /* reset and initialize the h/w */
1902         err = init_device(dev);
1903         if (err) {
1904                 dev_err(dev, "failed to initialize device\n");
1905                 goto err_out;
1906         }
1907
1908         /* register the RNG, if available */
1909         if (hw_supports(dev, DESC_HDR_SEL0_RNG)) {
1910                 err = talitos_register_rng(dev);
1911                 if (err) {
1912                         dev_err(dev, "failed to register hwrng: %d\n", err);
1913                         goto err_out;
1914                 } else
1915                         dev_info(dev, "hwrng\n");
1916         }
1917
1918         /* register crypto algorithms the device supports */
1919         for (i = 0; i < ARRAY_SIZE(driver_algs); i++) {
1920                 if (hw_supports(dev, driver_algs[i].desc_hdr_template)) {
1921                         struct talitos_crypto_alg *t_alg;
1922
1923                         t_alg = talitos_alg_alloc(dev, &driver_algs[i]);
1924                         if (IS_ERR(t_alg)) {
1925                                 err = PTR_ERR(t_alg);
1926                                 goto err_out;
1927                         }
1928
1929                         err = crypto_register_alg(&t_alg->crypto_alg);
1930                         if (err) {
1931                                 dev_err(dev, "%s alg registration failed\n",
1932                                         t_alg->crypto_alg.cra_driver_name);
1933                                 kfree(t_alg);
1934                         } else {
1935                                 list_add_tail(&t_alg->entry, &priv->alg_list);
1936                                 dev_info(dev, "%s\n",
1937                                          t_alg->crypto_alg.cra_driver_name);
1938                         }
1939                 }
1940         }
1941
1942         return 0;
1943
1944 err_out:
1945         talitos_remove(ofdev);
1946
1947         return err;
1948 }
1949
1950 static struct of_device_id talitos_match[] = {
1951         {
1952                 .compatible = "fsl,sec2.0",
1953         },
1954         {},
1955 };
1956 MODULE_DEVICE_TABLE(of, talitos_match);
1957
1958 static struct of_platform_driver talitos_driver = {
1959         .name = "talitos",
1960         .match_table = talitos_match,
1961         .probe = talitos_probe,
1962         .remove = talitos_remove,
1963 };
1964
1965 static int __init talitos_init(void)
1966 {
1967         return of_register_platform_driver(&talitos_driver);
1968 }
1969 module_init(talitos_init);
1970
1971 static void __exit talitos_exit(void)
1972 {
1973         of_unregister_platform_driver(&talitos_driver);
1974 }
1975 module_exit(talitos_exit);
1976
1977 MODULE_LICENSE("GPL");
1978 MODULE_AUTHOR("Kim Phillips <kim.phillips@freescale.com>");
1979 MODULE_DESCRIPTION("Freescale integrated security engine (SEC) driver");