SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/scsi/aic7xxx/aic7xxx_core.c

   1 /*
   2  * Core routines and tables shareable across OS platforms.
   3  *
   4  * Copyright (c) 1994-2002 Justin T. Gibbs.
   5  * Copyright (c) 2000-2002 Adaptec Inc.
   6  * All rights reserved.
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions, and the following disclaimer,
  13  *    without modification.
  14  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
  15  *    substantially similar to the "NO WARRANTY" disclaimer below
  16  *    ("Disclaimer") and any redistribution must be conditioned upon
  17  *    including a substantially similar Disclaimer requirement for further
  18  *    binary redistribution.
  19  * 3. Neither the names of the above-listed copyright holders nor the names
  20  *    of any contributors may be used to endorse or promote products derived
  21  *    from this software without specific prior written permission.
  22  *
  23  * Alternatively, this software may be distributed under the terms of the
  24  * GNU General Public License ("GPL") version 2 as published by the Free
  25  * Software Foundation.
  26  *
  27  * NO WARRANTY
  28  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  29  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  30  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
  31  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  32  * HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  33  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  34  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  35  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
  36  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
  37  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  38  * POSSIBILITY OF SUCH DAMAGES.
  39  *
  40  * $Id: //depot/aic7xxx/aic7xxx/aic7xxx.c#155 $
  41  */
  42
  43 #ifdef __linux__
  44 #include "aic7xxx_osm.h"
  45 #include "aic7xxx_inline.h"
  46 #include "aicasm/aicasm_insformat.h"
  47 #else
  48 #include <dev/aic7xxx/aic7xxx_osm.h>
  49 #include <dev/aic7xxx/aic7xxx_inline.h>
  50 #include <dev/aic7xxx/aicasm/aicasm_insformat.h>
  51 #endif
  52
  53 /***************************** Lookup Tables **********************************/
  54 char *ahc_chip_names[] =
  55 {
  56         "NONE",
  57         "aic7770",
  58         "aic7850",
  59         "aic7855",
  60         "aic7859",
  61         "aic7860",
  62         "aic7870",
  63         "aic7880",
  64         "aic7895",
  65         "aic7895C",
  66         "aic7890/91",
  67         "aic7896/97",
  68         "aic7892",
  69         "aic7899"
  70 };
  71 static const u_int num_chip_names = ARRAY_SIZE(ahc_chip_names);
  72
  73 /*
  74  * Hardware error codes.
  75  */
  76 struct ahc_hard_error_entry {
  77         uint8_t errno;
  78         char *errmesg;
  79 };
  80
  81 static struct ahc_hard_error_entry ahc_hard_errors[] = {
  82         { ILLHADDR,     "Illegal Host Access" },
  83         { ILLSADDR,     "Illegal Sequencer Address referrenced" },
  84         { ILLOPCODE,    "Illegal Opcode in sequencer program" },
  85         { SQPARERR,     "Sequencer Parity Error" },
  86         { DPARERR,      "Data-path Parity Error" },
  87         { MPARERR,      "Scratch or SCB Memory Parity Error" },
  88         { PCIERRSTAT,   "PCI Error detected" },
  89         { CIOPARERR,    "CIOBUS Parity Error" },
  90 };
  91 static const u_int num_errors = ARRAY_SIZE(ahc_hard_errors);
  92
  93 static struct ahc_phase_table_entry ahc_phase_table[] =
  94 {
  95         { P_DATAOUT,    MSG_NOOP,               "in Data-out phase"     },
  96         { P_DATAIN,     MSG_INITIATOR_DET_ERR,  "in Data-in phase"      },
  97         { P_DATAOUT_DT, MSG_NOOP,               "in DT Data-out phase"  },
  98         { P_DATAIN_DT,  MSG_INITIATOR_DET_ERR,  "in DT Data-in phase"   },
  99         { P_COMMAND,    MSG_NOOP,               "in Command phase"      },
 100         { P_MESGOUT,    MSG_NOOP,               "in Message-out phase"  },
 101         { P_STATUS,     MSG_INITIATOR_DET_ERR,  "in Status phase"       },
 102         { P_MESGIN,     MSG_PARITY_ERROR,       "in Message-in phase"   },
 103         { P_BUSFREE,    MSG_NOOP,               "while idle"            },
 104         { 0,            MSG_NOOP,               "in unknown phase"      }
 105 };
 106
 107 /*
 108  * In most cases we only wish to itterate over real phases, so
 109  * exclude the last element from the count.
 110  */
 111 static const u_int num_phases = ARRAY_SIZE(ahc_phase_table) - 1;
 112
 113 /*
 114  * Valid SCSIRATE values.  (p. 3-17)
 115  * Provides a mapping of tranfer periods in ns to the proper value to
 116  * stick in the scsixfer reg.
 117  */
 118 static struct ahc_syncrate ahc_syncrates[] =
 119 {
 120       /* ultra2    fast/ultra  period     rate */
 121         { 0x42,      0x000,      9,      "80.0" },
 122         { 0x03,      0x000,     10,      "40.0" },
 123         { 0x04,      0x000,     11,      "33.0" },
 124         { 0x05,      0x100,     12,      "20.0" },
 125         { 0x06,      0x110,     15,      "16.0" },
 126         { 0x07,      0x120,     18,      "13.4" },
 127         { 0x08,      0x000,     25,      "10.0" },
 128         { 0x19,      0x010,     31,      "8.0"  },
 129         { 0x1a,      0x020,     37,      "6.67" },
 130         { 0x1b,      0x030,     43,      "5.7"  },
 131         { 0x1c,      0x040,     50,      "5.0"  },
 132         { 0x00,      0x050,     56,      "4.4"  },
 133         { 0x00,      0x060,     62,      "4.0"  },
 134         { 0x00,      0x070,     68,      "3.6"  },
 135         { 0x00,      0x000,      0,      NULL   }
 136 };
 137
 138 /* Our Sequencer Program */
 139 #include "aic7xxx_seq.h"
 140
 141 /**************************** Function Declarations ***************************/
 142 static void             ahc_force_renegotiation(struct ahc_softc *ahc,
 143                                                 struct ahc_devinfo *devinfo);
 144 static struct ahc_tmode_tstate*
 145                         ahc_alloc_tstate(struct ahc_softc *ahc,
 146                                          u_int scsi_id, char channel);
 147 #ifdef AHC_TARGET_MODE
 148 static void             ahc_free_tstate(struct ahc_softc *ahc,
 149                                         u_int scsi_id, char channel, int force);
 150 #endif
 151 static struct ahc_syncrate*
 152                         ahc_devlimited_syncrate(struct ahc_softc *ahc,
 153                                                 struct ahc_initiator_tinfo *,
 154                                                 u_int *period,
 155                                                 u_int *ppr_options,
 156                                                 role_t role);
 157 static void             ahc_update_pending_scbs(struct ahc_softc *ahc);
 158 static void             ahc_fetch_devinfo(struct ahc_softc *ahc,
 159                                           struct ahc_devinfo *devinfo);
 160 static void             ahc_scb_devinfo(struct ahc_softc *ahc,
 161                                         struct ahc_devinfo *devinfo,
 162                                         struct scb *scb);
 163 static void             ahc_assert_atn(struct ahc_softc *ahc);
 164 static void             ahc_setup_initiator_msgout(struct ahc_softc *ahc,
 165                                                    struct ahc_devinfo *devinfo,
 166                                                    struct scb *scb);
 167 static void             ahc_build_transfer_msg(struct ahc_softc *ahc,
 168                                                struct ahc_devinfo *devinfo);
 169 static void             ahc_construct_sdtr(struct ahc_softc *ahc,
 170                                            struct ahc_devinfo *devinfo,
 171                                            u_int period, u_int offset);
 172 static void             ahc_construct_wdtr(struct ahc_softc *ahc,
 173                                            struct ahc_devinfo *devinfo,
 174                                            u_int bus_width);
 175 static void             ahc_construct_ppr(struct ahc_softc *ahc,
 176                                           struct ahc_devinfo *devinfo,
 177                                           u_int period, u_int offset,
 178                                           u_int bus_width, u_int ppr_options);
 179 static void             ahc_clear_msg_state(struct ahc_softc *ahc);
 180 static void             ahc_handle_proto_violation(struct ahc_softc *ahc);
 181 static void             ahc_handle_message_phase(struct ahc_softc *ahc);
 182 typedef enum {
 183         AHCMSG_1B,
 184         AHCMSG_2B,
 185         AHCMSG_EXT
 186 } ahc_msgtype;
 187 static int              ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type,
 188                                      u_int msgval, int full);
 189 static int              ahc_parse_msg(struct ahc_softc *ahc,
 190                                       struct ahc_devinfo *devinfo);
 191 static int              ahc_handle_msg_reject(struct ahc_softc *ahc,
 192                                               struct ahc_devinfo *devinfo);
 193 static void             ahc_handle_ign_wide_residue(struct ahc_softc *ahc,
 194                                                 struct ahc_devinfo *devinfo);
 195 static void             ahc_reinitialize_dataptrs(struct ahc_softc *ahc);
 196 static void             ahc_handle_devreset(struct ahc_softc *ahc,
 197                                             struct ahc_devinfo *devinfo,
 198                                             cam_status status, char *message,
 199                                             int verbose_level);
 200 #ifdef AHC_TARGET_MODE
 201 static void             ahc_setup_target_msgin(struct ahc_softc *ahc,
 202                                                struct ahc_devinfo *devinfo,
 203                                                struct scb *scb);
 204 #endif
 205
 206 static bus_dmamap_callback_t    ahc_dmamap_cb;
 207 static void                     ahc_build_free_scb_list(struct ahc_softc *ahc);
 208 static int                      ahc_init_scbdata(struct ahc_softc *ahc);
 209 static void                     ahc_fini_scbdata(struct ahc_softc *ahc);
 210 static void             ahc_qinfifo_requeue(struct ahc_softc *ahc,
 211                                             struct scb *prev_scb,
 212                                             struct scb *scb);
 213 static int              ahc_qinfifo_count(struct ahc_softc *ahc);
 214 static u_int            ahc_rem_scb_from_disc_list(struct ahc_softc *ahc,
 215                                                    u_int prev, u_int scbptr);
 216 static void             ahc_add_curscb_to_free_list(struct ahc_softc *ahc);
 217 static u_int            ahc_rem_wscb(struct ahc_softc *ahc,
 218                                      u_int scbpos, u_int prev);
 219 static void             ahc_reset_current_bus(struct ahc_softc *ahc);
 220 #ifdef AHC_DUMP_SEQ
 221 static void             ahc_dumpseq(struct ahc_softc *ahc);
 222 #endif
 223 static int              ahc_loadseq(struct ahc_softc *ahc);
 224 static int              ahc_check_patch(struct ahc_softc *ahc,
 225                                         struct patch **start_patch,
 226                                         u_int start_instr, u_int *skip_addr);
 227 static void             ahc_download_instr(struct ahc_softc *ahc,
 228                                            u_int instrptr, uint8_t *dconsts);
 229 #ifdef AHC_TARGET_MODE
 230 static void             ahc_queue_lstate_event(struct ahc_softc *ahc,
 231                                                struct ahc_tmode_lstate *lstate,
 232                                                u_int initiator_id,
 233                                                u_int event_type,
 234                                                u_int event_arg);
 235 static void             ahc_update_scsiid(struct ahc_softc *ahc,
 236                                           u_int targid_mask);
 237 static int              ahc_handle_target_cmd(struct ahc_softc *ahc,
 238                                               struct target_cmd *cmd);
 239 #endif
 240
 241 /************************* Sequencer Execution Control ************************/
 242 /*
 243  * Work around any chip bugs related to halting sequencer execution.
 244  * On Ultra2 controllers, we must clear the CIOBUS stretch signal by
 245  * reading a register that will set this signal and deassert it.
 246  * Without this workaround, if the chip is paused, by an interrupt or
 247  * manual pause while accessing scb ram, accesses to certain registers
 248  * will hang the system (infinite pci retries).
 249  */
 250 void
 251 ahc_pause_bug_fix(struct ahc_softc *ahc)
 252 {
 253         if ((ahc->features & AHC_ULTRA2) != 0)
 254                 (void)ahc_inb(ahc, CCSCBCTL);
 255 }
 256
 257 /*
 258  * Determine whether the sequencer has halted code execution.
 259  * Returns non-zero status if the sequencer is stopped.
 260  */
 261 int
 262 ahc_is_paused(struct ahc_softc *ahc)
 263 {
 264         return ((ahc_inb(ahc, HCNTRL) & PAUSE) != 0);
 265 }
 266
 267 /*
 268  * Request that the sequencer stop and wait, indefinitely, for it
 269  * to stop.  The sequencer will only acknowledge that it is paused
 270  * once it has reached an instruction boundary and PAUSEDIS is
 271  * cleared in the SEQCTL register.  The sequencer may use PAUSEDIS
 272  * for critical sections.
 273  */
 274 void
 275 ahc_pause(struct ahc_softc *ahc)
 276 {
 277         ahc_outb(ahc, HCNTRL, ahc->pause);
 278
 279         /*
 280          * Since the sequencer can disable pausing in a critical section, we
 281          * must loop until it actually stops.
 282          */
 283         while (ahc_is_paused(ahc) == 0)
 284                 ;
 285
 286         ahc_pause_bug_fix(ahc);
 287 }
 288
 289 /*
 290  * Allow the sequencer to continue program execution.
 291  * We check here to ensure that no additional interrupt
 292  * sources that would cause the sequencer to halt have been
 293  * asserted.  If, for example, a SCSI bus reset is detected
 294  * while we are fielding a different, pausing, interrupt type,
 295  * we don't want to release the sequencer before going back
 296  * into our interrupt handler and dealing with this new
 297  * condition.
 298  */
 299 void
 300 ahc_unpause(struct ahc_softc *ahc)
 301 {
 302         if ((ahc_inb(ahc, INTSTAT) & (SCSIINT | SEQINT | BRKADRINT)) == 0)
 303                 ahc_outb(ahc, HCNTRL, ahc->unpause);
 304 }
 305
 306 /************************** Memory mapping routines ***************************/
 307 struct ahc_dma_seg *
 308 ahc_sg_bus_to_virt(struct scb *scb, uint32_t sg_busaddr)
 309 {
 310         int sg_index;
 311
 312         sg_index = (sg_busaddr - scb->sg_list_phys)/sizeof(struct ahc_dma_seg);
 313         /* sg_list_phys points to entry 1, not 0 */
 314         sg_index++;
 315
 316         return (&scb->sg_list[sg_index]);
 317 }
 318
 319 uint32_t
 320 ahc_sg_virt_to_bus(struct scb *scb, struct ahc_dma_seg *sg)
 321 {
 322         int sg_index;
 323
 324         /* sg_list_phys points to entry 1, not 0 */
 325         sg_index = sg - &scb->sg_list[1];
 326
 327         return (scb->sg_list_phys + (sg_index * sizeof(*scb->sg_list)));
 328 }
 329
 330 uint32_t
 331 ahc_hscb_busaddr(struct ahc_softc *ahc, u_int index)
 332 {
 333         return (ahc->scb_data->hscb_busaddr
 334                 + (sizeof(struct hardware_scb) * index));
 335 }
 336
 337 void
 338 ahc_sync_scb(struct ahc_softc *ahc, struct scb *scb, int op)
 339 {
 340         ahc_dmamap_sync(ahc, ahc->scb_data->hscb_dmat,
 341                         ahc->scb_data->hscb_dmamap,
 342                         /*offset*/(scb->hscb - ahc->hscbs) * sizeof(*scb->hscb),
 343                         /*len*/sizeof(*scb->hscb), op);
 344 }
 345
 346 void
 347 ahc_sync_sglist(struct ahc_softc *ahc, struct scb *scb, int op)
 348 {
 349         if (scb->sg_count == 0)
 350                 return;
 351
 352         ahc_dmamap_sync(ahc, ahc->scb_data->sg_dmat, scb->sg_map->sg_dmamap,
 353                         /*offset*/(scb->sg_list - scb->sg_map->sg_vaddr)
 354                                 * sizeof(struct ahc_dma_seg),
 355                         /*len*/sizeof(struct ahc_dma_seg) * scb->sg_count, op);
 356 }
 357
 358 uint32_t
 359 ahc_targetcmd_offset(struct ahc_softc *ahc, u_int index)
 360 {
 361         return (((uint8_t *)&ahc->targetcmds[index]) - ahc->qoutfifo);
 362 }
 363
 364 /*********************** Miscelaneous Support Functions ***********************/
 365 /*
 366  * Determine whether the sequencer reported a residual
 367  * for this SCB/transaction.
 368  */
 369 void
 370 ahc_update_residual(struct ahc_softc *ahc, struct scb *scb)
 371 {
 372         uint32_t sgptr;
 373
 374         sgptr = ahc_le32toh(scb->hscb->sgptr);
 375         if ((sgptr & SG_RESID_VALID) != 0)
 376                 ahc_calc_residual(ahc, scb);
 377 }
 378
 379 /*
 380  * Return pointers to the transfer negotiation information
 381  * for the specified our_id/remote_id pair.
 382  */
 383 struct ahc_initiator_tinfo *
 384 ahc_fetch_transinfo(struct ahc_softc *ahc, char channel, u_int our_id,
 385                     u_int remote_id, struct ahc_tmode_tstate **tstate)
 386 {
 387         /*
 388          * Transfer data structures are stored from the perspective
 389          * of the target role.  Since the parameters for a connection
 390          * in the initiator role to a given target are the same as
 391          * when the roles are reversed, we pretend we are the target.
 392          */
 393         if (channel == 'B')
 394                 our_id += 8;
 395         *tstate = ahc->enabled_targets[our_id];
 396         return (&(*tstate)->transinfo[remote_id]);
 397 }
 398
 399 uint16_t
 400 ahc_inw(struct ahc_softc *ahc, u_int port)
 401 {
 402         uint16_t r = ahc_inb(ahc, port+1) << 8;
 403         return r | ahc_inb(ahc, port);
 404 }
 405
 406 void
 407 ahc_outw(struct ahc_softc *ahc, u_int port, u_int value)
 408 {
 409         ahc_outb(ahc, port, value & 0xFF);
 410         ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 411 }
 412
 413 uint32_t
 414 ahc_inl(struct ahc_softc *ahc, u_int port)
 415 {
 416         return ((ahc_inb(ahc, port))
 417               | (ahc_inb(ahc, port+1) << 8)
 418               | (ahc_inb(ahc, port+2) << 16)
 419               | (ahc_inb(ahc, port+3) << 24));
 420 }
 421
 422 void
 423 ahc_outl(struct ahc_softc *ahc, u_int port, uint32_t value)
 424 {
 425         ahc_outb(ahc, port, (value) & 0xFF);
 426         ahc_outb(ahc, port+1, ((value) >> 8) & 0xFF);
 427         ahc_outb(ahc, port+2, ((value) >> 16) & 0xFF);
 428         ahc_outb(ahc, port+3, ((value) >> 24) & 0xFF);
 429 }
 430
 431 uint64_t
 432 ahc_inq(struct ahc_softc *ahc, u_int port)
 433 {
 434         return ((ahc_inb(ahc, port))
 435               | (ahc_inb(ahc, port+1) << 8)
 436               | (ahc_inb(ahc, port+2) << 16)
 437               | (ahc_inb(ahc, port+3) << 24)
 438               | (((uint64_t)ahc_inb(ahc, port+4)) << 32)
 439               | (((uint64_t)ahc_inb(ahc, port+5)) << 40)
 440               | (((uint64_t)ahc_inb(ahc, port+6)) << 48)
 441               | (((uint64_t)ahc_inb(ahc, port+7)) << 56));
 442 }
 443
 444 void
 445 ahc_outq(struct ahc_softc *ahc, u_int port, uint64_t value)
 446 {
 447         ahc_outb(ahc, port, value & 0xFF);
 448         ahc_outb(ahc, port+1, (value >> 8) & 0xFF);
 449         ahc_outb(ahc, port+2, (value >> 16) & 0xFF);
 450         ahc_outb(ahc, port+3, (value >> 24) & 0xFF);
 451         ahc_outb(ahc, port+4, (value >> 32) & 0xFF);
 452         ahc_outb(ahc, port+5, (value >> 40) & 0xFF);
 453         ahc_outb(ahc, port+6, (value >> 48) & 0xFF);
 454         ahc_outb(ahc, port+7, (value >> 56) & 0xFF);
 455 }
 456
 457 /*
 458  * Get a free scb. If there are none, see if we can allocate a new SCB.
 459  */
 460 struct scb *
 461 ahc_get_scb(struct ahc_softc *ahc)
 462 {
 463         struct scb *scb;
 464
 465         if ((scb = SLIST_FIRST(&ahc->scb_data->free_scbs)) == NULL) {
 466                 ahc_alloc_scbs(ahc);
 467                 scb = SLIST_FIRST(&ahc->scb_data->free_scbs);
 468                 if (scb == NULL)
 469                         return (NULL);
 470         }
 471         SLIST_REMOVE_HEAD(&ahc->scb_data->free_scbs, links.sle);
 472         return (scb);
 473 }
 474
 475 /*
 476  * Return an SCB resource to the free list.
 477  */
 478 void
 479 ahc_free_scb(struct ahc_softc *ahc, struct scb *scb)
 480 {
 481         struct hardware_scb *hscb;
 482
 483         hscb = scb->hscb;
 484         /* Clean up for the next user */
 485         ahc->scb_data->scbindex[hscb->tag] = NULL;
 486         scb->flags = SCB_FREE;
 487         hscb->control = 0;
 488
 489         SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs, scb, links.sle);
 490
 491         /* Notify the OSM that a resource is now available. */
 492         ahc_platform_scb_free(ahc, scb);
 493 }
 494
 495 struct scb *
 496 ahc_lookup_scb(struct ahc_softc *ahc, u_int tag)
 497 {
 498         struct scb* scb;
 499
 500         scb = ahc->scb_data->scbindex[tag];
 501         if (scb != NULL)
 502                 ahc_sync_scb(ahc, scb,
 503                              BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE);
 504         return (scb);
 505 }
 506
 507 void
 508 ahc_swap_with_next_hscb(struct ahc_softc *ahc, struct scb *scb)
 509 {
 510         struct hardware_scb *q_hscb;
 511         u_int  saved_tag;
 512
 513         /*
 514          * Our queuing method is a bit tricky.  The card
 515          * knows in advance which HSCB to download, and we
 516          * can't disappoint it.  To achieve this, the next
 517          * SCB to download is saved off in ahc->next_queued_scb.
 518          * When we are called to queue "an arbitrary scb",
 519          * we copy the contents of the incoming HSCB to the one
 520          * the sequencer knows about, swap HSCB pointers and
 521          * finally assign the SCB to the tag indexed location
 522          * in the scb_array.  This makes sure that we can still
 523          * locate the correct SCB by SCB_TAG.
 524          */
 525         q_hscb = ahc->next_queued_scb->hscb;
 526         saved_tag = q_hscb->tag;
 527         memcpy(q_hscb, scb->hscb, sizeof(*scb->hscb));
 528         if ((scb->flags & SCB_CDB32_PTR) != 0) {
 529                 q_hscb->shared_data.cdb_ptr =
 530                     ahc_htole32(ahc_hscb_busaddr(ahc, q_hscb->tag)
 531                               + offsetof(struct hardware_scb, cdb32));
 532         }
 533         q_hscb->tag = saved_tag;
 534         q_hscb->next = scb->hscb->tag;
 535
 536         /* Now swap HSCB pointers. */
 537         ahc->next_queued_scb->hscb = scb->hscb;
 538         scb->hscb = q_hscb;
 539
 540         /* Now define the mapping from tag to SCB in the scbindex */
 541         ahc->scb_data->scbindex[scb->hscb->tag] = scb;
 542 }
 543
 544 /*
 545  * Tell the sequencer about a new transaction to execute.
 546  */
 547 void
 548 ahc_queue_scb(struct ahc_softc *ahc, struct scb *scb)
 549 {
 550         ahc_swap_with_next_hscb(ahc, scb);
 551
 552         if (scb->hscb->tag == SCB_LIST_NULL
 553          || scb->hscb->next == SCB_LIST_NULL)
 554                 panic("Attempt to queue invalid SCB tag %x:%x\n",
 555                       scb->hscb->tag, scb->hscb->next);
 556
 557         /*
 558          * Setup data "oddness".
 559          */
 560         scb->hscb->lun &= LID;
 561         if (ahc_get_transfer_length(scb) & 0x1)
 562                 scb->hscb->lun |= SCB_XFERLEN_ODD;
 563
 564         /*
 565          * Keep a history of SCBs we've downloaded in the qinfifo.
 566          */
 567         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
 568
 569         /*
 570          * Make sure our data is consistent from the
 571          * perspective of the adapter.
 572          */
 573         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
 574
 575         /* Tell the adapter about the newly queued SCB */
 576         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
 577                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
 578         } else {
 579                 if ((ahc->features & AHC_AUTOPAUSE) == 0)
 580                         ahc_pause(ahc);
 581                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
 582                 if ((ahc->features & AHC_AUTOPAUSE) == 0)
 583                         ahc_unpause(ahc);
 584         }
 585 }
 586
 587 struct scsi_sense_data *
 588 ahc_get_sense_buf(struct ahc_softc *ahc, struct scb *scb)
 589 {
 590         int offset;
 591
 592         offset = scb - ahc->scb_data->scbarray;
 593         return (&ahc->scb_data->sense[offset]);
 594 }
 595
 596 uint32_t
 597 ahc_get_sense_bufaddr(struct ahc_softc *ahc, struct scb *scb)
 598 {
 599         int offset;
 600
 601         offset = scb - ahc->scb_data->scbarray;
 602         return (ahc->scb_data->sense_busaddr
 603               + (offset * sizeof(struct scsi_sense_data)));
 604 }
 605
 606 /************************** Interrupt Processing ******************************/
 607 void
 608 ahc_sync_qoutfifo(struct ahc_softc *ahc, int op)
 609 {
 610         ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 611                         /*offset*/0, /*len*/256, op);
 612 }
 613
 614 void
 615 ahc_sync_tqinfifo(struct ahc_softc *ahc, int op)
 616 {
 617 #ifdef AHC_TARGET_MODE
 618         if ((ahc->flags & AHC_TARGETROLE) != 0) {
 619                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 620                                 ahc->shared_data_dmamap,
 621                                 ahc_targetcmd_offset(ahc, 0),
 622                                 sizeof(struct target_cmd) * AHC_TMODE_CMDS,
 623                                 op);
 624         }
 625 #endif
 626 }
 627
 628 /*
 629  * See if the firmware has posted any completed commands
 630  * into our in-core command complete fifos.
 631  */
 632 #define AHC_RUN_QOUTFIFO 0x1
 633 #define AHC_RUN_TQINFIFO 0x2
 634 u_int
 635 ahc_check_cmdcmpltqueues(struct ahc_softc *ahc)
 636 {
 637         u_int retval;
 638
 639         retval = 0;
 640         ahc_dmamap_sync(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
 641                         /*offset*/ahc->qoutfifonext, /*len*/1,
 642                         BUS_DMASYNC_POSTREAD);
 643         if (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL)
 644                 retval |= AHC_RUN_QOUTFIFO;
 645 #ifdef AHC_TARGET_MODE
 646         if ((ahc->flags & AHC_TARGETROLE) != 0
 647          && (ahc->flags & AHC_TQINFIFO_BLOCKED) == 0) {
 648                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 649                                 ahc->shared_data_dmamap,
 650                                 ahc_targetcmd_offset(ahc, ahc->tqinfifofnext),
 651                                 /*len*/sizeof(struct target_cmd),
 652                                 BUS_DMASYNC_POSTREAD);
 653                 if (ahc->targetcmds[ahc->tqinfifonext].cmd_valid != 0)
 654                         retval |= AHC_RUN_TQINFIFO;
 655         }
 656 #endif
 657         return (retval);
 658 }
 659
 660 /*
 661  * Catch an interrupt from the adapter
 662  */
 663 int
 664 ahc_intr(struct ahc_softc *ahc)
 665 {
 666         u_int   intstat;
 667
 668         if ((ahc->pause & INTEN) == 0) {
 669                 /*
 670                  * Our interrupt is not enabled on the chip
 671                  * and may be disabled for re-entrancy reasons,
 672                  * so just return.  This is likely just a shared
 673                  * interrupt.
 674                  */
 675                 return (0);
 676         }
 677         /*
 678          * Instead of directly reading the interrupt status register,
 679          * infer the cause of the interrupt by checking our in-core
 680          * completion queues.  This avoids a costly PCI bus read in
 681          * most cases.
 682          */
 683         if ((ahc->flags & (AHC_ALL_INTERRUPTS|AHC_EDGE_INTERRUPT)) == 0
 684          && (ahc_check_cmdcmpltqueues(ahc) != 0))
 685                 intstat = CMDCMPLT;
 686         else {
 687                 intstat = ahc_inb(ahc, INTSTAT);
 688         }
 689
 690         if ((intstat & INT_PEND) == 0) {
 691 #if AHC_PCI_CONFIG > 0
 692                 if (ahc->unsolicited_ints > 500) {
 693                         ahc->unsolicited_ints = 0;
 694                         if ((ahc->chip & AHC_PCI) != 0
 695                          && (ahc_inb(ahc, ERROR) & PCIERRSTAT) != 0)
 696                                 ahc->bus_intr(ahc);
 697                 }
 698 #endif
 699                 ahc->unsolicited_ints++;
 700                 return (0);
 701         }
 702         ahc->unsolicited_ints = 0;
 703
 704         if (intstat & CMDCMPLT) {
 705                 ahc_outb(ahc, CLRINT, CLRCMDINT);
 706
 707                 /*
 708                  * Ensure that the chip sees that we've cleared
 709                  * this interrupt before we walk the output fifo.
 710                  * Otherwise, we may, due to posted bus writes,
 711                  * clear the interrupt after we finish the scan,
 712                  * and after the sequencer has added new entries
 713                  * and asserted the interrupt again.
 714                  */
 715                 ahc_flush_device_writes(ahc);
 716                 ahc_run_qoutfifo(ahc);
 717 #ifdef AHC_TARGET_MODE
 718                 if ((ahc->flags & AHC_TARGETROLE) != 0)
 719                         ahc_run_tqinfifo(ahc, /*paused*/FALSE);
 720 #endif
 721         }
 722
 723         /*
 724          * Handle statuses that may invalidate our cached
 725          * copy of INTSTAT separately.
 726          */
 727         if (intstat == 0xFF && (ahc->features & AHC_REMOVABLE) != 0) {
 728                 /* Hot eject.  Do nothing */
 729         } else if (intstat & BRKADRINT) {
 730                 ahc_handle_brkadrint(ahc);
 731         } else if ((intstat & (SEQINT|SCSIINT)) != 0) {
 732
 733                 ahc_pause_bug_fix(ahc);
 734
 735                 if ((intstat & SEQINT) != 0)
 736                         ahc_handle_seqint(ahc, intstat);
 737
 738                 if ((intstat & SCSIINT) != 0)
 739                         ahc_handle_scsiint(ahc, intstat);
 740         }
 741         return (1);
 742 }
 743
 744 /************************* Sequencer Execution Control ************************/
 745 /*
 746  * Restart the sequencer program from address zero
 747  */
 748 void
 749 ahc_restart(struct ahc_softc *ahc)
 750 {
 751
 752         ahc_pause(ahc);
 753
 754         /* No more pending messages. */
 755         ahc_clear_msg_state(ahc);
 756
 757         ahc_outb(ahc, SCSISIGO, 0);             /* De-assert BSY */
 758         ahc_outb(ahc, MSG_OUT, MSG_NOOP);       /* No message to send */
 759         ahc_outb(ahc, SXFRCTL1, ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
 760         ahc_outb(ahc, LASTPHASE, P_BUSFREE);
 761         ahc_outb(ahc, SAVED_SCSIID, 0xFF);
 762         ahc_outb(ahc, SAVED_LUN, 0xFF);
 763
 764         /*
 765          * Ensure that the sequencer's idea of TQINPOS
 766          * matches our own.  The sequencer increments TQINPOS
 767          * only after it sees a DMA complete and a reset could
 768          * occur before the increment leaving the kernel to believe
 769          * the command arrived but the sequencer to not.
 770          */
 771         ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
 772
 773         /* Always allow reselection */
 774         ahc_outb(ahc, SCSISEQ,
 775                  ahc_inb(ahc, SCSISEQ_TEMPLATE) & (ENSELI|ENRSELI|ENAUTOATNP));
 776         if ((ahc->features & AHC_CMD_CHAN) != 0) {
 777                 /* Ensure that no DMA operations are in progress */
 778                 ahc_outb(ahc, CCSCBCNT, 0);
 779                 ahc_outb(ahc, CCSGCTL, 0);
 780                 ahc_outb(ahc, CCSCBCTL, 0);
 781         }
 782         /*
 783          * If we were in the process of DMA'ing SCB data into
 784          * an SCB, replace that SCB on the free list.  This prevents
 785          * an SCB leak.
 786          */
 787         if ((ahc_inb(ahc, SEQ_FLAGS2) & SCB_DMA) != 0) {
 788                 ahc_add_curscb_to_free_list(ahc);
 789                 ahc_outb(ahc, SEQ_FLAGS2,
 790                          ahc_inb(ahc, SEQ_FLAGS2) & ~SCB_DMA);
 791         }
 792
 793         /*
 794          * Clear any pending sequencer interrupt.  It is no
 795          * longer relevant since we're resetting the Program
 796          * Counter.
 797          */
 798         ahc_outb(ahc, CLRINT, CLRSEQINT);
 799
 800         ahc_outb(ahc, MWI_RESIDUAL, 0);
 801         ahc_outb(ahc, SEQCTL, ahc->seqctl);
 802         ahc_outb(ahc, SEQADDR0, 0);
 803         ahc_outb(ahc, SEQADDR1, 0);
 804
 805         ahc_unpause(ahc);
 806 }
 807
 808 /************************* Input/Output Queues ********************************/
 809 void
 810 ahc_run_qoutfifo(struct ahc_softc *ahc)
 811 {
 812         struct scb *scb;
 813         u_int  scb_index;
 814
 815         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
 816         while (ahc->qoutfifo[ahc->qoutfifonext] != SCB_LIST_NULL) {
 817
 818                 scb_index = ahc->qoutfifo[ahc->qoutfifonext];
 819                 if ((ahc->qoutfifonext & 0x03) == 0x03) {
 820                         u_int modnext;
 821
 822                         /*
 823                          * Clear 32bits of QOUTFIFO at a time
 824                          * so that we don't clobber an incoming
 825                          * byte DMA to the array on architectures
 826                          * that only support 32bit load and store
 827                          * operations.
 828                          */
 829                         modnext = ahc->qoutfifonext & ~0x3;
 830                         *((uint32_t *)(&ahc->qoutfifo[modnext])) = 0xFFFFFFFFUL;
 831                         ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
 832                                         ahc->shared_data_dmamap,
 833                                         /*offset*/modnext, /*len*/4,
 834                                         BUS_DMASYNC_PREREAD);
 835                 }
 836                 ahc->qoutfifonext++;
 837
 838                 scb = ahc_lookup_scb(ahc, scb_index);
 839                 if (scb == NULL) {
 840                         printf("%s: WARNING no command for scb %d "
 841                                "(cmdcmplt)\nQOUTPOS = %d\n",
 842                                ahc_name(ahc), scb_index,
 843                                (ahc->qoutfifonext - 1) & 0xFF);
 844                         continue;
 845                 }
 846
 847                 /*
 848                  * Save off the residual
 849                  * if there is one.
 850                  */
 851                 ahc_update_residual(ahc, scb);
 852                 ahc_done(ahc, scb);
 853         }
 854 }
 855
 856 void
 857 ahc_run_untagged_queues(struct ahc_softc *ahc)
 858 {
 859         int i;
 860
 861         for (i = 0; i < 16; i++)
 862                 ahc_run_untagged_queue(ahc, &ahc->untagged_queues[i]);
 863 }
 864
 865 void
 866 ahc_run_untagged_queue(struct ahc_softc *ahc, struct scb_tailq *queue)
 867 {
 868         struct scb *scb;
 869
 870         if (ahc->untagged_queue_lock != 0)
 871                 return;
 872
 873         if ((scb = TAILQ_FIRST(queue)) != NULL
 874          && (scb->flags & SCB_ACTIVE) == 0) {
 875                 scb->flags |= SCB_ACTIVE;
 876                 ahc_queue_scb(ahc, scb);
 877         }
 878 }
 879
 880 /************************* Interrupt Handling *********************************/
 881 void
 882 ahc_handle_brkadrint(struct ahc_softc *ahc)
 883 {
 884         /*
 885          * We upset the sequencer :-(
 886          * Lookup the error message
 887          */
 888         int i;
 889         int error;
 890
 891         error = ahc_inb(ahc, ERROR);
 892         for (i = 0; error != 1 && i < num_errors; i++)
 893                 error >>= 1;
 894         printf("%s: brkadrint, %s at seqaddr = 0x%x\n",
 895                ahc_name(ahc), ahc_hard_errors[i].errmesg,
 896                ahc_inb(ahc, SEQADDR0) |
 897                (ahc_inb(ahc, SEQADDR1) << 8));
 898
 899         ahc_dump_card_state(ahc);
 900
 901         /* Tell everyone that this HBA is no longer available */
 902         ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, ALL_CHANNELS,
 903                        CAM_LUN_WILDCARD, SCB_LIST_NULL, ROLE_UNKNOWN,
 904                        CAM_NO_HBA);
 905
 906         /* Disable all interrupt sources by resetting the controller */
 907         ahc_shutdown(ahc);
 908 }
 909
 910 void
 911 ahc_handle_seqint(struct ahc_softc *ahc, u_int intstat)
 912 {
 913         struct scb *scb;
 914         struct ahc_devinfo devinfo;
 915
 916         ahc_fetch_devinfo(ahc, &devinfo);
 917
 918         /*
 919          * Clear the upper byte that holds SEQINT status
 920          * codes and clear the SEQINT bit. We will unpause
 921          * the sequencer, if appropriate, after servicing
 922          * the request.
 923          */
 924         ahc_outb(ahc, CLRINT, CLRSEQINT);
 925         switch (intstat & SEQINT_MASK) {
 926         case BAD_STATUS:
 927         {
 928                 u_int  scb_index;
 929                 struct hardware_scb *hscb;
 930
 931                 /*
 932                  * Set the default return value to 0 (don't
 933                  * send sense).  The sense code will change
 934                  * this if needed.
 935                  */
 936                 ahc_outb(ahc, RETURN_1, 0);
 937
 938                 /*
 939                  * The sequencer will notify us when a command
 940                  * has an error that would be of interest to
 941                  * the kernel.  This allows us to leave the sequencer
 942                  * running in the common case of command completes
 943                  * without error.  The sequencer will already have
 944                  * dma'd the SCB back up to us, so we can reference
 945                  * the in kernel copy directly.
 946                  */
 947                 scb_index = ahc_inb(ahc, SCB_TAG);
 948                 scb = ahc_lookup_scb(ahc, scb_index);
 949                 if (scb == NULL) {
 950                         ahc_print_devinfo(ahc, &devinfo);
 951                         printf("ahc_intr - referenced scb "
 952                                "not valid during seqint 0x%x scb(%d)\n",
 953                                intstat, scb_index);
 954                         ahc_dump_card_state(ahc);
 955                         panic("for safety");
 956                         goto unpause;
 957                 }
 958
 959                 hscb = scb->hscb;
 960
 961                 /* Don't want to clobber the original sense code */
 962                 if ((scb->flags & SCB_SENSE) != 0) {
 963                         /*
 964                          * Clear the SCB_SENSE Flag and have
 965                          * the sequencer do a normal command
 966                          * complete.
 967                          */
 968                         scb->flags &= ~SCB_SENSE;
 969                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
 970                         break;
 971                 }
 972                 ahc_set_transaction_status(scb, CAM_SCSI_STATUS_ERROR);
 973                 /* Freeze the queue until the client sees the error. */
 974                 ahc_freeze_devq(ahc, scb);
 975                 ahc_freeze_scb(scb);
 976                 ahc_set_scsi_status(scb, hscb->shared_data.status.scsi_status);
 977                 switch (hscb->shared_data.status.scsi_status) {
 978                 case SCSI_STATUS_OK:
 979                         printf("%s: Interrupted for staus of 0???\n",
 980                                ahc_name(ahc));
 981                         break;
 982                 case SCSI_STATUS_CMD_TERMINATED:
 983                 case SCSI_STATUS_CHECK_COND:
 984                 {
 985                         struct ahc_dma_seg *sg;
 986                         struct scsi_sense *sc;
 987                         struct ahc_initiator_tinfo *targ_info;
 988                         struct ahc_tmode_tstate *tstate;
 989                         struct ahc_transinfo *tinfo;
 990 #ifdef AHC_DEBUG
 991                         if (ahc_debug & AHC_SHOW_SENSE) {
 992                                 ahc_print_path(ahc, scb);
 993                                 printf("SCB %d: requests Check Status\n",
 994                                        scb->hscb->tag);
 995                         }
 996 #endif
 997
 998                         if (ahc_perform_autosense(scb) == 0)
 999                                 break;
1000
1001                         targ_info = ahc_fetch_transinfo(ahc,
1002                                                         devinfo.channel,
1003                                                         devinfo.our_scsiid,
1004                                                         devinfo.target,
1005                                                         &tstate);
1006                         tinfo = &targ_info->curr;
1007                         sg = scb->sg_list;
1008                         sc = (struct scsi_sense *)(&hscb->shared_data.cdb);
1009                         /*
1010                          * Save off the residual if there is one.
1011                          */
1012                         ahc_update_residual(ahc, scb);
1013 #ifdef AHC_DEBUG
1014                         if (ahc_debug & AHC_SHOW_SENSE) {
1015                                 ahc_print_path(ahc, scb);
1016                                 printf("Sending Sense\n");
1017                         }
1018 #endif
1019                         sg->addr = ahc_get_sense_bufaddr(ahc, scb);
1020                         sg->len = ahc_get_sense_bufsize(ahc, scb);
1021                         sg->len |= AHC_DMA_LAST_SEG;
1022
1023                         /* Fixup byte order */
1024                         sg->addr = ahc_htole32(sg->addr);
1025                         sg->len = ahc_htole32(sg->len);
1026
1027                         sc->opcode = REQUEST_SENSE;
1028                         sc->byte2 = 0;
1029                         if (tinfo->protocol_version <= SCSI_REV_2
1030                          && SCB_GET_LUN(scb) < 8)
1031                                 sc->byte2 = SCB_GET_LUN(scb) << 5;
1032                         sc->unused[0] = 0;
1033                         sc->unused[1] = 0;
1034                         sc->length = sg->len;
1035                         sc->control = 0;
1036
1037                         /*
1038                          * We can't allow the target to disconnect.
1039                          * This will be an untagged transaction and
1040                          * having the target disconnect will make this
1041                          * transaction indestinguishable from outstanding
1042                          * tagged transactions.
1043                          */
1044                         hscb->control = 0;
1045
1046                         /*
1047                          * This request sense could be because the
1048                          * the device lost power or in some other
1049                          * way has lost our transfer negotiations.
1050                          * Renegotiate if appropriate.  Unit attention
1051                          * errors will be reported before any data
1052                          * phases occur.
1053                          */
1054                         if (ahc_get_residual(scb)
1055                          == ahc_get_transfer_length(scb)) {
1056                                 ahc_update_neg_request(ahc, &devinfo,
1057                                                        tstate, targ_info,
1058                                                        AHC_NEG_IF_NON_ASYNC);
1059                         }
1060                         if (tstate->auto_negotiate & devinfo.target_mask) {
1061                                 hscb->control |= MK_MESSAGE;
1062                                 scb->flags &= ~SCB_NEGOTIATE;
1063                                 scb->flags |= SCB_AUTO_NEGOTIATE;
1064                         }
1065                         hscb->cdb_len = sizeof(*sc);
1066                         hscb->dataptr = sg->addr;
1067                         hscb->datacnt = sg->len;
1068                         hscb->sgptr = scb->sg_list_phys | SG_FULL_RESID;
1069                         hscb->sgptr = ahc_htole32(hscb->sgptr);
1070                         scb->sg_count = 1;
1071                         scb->flags |= SCB_SENSE;
1072                         ahc_qinfifo_requeue_tail(ahc, scb);
1073                         ahc_outb(ahc, RETURN_1, SEND_SENSE);
1074                         /*
1075                          * Ensure we have enough time to actually
1076                          * retrieve the sense.
1077                          */
1078                         ahc_scb_timer_reset(scb, 5 * 1000000);
1079                         break;
1080                 }
1081                 default:
1082                         break;
1083                 }
1084                 break;
1085         }
1086         case NO_MATCH:
1087         {
1088                 /* Ensure we don't leave the selection hardware on */
1089                 ahc_outb(ahc, SCSISEQ,
1090                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1091
1092                 printf("%s:%c:%d: no active SCB for reconnecting "
1093                        "target - issuing BUS DEVICE RESET\n",
1094                        ahc_name(ahc), devinfo.channel, devinfo.target);
1095                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1096                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
1097                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1098                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1099                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1100                        "SINDEX == 0x%x\n",
1101                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1102                        ahc_index_busy_tcl(ahc,
1103                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1104                                       ahc_inb(ahc, SAVED_LUN))),
1105                        ahc_inb(ahc, SINDEX));
1106                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1107                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1108                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1109                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1110                        ahc_inb(ahc, SCB_CONTROL));
1111                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1112                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1113                 printf("SXFRCTL0 == 0x%x\n", ahc_inb(ahc, SXFRCTL0));
1114                 printf("SEQCTL == 0x%x\n", ahc_inb(ahc, SEQCTL));
1115                 ahc_dump_card_state(ahc);
1116                 ahc->msgout_buf[0] = MSG_BUS_DEV_RESET;
1117                 ahc->msgout_len = 1;
1118                 ahc->msgout_index = 0;
1119                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
1120                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
1121                 ahc_assert_atn(ahc);
1122                 break;
1123         }
1124         case SEND_REJECT:
1125         {
1126                 u_int rejbyte = ahc_inb(ahc, ACCUM);
1127                 printf("%s:%c:%d: Warning - unknown message received from "
1128                        "target (0x%x).  Rejecting\n",
1129                        ahc_name(ahc), devinfo.channel, devinfo.target, rejbyte);
1130                 break;
1131         }
1132         case PROTO_VIOLATION:
1133         {
1134                 ahc_handle_proto_violation(ahc);
1135                 break;
1136         }
1137         case IGN_WIDE_RES:
1138                 ahc_handle_ign_wide_residue(ahc, &devinfo);
1139                 break;
1140         case PDATA_REINIT:
1141                 ahc_reinitialize_dataptrs(ahc);
1142                 break;
1143         case BAD_PHASE:
1144         {
1145                 u_int lastphase;
1146
1147                 lastphase = ahc_inb(ahc, LASTPHASE);
1148                 printf("%s:%c:%d: unknown scsi bus phase %x, "
1149                        "lastphase = 0x%x.  Attempting to continue\n",
1150                        ahc_name(ahc), devinfo.channel, devinfo.target,
1151                        lastphase, ahc_inb(ahc, SCSISIGI));
1152                 break;
1153         }
1154         case MISSED_BUSFREE:
1155         {
1156                 u_int lastphase;
1157
1158                 lastphase = ahc_inb(ahc, LASTPHASE);
1159                 printf("%s:%c:%d: Missed busfree. "
1160                        "Lastphase = 0x%x, Curphase = 0x%x\n",
1161                        ahc_name(ahc), devinfo.channel, devinfo.target,
1162                        lastphase, ahc_inb(ahc, SCSISIGI));
1163                 ahc_restart(ahc);
1164                 return;
1165         }
1166         case HOST_MSG_LOOP:
1167         {
1168                 /*
1169                  * The sequencer has encountered a message phase
1170                  * that requires host assistance for completion.
1171                  * While handling the message phase(s), we will be
1172                  * notified by the sequencer after each byte is
1173                  * transfered so we can track bus phase changes.
1174                  *
1175                  * If this is the first time we've seen a HOST_MSG_LOOP
1176                  * interrupt, initialize the state of the host message
1177                  * loop.
1178                  */
1179                 if (ahc->msg_type == MSG_TYPE_NONE) {
1180                         struct scb *scb;
1181                         u_int scb_index;
1182                         u_int bus_phase;
1183
1184                         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1185                         if (bus_phase != P_MESGIN
1186                          && bus_phase != P_MESGOUT) {
1187                                 printf("ahc_intr: HOST_MSG_LOOP bad "
1188                                        "phase 0x%x\n",
1189                                       bus_phase);
1190                                 /*
1191                                  * Probably transitioned to bus free before
1192                                  * we got here.  Just punt the message.
1193                                  */
1194                                 ahc_clear_intstat(ahc);
1195                                 ahc_restart(ahc);
1196                                 return;
1197                         }
1198
1199                         scb_index = ahc_inb(ahc, SCB_TAG);
1200                         scb = ahc_lookup_scb(ahc, scb_index);
1201                         if (devinfo.role == ROLE_INITIATOR) {
1202                                 if (bus_phase == P_MESGOUT) {
1203                                         if (scb == NULL)
1204                                                 panic("HOST_MSG_LOOP with "
1205                                                       "invalid SCB %x\n",
1206                                                       scb_index);
1207
1208                                         ahc_setup_initiator_msgout(ahc,
1209                                                                    &devinfo,
1210                                                                    scb);
1211                                 } else {
1212                                         ahc->msg_type =
1213                                             MSG_TYPE_INITIATOR_MSGIN;
1214                                         ahc->msgin_index = 0;
1215                                 }
1216                         }
1217 #ifdef AHC_TARGET_MODE
1218                         else {
1219                                 if (bus_phase == P_MESGOUT) {
1220                                         ahc->msg_type =
1221                                             MSG_TYPE_TARGET_MSGOUT;
1222                                         ahc->msgin_index = 0;
1223                                 }
1224                                 else
1225                                         ahc_setup_target_msgin(ahc,
1226                                                                &devinfo,
1227                                                                scb);
1228                         }
1229 #endif
1230                 }
1231
1232                 ahc_handle_message_phase(ahc);
1233                 break;
1234         }
1235         case PERR_DETECTED:
1236         {
1237                 /*
1238                  * If we've cleared the parity error interrupt
1239                  * but the sequencer still believes that SCSIPERR
1240                  * is true, it must be that the parity error is
1241                  * for the currently presented byte on the bus,
1242                  * and we are not in a phase (data-in) where we will
1243                  * eventually ack this byte.  Ack the byte and
1244                  * throw it away in the hope that the target will
1245                  * take us to message out to deliver the appropriate
1246                  * error message.
1247                  */
1248                 if ((intstat & SCSIINT) == 0
1249                  && (ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0) {
1250
1251                         if ((ahc->features & AHC_DT) == 0) {
1252                                 u_int curphase;
1253
1254                                 /*
1255                                  * The hardware will only let you ack bytes
1256                                  * if the expected phase in SCSISIGO matches
1257                                  * the current phase.  Make sure this is
1258                                  * currently the case.
1259                                  */
1260                                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1261                                 ahc_outb(ahc, LASTPHASE, curphase);
1262                                 ahc_outb(ahc, SCSISIGO, curphase);
1263                         }
1264                         if ((ahc_inb(ahc, SCSISIGI) & (CDI|MSGI)) == 0) {
1265                                 int wait;
1266
1267                                 /*
1268                                  * In a data phase.  Faster to bitbucket
1269                                  * the data than to individually ack each
1270                                  * byte.  This is also the only strategy
1271                                  * that will work with AUTOACK enabled.
1272                                  */
1273                                 ahc_outb(ahc, SXFRCTL1,
1274                                          ahc_inb(ahc, SXFRCTL1) | BITBUCKET);
1275                                 wait = 5000;
1276                                 while (--wait != 0) {
1277                                         if ((ahc_inb(ahc, SCSISIGI)
1278                                           & (CDI|MSGI)) != 0)
1279                                                 break;
1280                                         ahc_delay(100);
1281                                 }
1282                                 ahc_outb(ahc, SXFRCTL1,
1283                                          ahc_inb(ahc, SXFRCTL1) & ~BITBUCKET);
1284                                 if (wait == 0) {
1285                                         struct  scb *scb;
1286                                         u_int   scb_index;
1287
1288                                         ahc_print_devinfo(ahc, &devinfo);
1289                                         printf("Unable to clear parity error.  "
1290                                                "Resetting bus.\n");
1291                                         scb_index = ahc_inb(ahc, SCB_TAG);
1292                                         scb = ahc_lookup_scb(ahc, scb_index);
1293                                         if (scb != NULL)
1294                                                 ahc_set_transaction_status(scb,
1295                                                     CAM_UNCOR_PARITY);
1296                                         ahc_reset_channel(ahc, devinfo.channel,
1297                                                           /*init reset*/TRUE);
1298                                 }
1299                         } else {
1300                                 ahc_inb(ahc, SCSIDATL);
1301                         }
1302                 }
1303                 break;
1304         }
1305         case DATA_OVERRUN:
1306         {
1307                 /*
1308                  * When the sequencer detects an overrun, it
1309                  * places the controller in "BITBUCKET" mode
1310                  * and allows the target to complete its transfer.
1311                  * Unfortunately, none of the counters get updated
1312                  * when the controller is in this mode, so we have
1313                  * no way of knowing how large the overrun was.
1314                  */
1315                 u_int scbindex = ahc_inb(ahc, SCB_TAG);
1316                 u_int lastphase = ahc_inb(ahc, LASTPHASE);
1317                 u_int i;
1318
1319                 scb = ahc_lookup_scb(ahc, scbindex);
1320                 for (i = 0; i < num_phases; i++) {
1321                         if (lastphase == ahc_phase_table[i].phase)
1322                                 break;
1323                 }
1324                 ahc_print_path(ahc, scb);
1325                 printf("data overrun detected %s."
1326                        "  Tag == 0x%x.\n",
1327                        ahc_phase_table[i].phasemsg,
1328                        scb->hscb->tag);
1329                 ahc_print_path(ahc, scb);
1330                 printf("%s seen Data Phase.  Length = %ld.  NumSGs = %d.\n",
1331                        ahc_inb(ahc, SEQ_FLAGS) & DPHASE ? "Have" : "Haven't",
1332                        ahc_get_transfer_length(scb), scb->sg_count);
1333                 if (scb->sg_count > 0) {
1334                         for (i = 0; i < scb->sg_count; i++) {
1335
1336                                 printf("sg[%d] - Addr 0x%x%x : Length %d\n",
1337                                        i,
1338                                        (ahc_le32toh(scb->sg_list[i].len) >> 24
1339                                         & SG_HIGH_ADDR_BITS),
1340                                        ahc_le32toh(scb->sg_list[i].addr),
1341                                        ahc_le32toh(scb->sg_list[i].len)
1342                                        & AHC_SG_LEN_MASK);
1343                         }
1344                 }
1345                 /*
1346                  * Set this and it will take effect when the
1347                  * target does a command complete.
1348                  */
1349                 ahc_freeze_devq(ahc, scb);
1350                 if ((scb->flags & SCB_SENSE) == 0) {
1351                         ahc_set_transaction_status(scb, CAM_DATA_RUN_ERR);
1352                 } else {
1353                         scb->flags &= ~SCB_SENSE;
1354                         ahc_set_transaction_status(scb, CAM_AUTOSENSE_FAIL);
1355                 }
1356                 ahc_freeze_scb(scb);
1357
1358                 if ((ahc->features & AHC_ULTRA2) != 0) {
1359                         /*
1360                          * Clear the channel in case we return
1361                          * to data phase later.
1362                          */
1363                         ahc_outb(ahc, SXFRCTL0,
1364                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1365                         ahc_outb(ahc, SXFRCTL0,
1366                                  ahc_inb(ahc, SXFRCTL0) | CLRSTCNT|CLRCHN);
1367                 }
1368                 if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
1369                         u_int dscommand1;
1370
1371                         /* Ensure HHADDR is 0 for future DMA operations. */
1372                         dscommand1 = ahc_inb(ahc, DSCOMMAND1);
1373                         ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
1374                         ahc_outb(ahc, HADDR, 0);
1375                         ahc_outb(ahc, DSCOMMAND1, dscommand1);
1376                 }
1377                 break;
1378         }
1379         case MKMSG_FAILED:
1380         {
1381                 u_int scbindex;
1382
1383                 printf("%s:%c:%d:%d: Attempt to issue message failed\n",
1384                        ahc_name(ahc), devinfo.channel, devinfo.target,
1385                        devinfo.lun);
1386                 scbindex = ahc_inb(ahc, SCB_TAG);
1387                 scb = ahc_lookup_scb(ahc, scbindex);
1388                 if (scb != NULL
1389                  && (scb->flags & SCB_RECOVERY_SCB) != 0)
1390                         /*
1391                          * Ensure that we didn't put a second instance of this
1392                          * SCB into the QINFIFO.
1393                          */
1394                         ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
1395                                            SCB_GET_CHANNEL(ahc, scb),
1396                                            SCB_GET_LUN(scb), scb->hscb->tag,
1397                                            ROLE_INITIATOR, /*status*/0,
1398                                            SEARCH_REMOVE);
1399                 break;
1400         }
1401         case NO_FREE_SCB:
1402         {
1403                 printf("%s: No free or disconnected SCBs\n", ahc_name(ahc));
1404                 ahc_dump_card_state(ahc);
1405                 panic("for safety");
1406                 break;
1407         }
1408         case SCB_MISMATCH:
1409         {
1410                 u_int scbptr;
1411
1412                 scbptr = ahc_inb(ahc, SCBPTR);
1413                 printf("Bogus TAG after DMA.  SCBPTR %d, tag %d, our tag %d\n",
1414                        scbptr, ahc_inb(ahc, ARG_1),
1415                        ahc->scb_data->hscbs[scbptr].tag);
1416                 ahc_dump_card_state(ahc);
1417                 panic("for saftey");
1418                 break;
1419         }
1420         case OUT_OF_RANGE:
1421         {
1422                 printf("%s: BTT calculation out of range\n", ahc_name(ahc));
1423                 printf("SAVED_SCSIID == 0x%x, SAVED_LUN == 0x%x, "
1424                        "ARG_1 == 0x%x ACCUM = 0x%x\n",
1425                        ahc_inb(ahc, SAVED_SCSIID), ahc_inb(ahc, SAVED_LUN),
1426                        ahc_inb(ahc, ARG_1), ahc_inb(ahc, ACCUM));
1427                 printf("SEQ_FLAGS == 0x%x, SCBPTR == 0x%x, BTT == 0x%x, "
1428                        "SINDEX == 0x%x\n, A == 0x%x\n",
1429                        ahc_inb(ahc, SEQ_FLAGS), ahc_inb(ahc, SCBPTR),
1430                        ahc_index_busy_tcl(ahc,
1431                             BUILD_TCL(ahc_inb(ahc, SAVED_SCSIID),
1432                                       ahc_inb(ahc, SAVED_LUN))),
1433                        ahc_inb(ahc, SINDEX),
1434                        ahc_inb(ahc, ACCUM));
1435                 printf("SCSIID == 0x%x, SCB_SCSIID == 0x%x, SCB_LUN == 0x%x, "
1436                        "SCB_TAG == 0x%x, SCB_CONTROL == 0x%x\n",
1437                        ahc_inb(ahc, SCSIID), ahc_inb(ahc, SCB_SCSIID),
1438                        ahc_inb(ahc, SCB_LUN), ahc_inb(ahc, SCB_TAG),
1439                        ahc_inb(ahc, SCB_CONTROL));
1440                 printf("SCSIBUSL == 0x%x, SCSISIGI == 0x%x\n",
1441                        ahc_inb(ahc, SCSIBUSL), ahc_inb(ahc, SCSISIGI));
1442                 ahc_dump_card_state(ahc);
1443                 panic("for safety");
1444                 break;
1445         }
1446         default:
1447                 printf("ahc_intr: seqint, "
1448                        "intstat == 0x%x, scsisigi = 0x%x\n",
1449                        intstat, ahc_inb(ahc, SCSISIGI));
1450                 break;
1451         }
1452 unpause:
1453         /*
1454          *  The sequencer is paused immediately on
1455          *  a SEQINT, so we should restart it when
1456          *  we're done.
1457          */
1458         ahc_unpause(ahc);
1459 }
1460
1461 void
1462 ahc_handle_scsiint(struct ahc_softc *ahc, u_int intstat)
1463 {
1464         u_int   scb_index;
1465         u_int   status0;
1466         u_int   status;
1467         struct  scb *scb;
1468         char    cur_channel;
1469         char    intr_channel;
1470
1471         if ((ahc->features & AHC_TWIN) != 0
1472          && ((ahc_inb(ahc, SBLKCTL) & SELBUSB) != 0))
1473                 cur_channel = 'B';
1474         else
1475                 cur_channel = 'A';
1476         intr_channel = cur_channel;
1477
1478         if ((ahc->features & AHC_ULTRA2) != 0)
1479                 status0 = ahc_inb(ahc, SSTAT0) & IOERR;
1480         else
1481                 status0 = 0;
1482         status = ahc_inb(ahc, SSTAT1) & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1483         if (status == 0 && status0 == 0) {
1484                 if ((ahc->features & AHC_TWIN) != 0) {
1485                         /* Try the other channel */
1486                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1487                         status = ahc_inb(ahc, SSTAT1)
1488                                & (SELTO|SCSIRSTI|BUSFREE|SCSIPERR);
1489                         intr_channel = (cur_channel == 'A') ? 'B' : 'A';
1490                 }
1491                 if (status == 0) {
1492                         printf("%s: Spurious SCSI interrupt\n", ahc_name(ahc));
1493                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1494                         ahc_unpause(ahc);
1495                         return;
1496                 }
1497         }
1498
1499         /* Make sure the sequencer is in a safe location. */
1500         ahc_clear_critical_section(ahc);
1501
1502         scb_index = ahc_inb(ahc, SCB_TAG);
1503         scb = ahc_lookup_scb(ahc, scb_index);
1504         if (scb != NULL
1505          && (ahc_inb(ahc, SEQ_FLAGS) & NOT_IDENTIFIED) != 0)
1506                 scb = NULL;
1507
1508         if ((ahc->features & AHC_ULTRA2) != 0
1509          && (status0 & IOERR) != 0) {
1510                 int now_lvd;
1511
1512                 now_lvd = ahc_inb(ahc, SBLKCTL) & ENAB40;
1513                 printf("%s: Transceiver State Has Changed to %s mode\n",
1514                        ahc_name(ahc), now_lvd ? "LVD" : "SE");
1515                 ahc_outb(ahc, CLRSINT0, CLRIOERR);
1516                 /*
1517                  * When transitioning to SE mode, the reset line
1518                  * glitches, triggering an arbitration bug in some
1519                  * Ultra2 controllers.  This bug is cleared when we
1520                  * assert the reset line.  Since a reset glitch has
1521                  * already occurred with this transition and a
1522                  * transceiver state change is handled just like
1523                  * a bus reset anyway, asserting the reset line
1524                  * ourselves is safe.
1525                  */
1526                 ahc_reset_channel(ahc, intr_channel,
1527                                  /*Initiate Reset*/now_lvd == 0);
1528         } else if ((status & SCSIRSTI) != 0) {
1529                 printf("%s: Someone reset channel %c\n",
1530                         ahc_name(ahc), intr_channel);
1531                 if (intr_channel != cur_channel)
1532                         ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) ^ SELBUSB);
1533                 ahc_reset_channel(ahc, intr_channel, /*Initiate Reset*/FALSE);
1534         } else if ((status & SCSIPERR) != 0) {
1535                 /*
1536                  * Determine the bus phase and queue an appropriate message.
1537                  * SCSIPERR is latched true as soon as a parity error
1538                  * occurs.  If the sequencer acked the transfer that
1539                  * caused the parity error and the currently presented
1540                  * transfer on the bus has correct parity, SCSIPERR will
1541                  * be cleared by CLRSCSIPERR.  Use this to determine if
1542                  * we should look at the last phase the sequencer recorded,
1543                  * or the current phase presented on the bus.
1544                  */
1545                 struct  ahc_devinfo devinfo;
1546                 u_int   mesg_out;
1547                 u_int   curphase;
1548                 u_int   errorphase;
1549                 u_int   lastphase;
1550                 u_int   scsirate;
1551                 u_int   i;
1552                 u_int   sstat2;
1553                 int     silent;
1554
1555                 lastphase = ahc_inb(ahc, LASTPHASE);
1556                 curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
1557                 sstat2 = ahc_inb(ahc, SSTAT2);
1558                 ahc_outb(ahc, CLRSINT1, CLRSCSIPERR);
1559                 /*
1560                  * For all phases save DATA, the sequencer won't
1561                  * automatically ack a byte that has a parity error
1562                  * in it.  So the only way that the current phase
1563                  * could be 'data-in' is if the parity error is for
1564                  * an already acked byte in the data phase.  During
1565                  * synchronous data-in transfers, we may actually
1566                  * ack bytes before latching the current phase in
1567                  * LASTPHASE, leading to the discrepancy between
1568                  * curphase and lastphase.
1569                  */
1570                 if ((ahc_inb(ahc, SSTAT1) & SCSIPERR) != 0
1571                  || curphase == P_DATAIN || curphase == P_DATAIN_DT)
1572                         errorphase = curphase;
1573                 else
1574                         errorphase = lastphase;
1575
1576                 for (i = 0; i < num_phases; i++) {
1577                         if (errorphase == ahc_phase_table[i].phase)
1578                                 break;
1579                 }
1580                 mesg_out = ahc_phase_table[i].mesg_out;
1581                 silent = FALSE;
1582                 if (scb != NULL) {
1583                         if (SCB_IS_SILENT(scb))
1584                                 silent = TRUE;
1585                         else
1586                                 ahc_print_path(ahc, scb);
1587                         scb->flags |= SCB_TRANSMISSION_ERROR;
1588                 } else
1589                         printf("%s:%c:%d: ", ahc_name(ahc), intr_channel,
1590                                SCSIID_TARGET(ahc, ahc_inb(ahc, SAVED_SCSIID)));
1591                 scsirate = ahc_inb(ahc, SCSIRATE);
1592                 if (silent == FALSE) {
1593                         printf("parity error detected %s. "
1594                                "SEQADDR(0x%x) SCSIRATE(0x%x)\n",
1595                                ahc_phase_table[i].phasemsg,
1596                                ahc_inw(ahc, SEQADDR0),
1597                                scsirate);
1598                         if ((ahc->features & AHC_DT) != 0) {
1599                                 if ((sstat2 & CRCVALERR) != 0)
1600                                         printf("\tCRC Value Mismatch\n");
1601                                 if ((sstat2 & CRCENDERR) != 0)
1602                                         printf("\tNo terminal CRC packet "
1603                                                "recevied\n");
1604                                 if ((sstat2 & CRCREQERR) != 0)
1605                                         printf("\tIllegal CRC packet "
1606                                                "request\n");
1607                                 if ((sstat2 & DUAL_EDGE_ERR) != 0)
1608                                         printf("\tUnexpected %sDT Data Phase\n",
1609                                                (scsirate & SINGLE_EDGE)
1610                                              ? "" : "non-");
1611                         }
1612                 }
1613
1614                 if ((ahc->features & AHC_DT) != 0
1615                  && (sstat2 & DUAL_EDGE_ERR) != 0) {
1616                         /*
1617                          * This error applies regardless of
1618                          * data direction, so ignore the value
1619                          * in the phase table.
1620                          */
1621                         mesg_out = MSG_INITIATOR_DET_ERR;
1622                 }
1623
1624                 /*
1625                  * We've set the hardware to assert ATN if we
1626                  * get a parity error on "in" phases, so all we
1627                  * need to do is stuff the message buffer with
1628                  * the appropriate message.  "In" phases have set
1629                  * mesg_out to something other than MSG_NOP.
1630                  */
1631                 if (mesg_out != MSG_NOOP) {
1632                         if (ahc->msg_type != MSG_TYPE_NONE)
1633                                 ahc->send_msg_perror = TRUE;
1634                         else
1635                                 ahc_outb(ahc, MSG_OUT, mesg_out);
1636                 }
1637                 /*
1638                  * Force a renegotiation with this target just in
1639                  * case we are out of sync for some external reason
1640                  * unknown (or unreported) by the target.
1641                  */
1642                 ahc_fetch_devinfo(ahc, &devinfo);
1643                 ahc_force_renegotiation(ahc, &devinfo);
1644
1645                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1646                 ahc_unpause(ahc);
1647         } else if ((status & SELTO) != 0) {
1648                 u_int   scbptr;
1649
1650                 /* Stop the selection */
1651                 ahc_outb(ahc, SCSISEQ, 0);
1652
1653                 /* No more pending messages */
1654                 ahc_clear_msg_state(ahc);
1655
1656                 /* Clear interrupt state */
1657                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1658                 ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRBUSFREE|CLRSCSIPERR);
1659
1660                 /*
1661                  * Although the driver does not care about the
1662                  * 'Selection in Progress' status bit, the busy
1663                  * LED does.  SELINGO is only cleared by a sucessfull
1664                  * selection, so we must manually clear it to insure
1665                  * the LED turns off just incase no future successful
1666                  * selections occur (e.g. no devices on the bus).
1667                  */
1668                 ahc_outb(ahc, CLRSINT0, CLRSELINGO);
1669
1670                 scbptr = ahc_inb(ahc, WAITING_SCBH);
1671                 ahc_outb(ahc, SCBPTR, scbptr);
1672                 scb_index = ahc_inb(ahc, SCB_TAG);
1673
1674                 scb = ahc_lookup_scb(ahc, scb_index);
1675                 if (scb == NULL) {
1676                         printf("%s: ahc_intr - referenced scb not "
1677                                "valid during SELTO scb(%d, %d)\n",
1678                                ahc_name(ahc), scbptr, scb_index);
1679                         ahc_dump_card_state(ahc);
1680                 } else {
1681                         struct ahc_devinfo devinfo;
1682 #ifdef AHC_DEBUG
1683                         if ((ahc_debug & AHC_SHOW_SELTO) != 0) {
1684                                 ahc_print_path(ahc, scb);
1685                                 printf("Saw Selection Timeout for SCB 0x%x\n",
1686                                        scb_index);
1687                         }
1688 #endif
1689                         ahc_scb_devinfo(ahc, &devinfo, scb);
1690                         ahc_set_transaction_status(scb, CAM_SEL_TIMEOUT);
1691                         ahc_freeze_devq(ahc, scb);
1692
1693                         /*
1694                          * Cancel any pending transactions on the device
1695                          * now that it seems to be missing.  This will
1696                          * also revert us to async/narrow transfers until
1697                          * we can renegotiate with the device.
1698                          */
1699                         ahc_handle_devreset(ahc, &devinfo,
1700                                             CAM_SEL_TIMEOUT,
1701                                             "Selection Timeout",
1702                                             /*verbose_level*/1);
1703                 }
1704                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1705                 ahc_restart(ahc);
1706         } else if ((status & BUSFREE) != 0
1707                 && (ahc_inb(ahc, SIMODE1) & ENBUSFREE) != 0) {
1708                 struct  ahc_devinfo devinfo;
1709                 u_int   lastphase;
1710                 u_int   saved_scsiid;
1711                 u_int   saved_lun;
1712                 u_int   target;
1713                 u_int   initiator_role_id;
1714                 char    channel;
1715                 int     printerror;
1716
1717                 /*
1718                  * Clear our selection hardware as soon as possible.
1719                  * We may have an entry in the waiting Q for this target,
1720                  * that is affected by this busfree and we don't want to
1721                  * go about selecting the target while we handle the event.
1722                  */
1723                 ahc_outb(ahc, SCSISEQ,
1724                          ahc_inb(ahc, SCSISEQ) & (ENSELI|ENRSELI|ENAUTOATNP));
1725
1726                 /*
1727                  * Disable busfree interrupts and clear the busfree
1728                  * interrupt status.  We do this here so that several
1729                  * bus transactions occur prior to clearing the SCSIINT
1730                  * latch.  It can take a bit for the clearing to take effect.
1731                  */
1732                 ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENBUSFREE);
1733                 ahc_outb(ahc, CLRSINT1, CLRBUSFREE|CLRSCSIPERR);
1734
1735                 /*
1736                  * Look at what phase we were last in.
1737                  * If its message out, chances are pretty good
1738                  * that the busfree was in response to one of
1739                  * our abort requests.
1740                  */
1741                 lastphase = ahc_inb(ahc, LASTPHASE);
1742                 saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
1743                 saved_lun = ahc_inb(ahc, SAVED_LUN);
1744                 target = SCSIID_TARGET(ahc, saved_scsiid);
1745                 initiator_role_id = SCSIID_OUR_ID(saved_scsiid);
1746                 channel = SCSIID_CHANNEL(ahc, saved_scsiid);
1747                 ahc_compile_devinfo(&devinfo, initiator_role_id,
1748                                     target, saved_lun, channel, ROLE_INITIATOR);
1749                 printerror = 1;
1750
1751                 if (lastphase == P_MESGOUT) {
1752                         u_int tag;
1753
1754                         tag = SCB_LIST_NULL;
1755                         if (ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT_TAG, TRUE)
1756                          || ahc_sent_msg(ahc, AHCMSG_1B, MSG_ABORT, TRUE)) {
1757                                 if (ahc->msgout_buf[ahc->msgout_index - 1]
1758                                  == MSG_ABORT_TAG)
1759                                         tag = scb->hscb->tag;
1760                                 ahc_print_path(ahc, scb);
1761                                 printf("SCB %d - Abort%s Completed.\n",
1762                                        scb->hscb->tag, tag == SCB_LIST_NULL ?
1763                                        "" : " Tag");
1764                                 ahc_abort_scbs(ahc, target, channel,
1765                                                saved_lun, tag,
1766                                                ROLE_INITIATOR,
1767                                                CAM_REQ_ABORTED);
1768                                 printerror = 0;
1769                         } else if (ahc_sent_msg(ahc, AHCMSG_1B,
1770                                                 MSG_BUS_DEV_RESET, TRUE)) {
1771 #ifdef __FreeBSD__
1772                                 /*
1773                                  * Don't mark the user's request for this BDR
1774                                  * as completing with CAM_BDR_SENT.  CAM3
1775                                  * specifies CAM_REQ_CMP.
1776                                  */
1777                                 if (scb != NULL
1778                                  && scb->io_ctx->ccb_h.func_code== XPT_RESET_DEV
1779                                  && ahc_match_scb(ahc, scb, target, channel,
1780                                                   CAM_LUN_WILDCARD,
1781                                                   SCB_LIST_NULL,
1782                                                   ROLE_INITIATOR)) {
1783                                         ahc_set_transaction_status(scb, CAM_REQ_CMP);
1784                                 }
1785 #endif
1786                                 ahc_compile_devinfo(&devinfo,
1787                                                     initiator_role_id,
1788                                                     target,
1789                                                     CAM_LUN_WILDCARD,
1790                                                     channel,
1791                                                     ROLE_INITIATOR);
1792                                 ahc_handle_devreset(ahc, &devinfo,
1793                                                     CAM_BDR_SENT,
1794                                                     "Bus Device Reset",
1795                                                     /*verbose_level*/0);
1796                                 printerror = 0;
1797                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1798                                                 MSG_EXT_PPR, FALSE)) {
1799                                 struct ahc_initiator_tinfo *tinfo;
1800                                 struct ahc_tmode_tstate *tstate;
1801
1802                                 /*
1803                                  * PPR Rejected.  Try non-ppr negotiation
1804                                  * and retry command.
1805                                  */
1806                                 tinfo = ahc_fetch_transinfo(ahc,
1807                                                             devinfo.channel,
1808                                                             devinfo.our_scsiid,
1809                                                             devinfo.target,
1810                                                             &tstate);
1811                                 tinfo->curr.transport_version = 2;
1812                                 tinfo->goal.transport_version = 2;
1813                                 tinfo->goal.ppr_options = 0;
1814                                 ahc_qinfifo_requeue_tail(ahc, scb);
1815                                 printerror = 0;
1816                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1817                                                 MSG_EXT_WDTR, FALSE)) {
1818                                 /*
1819                                  * Negotiation Rejected.  Go-narrow and
1820                                  * retry command.
1821                                  */
1822                                 ahc_set_width(ahc, &devinfo,
1823                                               MSG_EXT_WDTR_BUS_8_BIT,
1824                                               AHC_TRANS_CUR|AHC_TRANS_GOAL,
1825                                               /*paused*/TRUE);
1826                                 ahc_qinfifo_requeue_tail(ahc, scb);
1827                                 printerror = 0;
1828                         } else if (ahc_sent_msg(ahc, AHCMSG_EXT,
1829                                                 MSG_EXT_SDTR, FALSE)) {
1830                                 /*
1831                                  * Negotiation Rejected.  Go-async and
1832                                  * retry command.
1833                                  */
1834                                 ahc_set_syncrate(ahc, &devinfo,
1835                                                 /*syncrate*/NULL,
1836                                                 /*period*/0, /*offset*/0,
1837                                                 /*ppr_options*/0,
1838                                                 AHC_TRANS_CUR|AHC_TRANS_GOAL,
1839                                                 /*paused*/TRUE);
1840                                 ahc_qinfifo_requeue_tail(ahc, scb);
1841                                 printerror = 0;
1842                         }
1843                 }
1844                 if (printerror != 0) {
1845                         u_int i;
1846
1847                         if (scb != NULL) {
1848                                 u_int tag;
1849
1850                                 if ((scb->hscb->control & TAG_ENB) != 0)
1851                                         tag = scb->hscb->tag;
1852                                 else
1853                                         tag = SCB_LIST_NULL;
1854                                 ahc_print_path(ahc, scb);
1855                                 ahc_abort_scbs(ahc, target, channel,
1856                                                SCB_GET_LUN(scb), tag,
1857                                                ROLE_INITIATOR,
1858                                                CAM_UNEXP_BUSFREE);
1859                         } else {
1860                                 /*
1861                                  * We had not fully identified this connection,
1862                                  * so we cannot abort anything.
1863                                  */
1864                                 printf("%s: ", ahc_name(ahc));
1865                         }
1866                         for (i = 0; i < num_phases; i++) {
1867                                 if (lastphase == ahc_phase_table[i].phase)
1868                                         break;
1869                         }
1870                         if (lastphase != P_BUSFREE) {
1871                                 /*
1872                                  * Renegotiate with this device at the
1873                                  * next oportunity just in case this busfree
1874                                  * is due to a negotiation mismatch with the
1875                                  * device.
1876                                  */
1877                                 ahc_force_renegotiation(ahc, &devinfo);
1878                         }
1879                         printf("Unexpected busfree %s\n"
1880                                "SEQADDR == 0x%x\n",
1881                                ahc_phase_table[i].phasemsg,
1882                                ahc_inb(ahc, SEQADDR0)
1883                                 | (ahc_inb(ahc, SEQADDR1) << 8));
1884                 }
1885                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1886                 ahc_restart(ahc);
1887         } else {
1888                 printf("%s: Missing case in ahc_handle_scsiint. status = %x\n",
1889                        ahc_name(ahc), status);
1890                 ahc_outb(ahc, CLRINT, CLRSCSIINT);
1891         }
1892 }
1893
1894 /*
1895  * Force renegotiation to occur the next time we initiate
1896  * a command to the current device.
1897  */
1898 static void
1899 ahc_force_renegotiation(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
1900 {
1901         struct  ahc_initiator_tinfo *targ_info;
1902         struct  ahc_tmode_tstate *tstate;
1903
1904         targ_info = ahc_fetch_transinfo(ahc,
1905                                         devinfo->channel,
1906                                         devinfo->our_scsiid,
1907                                         devinfo->target,
1908                                         &tstate);
1909         ahc_update_neg_request(ahc, devinfo, tstate,
1910                                targ_info, AHC_NEG_IF_NON_ASYNC);
1911 }
1912
1913 #define AHC_MAX_STEPS 2000
1914 void
1915 ahc_clear_critical_section(struct ahc_softc *ahc)
1916 {
1917         int     stepping;
1918         int     steps;
1919         u_int   simode0;
1920         u_int   simode1;
1921
1922         if (ahc->num_critical_sections == 0)
1923                 return;
1924
1925         stepping = FALSE;
1926         steps = 0;
1927         simode0 = 0;
1928         simode1 = 0;
1929         for (;;) {
1930                 struct  cs *cs;
1931                 u_int   seqaddr;
1932                 u_int   i;
1933
1934                 seqaddr = ahc_inb(ahc, SEQADDR0)
1935                         | (ahc_inb(ahc, SEQADDR1) << 8);
1936
1937                 /*
1938                  * Seqaddr represents the next instruction to execute,
1939                  * so we are really executing the instruction just
1940                  * before it.
1941                  */
1942                 if (seqaddr != 0)
1943                         seqaddr -= 1;
1944                 cs = ahc->critical_sections;
1945                 for (i = 0; i < ahc->num_critical_sections; i++, cs++) {
1946
1947                         if (cs->begin < seqaddr && cs->end >= seqaddr)
1948                                 break;
1949                 }
1950
1951                 if (i == ahc->num_critical_sections)
1952                         break;
1953
1954                 if (steps > AHC_MAX_STEPS) {
1955                         printf("%s: Infinite loop in critical section\n",
1956                                ahc_name(ahc));
1957                         ahc_dump_card_state(ahc);
1958                         panic("critical section loop");
1959                 }
1960
1961                 steps++;
1962                 if (stepping == FALSE) {
1963
1964                         /*
1965                          * Disable all interrupt sources so that the
1966                          * sequencer will not be stuck by a pausing
1967                          * interrupt condition while we attempt to
1968                          * leave a critical section.
1969                          */
1970                         simode0 = ahc_inb(ahc, SIMODE0);
1971                         ahc_outb(ahc, SIMODE0, 0);
1972                         simode1 = ahc_inb(ahc, SIMODE1);
1973                         if ((ahc->features & AHC_DT) != 0)
1974                                 /*
1975                                  * On DT class controllers, we
1976                                  * use the enhanced busfree logic.
1977                                  * Unfortunately we cannot re-enable
1978                                  * busfree detection within the
1979                                  * current connection, so we must
1980                                  * leave it on while single stepping.
1981                                  */
1982                                 ahc_outb(ahc, SIMODE1, simode1 & ENBUSFREE);
1983                         else
1984                                 ahc_outb(ahc, SIMODE1, 0);
1985                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1986                         ahc_outb(ahc, SEQCTL, ahc->seqctl | STEP);
1987                         stepping = TRUE;
1988                 }
1989                 if ((ahc->features & AHC_DT) != 0) {
1990                         ahc_outb(ahc, CLRSINT1, CLRBUSFREE);
1991                         ahc_outb(ahc, CLRINT, CLRSCSIINT);
1992                 }
1993                 ahc_outb(ahc, HCNTRL, ahc->unpause);
1994                 while (!ahc_is_paused(ahc))
1995                         ahc_delay(200);
1996         }
1997         if (stepping) {
1998                 ahc_outb(ahc, SIMODE0, simode0);
1999                 ahc_outb(ahc, SIMODE1, simode1);
2000                 ahc_outb(ahc, SEQCTL, ahc->seqctl);
2001         }
2002 }
2003
2004 /*
2005  * Clear any pending interrupt status.
2006  */
2007 void
2008 ahc_clear_intstat(struct ahc_softc *ahc)
2009 {
2010         /* Clear any interrupt conditions this may have caused */
2011         ahc_outb(ahc, CLRSINT1, CLRSELTIMEO|CLRATNO|CLRSCSIRSTI
2012                                 |CLRBUSFREE|CLRSCSIPERR|CLRPHASECHG|
2013                                 CLRREQINIT);
2014         ahc_flush_device_writes(ahc);
2015         ahc_outb(ahc, CLRSINT0, CLRSELDO|CLRSELDI|CLRSELINGO);
2016         ahc_flush_device_writes(ahc);
2017         ahc_outb(ahc, CLRINT, CLRSCSIINT);
2018         ahc_flush_device_writes(ahc);
2019 }
2020
2021 /**************************** Debugging Routines ******************************/
2022 #ifdef AHC_DEBUG
2023 uint32_t ahc_debug = AHC_DEBUG_OPTS;
2024 #endif
2025
2026 void
2027 ahc_print_scb(struct scb *scb)
2028 {
2029         int i;
2030
2031         struct hardware_scb *hscb = scb->hscb;
2032
2033         printf("scb:%p control:0x%x scsiid:0x%x lun:%d cdb_len:%d\n",
2034                (void *)scb,
2035                hscb->control,
2036                hscb->scsiid,
2037                hscb->lun,
2038                hscb->cdb_len);
2039         printf("Shared Data: ");
2040         for (i = 0; i < sizeof(hscb->shared_data.cdb); i++)
2041                 printf("%#02x", hscb->shared_data.cdb[i]);
2042         printf("        dataptr:%#x datacnt:%#x sgptr:%#x tag:%#x\n",
2043                 ahc_le32toh(hscb->dataptr),
2044                 ahc_le32toh(hscb->datacnt),
2045                 ahc_le32toh(hscb->sgptr),
2046                 hscb->tag);
2047         if (scb->sg_count > 0) {
2048                 for (i = 0; i < scb->sg_count; i++) {
2049                         printf("sg[%d] - Addr 0x%x%x : Length %d\n",
2050                                i,
2051                                (ahc_le32toh(scb->sg_list[i].len) >> 24
2052                                 & SG_HIGH_ADDR_BITS),
2053                                ahc_le32toh(scb->sg_list[i].addr),
2054                                ahc_le32toh(scb->sg_list[i].len));
2055                 }
2056         }
2057 }
2058
2059 /************************* Transfer Negotiation *******************************/
2060 /*
2061  * Allocate per target mode instance (ID we respond to as a target)
2062  * transfer negotiation data structures.
2063  */
2064 static struct ahc_tmode_tstate *
2065 ahc_alloc_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel)
2066 {
2067         struct ahc_tmode_tstate *master_tstate;
2068         struct ahc_tmode_tstate *tstate;
2069         int i;
2070
2071         master_tstate = ahc->enabled_targets[ahc->our_id];
2072         if (channel == 'B') {
2073                 scsi_id += 8;
2074                 master_tstate = ahc->enabled_targets[ahc->our_id_b + 8];
2075         }
2076         if (ahc->enabled_targets[scsi_id] != NULL
2077          && ahc->enabled_targets[scsi_id] != master_tstate)
2078                 panic("%s: ahc_alloc_tstate - Target already allocated",
2079                       ahc_name(ahc));
2080         tstate = (struct ahc_tmode_tstate*)malloc(sizeof(*tstate),
2081                                                    M_DEVBUF, M_NOWAIT);
2082         if (tstate == NULL)
2083                 return (NULL);
2084
2085         /*
2086          * If we have allocated a master tstate, copy user settings from
2087          * the master tstate (taken from SRAM or the EEPROM) for this
2088          * channel, but reset our current and goal settings to async/narrow
2089          * until an initiator talks to us.
2090          */
2091         if (master_tstate != NULL) {
2092                 memcpy(tstate, master_tstate, sizeof(*tstate));
2093                 memset(tstate->enabled_luns, 0, sizeof(tstate->enabled_luns));
2094                 tstate->ultraenb = 0;
2095                 for (i = 0; i < AHC_NUM_TARGETS; i++) {
2096                         memset(&tstate->transinfo[i].curr, 0,
2097                               sizeof(tstate->transinfo[i].curr));
2098                         memset(&tstate->transinfo[i].goal, 0,
2099                               sizeof(tstate->transinfo[i].goal));
2100                 }
2101         } else
2102                 memset(tstate, 0, sizeof(*tstate));
2103         ahc->enabled_targets[scsi_id] = tstate;
2104         return (tstate);
2105 }
2106
2107 #ifdef AHC_TARGET_MODE
2108 /*
2109  * Free per target mode instance (ID we respond to as a target)
2110  * transfer negotiation data structures.
2111  */
2112 static void
2113 ahc_free_tstate(struct ahc_softc *ahc, u_int scsi_id, char channel, int force)
2114 {
2115         struct ahc_tmode_tstate *tstate;
2116
2117         /*
2118          * Don't clean up our "master" tstate.
2119          * It has our default user settings.
2120          */
2121         if (((channel == 'B' && scsi_id == ahc->our_id_b)
2122           || (channel == 'A' && scsi_id == ahc->our_id))
2123          && force == FALSE)
2124                 return;
2125
2126         if (channel == 'B')
2127                 scsi_id += 8;
2128         tstate = ahc->enabled_targets[scsi_id];
2129         if (tstate != NULL)
2130                 free(tstate, M_DEVBUF);
2131         ahc->enabled_targets[scsi_id] = NULL;
2132 }
2133 #endif
2134
2135 /*
2136  * Called when we have an active connection to a target on the bus,
2137  * this function finds the nearest syncrate to the input period limited
2138  * by the capabilities of the bus connectivity of and sync settings for
2139  * the target.
2140  */
2141 struct ahc_syncrate *
2142 ahc_devlimited_syncrate(struct ahc_softc *ahc,
2143                         struct ahc_initiator_tinfo *tinfo,
2144                         u_int *period, u_int *ppr_options, role_t role)
2145 {
2146         struct  ahc_transinfo *transinfo;
2147         u_int   maxsync;
2148
2149         if ((ahc->features & AHC_ULTRA2) != 0) {
2150                 if ((ahc_inb(ahc, SBLKCTL) & ENAB40) != 0
2151                  && (ahc_inb(ahc, SSTAT2) & EXP_ACTIVE) == 0) {
2152                         maxsync = AHC_SYNCRATE_DT;
2153                 } else {
2154                         maxsync = AHC_SYNCRATE_ULTRA;
2155                         /* Can't do DT on an SE bus */
2156                         *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2157                 }
2158         } else if ((ahc->features & AHC_ULTRA) != 0) {
2159                 maxsync = AHC_SYNCRATE_ULTRA;
2160         } else {
2161                 maxsync = AHC_SYNCRATE_FAST;
2162         }
2163         /*
2164          * Never allow a value higher than our current goal
2165          * period otherwise we may allow a target initiated
2166          * negotiation to go above the limit as set by the
2167          * user.  In the case of an initiator initiated
2168          * sync negotiation, we limit based on the user
2169          * setting.  This allows the system to still accept
2170          * incoming negotiations even if target initiated
2171          * negotiation is not performed.
2172          */
2173         if (role == ROLE_TARGET)
2174                 transinfo = &tinfo->user;
2175         else
2176                 transinfo = &tinfo->goal;
2177         *ppr_options &= transinfo->ppr_options;
2178         if (transinfo->width == MSG_EXT_WDTR_BUS_8_BIT) {
2179                 maxsync = max(maxsync, (u_int)AHC_SYNCRATE_ULTRA2);
2180                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2181         }
2182         if (transinfo->period == 0) {
2183                 *period = 0;
2184                 *ppr_options = 0;
2185                 return (NULL);
2186         }
2187         *period = max(*period, (u_int)transinfo->period);
2188         return (ahc_find_syncrate(ahc, period, ppr_options, maxsync));
2189 }
2190
2191 /*
2192  * Look up the valid period to SCSIRATE conversion in our table.
2193  * Return the period and offset that should be sent to the target
2194  * if this was the beginning of an SDTR.
2195  */
2196 struct ahc_syncrate *
2197 ahc_find_syncrate(struct ahc_softc *ahc, u_int *period,
2198                   u_int *ppr_options, u_int maxsync)
2199 {
2200         struct ahc_syncrate *syncrate;
2201
2202         if ((ahc->features & AHC_DT) == 0)
2203                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2204
2205         /* Skip all DT only entries if DT is not available */
2206         if ((*ppr_options & MSG_EXT_PPR_DT_REQ) == 0
2207          && maxsync < AHC_SYNCRATE_ULTRA2)
2208                 maxsync = AHC_SYNCRATE_ULTRA2;
2209
2210         /* Now set the maxsync based on the card capabilities
2211          * DT is already done above */
2212         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2213             && maxsync < AHC_SYNCRATE_ULTRA)
2214                 maxsync = AHC_SYNCRATE_ULTRA;
2215         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2216             && maxsync < AHC_SYNCRATE_FAST)
2217                 maxsync = AHC_SYNCRATE_FAST;
2218
2219         for (syncrate = &ahc_syncrates[maxsync];
2220              syncrate->rate != NULL;
2221              syncrate++) {
2222
2223                 /*
2224                  * The Ultra2 table doesn't go as low
2225                  * as for the Fast/Ultra cards.
2226                  */
2227                 if ((ahc->features & AHC_ULTRA2) != 0
2228                  && (syncrate->sxfr_u2 == 0))
2229                         break;
2230
2231                 if (*period <= syncrate->period) {
2232                         /*
2233                          * When responding to a target that requests
2234                          * sync, the requested rate may fall between
2235                          * two rates that we can output, but still be
2236                          * a rate that we can receive.  Because of this,
2237                          * we want to respond to the target with
2238                          * the same rate that it sent to us even
2239                          * if the period we use to send data to it
2240                          * is lower.  Only lower the response period
2241                          * if we must.
2242                          */
2243                         if (syncrate == &ahc_syncrates[maxsync])
2244                                 *period = syncrate->period;
2245
2246                         /*
2247                          * At some speeds, we only support
2248                          * ST transfers.
2249                          */
2250                         if ((syncrate->sxfr_u2 & ST_SXFR) != 0)
2251                                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2252                         break;
2253                 }
2254         }
2255
2256         if ((*period == 0)
2257          || (syncrate->rate == NULL)
2258          || ((ahc->features & AHC_ULTRA2) != 0
2259           && (syncrate->sxfr_u2 == 0))) {
2260                 /* Use asynchronous transfers. */
2261                 *period = 0;
2262                 syncrate = NULL;
2263                 *ppr_options &= ~MSG_EXT_PPR_DT_REQ;
2264         }
2265         return (syncrate);
2266 }
2267
2268 /*
2269  * Convert from an entry in our syncrate table to the SCSI equivalent
2270  * sync "period" factor.
2271  */
2272 u_int
2273 ahc_find_period(struct ahc_softc *ahc, u_int scsirate, u_int maxsync)
2274 {
2275         struct ahc_syncrate *syncrate;
2276
2277         if ((ahc->features & AHC_ULTRA2) != 0)
2278                 scsirate &= SXFR_ULTRA2;
2279         else
2280                 scsirate &= SXFR;
2281
2282         /* now set maxsync based on card capabilities */
2283         if ((ahc->features & AHC_DT) == 0 && maxsync < AHC_SYNCRATE_ULTRA2)
2284                 maxsync = AHC_SYNCRATE_ULTRA2;
2285         if ((ahc->features & (AHC_DT | AHC_ULTRA2)) == 0
2286             && maxsync < AHC_SYNCRATE_ULTRA)
2287                 maxsync = AHC_SYNCRATE_ULTRA;
2288         if ((ahc->features & (AHC_DT | AHC_ULTRA2 | AHC_ULTRA)) == 0
2289             && maxsync < AHC_SYNCRATE_FAST)
2290                 maxsync = AHC_SYNCRATE_FAST;
2291
2292
2293         syncrate = &ahc_syncrates[maxsync];
2294         while (syncrate->rate != NULL) {
2295
2296                 if ((ahc->features & AHC_ULTRA2) != 0) {
2297                         if (syncrate->sxfr_u2 == 0)
2298                                 break;
2299                         else if (scsirate == (syncrate->sxfr_u2 & SXFR_ULTRA2))
2300                                 return (syncrate->period);
2301                 } else if (scsirate == (syncrate->sxfr & SXFR)) {
2302                                 return (syncrate->period);
2303                 }
2304                 syncrate++;
2305         }
2306         return (0); /* async */
2307 }
2308
2309 /*
2310  * Truncate the given synchronous offset to a value the
2311  * current adapter type and syncrate are capable of.
2312  */
2313 void
2314 ahc_validate_offset(struct ahc_softc *ahc,
2315                     struct ahc_initiator_tinfo *tinfo,
2316                     struct ahc_syncrate *syncrate,
2317                     u_int *offset, int wide, role_t role)
2318 {
2319         u_int maxoffset;
2320
2321         /* Limit offset to what we can do */
2322         if (syncrate == NULL) {
2323                 maxoffset = 0;
2324         } else if ((ahc->features & AHC_ULTRA2) != 0) {
2325                 maxoffset = MAX_OFFSET_ULTRA2;
2326         } else {
2327                 if (wide)
2328                         maxoffset = MAX_OFFSET_16BIT;
2329                 else
2330                         maxoffset = MAX_OFFSET_8BIT;
2331         }
2332         *offset = min(*offset, maxoffset);
2333         if (tinfo != NULL) {
2334                 if (role == ROLE_TARGET)
2335                         *offset = min(*offset, (u_int)tinfo->user.offset);
2336                 else
2337                         *offset = min(*offset, (u_int)tinfo->goal.offset);
2338         }
2339 }
2340
2341 /*
2342  * Truncate the given transfer width parameter to a value the
2343  * current adapter type is capable of.
2344  */
2345 void
2346 ahc_validate_width(struct ahc_softc *ahc, struct ahc_initiator_tinfo *tinfo,
2347                    u_int *bus_width, role_t role)
2348 {
2349         switch (*bus_width) {
2350         default:
2351                 if (ahc->features & AHC_WIDE) {
2352                         /* Respond Wide */
2353                         *bus_width = MSG_EXT_WDTR_BUS_16_BIT;
2354                         break;
2355                 }
2356                 /* FALLTHROUGH */
2357         case MSG_EXT_WDTR_BUS_8_BIT:
2358                 *bus_width = MSG_EXT_WDTR_BUS_8_BIT;
2359                 break;
2360         }
2361         if (tinfo != NULL) {
2362                 if (role == ROLE_TARGET)
2363                         *bus_width = min((u_int)tinfo->user.width, *bus_width);
2364                 else
2365                         *bus_width = min((u_int)tinfo->goal.width, *bus_width);
2366         }
2367 }
2368
2369 /*
2370  * Update the bitmask of targets for which the controller should
2371  * negotiate with at the next convenient oportunity.  This currently
2372  * means the next time we send the initial identify messages for
2373  * a new transaction.
2374  */
2375 int
2376 ahc_update_neg_request(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2377                        struct ahc_tmode_tstate *tstate,
2378                        struct ahc_initiator_tinfo *tinfo, ahc_neg_type neg_type)
2379 {
2380         u_int auto_negotiate_orig;
2381
2382         auto_negotiate_orig = tstate->auto_negotiate;
2383         if (neg_type == AHC_NEG_ALWAYS) {
2384                 /*
2385                  * Force our "current" settings to be
2386                  * unknown so that unless a bus reset
2387                  * occurs the need to renegotiate is
2388                  * recorded persistently.
2389                  */
2390                 if ((ahc->features & AHC_WIDE) != 0)
2391                         tinfo->curr.width = AHC_WIDTH_UNKNOWN;
2392                 tinfo->curr.period = AHC_PERIOD_UNKNOWN;
2393                 tinfo->curr.offset = AHC_OFFSET_UNKNOWN;
2394         }
2395         if (tinfo->curr.period != tinfo->goal.period
2396          || tinfo->curr.width != tinfo->goal.width
2397          || tinfo->curr.offset != tinfo->goal.offset
2398          || tinfo->curr.ppr_options != tinfo->goal.ppr_options
2399          || (neg_type == AHC_NEG_IF_NON_ASYNC
2400           && (tinfo->goal.offset != 0
2401            || tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT
2402            || tinfo->goal.ppr_options != 0)))
2403                 tstate->auto_negotiate |= devinfo->target_mask;
2404         else
2405                 tstate->auto_negotiate &= ~devinfo->target_mask;
2406
2407         return (auto_negotiate_orig != tstate->auto_negotiate);
2408 }
2409
2410 /*
2411  * Update the user/goal/curr tables of synchronous negotiation
2412  * parameters as well as, in the case of a current or active update,
2413  * any data structures on the host controller.  In the case of an
2414  * active update, the specified target is currently talking to us on
2415  * the bus, so the transfer parameter update must take effect
2416  * immediately.
2417  */
2418 void
2419 ahc_set_syncrate(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2420                  struct ahc_syncrate *syncrate, u_int period,
2421                  u_int offset, u_int ppr_options, u_int type, int paused)
2422 {
2423         struct  ahc_initiator_tinfo *tinfo;
2424         struct  ahc_tmode_tstate *tstate;
2425         u_int   old_period;
2426         u_int   old_offset;
2427         u_int   old_ppr;
2428         int     active;
2429         int     update_needed;
2430
2431         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2432         update_needed = 0;
2433
2434         if (syncrate == NULL) {
2435                 period = 0;
2436                 offset = 0;
2437         }
2438
2439         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2440                                     devinfo->target, &tstate);
2441
2442         if ((type & AHC_TRANS_USER) != 0) {
2443                 tinfo->user.period = period;
2444                 tinfo->user.offset = offset;
2445                 tinfo->user.ppr_options = ppr_options;
2446         }
2447
2448         if ((type & AHC_TRANS_GOAL) != 0) {
2449                 tinfo->goal.period = period;
2450                 tinfo->goal.offset = offset;
2451                 tinfo->goal.ppr_options = ppr_options;
2452         }
2453
2454         old_period = tinfo->curr.period;
2455         old_offset = tinfo->curr.offset;
2456         old_ppr    = tinfo->curr.ppr_options;
2457
2458         if ((type & AHC_TRANS_CUR) != 0
2459          && (old_period != period
2460           || old_offset != offset
2461           || old_ppr != ppr_options)) {
2462                 u_int   scsirate;
2463
2464                 update_needed++;
2465                 scsirate = tinfo->scsirate;
2466                 if ((ahc->features & AHC_ULTRA2) != 0) {
2467
2468                         scsirate &= ~(SXFR_ULTRA2|SINGLE_EDGE|ENABLE_CRC);
2469                         if (syncrate != NULL) {
2470                                 scsirate |= syncrate->sxfr_u2;
2471                                 if ((ppr_options & MSG_EXT_PPR_DT_REQ) != 0)
2472                                         scsirate |= ENABLE_CRC;
2473                                 else
2474                                         scsirate |= SINGLE_EDGE;
2475                         }
2476                 } else {
2477
2478                         scsirate &= ~(SXFR|SOFS);
2479                         /*
2480                          * Ensure Ultra mode is set properly for
2481                          * this target.
2482                          */
2483                         tstate->ultraenb &= ~devinfo->target_mask;
2484                         if (syncrate != NULL) {
2485                                 if (syncrate->sxfr & ULTRA_SXFR) {
2486                                         tstate->ultraenb |=
2487                                                 devinfo->target_mask;
2488                                 }
2489                                 scsirate |= syncrate->sxfr & SXFR;
2490                                 scsirate |= offset & SOFS;
2491                         }
2492                         if (active) {
2493                                 u_int sxfrctl0;
2494
2495                                 sxfrctl0 = ahc_inb(ahc, SXFRCTL0);
2496                                 sxfrctl0 &= ~FAST20;
2497                                 if (tstate->ultraenb & devinfo->target_mask)
2498                                         sxfrctl0 |= FAST20;
2499                                 ahc_outb(ahc, SXFRCTL0, sxfrctl0);
2500                         }
2501                 }
2502                 if (active) {
2503                         ahc_outb(ahc, SCSIRATE, scsirate);
2504                         if ((ahc->features & AHC_ULTRA2) != 0)
2505                                 ahc_outb(ahc, SCSIOFFSET, offset);
2506                 }
2507
2508                 tinfo->scsirate = scsirate;
2509                 tinfo->curr.period = period;
2510                 tinfo->curr.offset = offset;
2511                 tinfo->curr.ppr_options = ppr_options;
2512
2513                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2514                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2515                 if (bootverbose) {
2516                         if (offset != 0) {
2517                                 printf("%s: target %d synchronous at %sMHz%s, "
2518                                        "offset = 0x%x\n", ahc_name(ahc),
2519                                        devinfo->target, syncrate->rate,
2520                                        (ppr_options & MSG_EXT_PPR_DT_REQ)
2521                                        ? " DT" : "", offset);
2522                         } else {
2523                                 printf("%s: target %d using "
2524                                        "asynchronous transfers\n",
2525                                        ahc_name(ahc), devinfo->target);
2526                         }
2527                 }
2528         }
2529
2530         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2531                                                 tinfo, AHC_NEG_TO_GOAL);
2532
2533         if (update_needed)
2534                 ahc_update_pending_scbs(ahc);
2535 }
2536
2537 /*
2538  * Update the user/goal/curr tables of wide negotiation
2539  * parameters as well as, in the case of a current or active update,
2540  * any data structures on the host controller.  In the case of an
2541  * active update, the specified target is currently talking to us on
2542  * the bus, so the transfer parameter update must take effect
2543  * immediately.
2544  */
2545 void
2546 ahc_set_width(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2547               u_int width, u_int type, int paused)
2548 {
2549         struct  ahc_initiator_tinfo *tinfo;
2550         struct  ahc_tmode_tstate *tstate;
2551         u_int   oldwidth;
2552         int     active;
2553         int     update_needed;
2554
2555         active = (type & AHC_TRANS_ACTIVE) == AHC_TRANS_ACTIVE;
2556         update_needed = 0;
2557         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2558                                     devinfo->target, &tstate);
2559
2560         if ((type & AHC_TRANS_USER) != 0)
2561                 tinfo->user.width = width;
2562
2563         if ((type & AHC_TRANS_GOAL) != 0)
2564                 tinfo->goal.width = width;
2565
2566         oldwidth = tinfo->curr.width;
2567         if ((type & AHC_TRANS_CUR) != 0 && oldwidth != width) {
2568                 u_int   scsirate;
2569
2570                 update_needed++;
2571                 scsirate =  tinfo->scsirate;
2572                 scsirate &= ~WIDEXFER;
2573                 if (width == MSG_EXT_WDTR_BUS_16_BIT)
2574                         scsirate |= WIDEXFER;
2575
2576                 tinfo->scsirate = scsirate;
2577
2578                 if (active)
2579                         ahc_outb(ahc, SCSIRATE, scsirate);
2580
2581                 tinfo->curr.width = width;
2582
2583                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
2584                                CAM_LUN_WILDCARD, AC_TRANSFER_NEG);
2585                 if (bootverbose) {
2586                         printf("%s: target %d using %dbit transfers\n",
2587                                ahc_name(ahc), devinfo->target,
2588                                8 * (0x01 << width));
2589                 }
2590         }
2591
2592         update_needed += ahc_update_neg_request(ahc, devinfo, tstate,
2593                                                 tinfo, AHC_NEG_TO_GOAL);
2594         if (update_needed)
2595                 ahc_update_pending_scbs(ahc);
2596 }
2597
2598 /*
2599  * Update the current state of tagged queuing for a given target.
2600  */
2601 static void
2602 ahc_set_tags(struct ahc_softc *ahc, struct scsi_cmnd *cmd,
2603              struct ahc_devinfo *devinfo, ahc_queue_alg alg)
2604 {
2605         struct scsi_device *sdev = cmd->device;
2606
2607         ahc_platform_set_tags(ahc, sdev, devinfo, alg);
2608         ahc_send_async(ahc, devinfo->channel, devinfo->target,
2609                        devinfo->lun, AC_TRANSFER_NEG);
2610 }
2611
2612 /*
2613  * When the transfer settings for a connection change, update any
2614  * in-transit SCBs to contain the new data so the hardware will
2615  * be set correctly during future (re)selections.
2616  */
2617 static void
2618 ahc_update_pending_scbs(struct ahc_softc *ahc)
2619 {
2620         struct  scb *pending_scb;
2621         int     pending_scb_count;
2622         int     i;
2623         int     paused;
2624         u_int   saved_scbptr;
2625
2626         /*
2627          * Traverse the pending SCB list and ensure that all of the
2628          * SCBs there have the proper settings.
2629          */
2630         pending_scb_count = 0;
2631         LIST_FOREACH(pending_scb, &ahc->pending_scbs, pending_links) {
2632                 struct ahc_devinfo devinfo;
2633                 struct hardware_scb *pending_hscb;
2634                 struct ahc_initiator_tinfo *tinfo;
2635                 struct ahc_tmode_tstate *tstate;
2636
2637                 ahc_scb_devinfo(ahc, &devinfo, pending_scb);
2638                 tinfo = ahc_fetch_transinfo(ahc, devinfo.channel,
2639                                             devinfo.our_scsiid,
2640                                             devinfo.target, &tstate);
2641                 pending_hscb = pending_scb->hscb;
2642                 pending_hscb->control &= ~ULTRAENB;
2643                 if ((tstate->ultraenb & devinfo.target_mask) != 0)
2644                         pending_hscb->control |= ULTRAENB;
2645                 pending_hscb->scsirate = tinfo->scsirate;
2646                 pending_hscb->scsioffset = tinfo->curr.offset;
2647                 if ((tstate->auto_negotiate & devinfo.target_mask) == 0
2648                  && (pending_scb->flags & SCB_AUTO_NEGOTIATE) != 0) {
2649                         pending_scb->flags &= ~SCB_AUTO_NEGOTIATE;
2650                         pending_hscb->control &= ~MK_MESSAGE;
2651                 }
2652                 ahc_sync_scb(ahc, pending_scb,
2653                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
2654                 pending_scb_count++;
2655         }
2656
2657         if (pending_scb_count == 0)
2658                 return;
2659
2660         if (ahc_is_paused(ahc)) {
2661                 paused = 1;
2662         } else {
2663                 paused = 0;
2664                 ahc_pause(ahc);
2665         }
2666
2667         saved_scbptr = ahc_inb(ahc, SCBPTR);
2668         /* Ensure that the hscbs down on the card match the new information */
2669         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
2670                 struct  hardware_scb *pending_hscb;
2671                 u_int   control;
2672                 u_int   scb_tag;
2673
2674                 ahc_outb(ahc, SCBPTR, i);
2675                 scb_tag = ahc_inb(ahc, SCB_TAG);
2676                 pending_scb = ahc_lookup_scb(ahc, scb_tag);
2677                 if (pending_scb == NULL)
2678                         continue;
2679
2680                 pending_hscb = pending_scb->hscb;
2681                 control = ahc_inb(ahc, SCB_CONTROL);
2682                 control &= ~(ULTRAENB|MK_MESSAGE);
2683                 control |= pending_hscb->control & (ULTRAENB|MK_MESSAGE);
2684                 ahc_outb(ahc, SCB_CONTROL, control);
2685                 ahc_outb(ahc, SCB_SCSIRATE, pending_hscb->scsirate);
2686                 ahc_outb(ahc, SCB_SCSIOFFSET, pending_hscb->scsioffset);
2687         }
2688         ahc_outb(ahc, SCBPTR, saved_scbptr);
2689
2690         if (paused == 0)
2691                 ahc_unpause(ahc);
2692 }
2693
2694 /**************************** Pathing Information *****************************/
2695 static void
2696 ahc_fetch_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2697 {
2698         u_int   saved_scsiid;
2699         role_t  role;
2700         int     our_id;
2701
2702         if (ahc_inb(ahc, SSTAT0) & TARGET)
2703                 role = ROLE_TARGET;
2704         else
2705                 role = ROLE_INITIATOR;
2706
2707         if (role == ROLE_TARGET
2708          && (ahc->features & AHC_MULTI_TID) != 0
2709          && (ahc_inb(ahc, SEQ_FLAGS)
2710            & (CMDPHASE_PENDING|TARG_CMD_PENDING|NO_DISCONNECT)) != 0) {
2711                 /* We were selected, so pull our id from TARGIDIN */
2712                 our_id = ahc_inb(ahc, TARGIDIN) & OID;
2713         } else if ((ahc->features & AHC_ULTRA2) != 0)
2714                 our_id = ahc_inb(ahc, SCSIID_ULTRA2) & OID;
2715         else
2716                 our_id = ahc_inb(ahc, SCSIID) & OID;
2717
2718         saved_scsiid = ahc_inb(ahc, SAVED_SCSIID);
2719         ahc_compile_devinfo(devinfo,
2720                             our_id,
2721                             SCSIID_TARGET(ahc, saved_scsiid),
2722                             ahc_inb(ahc, SAVED_LUN),
2723                             SCSIID_CHANNEL(ahc, saved_scsiid),
2724                             role);
2725 }
2726
2727 struct ahc_phase_table_entry*
2728 ahc_lookup_phase_entry(int phase)
2729 {
2730         struct ahc_phase_table_entry *entry;
2731         struct ahc_phase_table_entry *last_entry;
2732
2733         /*
2734          * num_phases doesn't include the default entry which
2735          * will be returned if the phase doesn't match.
2736          */
2737         last_entry = &ahc_phase_table[num_phases];
2738         for (entry = ahc_phase_table; entry < last_entry; entry++) {
2739                 if (phase == entry->phase)
2740                         break;
2741         }
2742         return (entry);
2743 }
2744
2745 void
2746 ahc_compile_devinfo(struct ahc_devinfo *devinfo, u_int our_id, u_int target,
2747                     u_int lun, char channel, role_t role)
2748 {
2749         devinfo->our_scsiid = our_id;
2750         devinfo->target = target;
2751         devinfo->lun = lun;
2752         devinfo->target_offset = target;
2753         devinfo->channel = channel;
2754         devinfo->role = role;
2755         if (channel == 'B')
2756                 devinfo->target_offset += 8;
2757         devinfo->target_mask = (0x01 << devinfo->target_offset);
2758 }
2759
2760 void
2761 ahc_print_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2762 {
2763         printf("%s:%c:%d:%d: ", ahc_name(ahc), devinfo->channel,
2764                devinfo->target, devinfo->lun);
2765 }
2766
2767 static void
2768 ahc_scb_devinfo(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2769                 struct scb *scb)
2770 {
2771         role_t  role;
2772         int     our_id;
2773
2774         our_id = SCSIID_OUR_ID(scb->hscb->scsiid);
2775         role = ROLE_INITIATOR;
2776         if ((scb->flags & SCB_TARGET_SCB) != 0)
2777                 role = ROLE_TARGET;
2778         ahc_compile_devinfo(devinfo, our_id, SCB_GET_TARGET(ahc, scb),
2779                             SCB_GET_LUN(scb), SCB_GET_CHANNEL(ahc, scb), role);
2780 }
2781
2782
2783 /************************ Message Phase Processing ****************************/
2784 static void
2785 ahc_assert_atn(struct ahc_softc *ahc)
2786 {
2787         u_int scsisigo;
2788
2789         scsisigo = ATNO;
2790         if ((ahc->features & AHC_DT) == 0)
2791                 scsisigo |= ahc_inb(ahc, SCSISIGI);
2792         ahc_outb(ahc, SCSISIGO, scsisigo);
2793 }
2794
2795 /*
2796  * When an initiator transaction with the MK_MESSAGE flag either reconnects
2797  * or enters the initial message out phase, we are interrupted.  Fill our
2798  * outgoing message buffer with the appropriate message and beging handing
2799  * the message phase(s) manually.
2800  */
2801 static void
2802 ahc_setup_initiator_msgout(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2803                            struct scb *scb)
2804 {
2805         /*
2806          * To facilitate adding multiple messages together,
2807          * each routine should increment the index and len
2808          * variables instead of setting them explicitly.
2809          */
2810         ahc->msgout_index = 0;
2811         ahc->msgout_len = 0;
2812
2813         if ((scb->flags & SCB_DEVICE_RESET) == 0
2814          && ahc_inb(ahc, MSG_OUT) == MSG_IDENTIFYFLAG) {
2815                 u_int identify_msg;
2816
2817                 identify_msg = MSG_IDENTIFYFLAG | SCB_GET_LUN(scb);
2818                 if ((scb->hscb->control & DISCENB) != 0)
2819                         identify_msg |= MSG_IDENTIFY_DISCFLAG;
2820                 ahc->msgout_buf[ahc->msgout_index++] = identify_msg;
2821                 ahc->msgout_len++;
2822
2823                 if ((scb->hscb->control & TAG_ENB) != 0) {
2824                         ahc->msgout_buf[ahc->msgout_index++] =
2825                             scb->hscb->control & (TAG_ENB|SCB_TAG_TYPE);
2826                         ahc->msgout_buf[ahc->msgout_index++] = scb->hscb->tag;
2827                         ahc->msgout_len += 2;
2828                 }
2829         }
2830
2831         if (scb->flags & SCB_DEVICE_RESET) {
2832                 ahc->msgout_buf[ahc->msgout_index++] = MSG_BUS_DEV_RESET;
2833                 ahc->msgout_len++;
2834                 ahc_print_path(ahc, scb);
2835                 printf("Bus Device Reset Message Sent\n");
2836                 /*
2837                  * Clear our selection hardware in advance of
2838                  * the busfree.  We may have an entry in the waiting
2839                  * Q for this target, and we don't want to go about
2840                  * selecting while we handle the busfree and blow it
2841                  * away.
2842                  */
2843                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2844         } else if ((scb->flags & SCB_ABORT) != 0) {
2845                 if ((scb->hscb->control & TAG_ENB) != 0)
2846                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT_TAG;
2847                 else
2848                         ahc->msgout_buf[ahc->msgout_index++] = MSG_ABORT;
2849                 ahc->msgout_len++;
2850                 ahc_print_path(ahc, scb);
2851                 printf("Abort%s Message Sent\n",
2852                        (scb->hscb->control & TAG_ENB) != 0 ? " Tag" : "");
2853                 /*
2854                  * Clear our selection hardware in advance of
2855                  * the busfree.  We may have an entry in the waiting
2856                  * Q for this target, and we don't want to go about
2857                  * selecting while we handle the busfree and blow it
2858                  * away.
2859                  */
2860                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
2861         } else if ((scb->flags & (SCB_AUTO_NEGOTIATE|SCB_NEGOTIATE)) != 0) {
2862                 ahc_build_transfer_msg(ahc, devinfo);
2863         } else {
2864                 printf("ahc_intr: AWAITING_MSG for an SCB that "
2865                        "does not have a waiting message\n");
2866                 printf("SCSIID = %x, target_mask = %x\n", scb->hscb->scsiid,
2867                        devinfo->target_mask);
2868                 panic("SCB = %d, SCB Control = %x, MSG_OUT = %x "
2869                       "SCB flags = %x", scb->hscb->tag, scb->hscb->control,
2870                       ahc_inb(ahc, MSG_OUT), scb->flags);
2871         }
2872
2873         /*
2874          * Clear the MK_MESSAGE flag from the SCB so we aren't
2875          * asked to send this message again.
2876          */
2877         ahc_outb(ahc, SCB_CONTROL, ahc_inb(ahc, SCB_CONTROL) & ~MK_MESSAGE);
2878         scb->hscb->control &= ~MK_MESSAGE;
2879         ahc->msgout_index = 0;
2880         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
2881 }
2882
2883 /*
2884  * Build an appropriate transfer negotiation message for the
2885  * currently active target.
2886  */
2887 static void
2888 ahc_build_transfer_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
2889 {
2890         /*
2891          * We need to initiate transfer negotiations.
2892          * If our current and goal settings are identical,
2893          * we want to renegotiate due to a check condition.
2894          */
2895         struct  ahc_initiator_tinfo *tinfo;
2896         struct  ahc_tmode_tstate *tstate;
2897         struct  ahc_syncrate *rate;
2898         int     dowide;
2899         int     dosync;
2900         int     doppr;
2901         u_int   period;
2902         u_int   ppr_options;
2903         u_int   offset;
2904
2905         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
2906                                     devinfo->target, &tstate);
2907         /*
2908          * Filter our period based on the current connection.
2909          * If we can't perform DT transfers on this segment (not in LVD
2910          * mode for instance), then our decision to issue a PPR message
2911          * may change.
2912          */
2913         period = tinfo->goal.period;
2914         offset = tinfo->goal.offset;
2915         ppr_options = tinfo->goal.ppr_options;
2916         /* Target initiated PPR is not allowed in the SCSI spec */
2917         if (devinfo->role == ROLE_TARGET)
2918                 ppr_options = 0;
2919         rate = ahc_devlimited_syncrate(ahc, tinfo, &period,
2920                                        &ppr_options, devinfo->role);
2921         dowide = tinfo->curr.width != tinfo->goal.width;
2922         dosync = tinfo->curr.offset != offset || tinfo->curr.period != period;
2923         /*
2924          * Only use PPR if we have options that need it, even if the device
2925          * claims to support it.  There might be an expander in the way
2926          * that doesn't.
2927          */
2928         doppr = ppr_options != 0;
2929
2930         if (!dowide && !dosync && !doppr) {
2931                 dowide = tinfo->goal.width != MSG_EXT_WDTR_BUS_8_BIT;
2932                 dosync = tinfo->goal.offset != 0;
2933         }
2934
2935         if (!dowide && !dosync && !doppr) {
2936                 /*
2937                  * Force async with a WDTR message if we have a wide bus,
2938                  * or just issue an SDTR with a 0 offset.
2939                  */
2940                 if ((ahc->features & AHC_WIDE) != 0)
2941                         dowide = 1;
2942                 else
2943                         dosync = 1;
2944
2945                 if (bootverbose) {
2946                         ahc_print_devinfo(ahc, devinfo);
2947                         printf("Ensuring async\n");
2948                 }
2949         }
2950
2951         /* Target initiated PPR is not allowed in the SCSI spec */
2952         if (devinfo->role == ROLE_TARGET)
2953                 doppr = 0;
2954
2955         /*
2956          * Both the PPR message and SDTR message require the
2957          * goal syncrate to be limited to what the target device
2958          * is capable of handling (based on whether an LVD->SE
2959          * expander is on the bus), so combine these two cases.
2960          * Regardless, guarantee that if we are using WDTR and SDTR
2961          * messages that WDTR comes first.
2962          */
2963         if (doppr || (dosync && !dowide)) {
2964
2965                 offset = tinfo->goal.offset;
2966                 ahc_validate_offset(ahc, tinfo, rate, &offset,
2967                                     doppr ? tinfo->goal.width
2968                                           : tinfo->curr.width,
2969                                     devinfo->role);
2970                 if (doppr) {
2971                         ahc_construct_ppr(ahc, devinfo, period, offset,
2972                                           tinfo->goal.width, ppr_options);
2973                 } else {
2974                         ahc_construct_sdtr(ahc, devinfo, period, offset);
2975                 }
2976         } else {
2977                 ahc_construct_wdtr(ahc, devinfo, tinfo->goal.width);
2978         }
2979 }
2980
2981 /*
2982  * Build a synchronous negotiation message in our message
2983  * buffer based on the input parameters.
2984  */
2985 static void
2986 ahc_construct_sdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
2987                    u_int period, u_int offset)
2988 {
2989         if (offset == 0)
2990                 period = AHC_ASYNC_XFER_PERIOD;
2991         ahc->msgout_index += spi_populate_sync_msg(
2992                         ahc->msgout_buf + ahc->msgout_index, period, offset);
2993         ahc->msgout_len += 5;
2994         if (bootverbose) {
2995                 printf("(%s:%c:%d:%d): Sending SDTR period %x, offset %x\n",
2996                        ahc_name(ahc), devinfo->channel, devinfo->target,
2997                        devinfo->lun, period, offset);
2998         }
2999 }
3000
3001 /*
3002  * Build a wide negotiation message in our message
3003  * buffer based on the input parameters.
3004  */
3005 static void
3006 ahc_construct_wdtr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3007                    u_int bus_width)
3008 {
3009         ahc->msgout_index += spi_populate_width_msg(
3010                         ahc->msgout_buf + ahc->msgout_index, bus_width);
3011         ahc->msgout_len += 4;
3012         if (bootverbose) {
3013                 printf("(%s:%c:%d:%d): Sending WDTR %x\n",
3014                        ahc_name(ahc), devinfo->channel, devinfo->target,
3015                        devinfo->lun, bus_width);
3016         }
3017 }
3018
3019 /*
3020  * Build a parallel protocol request message in our message
3021  * buffer based on the input parameters.
3022  */
3023 static void
3024 ahc_construct_ppr(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
3025                   u_int period, u_int offset, u_int bus_width,
3026                   u_int ppr_options)
3027 {
3028         if (offset == 0)
3029                 period = AHC_ASYNC_XFER_PERIOD;
3030         ahc->msgout_index += spi_populate_ppr_msg(
3031                         ahc->msgout_buf + ahc->msgout_index, period, offset,
3032                         bus_width, ppr_options);
3033         ahc->msgout_len += 8;
3034         if (bootverbose) {
3035                 printf("(%s:%c:%d:%d): Sending PPR bus_width %x, period %x, "
3036                        "offset %x, ppr_options %x\n", ahc_name(ahc),
3037                        devinfo->channel, devinfo->target, devinfo->lun,
3038                        bus_width, period, offset, ppr_options);
3039         }
3040 }
3041
3042 /*
3043  * Clear any active message state.
3044  */
3045 static void
3046 ahc_clear_msg_state(struct ahc_softc *ahc)
3047 {
3048         ahc->msgout_len = 0;
3049         ahc->msgin_index = 0;
3050         ahc->msg_type = MSG_TYPE_NONE;
3051         if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0) {
3052                 /*
3053                  * The target didn't care to respond to our
3054                  * message request, so clear ATN.
3055                  */
3056                 ahc_outb(ahc, CLRSINT1, CLRATNO);
3057         }
3058         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
3059         ahc_outb(ahc, SEQ_FLAGS2,
3060                  ahc_inb(ahc, SEQ_FLAGS2) & ~TARGET_MSG_PENDING);
3061 }
3062
3063 static void
3064 ahc_handle_proto_violation(struct ahc_softc *ahc)
3065 {
3066         struct  ahc_devinfo devinfo;
3067         struct  scb *scb;
3068         u_int   scbid;
3069         u_int   seq_flags;
3070         u_int   curphase;
3071         u_int   lastphase;
3072         int     found;
3073
3074         ahc_fetch_devinfo(ahc, &devinfo);
3075         scbid = ahc_inb(ahc, SCB_TAG);
3076         scb = ahc_lookup_scb(ahc, scbid);
3077         seq_flags = ahc_inb(ahc, SEQ_FLAGS);
3078         curphase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3079         lastphase = ahc_inb(ahc, LASTPHASE);
3080         if ((seq_flags & NOT_IDENTIFIED) != 0) {
3081
3082                 /*
3083                  * The reconnecting target either did not send an
3084                  * identify message, or did, but we didn't find an SCB
3085                  * to match.
3086                  */
3087                 ahc_print_devinfo(ahc, &devinfo);
3088                 printf("Target did not send an IDENTIFY message. "
3089                        "LASTPHASE = 0x%x.\n", lastphase);
3090                 scb = NULL;
3091         } else if (scb == NULL) {
3092                 /*
3093                  * We don't seem to have an SCB active for this
3094                  * transaction.  Print an error and reset the bus.
3095                  */
3096                 ahc_print_devinfo(ahc, &devinfo);
3097                 printf("No SCB found during protocol violation\n");
3098                 goto proto_violation_reset;
3099         } else {
3100                 ahc_set_transaction_status(scb, CAM_SEQUENCE_FAIL);
3101                 if ((seq_flags & NO_CDB_SENT) != 0) {
3102                         ahc_print_path(ahc, scb);
3103                         printf("No or incomplete CDB sent to device.\n");
3104                 } else if ((ahc_inb(ahc, SCB_CONTROL) & STATUS_RCVD) == 0) {
3105                         /*
3106                          * The target never bothered to provide status to
3107                          * us prior to completing the command.  Since we don't
3108                          * know the disposition of this command, we must attempt
3109                          * to abort it.  Assert ATN and prepare to send an abort
3110                          * message.
3111                          */
3112                         ahc_print_path(ahc, scb);
3113                         printf("Completed command without status.\n");
3114                 } else {
3115                         ahc_print_path(ahc, scb);
3116                         printf("Unknown protocol violation.\n");
3117                         ahc_dump_card_state(ahc);
3118                 }
3119         }
3120         if ((lastphase & ~P_DATAIN_DT) == 0
3121          || lastphase == P_COMMAND) {
3122 proto_violation_reset:
3123                 /*
3124                  * Target either went directly to data/command
3125                  * phase or didn't respond to our ATN.
3126                  * The only safe thing to do is to blow
3127                  * it away with a bus reset.
3128                  */
3129                 found = ahc_reset_channel(ahc, 'A', TRUE);
3130                 printf("%s: Issued Channel %c Bus Reset. "
3131                        "%d SCBs aborted\n", ahc_name(ahc), 'A', found);
3132         } else {
3133                 /*
3134                  * Leave the selection hardware off in case
3135                  * this abort attempt will affect yet to
3136                  * be sent commands.
3137                  */
3138                 ahc_outb(ahc, SCSISEQ,
3139                          ahc_inb(ahc, SCSISEQ) & ~ENSELO);
3140                 ahc_assert_atn(ahc);
3141                 ahc_outb(ahc, MSG_OUT, HOST_MSG);
3142                 if (scb == NULL) {
3143                         ahc_print_devinfo(ahc, &devinfo);
3144                         ahc->msgout_buf[0] = MSG_ABORT_TASK;
3145                         ahc->msgout_len = 1;
3146                         ahc->msgout_index = 0;
3147                         ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3148                 } else {
3149                         ahc_print_path(ahc, scb);
3150                         scb->flags |= SCB_ABORT;
3151                 }
3152                 printf("Protocol violation %s.  Attempting to abort.\n",
3153                        ahc_lookup_phase_entry(curphase)->phasemsg);
3154         }
3155 }
3156
3157 /*
3158  * Manual message loop handler.
3159  */
3160 static void
3161 ahc_handle_message_phase(struct ahc_softc *ahc)
3162 {
3163         struct  ahc_devinfo devinfo;
3164         u_int   bus_phase;
3165         int     end_session;
3166
3167         ahc_fetch_devinfo(ahc, &devinfo);
3168         end_session = FALSE;
3169         bus_phase = ahc_inb(ahc, SCSISIGI) & PHASE_MASK;
3170
3171 reswitch:
3172         switch (ahc->msg_type) {
3173         case MSG_TYPE_INITIATOR_MSGOUT:
3174         {
3175                 int lastbyte;
3176                 int phasemis;
3177                 int msgdone;
3178
3179                 if (ahc->msgout_len == 0)
3180                         panic("HOST_MSG_LOOP interrupt with no active message");
3181
3182 #ifdef AHC_DEBUG
3183                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3184                         ahc_print_devinfo(ahc, &devinfo);
3185                         printf("INITIATOR_MSG_OUT");
3186                 }
3187 #endif
3188                 phasemis = bus_phase != P_MESGOUT;
3189                 if (phasemis) {
3190 #ifdef AHC_DEBUG
3191                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3192                                 printf(" PHASEMIS %s\n",
3193                                        ahc_lookup_phase_entry(bus_phase)
3194                                                              ->phasemsg);
3195                         }
3196 #endif
3197                         if (bus_phase == P_MESGIN) {
3198                                 /*
3199                                  * Change gears and see if
3200                                  * this messages is of interest to
3201                                  * us or should be passed back to
3202                                  * the sequencer.
3203                                  */
3204                                 ahc_outb(ahc, CLRSINT1, CLRATNO);
3205                                 ahc->send_msg_perror = FALSE;
3206                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGIN;
3207                                 ahc->msgin_index = 0;
3208                                 goto reswitch;
3209                         }
3210                         end_session = TRUE;
3211                         break;
3212                 }
3213
3214                 if (ahc->send_msg_perror) {
3215                         ahc_outb(ahc, CLRSINT1, CLRATNO);
3216                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3217 #ifdef AHC_DEBUG
3218                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3219                                 printf(" byte 0x%x\n", ahc->send_msg_perror);
3220 #endif
3221                         ahc_outb(ahc, SCSIDATL, MSG_PARITY_ERROR);
3222                         break;
3223                 }
3224
3225                 msgdone = ahc->msgout_index == ahc->msgout_len;
3226                 if (msgdone) {
3227                         /*
3228                          * The target has requested a retry.
3229                          * Re-assert ATN, reset our message index to
3230                          * 0, and try again.
3231                          */
3232                         ahc->msgout_index = 0;
3233                         ahc_assert_atn(ahc);
3234                 }
3235
3236                 lastbyte = ahc->msgout_index == (ahc->msgout_len - 1);
3237                 if (lastbyte) {
3238                         /* Last byte is signified by dropping ATN */
3239                         ahc_outb(ahc, CLRSINT1, CLRATNO);
3240                 }
3241
3242                 /*
3243                  * Clear our interrupt status and present
3244                  * the next byte on the bus.
3245                  */
3246                 ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3247 #ifdef AHC_DEBUG
3248                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3249                         printf(" byte 0x%x\n",
3250                                ahc->msgout_buf[ahc->msgout_index]);
3251 #endif
3252                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3253                 break;
3254         }
3255         case MSG_TYPE_INITIATOR_MSGIN:
3256         {
3257                 int phasemis;
3258                 int message_done;
3259
3260 #ifdef AHC_DEBUG
3261                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3262                         ahc_print_devinfo(ahc, &devinfo);
3263                         printf("INITIATOR_MSG_IN");
3264                 }
3265 #endif
3266                 phasemis = bus_phase != P_MESGIN;
3267                 if (phasemis) {
3268 #ifdef AHC_DEBUG
3269                         if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3270                                 printf(" PHASEMIS %s\n",
3271                                        ahc_lookup_phase_entry(bus_phase)
3272                                                              ->phasemsg);
3273                         }
3274 #endif
3275                         ahc->msgin_index = 0;
3276                         if (bus_phase == P_MESGOUT
3277                          && (ahc->send_msg_perror == TRUE
3278                           || (ahc->msgout_len != 0
3279                            && ahc->msgout_index == 0))) {
3280                                 ahc->msg_type = MSG_TYPE_INITIATOR_MSGOUT;
3281                                 goto reswitch;
3282                         }
3283                         end_session = TRUE;
3284                         break;
3285                 }
3286
3287                 /* Pull the byte in without acking it */
3288                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIBUSL);
3289 #ifdef AHC_DEBUG
3290                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0)
3291                         printf(" byte 0x%x\n",
3292                                ahc->msgin_buf[ahc->msgin_index]);
3293 #endif
3294
3295                 message_done = ahc_parse_msg(ahc, &devinfo);
3296
3297                 if (message_done) {
3298                         /*
3299                          * Clear our incoming message buffer in case there
3300                          * is another message following this one.
3301                          */
3302                         ahc->msgin_index = 0;
3303
3304                         /*
3305                          * If this message illicited a response,
3306                          * assert ATN so the target takes us to the
3307                          * message out phase.
3308                          */
3309                         if (ahc->msgout_len != 0) {
3310 #ifdef AHC_DEBUG
3311                                 if ((ahc_debug & AHC_SHOW_MESSAGES) != 0) {
3312                                         ahc_print_devinfo(ahc, &devinfo);
3313                                         printf("Asserting ATN for response\n");
3314                                 }
3315 #endif
3316                                 ahc_assert_atn(ahc);
3317                         }
3318                 } else
3319                         ahc->msgin_index++;
3320
3321                 if (message_done == MSGLOOP_TERMINATED) {
3322                         end_session = TRUE;
3323                 } else {
3324                         /* Ack the byte */
3325                         ahc_outb(ahc, CLRSINT1, CLRREQINIT);
3326                         ahc_inb(ahc, SCSIDATL);
3327                 }
3328                 break;
3329         }
3330         case MSG_TYPE_TARGET_MSGIN:
3331         {
3332                 int msgdone;
3333                 int msgout_request;
3334
3335                 if (ahc->msgout_len == 0)
3336                         panic("Target MSGIN with no active message");
3337
3338                 /*
3339                  * If we interrupted a mesgout session, the initiator
3340                  * will not know this until our first REQ.  So, we
3341                  * only honor mesgout requests after we've sent our
3342                  * first byte.
3343                  */
3344                 if ((ahc_inb(ahc, SCSISIGI) & ATNI) != 0
3345                  && ahc->msgout_index > 0)
3346                         msgout_request = TRUE;
3347                 else
3348                         msgout_request = FALSE;
3349
3350                 if (msgout_request) {
3351
3352                         /*
3353                          * Change gears and see if
3354                          * this messages is of interest to
3355                          * us or should be passed back to
3356                          * the sequencer.
3357                          */
3358                         ahc->msg_type = MSG_TYPE_TARGET_MSGOUT;
3359                         ahc_outb(ahc, SCSISIGO, P_MESGOUT | BSYO);
3360                         ahc->msgin_index = 0;
3361                         /* Dummy read to REQ for first byte */
3362                         ahc_inb(ahc, SCSIDATL);
3363                         ahc_outb(ahc, SXFRCTL0,
3364                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3365                         break;
3366                 }
3367
3368                 msgdone = ahc->msgout_index == ahc->msgout_len;
3369                 if (msgdone) {
3370                         ahc_outb(ahc, SXFRCTL0,
3371                                  ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3372                         end_session = TRUE;
3373                         break;
3374                 }
3375
3376                 /*
3377                  * Present the next byte on the bus.
3378                  */
3379                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3380                 ahc_outb(ahc, SCSIDATL, ahc->msgout_buf[ahc->msgout_index++]);
3381                 break;
3382         }
3383         case MSG_TYPE_TARGET_MSGOUT:
3384         {
3385                 int lastbyte;
3386                 int msgdone;
3387
3388                 /*
3389                  * The initiator signals that this is
3390                  * the last byte by dropping ATN.
3391                  */
3392                 lastbyte = (ahc_inb(ahc, SCSISIGI) & ATNI) == 0;
3393
3394                 /*
3395                  * Read the latched byte, but turn off SPIOEN first
3396                  * so that we don't inadvertently cause a REQ for the
3397                  * next byte.
3398                  */
3399                 ahc_outb(ahc, SXFRCTL0, ahc_inb(ahc, SXFRCTL0) & ~SPIOEN);
3400                 ahc->msgin_buf[ahc->msgin_index] = ahc_inb(ahc, SCSIDATL);
3401                 msgdone = ahc_parse_msg(ahc, &devinfo);
3402                 if (msgdone == MSGLOOP_TERMINATED) {
3403                         /*
3404                          * The message is *really* done in that it caused
3405                          * us to go to bus free.  The sequencer has already
3406                          * been reset at this point, so pull the ejection
3407                          * handle.
3408                          */
3409                         return;
3410                 }
3411
3412                 ahc->msgin_index++;
3413
3414                 /*
3415                  * XXX Read spec about initiator dropping ATN too soon
3416                  *     and use msgdone to detect it.
3417                  */
3418                 if (msgdone == MSGLOOP_MSGCOMPLETE) {
3419                         ahc->msgin_index = 0;
3420
3421                         /*
3422                          * If this message illicited a response, transition
3423                          * to the Message in phase and send it.
3424                          */
3425                         if (ahc->msgout_len != 0) {
3426                                 ahc_outb(ahc, SCSISIGO, P_MESGIN | BSYO);
3427                                 ahc_outb(ahc, SXFRCTL0,
3428                                          ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3429                                 ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
3430                                 ahc->msgin_index = 0;
3431                                 break;
3432                         }
3433                 }
3434
3435                 if (lastbyte)
3436                         end_session = TRUE;
3437                 else {
3438                         /* Ask for the next byte. */
3439                         ahc_outb(ahc, SXFRCTL0,
3440                                  ahc_inb(ahc, SXFRCTL0) | SPIOEN);
3441                 }
3442
3443                 break;
3444         }
3445         default:
3446                 panic("Unknown REQINIT message type");
3447         }
3448
3449         if (end_session) {
3450                 ahc_clear_msg_state(ahc);
3451                 ahc_outb(ahc, RETURN_1, EXIT_MSG_LOOP);
3452         } else
3453                 ahc_outb(ahc, RETURN_1, CONT_MSG_LOOP);
3454 }
3455
3456 /*
3457  * See if we sent a particular extended message to the target.
3458  * If "full" is true, return true only if the target saw the full
3459  * message.  If "full" is false, return true if the target saw at
3460  * least the first byte of the message.
3461  */
3462 static int
3463 ahc_sent_msg(struct ahc_softc *ahc, ahc_msgtype type, u_int msgval, int full)
3464 {
3465         int found;
3466         u_int index;
3467
3468         found = FALSE;
3469         index = 0;
3470
3471         while (index < ahc->msgout_len) {
3472                 if (ahc->msgout_buf[index] == MSG_EXTENDED) {
3473                         u_int end_index;
3474
3475                         end_index = index + 1 + ahc->msgout_buf[index + 1];
3476                         if (ahc->msgout_buf[index+2] == msgval
3477                          && type == AHCMSG_EXT) {
3478
3479                                 if (full) {
3480                                         if (ahc->msgout_index > end_index)
3481                                                 found = TRUE;
3482                                 } else if (ahc->msgout_index > index)
3483                                         found = TRUE;
3484                         }
3485                         index = end_index;
3486                 } else if (ahc->msgout_buf[index] >= MSG_SIMPLE_TASK
3487                         && ahc->msgout_buf[index] <= MSG_IGN_WIDE_RESIDUE) {
3488
3489                         /* Skip tag type and tag id or residue param*/
3490                         index += 2;
3491                 } else {
3492                         /* Single byte message */
3493                         if (type == AHCMSG_1B
3494                          && ahc->msgout_buf[index] == msgval
3495                          && ahc->msgout_index > index)
3496                                 found = TRUE;
3497                         index++;
3498                 }
3499
3500                 if (found)
3501                         break;
3502         }
3503         return (found);
3504 }
3505
3506 /*
3507  * Wait for a complete incoming message, parse it, and respond accordingly.
3508  */
3509 static int
3510 ahc_parse_msg(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3511 {
3512         struct  ahc_initiator_tinfo *tinfo;
3513         struct  ahc_tmode_tstate *tstate;
3514         int     reject;
3515         int     done;
3516         int     response;
3517         u_int   targ_scsirate;
3518
3519         done = MSGLOOP_IN_PROG;
3520         response = FALSE;
3521         reject = FALSE;
3522         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel, devinfo->our_scsiid,
3523                                     devinfo->target, &tstate);
3524         targ_scsirate = tinfo->scsirate;
3525
3526         /*
3527          * Parse as much of the message as is available,
3528          * rejecting it if we don't support it.  When
3529          * the entire message is available and has been
3530          * handled, return MSGLOOP_MSGCOMPLETE, indicating
3531          * that we have parsed an entire message.
3532          *
3533          * In the case of extended messages, we accept the length
3534          * byte outright and perform more checking once we know the
3535          * extended message type.
3536          */
3537         switch (ahc->msgin_buf[0]) {
3538         case MSG_DISCONNECT:
3539         case MSG_SAVEDATAPOINTER:
3540         case MSG_CMDCOMPLETE:
3541         case MSG_RESTOREPOINTERS:
3542         case MSG_IGN_WIDE_RESIDUE:
3543                 /*
3544                  * End our message loop as these are messages
3545                  * the sequencer handles on its own.
3546                  */
3547                 done = MSGLOOP_TERMINATED;
3548                 break;
3549         case MSG_MESSAGE_REJECT:
3550                 response = ahc_handle_msg_reject(ahc, devinfo);
3551                 /* FALLTHROUGH */
3552         case MSG_NOOP:
3553                 done = MSGLOOP_MSGCOMPLETE;
3554                 break;
3555         case MSG_EXTENDED:
3556         {
3557                 /* Wait for enough of the message to begin validation */
3558                 if (ahc->msgin_index < 2)
3559                         break;
3560                 switch (ahc->msgin_buf[2]) {
3561                 case MSG_EXT_SDTR:
3562                 {
3563                         struct   ahc_syncrate *syncrate;
3564                         u_int    period;
3565                         u_int    ppr_options;
3566                         u_int    offset;
3567                         u_int    saved_offset;
3568
3569                         if (ahc->msgin_buf[1] != MSG_EXT_SDTR_LEN) {
3570                                 reject = TRUE;
3571                                 break;
3572                         }
3573
3574                         /*
3575                          * Wait until we have both args before validating
3576                          * and acting on this message.
3577                          *
3578                          * Add one to MSG_EXT_SDTR_LEN to account for
3579                          * the extended message preamble.
3580                          */
3581                         if (ahc->msgin_index < (MSG_EXT_SDTR_LEN + 1))
3582                                 break;
3583
3584                         period = ahc->msgin_buf[3];
3585                         ppr_options = 0;
3586                         saved_offset = offset = ahc->msgin_buf[4];
3587                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3588                                                            &ppr_options,
3589                                                            devinfo->role);
3590                         ahc_validate_offset(ahc, tinfo, syncrate, &offset,
3591                                             targ_scsirate & WIDEXFER,
3592                                             devinfo->role);
3593                         if (bootverbose) {
3594                                 printf("(%s:%c:%d:%d): Received "
3595                                        "SDTR period %x, offset %x\n\t"
3596                                        "Filtered to period %x, offset %x\n",
3597                                        ahc_name(ahc), devinfo->channel,
3598                                        devinfo->target, devinfo->lun,
3599                                        ahc->msgin_buf[3], saved_offset,
3600                                        period, offset);
3601                         }
3602                         ahc_set_syncrate(ahc, devinfo,
3603                                          syncrate, period,
3604                                          offset, ppr_options,
3605                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3606                                          /*paused*/TRUE);
3607
3608                         /*
3609                          * See if we initiated Sync Negotiation
3610                          * and didn't have to fall down to async
3611                          * transfers.
3612                          */
3613                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, TRUE)) {
3614                                 /* We started it */
3615                                 if (saved_offset != offset) {
3616                                         /* Went too low - force async */
3617                                         reject = TRUE;
3618                                 }
3619                         } else {
3620                                 /*
3621                                  * Send our own SDTR in reply
3622                                  */
3623                                 if (bootverbose
3624                                  && devinfo->role == ROLE_INITIATOR) {
3625                                         printf("(%s:%c:%d:%d): Target "
3626                                                "Initiated SDTR\n",
3627                                                ahc_name(ahc), devinfo->channel,
3628                                                devinfo->target, devinfo->lun);
3629                                 }
3630                                 ahc->msgout_index = 0;
3631                                 ahc->msgout_len = 0;
3632                                 ahc_construct_sdtr(ahc, devinfo,
3633                                                    period, offset);
3634                                 ahc->msgout_index = 0;
3635                                 response = TRUE;
3636                         }
3637                         done = MSGLOOP_MSGCOMPLETE;
3638                         break;
3639                 }
3640                 case MSG_EXT_WDTR:
3641                 {
3642                         u_int bus_width;
3643                         u_int saved_width;
3644                         u_int sending_reply;
3645
3646                         sending_reply = FALSE;
3647                         if (ahc->msgin_buf[1] != MSG_EXT_WDTR_LEN) {
3648                                 reject = TRUE;
3649                                 break;
3650                         }
3651
3652                         /*
3653                          * Wait until we have our arg before validating
3654                          * and acting on this message.
3655                          *
3656                          * Add one to MSG_EXT_WDTR_LEN to account for
3657                          * the extended message preamble.
3658                          */
3659                         if (ahc->msgin_index < (MSG_EXT_WDTR_LEN + 1))
3660                                 break;
3661
3662                         bus_width = ahc->msgin_buf[3];
3663                         saved_width = bus_width;
3664                         ahc_validate_width(ahc, tinfo, &bus_width,
3665                                            devinfo->role);
3666                         if (bootverbose) {
3667                                 printf("(%s:%c:%d:%d): Received WDTR "
3668                                        "%x filtered to %x\n",
3669                                        ahc_name(ahc), devinfo->channel,
3670                                        devinfo->target, devinfo->lun,
3671                                        saved_width, bus_width);
3672                         }
3673
3674                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, TRUE)) {
3675                                 /*
3676                                  * Don't send a WDTR back to the
3677                                  * target, since we asked first.
3678                                  * If the width went higher than our
3679                                  * request, reject it.
3680                                  */
3681                                 if (saved_width > bus_width) {
3682                                         reject = TRUE;
3683                                         printf("(%s:%c:%d:%d): requested %dBit "
3684                                                "transfers.  Rejecting...\n",
3685                                                ahc_name(ahc), devinfo->channel,
3686                                                devinfo->target, devinfo->lun,
3687                                                8 * (0x01 << bus_width));
3688                                         bus_width = 0;
3689                                 }
3690                         } else {
3691                                 /*
3692                                  * Send our own WDTR in reply
3693                                  */
3694                                 if (bootverbose
3695                                  && devinfo->role == ROLE_INITIATOR) {
3696                                         printf("(%s:%c:%d:%d): Target "
3697                                                "Initiated WDTR\n",
3698                                                ahc_name(ahc), devinfo->channel,
3699                                                devinfo->target, devinfo->lun);
3700                                 }
3701                                 ahc->msgout_index = 0;
3702                                 ahc->msgout_len = 0;
3703                                 ahc_construct_wdtr(ahc, devinfo, bus_width);
3704                                 ahc->msgout_index = 0;
3705                                 response = TRUE;
3706                                 sending_reply = TRUE;
3707                         }
3708                         /*
3709                          * After a wide message, we are async, but
3710                          * some devices don't seem to honor this portion
3711                          * of the spec.  Force a renegotiation of the
3712                          * sync component of our transfer agreement even
3713                          * if our goal is async.  By updating our width
3714                          * after forcing the negotiation, we avoid
3715                          * renegotiating for width.
3716                          */
3717                         ahc_update_neg_request(ahc, devinfo, tstate,
3718                                                tinfo, AHC_NEG_ALWAYS);
3719                         ahc_set_width(ahc, devinfo, bus_width,
3720                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3721                                       /*paused*/TRUE);
3722                         if (sending_reply == FALSE && reject == FALSE) {
3723
3724                                 /*
3725                                  * We will always have an SDTR to send.
3726                                  */
3727                                 ahc->msgout_index = 0;
3728                                 ahc->msgout_len = 0;
3729                                 ahc_build_transfer_msg(ahc, devinfo);
3730                                 ahc->msgout_index = 0;
3731                                 response = TRUE;
3732                         }
3733                         done = MSGLOOP_MSGCOMPLETE;
3734                         break;
3735                 }
3736                 case MSG_EXT_PPR:
3737                 {
3738                         struct  ahc_syncrate *syncrate;
3739                         u_int   period;
3740                         u_int   offset;
3741                         u_int   bus_width;
3742                         u_int   ppr_options;
3743                         u_int   saved_width;
3744                         u_int   saved_offset;
3745                         u_int   saved_ppr_options;
3746
3747                         if (ahc->msgin_buf[1] != MSG_EXT_PPR_LEN) {
3748                                 reject = TRUE;
3749                                 break;
3750                         }
3751
3752                         /*
3753                          * Wait until we have all args before validating
3754                          * and acting on this message.
3755                          *
3756                          * Add one to MSG_EXT_PPR_LEN to account for
3757                          * the extended message preamble.
3758                          */
3759                         if (ahc->msgin_index < (MSG_EXT_PPR_LEN + 1))
3760                                 break;
3761
3762                         period = ahc->msgin_buf[3];
3763                         offset = ahc->msgin_buf[5];
3764                         bus_width = ahc->msgin_buf[6];
3765                         saved_width = bus_width;
3766                         ppr_options = ahc->msgin_buf[7];
3767                         /*
3768                          * According to the spec, a DT only
3769                          * period factor with no DT option
3770                          * set implies async.
3771                          */
3772                         if ((ppr_options & MSG_EXT_PPR_DT_REQ) == 0
3773                          && period == 9)
3774                                 offset = 0;
3775                         saved_ppr_options = ppr_options;
3776                         saved_offset = offset;
3777
3778                         /*
3779                          * Mask out any options we don't support
3780                          * on any controller.  Transfer options are
3781                          * only available if we are negotiating wide.
3782                          */
3783                         ppr_options &= MSG_EXT_PPR_DT_REQ;
3784                         if (bus_width == 0)
3785                                 ppr_options = 0;
3786
3787                         ahc_validate_width(ahc, tinfo, &bus_width,
3788                                            devinfo->role);
3789                         syncrate = ahc_devlimited_syncrate(ahc, tinfo, &period,
3790                                                            &ppr_options,
3791                                                            devinfo->role);
3792                         ahc_validate_offset(ahc, tinfo, syncrate,
3793                                             &offset, bus_width,
3794                                             devinfo->role);
3795
3796                         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, TRUE)) {
3797                                 /*
3798                                  * If we are unable to do any of the
3799                                  * requested options (we went too low),
3800                                  * then we'll have to reject the message.
3801                                  */
3802                                 if (saved_width > bus_width
3803                                  || saved_offset != offset
3804                                  || saved_ppr_options != ppr_options) {
3805                                         reject = TRUE;
3806                                         period = 0;
3807                                         offset = 0;
3808                                         bus_width = 0;
3809                                         ppr_options = 0;
3810                                         syncrate = NULL;
3811                                 }
3812                         } else {
3813                                 if (devinfo->role != ROLE_TARGET)
3814                                         printf("(%s:%c:%d:%d): Target "
3815                                                "Initiated PPR\n",
3816                                                ahc_name(ahc), devinfo->channel,
3817                                                devinfo->target, devinfo->lun);
3818                                 else
3819                                         printf("(%s:%c:%d:%d): Initiator "
3820                                                "Initiated PPR\n",
3821                                                ahc_name(ahc), devinfo->channel,
3822                                                devinfo->target, devinfo->lun);
3823                                 ahc->msgout_index = 0;
3824                                 ahc->msgout_len = 0;
3825                                 ahc_construct_ppr(ahc, devinfo, period, offset,
3826                                                   bus_width, ppr_options);
3827                                 ahc->msgout_index = 0;
3828                                 response = TRUE;
3829                         }
3830                         if (bootverbose) {
3831                                 printf("(%s:%c:%d:%d): Received PPR width %x, "
3832                                        "period %x, offset %x,options %x\n"
3833                                        "\tFiltered to width %x, period %x, "
3834                                        "offset %x, options %x\n",
3835                                        ahc_name(ahc), devinfo->channel,
3836                                        devinfo->target, devinfo->lun,
3837                                        saved_width, ahc->msgin_buf[3],
3838                                        saved_offset, saved_ppr_options,
3839                                        bus_width, period, offset, ppr_options);
3840                         }
3841                         ahc_set_width(ahc, devinfo, bus_width,
3842                                       AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3843                                       /*paused*/TRUE);
3844                         ahc_set_syncrate(ahc, devinfo,
3845                                          syncrate, period,
3846                                          offset, ppr_options,
3847                                          AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3848                                          /*paused*/TRUE);
3849                         done = MSGLOOP_MSGCOMPLETE;
3850                         break;
3851                 }
3852                 default:
3853                         /* Unknown extended message.  Reject it. */
3854                         reject = TRUE;
3855                         break;
3856                 }
3857                 break;
3858         }
3859 #ifdef AHC_TARGET_MODE
3860         case MSG_BUS_DEV_RESET:
3861                 ahc_handle_devreset(ahc, devinfo,
3862                                     CAM_BDR_SENT,
3863                                     "Bus Device Reset Received",
3864                                     /*verbose_level*/0);
3865                 ahc_restart(ahc);
3866                 done = MSGLOOP_TERMINATED;
3867                 break;
3868         case MSG_ABORT_TAG:
3869         case MSG_ABORT:
3870         case MSG_CLEAR_QUEUE:
3871         {
3872                 int tag;
3873
3874                 /* Target mode messages */
3875                 if (devinfo->role != ROLE_TARGET) {
3876                         reject = TRUE;
3877                         break;
3878                 }
3879                 tag = SCB_LIST_NULL;
3880                 if (ahc->msgin_buf[0] == MSG_ABORT_TAG)
3881                         tag = ahc_inb(ahc, INITIATOR_TAG);
3882                 ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
3883                                devinfo->lun, tag, ROLE_TARGET,
3884                                CAM_REQ_ABORTED);
3885
3886                 tstate = ahc->enabled_targets[devinfo->our_scsiid];
3887                 if (tstate != NULL) {
3888                         struct ahc_tmode_lstate* lstate;
3889
3890                         lstate = tstate->enabled_luns[devinfo->lun];
3891                         if (lstate != NULL) {
3892                                 ahc_queue_lstate_event(ahc, lstate,
3893                                                        devinfo->our_scsiid,
3894                                                        ahc->msgin_buf[0],
3895                                                        /*arg*/tag);
3896                                 ahc_send_lstate_events(ahc, lstate);
3897                         }
3898                 }
3899                 ahc_restart(ahc);
3900                 done = MSGLOOP_TERMINATED;
3901                 break;
3902         }
3903 #endif
3904         case MSG_TERM_IO_PROC:
3905         default:
3906                 reject = TRUE;
3907                 break;
3908         }
3909
3910         if (reject) {
3911                 /*
3912                  * Setup to reject the message.
3913                  */
3914                 ahc->msgout_index = 0;
3915                 ahc->msgout_len = 1;
3916                 ahc->msgout_buf[0] = MSG_MESSAGE_REJECT;
3917                 done = MSGLOOP_MSGCOMPLETE;
3918                 response = TRUE;
3919         }
3920
3921         if (done != MSGLOOP_IN_PROG && !response)
3922                 /* Clear the outgoing message buffer */
3923                 ahc->msgout_len = 0;
3924
3925         return (done);
3926 }
3927
3928 /*
3929  * Process a message reject message.
3930  */
3931 static int
3932 ahc_handle_msg_reject(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
3933 {
3934         /*
3935          * What we care about here is if we had an
3936          * outstanding SDTR or WDTR message for this
3937          * target.  If we did, this is a signal that
3938          * the target is refusing negotiation.
3939          */
3940         struct scb *scb;
3941         struct ahc_initiator_tinfo *tinfo;
3942         struct ahc_tmode_tstate *tstate;
3943         u_int scb_index;
3944         u_int last_msg;
3945         int   response = 0;
3946
3947         scb_index = ahc_inb(ahc, SCB_TAG);
3948         scb = ahc_lookup_scb(ahc, scb_index);
3949         tinfo = ahc_fetch_transinfo(ahc, devinfo->channel,
3950                                     devinfo->our_scsiid,
3951                                     devinfo->target, &tstate);
3952         /* Might be necessary */
3953         last_msg = ahc_inb(ahc, LAST_MSG);
3954
3955         if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_PPR, /*full*/FALSE)) {
3956                 /*
3957                  * Target does not support the PPR message.
3958                  * Attempt to negotiate SPI-2 style.
3959                  */
3960                 if (bootverbose) {
3961                         printf("(%s:%c:%d:%d): PPR Rejected. "
3962                                "Trying WDTR/SDTR\n",
3963                                ahc_name(ahc), devinfo->channel,
3964                                devinfo->target, devinfo->lun);
3965                 }
3966                 tinfo->goal.ppr_options = 0;
3967                 tinfo->curr.transport_version = 2;
3968                 tinfo->goal.transport_version = 2;
3969                 ahc->msgout_index = 0;
3970                 ahc->msgout_len = 0;
3971                 ahc_build_transfer_msg(ahc, devinfo);
3972                 ahc->msgout_index = 0;
3973                 response = 1;
3974         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_WDTR, /*full*/FALSE)) {
3975
3976                 /* note 8bit xfers */
3977                 printf("(%s:%c:%d:%d): refuses WIDE negotiation.  Using "
3978                        "8bit transfers\n", ahc_name(ahc),
3979                        devinfo->channel, devinfo->target, devinfo->lun);
3980                 ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
3981                               AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
3982                               /*paused*/TRUE);
3983                 /*
3984                  * No need to clear the sync rate.  If the target
3985                  * did not accept the command, our syncrate is
3986                  * unaffected.  If the target started the negotiation,
3987                  * but rejected our response, we already cleared the
3988                  * sync rate before sending our WDTR.
3989                  */
3990                 if (tinfo->goal.offset != tinfo->curr.offset) {
3991
3992                         /* Start the sync negotiation */
3993                         ahc->msgout_index = 0;
3994                         ahc->msgout_len = 0;
3995                         ahc_build_transfer_msg(ahc, devinfo);
3996                         ahc->msgout_index = 0;
3997                         response = 1;
3998                 }
3999         } else if (ahc_sent_msg(ahc, AHCMSG_EXT, MSG_EXT_SDTR, /*full*/FALSE)) {
4000                 /* note asynch xfers and clear flag */
4001                 ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL, /*period*/0,
4002                                  /*offset*/0, /*ppr_options*/0,
4003                                  AHC_TRANS_ACTIVE|AHC_TRANS_GOAL,
4004                                  /*paused*/TRUE);
4005                 printf("(%s:%c:%d:%d): refuses synchronous negotiation. "
4006                        "Using asynchronous transfers\n",
4007                        ahc_name(ahc), devinfo->channel,
4008                        devinfo->target, devinfo->lun);
4009         } else if ((scb->hscb->control & MSG_SIMPLE_TASK) != 0) {
4010                 int tag_type;
4011                 int mask;
4012
4013                 tag_type = (scb->hscb->control & MSG_SIMPLE_TASK);
4014
4015                 if (tag_type == MSG_SIMPLE_TASK) {
4016                         printf("(%s:%c:%d:%d): refuses tagged commands.  "
4017                                "Performing non-tagged I/O\n", ahc_name(ahc),
4018                                devinfo->channel, devinfo->target, devinfo->lun);
4019                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_NONE);
4020                         mask = ~0x23;
4021                 } else {
4022                         printf("(%s:%c:%d:%d): refuses %s tagged commands.  "
4023                                "Performing simple queue tagged I/O only\n",
4024                                ahc_name(ahc), devinfo->channel, devinfo->target,
4025                                devinfo->lun, tag_type == MSG_ORDERED_TASK
4026                                ? "ordered" : "head of queue");
4027                         ahc_set_tags(ahc, scb->io_ctx, devinfo, AHC_QUEUE_BASIC);
4028                         mask = ~0x03;
4029                 }
4030
4031                 /*
4032                  * Resend the identify for this CCB as the target
4033                  * may believe that the selection is invalid otherwise.
4034                  */
4035                 ahc_outb(ahc, SCB_CONTROL,
4036                          ahc_inb(ahc, SCB_CONTROL) & mask);
4037                 scb->hscb->control &= mask;
4038                 ahc_set_transaction_tag(scb, /*enabled*/FALSE,
4039                                         /*type*/MSG_SIMPLE_TASK);
4040                 ahc_outb(ahc, MSG_OUT, MSG_IDENTIFYFLAG);
4041                 ahc_assert_atn(ahc);
4042
4043                 /*
4044                  * This transaction is now at the head of
4045                  * the untagged queue for this target.
4046                  */
4047                 if ((ahc->flags & AHC_SCB_BTT) == 0) {
4048                         struct scb_tailq *untagged_q;
4049
4050                         untagged_q =
4051                             &(ahc->untagged_queues[devinfo->target_offset]);
4052                         TAILQ_INSERT_HEAD(untagged_q, scb, links.tqe);
4053                         scb->flags |= SCB_UNTAGGEDQ;
4054                 }
4055                 ahc_busy_tcl(ahc, BUILD_TCL(scb->hscb->scsiid, devinfo->lun),
4056                              scb->hscb->tag);
4057
4058                 /*
4059                  * Requeue all tagged commands for this target
4060                  * currently in our posession so they can be
4061                  * converted to untagged commands.
4062                  */
4063                 ahc_search_qinfifo(ahc, SCB_GET_TARGET(ahc, scb),
4064                                    SCB_GET_CHANNEL(ahc, scb),
4065                                    SCB_GET_LUN(scb), /*tag*/SCB_LIST_NULL,
4066                                    ROLE_INITIATOR, CAM_REQUEUE_REQ,
4067                                    SEARCH_COMPLETE);
4068         } else {
4069                 /*
4070                  * Otherwise, we ignore it.
4071                  */
4072                 printf("%s:%c:%d: Message reject for %x -- ignored\n",
4073                        ahc_name(ahc), devinfo->channel, devinfo->target,
4074                        last_msg);
4075         }
4076         return (response);
4077 }
4078
4079 /*
4080  * Process an ingnore wide residue message.
4081  */
4082 static void
4083 ahc_handle_ign_wide_residue(struct ahc_softc *ahc, struct ahc_devinfo *devinfo)
4084 {
4085         u_int scb_index;
4086         struct scb *scb;
4087
4088         scb_index = ahc_inb(ahc, SCB_TAG);
4089         scb = ahc_lookup_scb(ahc, scb_index);
4090         /*
4091          * XXX Actually check data direction in the sequencer?
4092          * Perhaps add datadir to some spare bits in the hscb?
4093          */
4094         if ((ahc_inb(ahc, SEQ_FLAGS) & DPHASE) == 0
4095          || ahc_get_transfer_dir(scb) != CAM_DIR_IN) {
4096                 /*
4097                  * Ignore the message if we haven't
4098                  * seen an appropriate data phase yet.
4099                  */
4100         } else {
4101                 /*
4102                  * If the residual occurred on the last
4103                  * transfer and the transfer request was
4104                  * expected to end on an odd count, do
4105                  * nothing.  Otherwise, subtract a byte
4106                  * and update the residual count accordingly.
4107                  */
4108                 uint32_t sgptr;
4109
4110                 sgptr = ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4111                 if ((sgptr & SG_LIST_NULL) != 0
4112                  && (ahc_inb(ahc, SCB_LUN) & SCB_XFERLEN_ODD) != 0) {
4113                         /*
4114                          * If the residual occurred on the last
4115                          * transfer and the transfer request was
4116                          * expected to end on an odd count, do
4117                          * nothing.
4118                          */
4119                 } else {
4120                         struct ahc_dma_seg *sg;
4121                         uint32_t data_cnt;
4122                         uint32_t data_addr;
4123                         uint32_t sglen;
4124
4125                         /* Pull in all of the sgptr */
4126                         sgptr = ahc_inl(ahc, SCB_RESIDUAL_SGPTR);
4127                         data_cnt = ahc_inl(ahc, SCB_RESIDUAL_DATACNT);
4128
4129                         if ((sgptr & SG_LIST_NULL) != 0) {
4130                                 /*
4131                                  * The residual data count is not updated
4132                                  * for the command run to completion case.
4133                                  * Explicitly zero the count.
4134                                  */
4135                                 data_cnt &= ~AHC_SG_LEN_MASK;
4136                         }
4137
4138                         data_addr = ahc_inl(ahc, SHADDR);
4139
4140                         data_cnt += 1;
4141                         data_addr -= 1;
4142                         sgptr &= SG_PTR_MASK;
4143
4144                         sg = ahc_sg_bus_to_virt(scb, sgptr);
4145
4146                         /*
4147                          * The residual sg ptr points to the next S/G
4148                          * to load so we must go back one.
4149                          */
4150                         sg--;
4151                         sglen = ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
4152                         if (sg != scb->sg_list
4153                          && sglen < (data_cnt & AHC_SG_LEN_MASK)) {
4154
4155                                 sg--;
4156                                 sglen = ahc_le32toh(sg->len);
4157                                 /*
4158                                  * Preserve High Address and SG_LIST bits
4159                                  * while setting the count to 1.
4160                                  */
4161                                 data_cnt = 1 | (sglen & (~AHC_SG_LEN_MASK));
4162                                 data_addr = ahc_le32toh(sg->addr)
4163                                           + (sglen & AHC_SG_LEN_MASK) - 1;
4164
4165                                 /*
4166                                  * Increment sg so it points to the
4167                                  * "next" sg.
4168                                  */
4169                                 sg++;
4170                                 sgptr = ahc_sg_virt_to_bus(scb, sg);
4171                         }
4172                         ahc_outl(ahc, SCB_RESIDUAL_SGPTR, sgptr);
4173                         ahc_outl(ahc, SCB_RESIDUAL_DATACNT, data_cnt);
4174                         /*
4175                          * Toggle the "oddness" of the transfer length
4176                          * to handle this mid-transfer ignore wide
4177                          * residue.  This ensures that the oddness is
4178                          * correct for subsequent data transfers.
4179                          */
4180                         ahc_outb(ahc, SCB_LUN,
4181                                  ahc_inb(ahc, SCB_LUN) ^ SCB_XFERLEN_ODD);
4182                 }
4183         }
4184 }
4185
4186
4187 /*
4188  * Reinitialize the data pointers for the active transfer
4189  * based on its current residual.
4190  */
4191 static void
4192 ahc_reinitialize_dataptrs(struct ahc_softc *ahc)
4193 {
4194         struct   scb *scb;
4195         struct   ahc_dma_seg *sg;
4196         u_int    scb_index;
4197         uint32_t sgptr;
4198         uint32_t resid;
4199         uint32_t dataptr;
4200
4201         scb_index = ahc_inb(ahc, SCB_TAG);
4202         scb = ahc_lookup_scb(ahc, scb_index);
4203         sgptr = (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 3) << 24)
4204               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 2) << 16)
4205               | (ahc_inb(ahc, SCB_RESIDUAL_SGPTR + 1) << 8)
4206               | ahc_inb(ahc, SCB_RESIDUAL_SGPTR);
4207
4208         sgptr &= SG_PTR_MASK;
4209         sg = ahc_sg_bus_to_virt(scb, sgptr);
4210
4211         /* The residual sg_ptr always points to the next sg */
4212         sg--;
4213
4214         resid = (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 2) << 16)
4215               | (ahc_inb(ahc, SCB_RESIDUAL_DATACNT + 1) << 8)
4216               | ahc_inb(ahc, SCB_RESIDUAL_DATACNT);
4217
4218         dataptr = ahc_le32toh(sg->addr)
4219                 + (ahc_le32toh(sg->len) & AHC_SG_LEN_MASK)
4220                 - resid;
4221         if ((ahc->flags & AHC_39BIT_ADDRESSING) != 0) {
4222                 u_int dscommand1;
4223
4224                 dscommand1 = ahc_inb(ahc, DSCOMMAND1);
4225                 ahc_outb(ahc, DSCOMMAND1, dscommand1 | HADDLDSEL0);
4226                 ahc_outb(ahc, HADDR,
4227                          (ahc_le32toh(sg->len) >> 24) & SG_HIGH_ADDR_BITS);
4228                 ahc_outb(ahc, DSCOMMAND1, dscommand1);
4229         }
4230         ahc_outb(ahc, HADDR + 3, dataptr >> 24);
4231         ahc_outb(ahc, HADDR + 2, dataptr >> 16);
4232         ahc_outb(ahc, HADDR + 1, dataptr >> 8);
4233         ahc_outb(ahc, HADDR, dataptr);
4234         ahc_outb(ahc, HCNT + 2, resid >> 16);
4235         ahc_outb(ahc, HCNT + 1, resid >> 8);
4236         ahc_outb(ahc, HCNT, resid);
4237         if ((ahc->features & AHC_ULTRA2) == 0) {
4238                 ahc_outb(ahc, STCNT + 2, resid >> 16);
4239                 ahc_outb(ahc, STCNT + 1, resid >> 8);
4240                 ahc_outb(ahc, STCNT, resid);
4241         }
4242 }
4243
4244 /*
4245  * Handle the effects of issuing a bus device reset message.
4246  */
4247 static void
4248 ahc_handle_devreset(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4249                     cam_status status, char *message, int verbose_level)
4250 {
4251 #ifdef AHC_TARGET_MODE
4252         struct ahc_tmode_tstate* tstate;
4253         u_int lun;
4254 #endif
4255         int found;
4256
4257         found = ahc_abort_scbs(ahc, devinfo->target, devinfo->channel,
4258                                CAM_LUN_WILDCARD, SCB_LIST_NULL, devinfo->role,
4259                                status);
4260
4261 #ifdef AHC_TARGET_MODE
4262         /*
4263          * Send an immediate notify ccb to all target mord peripheral
4264          * drivers affected by this action.
4265          */
4266         tstate = ahc->enabled_targets[devinfo->our_scsiid];
4267         if (tstate != NULL) {
4268                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
4269                         struct ahc_tmode_lstate* lstate;
4270
4271                         lstate = tstate->enabled_luns[lun];
4272                         if (lstate == NULL)
4273                                 continue;
4274
4275                         ahc_queue_lstate_event(ahc, lstate, devinfo->our_scsiid,
4276                                                MSG_BUS_DEV_RESET, /*arg*/0);
4277                         ahc_send_lstate_events(ahc, lstate);
4278                 }
4279         }
4280 #endif
4281
4282         /*
4283          * Go back to async/narrow transfers and renegotiate.
4284          */
4285         ahc_set_width(ahc, devinfo, MSG_EXT_WDTR_BUS_8_BIT,
4286                       AHC_TRANS_CUR, /*paused*/TRUE);
4287         ahc_set_syncrate(ahc, devinfo, /*syncrate*/NULL,
4288                          /*period*/0, /*offset*/0, /*ppr_options*/0,
4289                          AHC_TRANS_CUR, /*paused*/TRUE);
4290
4291         if (status != CAM_SEL_TIMEOUT)
4292                 ahc_send_async(ahc, devinfo->channel, devinfo->target,
4293                                CAM_LUN_WILDCARD, AC_SENT_BDR);
4294
4295         if (message != NULL
4296          && (verbose_level <= bootverbose))
4297                 printf("%s: %s on %c:%d. %d SCBs aborted\n", ahc_name(ahc),
4298                        message, devinfo->channel, devinfo->target, found);
4299 }
4300
4301 #ifdef AHC_TARGET_MODE
4302 static void
4303 ahc_setup_target_msgin(struct ahc_softc *ahc, struct ahc_devinfo *devinfo,
4304                        struct scb *scb)
4305 {
4306
4307         /*
4308          * To facilitate adding multiple messages together,
4309          * each routine should increment the index and len
4310          * variables instead of setting them explicitly.
4311          */
4312         ahc->msgout_index = 0;
4313         ahc->msgout_len = 0;
4314
4315         if (scb != NULL && (scb->flags & SCB_AUTO_NEGOTIATE) != 0)
4316                 ahc_build_transfer_msg(ahc, devinfo);
4317         else
4318                 panic("ahc_intr: AWAITING target message with no message");
4319
4320         ahc->msgout_index = 0;
4321         ahc->msg_type = MSG_TYPE_TARGET_MSGIN;
4322 }
4323 #endif
4324 /**************************** Initialization **********************************/
4325 /*
4326  * Allocate a controller structure for a new device
4327  * and perform initial initializion.
4328  */
4329 struct ahc_softc *
4330 ahc_alloc(void *platform_arg, char *name)
4331 {
4332         struct  ahc_softc *ahc;
4333         int     i;
4334
4335 #ifndef __FreeBSD__
4336         ahc = malloc(sizeof(*ahc), M_DEVBUF, M_NOWAIT);
4337         if (!ahc) {
4338                 printf("aic7xxx: cannot malloc softc!\n");
4339                 free(name, M_DEVBUF);
4340                 return NULL;
4341         }
4342 #else
4343         ahc = device_get_softc((device_t)platform_arg);
4344 #endif
4345         memset(ahc, 0, sizeof(*ahc));
4346         ahc->seep_config = malloc(sizeof(*ahc->seep_config),
4347                                   M_DEVBUF, M_NOWAIT);
4348         if (ahc->seep_config == NULL) {
4349 #ifndef __FreeBSD__
4350                 free(ahc, M_DEVBUF);
4351 #endif
4352                 free(name, M_DEVBUF);
4353                 return (NULL);
4354         }
4355         LIST_INIT(&ahc->pending_scbs);
4356         /* We don't know our unit number until the OSM sets it */
4357         ahc->name = name;
4358         ahc->unit = -1;
4359         ahc->description = NULL;
4360         ahc->channel = 'A';
4361         ahc->channel_b = 'B';
4362         ahc->chip = AHC_NONE;
4363         ahc->features = AHC_FENONE;
4364         ahc->bugs = AHC_BUGNONE;
4365         ahc->flags = AHC_FNONE;
4366         /*
4367          * Default to all error reporting enabled with the
4368          * sequencer operating at its fastest speed.
4369          * The bus attach code may modify this.
4370          */
4371         ahc->seqctl = FASTMODE;
4372
4373         for (i = 0; i < AHC_NUM_TARGETS; i++)
4374                 TAILQ_INIT(&ahc->untagged_queues[i]);
4375         if (ahc_platform_alloc(ahc, platform_arg) != 0) {
4376                 ahc_free(ahc);
4377                 ahc = NULL;
4378         }
4379         return (ahc);
4380 }
4381
4382 int
4383 ahc_softc_init(struct ahc_softc *ahc)
4384 {
4385
4386         /* The IRQMS bit is only valid on VL and EISA chips */
4387         if ((ahc->chip & AHC_PCI) == 0)
4388                 ahc->unpause = ahc_inb(ahc, HCNTRL) & IRQMS;
4389         else
4390                 ahc->unpause = 0;
4391         ahc->pause = ahc->unpause | PAUSE;
4392         /* XXX The shared scb data stuff should be deprecated */
4393         if (ahc->scb_data == NULL) {
4394                 ahc->scb_data = malloc(sizeof(*ahc->scb_data),
4395                                        M_DEVBUF, M_NOWAIT);
4396                 if (ahc->scb_data == NULL)
4397                         return (ENOMEM);
4398                 memset(ahc->scb_data, 0, sizeof(*ahc->scb_data));
4399         }
4400
4401         return (0);
4402 }
4403
4404 void
4405 ahc_set_unit(struct ahc_softc *ahc, int unit)
4406 {
4407         ahc->unit = unit;
4408 }
4409
4410 void
4411 ahc_set_name(struct ahc_softc *ahc, char *name)
4412 {
4413         if (ahc->name != NULL)
4414                 free(ahc->name, M_DEVBUF);
4415         ahc->name = name;
4416 }
4417
4418 void
4419 ahc_free(struct ahc_softc *ahc)
4420 {
4421         int i;
4422
4423         switch (ahc->init_level) {
4424         default:
4425         case 5:
4426                 ahc_shutdown(ahc);
4427                 /* FALLTHROUGH */
4428         case 4:
4429                 ahc_dmamap_unload(ahc, ahc->shared_data_dmat,
4430                                   ahc->shared_data_dmamap);
4431                 /* FALLTHROUGH */
4432         case 3:
4433                 ahc_dmamem_free(ahc, ahc->shared_data_dmat, ahc->qoutfifo,
4434                                 ahc->shared_data_dmamap);
4435                 ahc_dmamap_destroy(ahc, ahc->shared_data_dmat,
4436                                    ahc->shared_data_dmamap);
4437                 /* FALLTHROUGH */
4438         case 2:
4439                 ahc_dma_tag_destroy(ahc, ahc->shared_data_dmat);
4440         case 1:
4441 #ifndef __linux__
4442                 ahc_dma_tag_destroy(ahc, ahc->buffer_dmat);
4443 #endif
4444                 break;
4445         case 0:
4446                 break;
4447         }
4448
4449 #ifndef __linux__
4450         ahc_dma_tag_destroy(ahc, ahc->parent_dmat);
4451 #endif
4452         ahc_platform_free(ahc);
4453         ahc_fini_scbdata(ahc);
4454         for (i = 0; i < AHC_NUM_TARGETS; i++) {
4455                 struct ahc_tmode_tstate *tstate;
4456
4457                 tstate = ahc->enabled_targets[i];
4458                 if (tstate != NULL) {
4459 #ifdef AHC_TARGET_MODE
4460                         int j;
4461
4462                         for (j = 0; j < AHC_NUM_LUNS; j++) {
4463                                 struct ahc_tmode_lstate *lstate;
4464
4465                                 lstate = tstate->enabled_luns[j];
4466                                 if (lstate != NULL) {
4467                                         xpt_free_path(lstate->path);
4468                                         free(lstate, M_DEVBUF);
4469                                 }
4470                         }
4471 #endif
4472                         free(tstate, M_DEVBUF);
4473                 }
4474         }
4475 #ifdef AHC_TARGET_MODE
4476         if (ahc->black_hole != NULL) {
4477                 xpt_free_path(ahc->black_hole->path);
4478                 free(ahc->black_hole, M_DEVBUF);
4479         }
4480 #endif
4481         if (ahc->name != NULL)
4482                 free(ahc->name, M_DEVBUF);
4483         if (ahc->seep_config != NULL)
4484                 free(ahc->seep_config, M_DEVBUF);
4485 #ifndef __FreeBSD__
4486         free(ahc, M_DEVBUF);
4487 #endif
4488         return;
4489 }
4490
4491 void
4492 ahc_shutdown(void *arg)
4493 {
4494         struct  ahc_softc *ahc;
4495         int     i;
4496
4497         ahc = (struct ahc_softc *)arg;
4498
4499         /* This will reset most registers to 0, but not all */
4500         ahc_reset(ahc, /*reinit*/FALSE);
4501         ahc_outb(ahc, SCSISEQ, 0);
4502         ahc_outb(ahc, SXFRCTL0, 0);
4503         ahc_outb(ahc, DSPCISTATUS, 0);
4504
4505         for (i = TARG_SCSIRATE; i < SCSICONF; i++)
4506                 ahc_outb(ahc, i, 0);
4507 }
4508
4509 /*
4510  * Reset the controller and record some information about it
4511  * that is only available just after a reset.  If "reinit" is
4512  * non-zero, this reset occured after initial configuration
4513  * and the caller requests that the chip be fully reinitialized
4514  * to a runable state.  Chip interrupts are *not* enabled after
4515  * a reinitialization.  The caller must enable interrupts via
4516  * ahc_intr_enable().
4517  */
4518 int
4519 ahc_reset(struct ahc_softc *ahc, int reinit)
4520 {
4521         u_int   sblkctl;
4522         u_int   sxfrctl1_a, sxfrctl1_b;
4523         int     error;
4524         int     wait;
4525
4526         /*
4527          * Preserve the value of the SXFRCTL1 register for all channels.
4528          * It contains settings that affect termination and we don't want
4529          * to disturb the integrity of the bus.
4530          */
4531         ahc_pause(ahc);
4532         sxfrctl1_b = 0;
4533         if ((ahc->chip & AHC_CHIPID_MASK) == AHC_AIC7770) {
4534                 u_int sblkctl;
4535
4536                 /*
4537                  * Save channel B's settings in case this chip
4538                  * is setup for TWIN channel operation.
4539                  */
4540                 sblkctl = ahc_inb(ahc, SBLKCTL);
4541                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4542                 sxfrctl1_b = ahc_inb(ahc, SXFRCTL1);
4543                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4544         }
4545         sxfrctl1_a = ahc_inb(ahc, SXFRCTL1);
4546
4547         ahc_outb(ahc, HCNTRL, CHIPRST | ahc->pause);
4548
4549         /*
4550          * Ensure that the reset has finished.  We delay 1000us
4551          * prior to reading the register to make sure the chip
4552          * has sufficiently completed its reset to handle register
4553          * accesses.
4554          */
4555         wait = 1000;
4556         do {
4557                 ahc_delay(1000);
4558         } while (--wait && !(ahc_inb(ahc, HCNTRL) & CHIPRSTACK));
4559
4560         if (wait == 0) {
4561                 printf("%s: WARNING - Failed chip reset!  "
4562                        "Trying to initialize anyway.\n", ahc_name(ahc));
4563         }
4564         ahc_outb(ahc, HCNTRL, ahc->pause);
4565
4566         /* Determine channel configuration */
4567         sblkctl = ahc_inb(ahc, SBLKCTL) & (SELBUSB|SELWIDE);
4568         /* No Twin Channel PCI cards */
4569         if ((ahc->chip & AHC_PCI) != 0)
4570                 sblkctl &= ~SELBUSB;
4571         switch (sblkctl) {
4572         case 0:
4573                 /* Single Narrow Channel */
4574                 break;
4575         case 2:
4576                 /* Wide Channel */
4577                 ahc->features |= AHC_WIDE;
4578                 break;
4579         case 8:
4580                 /* Twin Channel */
4581                 ahc->features |= AHC_TWIN;
4582                 break;
4583         default:
4584                 printf(" Unsupported adapter type.  Ignoring\n");
4585                 return(-1);
4586         }
4587
4588         /*
4589          * Reload sxfrctl1.
4590          *
4591          * We must always initialize STPWEN to 1 before we
4592          * restore the saved values.  STPWEN is initialized
4593          * to a tri-state condition which can only be cleared
4594          * by turning it on.
4595          */
4596         if ((ahc->features & AHC_TWIN) != 0) {
4597                 u_int sblkctl;
4598
4599                 sblkctl = ahc_inb(ahc, SBLKCTL);
4600                 ahc_outb(ahc, SBLKCTL, sblkctl | SELBUSB);
4601                 ahc_outb(ahc, SXFRCTL1, sxfrctl1_b);
4602                 ahc_outb(ahc, SBLKCTL, sblkctl & ~SELBUSB);
4603         }
4604         ahc_outb(ahc, SXFRCTL1, sxfrctl1_a);
4605
4606         error = 0;
4607         if (reinit != 0)
4608                 /*
4609                  * If a recovery action has forced a chip reset,
4610                  * re-initialize the chip to our liking.
4611                  */
4612                 error = ahc->bus_chip_init(ahc);
4613 #ifdef AHC_DUMP_SEQ
4614         else
4615                 ahc_dumpseq(ahc);
4616 #endif
4617
4618         return (error);
4619 }
4620
4621 /*
4622  * Determine the number of SCBs available on the controller
4623  */
4624 int
4625 ahc_probe_scbs(struct ahc_softc *ahc) {
4626         int i;
4627
4628         for (i = 0; i < AHC_SCB_MAX; i++) {
4629
4630                 ahc_outb(ahc, SCBPTR, i);
4631                 ahc_outb(ahc, SCB_BASE, i);
4632                 if (ahc_inb(ahc, SCB_BASE) != i)
4633                         break;
4634                 ahc_outb(ahc, SCBPTR, 0);
4635                 if (ahc_inb(ahc, SCB_BASE) != 0)
4636                         break;
4637         }
4638         return (i);
4639 }
4640
4641 static void
4642 ahc_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error)
4643 {
4644         dma_addr_t *baddr;
4645
4646         baddr = (dma_addr_t *)arg;
4647         *baddr = segs->ds_addr;
4648 }
4649
4650 static void
4651 ahc_build_free_scb_list(struct ahc_softc *ahc)
4652 {
4653         int scbsize;
4654         int i;
4655
4656         scbsize = 32;
4657         if ((ahc->flags & AHC_LSCBS_ENABLED) != 0)
4658                 scbsize = 64;
4659
4660         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
4661                 int j;
4662
4663                 ahc_outb(ahc, SCBPTR, i);
4664
4665                 /*
4666                  * Touch all SCB bytes to avoid parity errors
4667                  * should one of our debugging routines read
4668                  * an otherwise uninitiatlized byte.
4669                  */
4670                 for (j = 0; j < scbsize; j++)
4671                         ahc_outb(ahc, SCB_BASE+j, 0xFF);
4672
4673                 /* Clear the control byte. */
4674                 ahc_outb(ahc, SCB_CONTROL, 0);
4675
4676                 /* Set the next pointer */
4677                 if ((ahc->flags & AHC_PAGESCBS) != 0)
4678                         ahc_outb(ahc, SCB_NEXT, i+1);
4679                 else
4680                         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4681
4682                 /* Make the tag number, SCSIID, and lun invalid */
4683                 ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
4684                 ahc_outb(ahc, SCB_SCSIID, 0xFF);
4685                 ahc_outb(ahc, SCB_LUN, 0xFF);
4686         }
4687
4688         if ((ahc->flags & AHC_PAGESCBS) != 0) {
4689                 /* SCB 0 heads the free list. */
4690                 ahc_outb(ahc, FREE_SCBH, 0);
4691         } else {
4692                 /* No free list. */
4693                 ahc_outb(ahc, FREE_SCBH, SCB_LIST_NULL);
4694         }
4695
4696         /* Make sure that the last SCB terminates the free list */
4697         ahc_outb(ahc, SCBPTR, i-1);
4698         ahc_outb(ahc, SCB_NEXT, SCB_LIST_NULL);
4699 }
4700
4701 static int
4702 ahc_init_scbdata(struct ahc_softc *ahc)
4703 {
4704         struct scb_data *scb_data;
4705
4706         scb_data = ahc->scb_data;
4707         SLIST_INIT(&scb_data->free_scbs);
4708         SLIST_INIT(&scb_data->sg_maps);
4709
4710         /* Allocate SCB resources */
4711         scb_data->scbarray =
4712             (struct scb *)malloc(sizeof(struct scb) * AHC_SCB_MAX_ALLOC,
4713                                  M_DEVBUF, M_NOWAIT);
4714         if (scb_data->scbarray == NULL)
4715                 return (ENOMEM);
4716         memset(scb_data->scbarray, 0, sizeof(struct scb) * AHC_SCB_MAX_ALLOC);
4717
4718         /* Determine the number of hardware SCBs and initialize them */
4719
4720         scb_data->maxhscbs = ahc_probe_scbs(ahc);
4721         if (ahc->scb_data->maxhscbs == 0) {
4722                 printf("%s: No SCB space found\n", ahc_name(ahc));
4723                 return (ENXIO);
4724         }
4725
4726         /*
4727          * Create our DMA tags.  These tags define the kinds of device
4728          * accessible memory allocations and memory mappings we will
4729          * need to perform during normal operation.
4730          *
4731          * Unless we need to further restrict the allocation, we rely
4732          * on the restrictions of the parent dmat, hence the common
4733          * use of MAXADDR and MAXSIZE.
4734          */
4735
4736         /* DMA tag for our hardware scb structures */
4737         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4738                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4739                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4740                                /*highaddr*/BUS_SPACE_MAXADDR,
4741                                /*filter*/NULL, /*filterarg*/NULL,
4742                                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4743                                /*nsegments*/1,
4744                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4745                                /*flags*/0, &scb_data->hscb_dmat) != 0) {
4746                 goto error_exit;
4747         }
4748
4749         scb_data->init_level++;
4750
4751         /* Allocation for our hscbs */
4752         if (ahc_dmamem_alloc(ahc, scb_data->hscb_dmat,
4753                              (void **)&scb_data->hscbs,
4754                              BUS_DMA_NOWAIT, &scb_data->hscb_dmamap) != 0) {
4755                 goto error_exit;
4756         }
4757
4758         scb_data->init_level++;
4759
4760         /* And permanently map them */
4761         ahc_dmamap_load(ahc, scb_data->hscb_dmat, scb_data->hscb_dmamap,
4762                         scb_data->hscbs,
4763                         AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb),
4764                         ahc_dmamap_cb, &scb_data->hscb_busaddr, /*flags*/0);
4765
4766         scb_data->init_level++;
4767
4768         /* DMA tag for our sense buffers */
4769         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
4770                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4771                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4772                                /*highaddr*/BUS_SPACE_MAXADDR,
4773                                /*filter*/NULL, /*filterarg*/NULL,
4774                                AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4775                                /*nsegments*/1,
4776                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4777                                /*flags*/0, &scb_data->sense_dmat) != 0) {
4778                 goto error_exit;
4779         }
4780
4781         scb_data->init_level++;
4782
4783         /* Allocate them */
4784         if (ahc_dmamem_alloc(ahc, scb_data->sense_dmat,
4785                              (void **)&scb_data->sense,
4786                              BUS_DMA_NOWAIT, &scb_data->sense_dmamap) != 0) {
4787                 goto error_exit;
4788         }
4789
4790         scb_data->init_level++;
4791
4792         /* And permanently map them */
4793         ahc_dmamap_load(ahc, scb_data->sense_dmat, scb_data->sense_dmamap,
4794                         scb_data->sense,
4795                         AHC_SCB_MAX_ALLOC * sizeof(struct scsi_sense_data),
4796                         ahc_dmamap_cb, &scb_data->sense_busaddr, /*flags*/0);
4797
4798         scb_data->init_level++;
4799
4800         /* DMA tag for our S/G structures.  We allocate in page sized chunks */
4801         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/8,
4802                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
4803                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
4804                                /*highaddr*/BUS_SPACE_MAXADDR,
4805                                /*filter*/NULL, /*filterarg*/NULL,
4806                                PAGE_SIZE, /*nsegments*/1,
4807                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
4808                                /*flags*/0, &scb_data->sg_dmat) != 0) {
4809                 goto error_exit;
4810         }
4811
4812         scb_data->init_level++;
4813
4814         /* Perform initial CCB allocation */
4815         memset(scb_data->hscbs, 0,
4816                AHC_SCB_MAX_ALLOC * sizeof(struct hardware_scb));
4817         ahc_alloc_scbs(ahc);
4818
4819         if (scb_data->numscbs == 0) {
4820                 printf("%s: ahc_init_scbdata - "
4821                        "Unable to allocate initial scbs\n",
4822                        ahc_name(ahc));
4823                 goto error_exit;
4824         }
4825
4826         /*
4827          * Reserve the next queued SCB.
4828          */
4829         ahc->next_queued_scb = ahc_get_scb(ahc);
4830
4831         /*
4832          * Note that we were successfull
4833          */
4834         return (0);
4835
4836 error_exit:
4837
4838         return (ENOMEM);
4839 }
4840
4841 static void
4842 ahc_fini_scbdata(struct ahc_softc *ahc)
4843 {
4844         struct scb_data *scb_data;
4845
4846         scb_data = ahc->scb_data;
4847         if (scb_data == NULL)
4848                 return;
4849
4850         switch (scb_data->init_level) {
4851         default:
4852         case 7:
4853         {
4854                 struct sg_map_node *sg_map;
4855
4856                 while ((sg_map = SLIST_FIRST(&scb_data->sg_maps))!= NULL) {
4857                         SLIST_REMOVE_HEAD(&scb_data->sg_maps, links);
4858                         ahc_dmamap_unload(ahc, scb_data->sg_dmat,
4859                                           sg_map->sg_dmamap);
4860                         ahc_dmamem_free(ahc, scb_data->sg_dmat,
4861                                         sg_map->sg_vaddr,
4862                                         sg_map->sg_dmamap);
4863                         free(sg_map, M_DEVBUF);
4864                 }
4865                 ahc_dma_tag_destroy(ahc, scb_data->sg_dmat);
4866         }
4867         case 6:
4868                 ahc_dmamap_unload(ahc, scb_data->sense_dmat,
4869                                   scb_data->sense_dmamap);
4870         case 5:
4871                 ahc_dmamem_free(ahc, scb_data->sense_dmat, scb_data->sense,
4872                                 scb_data->sense_dmamap);
4873                 ahc_dmamap_destroy(ahc, scb_data->sense_dmat,
4874                                    scb_data->sense_dmamap);
4875         case 4:
4876                 ahc_dma_tag_destroy(ahc, scb_data->sense_dmat);
4877         case 3:
4878                 ahc_dmamap_unload(ahc, scb_data->hscb_dmat,
4879                                   scb_data->hscb_dmamap);
4880         case 2:
4881                 ahc_dmamem_free(ahc, scb_data->hscb_dmat, scb_data->hscbs,
4882                                 scb_data->hscb_dmamap);
4883                 ahc_dmamap_destroy(ahc, scb_data->hscb_dmat,
4884                                    scb_data->hscb_dmamap);
4885         case 1:
4886                 ahc_dma_tag_destroy(ahc, scb_data->hscb_dmat);
4887                 break;
4888         case 0:
4889                 break;
4890         }
4891         if (scb_data->scbarray != NULL)
4892                 free(scb_data->scbarray, M_DEVBUF);
4893 }
4894
4895 void
4896 ahc_alloc_scbs(struct ahc_softc *ahc)
4897 {
4898         struct scb_data *scb_data;
4899         struct scb *next_scb;
4900         struct sg_map_node *sg_map;
4901         dma_addr_t physaddr;
4902         struct ahc_dma_seg *segs;
4903         int newcount;
4904         int i;
4905
4906         scb_data = ahc->scb_data;
4907         if (scb_data->numscbs >= AHC_SCB_MAX_ALLOC)
4908                 /* Can't allocate any more */
4909                 return;
4910
4911         next_scb = &scb_data->scbarray[scb_data->numscbs];
4912
4913         sg_map = malloc(sizeof(*sg_map), M_DEVBUF, M_NOWAIT);
4914
4915         if (sg_map == NULL)
4916                 return;
4917
4918         /* Allocate S/G space for the next batch of SCBS */
4919         if (ahc_dmamem_alloc(ahc, scb_data->sg_dmat,
4920                              (void **)&sg_map->sg_vaddr,
4921                              BUS_DMA_NOWAIT, &sg_map->sg_dmamap) != 0) {
4922                 free(sg_map, M_DEVBUF);
4923                 return;
4924         }
4925
4926         SLIST_INSERT_HEAD(&scb_data->sg_maps, sg_map, links);
4927
4928         ahc_dmamap_load(ahc, scb_data->sg_dmat, sg_map->sg_dmamap,
4929                         sg_map->sg_vaddr, PAGE_SIZE, ahc_dmamap_cb,
4930                         &sg_map->sg_physaddr, /*flags*/0);
4931
4932         segs = sg_map->sg_vaddr;
4933         physaddr = sg_map->sg_physaddr;
4934
4935         newcount = (PAGE_SIZE / (AHC_NSEG * sizeof(struct ahc_dma_seg)));
4936         newcount = min(newcount, (AHC_SCB_MAX_ALLOC - scb_data->numscbs));
4937         for (i = 0; i < newcount; i++) {
4938                 struct scb_platform_data *pdata;
4939 #ifndef __linux__
4940                 int error;
4941 #endif
4942                 pdata = (struct scb_platform_data *)malloc(sizeof(*pdata),
4943                                                            M_DEVBUF, M_NOWAIT);
4944                 if (pdata == NULL)
4945                         break;
4946                 next_scb->platform_data = pdata;
4947                 next_scb->sg_map = sg_map;
4948                 next_scb->sg_list = segs;
4949                 /*
4950                  * The sequencer always starts with the second entry.
4951                  * The first entry is embedded in the scb.
4952                  */
4953                 next_scb->sg_list_phys = physaddr + sizeof(struct ahc_dma_seg);
4954                 next_scb->ahc_softc = ahc;
4955                 next_scb->flags = SCB_FREE;
4956 #ifndef __linux__
4957                 error = ahc_dmamap_create(ahc, ahc->buffer_dmat, /*flags*/0,
4958                                           &next_scb->dmamap);
4959                 if (error != 0)
4960                         break;
4961 #endif
4962                 next_scb->hscb = &scb_data->hscbs[scb_data->numscbs];
4963                 next_scb->hscb->tag = ahc->scb_data->numscbs;
4964                 SLIST_INSERT_HEAD(&ahc->scb_data->free_scbs,
4965                                   next_scb, links.sle);
4966                 segs += AHC_NSEG;
4967                 physaddr += (AHC_NSEG * sizeof(struct ahc_dma_seg));
4968                 next_scb++;
4969                 ahc->scb_data->numscbs++;
4970         }
4971 }
4972
4973 void
4974 ahc_controller_info(struct ahc_softc *ahc, char *buf)
4975 {
4976         int len;
4977
4978         len = sprintf(buf, "%s: ", ahc_chip_names[ahc->chip & AHC_CHIPID_MASK]);
4979         buf += len;
4980         if ((ahc->features & AHC_TWIN) != 0)
4981                 len = sprintf(buf, "Twin Channel, A SCSI Id=%d, "
4982                               "B SCSI Id=%d, primary %c, ",
4983                               ahc->our_id, ahc->our_id_b,
4984                               (ahc->flags & AHC_PRIMARY_CHANNEL) + 'A');
4985         else {
4986                 const char *speed;
4987                 const char *type;
4988
4989                 speed = "";
4990                 if ((ahc->features & AHC_ULTRA) != 0) {
4991                         speed = "Ultra ";
4992                 } else if ((ahc->features & AHC_DT) != 0) {
4993                         speed = "Ultra160 ";
4994                 } else if ((ahc->features & AHC_ULTRA2) != 0) {
4995                         speed = "Ultra2 ";
4996                 }
4997                 if ((ahc->features & AHC_WIDE) != 0) {
4998                         type = "Wide";
4999                 } else {
5000                         type = "Single";
5001                 }
5002                 len = sprintf(buf, "%s%s Channel %c, SCSI Id=%d, ",
5003                               speed, type, ahc->channel, ahc->our_id);
5004         }
5005         buf += len;
5006
5007         if ((ahc->flags & AHC_PAGESCBS) != 0)
5008                 sprintf(buf, "%d/%d SCBs",
5009                         ahc->scb_data->maxhscbs, AHC_MAX_QUEUE);
5010         else
5011                 sprintf(buf, "%d SCBs", ahc->scb_data->maxhscbs);
5012 }
5013
5014 int
5015 ahc_chip_init(struct ahc_softc *ahc)
5016 {
5017         int      term;
5018         int      error;
5019         u_int    i;
5020         u_int    scsi_conf;
5021         u_int    scsiseq_template;
5022         uint32_t physaddr;
5023
5024         ahc_outb(ahc, SEQ_FLAGS, 0);
5025         ahc_outb(ahc, SEQ_FLAGS2, 0);
5026
5027         /* Set the SCSI Id, SXFRCTL0, SXFRCTL1, and SIMODE1, for both channels*/
5028         if (ahc->features & AHC_TWIN) {
5029
5030                 /*
5031                  * Setup Channel B first.
5032                  */
5033                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) | SELBUSB);
5034                 term = (ahc->flags & AHC_TERM_ENB_B) != 0 ? STPWEN : 0;
5035                 ahc_outb(ahc, SCSIID, ahc->our_id_b);
5036                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5037                 ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5038                                         |term|ahc->seltime_b|ENSTIMER|ACTNEGEN);
5039                 if ((ahc->features & AHC_ULTRA2) != 0)
5040                         ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5041                 ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5042                 ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5043
5044                 /* Select Channel A */
5045                 ahc_outb(ahc, SBLKCTL, ahc_inb(ahc, SBLKCTL) & ~SELBUSB);
5046         }
5047         term = (ahc->flags & AHC_TERM_ENB_A) != 0 ? STPWEN : 0;
5048         if ((ahc->features & AHC_ULTRA2) != 0)
5049                 ahc_outb(ahc, SCSIID_ULTRA2, ahc->our_id);
5050         else
5051                 ahc_outb(ahc, SCSIID, ahc->our_id);
5052         scsi_conf = ahc_inb(ahc, SCSICONF);
5053         ahc_outb(ahc, SXFRCTL1, (scsi_conf & (ENSPCHK|STIMESEL))
5054                                 |term|ahc->seltime
5055                                 |ENSTIMER|ACTNEGEN);
5056         if ((ahc->features & AHC_ULTRA2) != 0)
5057                 ahc_outb(ahc, SIMODE0, ahc_inb(ahc, SIMODE0)|ENIOERR);
5058         ahc_outb(ahc, SIMODE1, ENSELTIMO|ENSCSIRST|ENSCSIPERR);
5059         ahc_outb(ahc, SXFRCTL0, DFON|SPIOEN);
5060
5061         /* There are no untagged SCBs active yet. */
5062         for (i = 0; i < 16; i++) {
5063                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, 0));
5064                 if ((ahc->flags & AHC_SCB_BTT) != 0) {
5065                         int lun;
5066
5067                         /*
5068                          * The SCB based BTT allows an entry per
5069                          * target and lun pair.
5070                          */
5071                         for (lun = 1; lun < AHC_NUM_LUNS; lun++)
5072                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, lun));
5073                 }
5074         }
5075
5076         /* All of our queues are empty */
5077         for (i = 0; i < 256; i++)
5078                 ahc->qoutfifo[i] = SCB_LIST_NULL;
5079         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_PREREAD);
5080
5081         for (i = 0; i < 256; i++)
5082                 ahc->qinfifo[i] = SCB_LIST_NULL;
5083
5084         if ((ahc->features & AHC_MULTI_TID) != 0) {
5085                 ahc_outb(ahc, TARGID, 0);
5086                 ahc_outb(ahc, TARGID + 1, 0);
5087         }
5088
5089         /*
5090          * Tell the sequencer where it can find our arrays in memory.
5091          */
5092         physaddr = ahc->scb_data->hscb_busaddr;
5093         ahc_outb(ahc, HSCB_ADDR, physaddr & 0xFF);
5094         ahc_outb(ahc, HSCB_ADDR + 1, (physaddr >> 8) & 0xFF);
5095         ahc_outb(ahc, HSCB_ADDR + 2, (physaddr >> 16) & 0xFF);
5096         ahc_outb(ahc, HSCB_ADDR + 3, (physaddr >> 24) & 0xFF);
5097
5098         physaddr = ahc->shared_data_busaddr;
5099         ahc_outb(ahc, SHARED_DATA_ADDR, physaddr & 0xFF);
5100         ahc_outb(ahc, SHARED_DATA_ADDR + 1, (physaddr >> 8) & 0xFF);
5101         ahc_outb(ahc, SHARED_DATA_ADDR + 2, (physaddr >> 16) & 0xFF);
5102         ahc_outb(ahc, SHARED_DATA_ADDR + 3, (physaddr >> 24) & 0xFF);
5103
5104         /*
5105          * Initialize the group code to command length table.
5106          * This overrides the values in TARG_SCSIRATE, so only
5107          * setup the table after we have processed that information.
5108          */
5109         ahc_outb(ahc, CMDSIZE_TABLE, 5);
5110         ahc_outb(ahc, CMDSIZE_TABLE + 1, 9);
5111         ahc_outb(ahc, CMDSIZE_TABLE + 2, 9);
5112         ahc_outb(ahc, CMDSIZE_TABLE + 3, 0);
5113         ahc_outb(ahc, CMDSIZE_TABLE + 4, 15);
5114         ahc_outb(ahc, CMDSIZE_TABLE + 5, 11);
5115         ahc_outb(ahc, CMDSIZE_TABLE + 6, 0);
5116         ahc_outb(ahc, CMDSIZE_TABLE + 7, 0);
5117
5118         if ((ahc->features & AHC_HS_MAILBOX) != 0)
5119                 ahc_outb(ahc, HS_MAILBOX, 0);
5120
5121         /* Tell the sequencer of our initial queue positions */
5122         if ((ahc->features & AHC_TARGETMODE) != 0) {
5123                 ahc->tqinfifonext = 1;
5124                 ahc_outb(ahc, KERNEL_TQINPOS, ahc->tqinfifonext - 1);
5125                 ahc_outb(ahc, TQINPOS, ahc->tqinfifonext);
5126         }
5127         ahc->qinfifonext = 0;
5128         ahc->qoutfifonext = 0;
5129         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5130                 ahc_outb(ahc, QOFF_CTLSTA, SCB_QSIZE_256);
5131                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5132                 ahc_outb(ahc, SNSCB_QOFF, ahc->qinfifonext);
5133                 ahc_outb(ahc, SDSCB_QOFF, 0);
5134         } else {
5135                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5136                 ahc_outb(ahc, QINPOS, ahc->qinfifonext);
5137                 ahc_outb(ahc, QOUTPOS, ahc->qoutfifonext);
5138         }
5139
5140         /* We don't have any waiting selections */
5141         ahc_outb(ahc, WAITING_SCBH, SCB_LIST_NULL);
5142
5143         /* Our disconnection list is empty too */
5144         ahc_outb(ahc, DISCONNECTED_SCBH, SCB_LIST_NULL);
5145
5146         /* Message out buffer starts empty */
5147         ahc_outb(ahc, MSG_OUT, MSG_NOOP);
5148
5149         /*
5150          * Setup the allowed SCSI Sequences based on operational mode.
5151          * If we are a target, we'll enalbe select in operations once
5152          * we've had a lun enabled.
5153          */
5154         scsiseq_template = ENSELO|ENAUTOATNO|ENAUTOATNP;
5155         if ((ahc->flags & AHC_INITIATORROLE) != 0)
5156                 scsiseq_template |= ENRSELI;
5157         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq_template);
5158
5159         /* Initialize our list of free SCBs. */
5160         ahc_build_free_scb_list(ahc);
5161
5162         /*
5163          * Tell the sequencer which SCB will be the next one it receives.
5164          */
5165         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5166
5167         /*
5168          * Load the Sequencer program and Enable the adapter
5169          * in "fast" mode.
5170          */
5171         if (bootverbose)
5172                 printf("%s: Downloading Sequencer Program...",
5173                        ahc_name(ahc));
5174
5175         error = ahc_loadseq(ahc);
5176         if (error != 0)
5177                 return (error);
5178
5179         if ((ahc->features & AHC_ULTRA2) != 0) {
5180                 int wait;
5181
5182                 /*
5183                  * Wait for up to 500ms for our transceivers
5184                  * to settle.  If the adapter does not have
5185                  * a cable attached, the transceivers may
5186                  * never settle, so don't complain if we
5187                  * fail here.
5188                  */
5189                 for (wait = 5000;
5190                      (ahc_inb(ahc, SBLKCTL) & (ENAB40|ENAB20)) == 0 && wait;
5191                      wait--)
5192                         ahc_delay(100);
5193         }
5194         ahc_restart(ahc);
5195         return (0);
5196 }
5197
5198 /*
5199  * Start the board, ready for normal operation
5200  */
5201 int
5202 ahc_init(struct ahc_softc *ahc)
5203 {
5204         int      max_targ;
5205         u_int    i;
5206         u_int    scsi_conf;
5207         u_int    ultraenb;
5208         u_int    discenable;
5209         u_int    tagenable;
5210         size_t   driver_data_size;
5211
5212 #ifdef AHC_DEBUG
5213         if ((ahc_debug & AHC_DEBUG_SEQUENCER) != 0)
5214                 ahc->flags |= AHC_SEQUENCER_DEBUG;
5215 #endif
5216
5217 #ifdef AHC_PRINT_SRAM
5218         printf("Scratch Ram:");
5219         for (i = 0x20; i < 0x5f; i++) {
5220                 if (((i % 8) == 0) && (i != 0)) {
5221                         printf ("\n              ");
5222                 }
5223                 printf (" 0x%x", ahc_inb(ahc, i));
5224         }
5225         if ((ahc->features & AHC_MORE_SRAM) != 0) {
5226                 for (i = 0x70; i < 0x7f; i++) {
5227                         if (((i % 8) == 0) && (i != 0)) {
5228                                 printf ("\n              ");
5229                         }
5230                         printf (" 0x%x", ahc_inb(ahc, i));
5231                 }
5232         }
5233         printf ("\n");
5234         /*
5235          * Reading uninitialized scratch ram may
5236          * generate parity errors.
5237          */
5238         ahc_outb(ahc, CLRINT, CLRPARERR);
5239         ahc_outb(ahc, CLRINT, CLRBRKADRINT);
5240 #endif
5241         max_targ = 15;
5242
5243         /*
5244          * Assume we have a board at this stage and it has been reset.
5245          */
5246         if ((ahc->flags & AHC_USEDEFAULTS) != 0)
5247                 ahc->our_id = ahc->our_id_b = 7;
5248
5249         /*
5250          * Default to allowing initiator operations.
5251          */
5252         ahc->flags |= AHC_INITIATORROLE;
5253
5254         /*
5255          * Only allow target mode features if this unit has them enabled.
5256          */
5257         if ((AHC_TMODE_ENABLE & (0x1 << ahc->unit)) == 0)
5258                 ahc->features &= ~AHC_TARGETMODE;
5259
5260 #ifndef __linux__
5261         /* DMA tag for mapping buffers into device visible space. */
5262         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5263                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5264                                /*lowaddr*/ahc->flags & AHC_39BIT_ADDRESSING
5265                                         ? (dma_addr_t)0x7FFFFFFFFFULL
5266                                         : BUS_SPACE_MAXADDR_32BIT,
5267                                /*highaddr*/BUS_SPACE_MAXADDR,
5268                                /*filter*/NULL, /*filterarg*/NULL,
5269                                /*maxsize*/(AHC_NSEG - 1) * PAGE_SIZE,
5270                                /*nsegments*/AHC_NSEG,
5271                                /*maxsegsz*/AHC_MAXTRANSFER_SIZE,
5272                                /*flags*/BUS_DMA_ALLOCNOW,
5273                                &ahc->buffer_dmat) != 0) {
5274                 return (ENOMEM);
5275         }
5276 #endif
5277
5278         ahc->init_level++;
5279
5280         /*
5281          * DMA tag for our command fifos and other data in system memory
5282          * the card's sequencer must be able to access.  For initiator
5283          * roles, we need to allocate space for the qinfifo and qoutfifo.
5284          * The qinfifo and qoutfifo are composed of 256 1 byte elements.
5285          * When providing for the target mode role, we must additionally
5286          * provide space for the incoming target command fifo and an extra
5287          * byte to deal with a dma bug in some chip versions.
5288          */
5289         driver_data_size = 2 * 256 * sizeof(uint8_t);
5290         if ((ahc->features & AHC_TARGETMODE) != 0)
5291                 driver_data_size += AHC_TMODE_CMDS * sizeof(struct target_cmd)
5292                                  + /*DMA WideOdd Bug Buffer*/1;
5293         if (ahc_dma_tag_create(ahc, ahc->parent_dmat, /*alignment*/1,
5294                                /*boundary*/BUS_SPACE_MAXADDR_32BIT + 1,
5295                                /*lowaddr*/BUS_SPACE_MAXADDR_32BIT,
5296                                /*highaddr*/BUS_SPACE_MAXADDR,
5297                                /*filter*/NULL, /*filterarg*/NULL,
5298                                driver_data_size,
5299                                /*nsegments*/1,
5300                                /*maxsegsz*/BUS_SPACE_MAXSIZE_32BIT,
5301                                /*flags*/0, &ahc->shared_data_dmat) != 0) {
5302                 return (ENOMEM);
5303         }
5304
5305         ahc->init_level++;
5306
5307         /* Allocation of driver data */
5308         if (ahc_dmamem_alloc(ahc, ahc->shared_data_dmat,
5309                              (void **)&ahc->qoutfifo,
5310                              BUS_DMA_NOWAIT, &ahc->shared_data_dmamap) != 0) {
5311                 return (ENOMEM);
5312         }
5313
5314         ahc->init_level++;
5315
5316         /* And permanently map it in */
5317         ahc_dmamap_load(ahc, ahc->shared_data_dmat, ahc->shared_data_dmamap,
5318                         ahc->qoutfifo, driver_data_size, ahc_dmamap_cb,
5319                         &ahc->shared_data_busaddr, /*flags*/0);
5320
5321         if ((ahc->features & AHC_TARGETMODE) != 0) {
5322                 ahc->targetcmds = (struct target_cmd *)ahc->qoutfifo;
5323                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[AHC_TMODE_CMDS];
5324                 ahc->dma_bug_buf = ahc->shared_data_busaddr
5325                                  + driver_data_size - 1;
5326                 /* All target command blocks start out invalid. */
5327                 for (i = 0; i < AHC_TMODE_CMDS; i++)
5328                         ahc->targetcmds[i].cmd_valid = 0;
5329                 ahc_sync_tqinfifo(ahc, BUS_DMASYNC_PREREAD);
5330                 ahc->qoutfifo = (uint8_t *)&ahc->targetcmds[256];
5331         }
5332         ahc->qinfifo = &ahc->qoutfifo[256];
5333
5334         ahc->init_level++;
5335
5336         /* Allocate SCB data now that buffer_dmat is initialized */
5337         if (ahc->scb_data->maxhscbs == 0)
5338                 if (ahc_init_scbdata(ahc) != 0)
5339                         return (ENOMEM);
5340
5341         /*
5342          * Allocate a tstate to house information for our
5343          * initiator presence on the bus as well as the user
5344          * data for any target mode initiator.
5345          */
5346         if (ahc_alloc_tstate(ahc, ahc->our_id, 'A') == NULL) {
5347                 printf("%s: unable to allocate ahc_tmode_tstate.  "
5348                        "Failing attach\n", ahc_name(ahc));
5349                 return (ENOMEM);
5350         }
5351
5352         if ((ahc->features & AHC_TWIN) != 0) {
5353                 if (ahc_alloc_tstate(ahc, ahc->our_id_b, 'B') == NULL) {
5354                         printf("%s: unable to allocate ahc_tmode_tstate.  "
5355                                "Failing attach\n", ahc_name(ahc));
5356                         return (ENOMEM);
5357                 }
5358         }
5359
5360         if (ahc->scb_data->maxhscbs < AHC_SCB_MAX_ALLOC) {
5361                 ahc->flags |= AHC_PAGESCBS;
5362         } else {
5363                 ahc->flags &= ~AHC_PAGESCBS;
5364         }
5365
5366 #ifdef AHC_DEBUG
5367         if (ahc_debug & AHC_SHOW_MISC) {
5368                 printf("%s: hardware scb %u bytes; kernel scb %u bytes; "
5369                        "ahc_dma %u bytes\n",
5370                         ahc_name(ahc),
5371                         (u_int)sizeof(struct hardware_scb),
5372                         (u_int)sizeof(struct scb),
5373                         (u_int)sizeof(struct ahc_dma_seg));
5374         }
5375 #endif /* AHC_DEBUG */
5376
5377         /*
5378          * Look at the information that board initialization or
5379          * the board bios has left us.
5380          */
5381         if (ahc->features & AHC_TWIN) {
5382                 scsi_conf = ahc_inb(ahc, SCSICONF + 1);
5383                 if ((scsi_conf & RESET_SCSI) != 0
5384                  && (ahc->flags & AHC_INITIATORROLE) != 0)
5385                         ahc->flags |= AHC_RESET_BUS_B;
5386         }
5387
5388         scsi_conf = ahc_inb(ahc, SCSICONF);
5389         if ((scsi_conf & RESET_SCSI) != 0
5390          && (ahc->flags & AHC_INITIATORROLE) != 0)
5391                 ahc->flags |= AHC_RESET_BUS_A;
5392
5393         ultraenb = 0;
5394         tagenable = ALL_TARGETS_MASK;
5395
5396         /* Grab the disconnection disable table and invert it for our needs */
5397         if ((ahc->flags & AHC_USEDEFAULTS) != 0) {
5398                 printf("%s: Host Adapter Bios disabled.  Using default SCSI "
5399                         "device parameters\n", ahc_name(ahc));
5400                 ahc->flags |= AHC_EXTENDED_TRANS_A|AHC_EXTENDED_TRANS_B|
5401                               AHC_TERM_ENB_A|AHC_TERM_ENB_B;
5402                 discenable = ALL_TARGETS_MASK;
5403                 if ((ahc->features & AHC_ULTRA) != 0)
5404                         ultraenb = ALL_TARGETS_MASK;
5405         } else {
5406                 discenable = ~((ahc_inb(ahc, DISC_DSB + 1) << 8)
5407                            | ahc_inb(ahc, DISC_DSB));
5408                 if ((ahc->features & (AHC_ULTRA|AHC_ULTRA2)) != 0)
5409                         ultraenb = (ahc_inb(ahc, ULTRA_ENB + 1) << 8)
5410                                       | ahc_inb(ahc, ULTRA_ENB);
5411         }
5412
5413         if ((ahc->features & (AHC_WIDE|AHC_TWIN)) == 0)
5414                 max_targ = 7;
5415
5416         for (i = 0; i <= max_targ; i++) {
5417                 struct ahc_initiator_tinfo *tinfo;
5418                 struct ahc_tmode_tstate *tstate;
5419                 u_int our_id;
5420                 u_int target_id;
5421                 char channel;
5422
5423                 channel = 'A';
5424                 our_id = ahc->our_id;
5425                 target_id = i;
5426                 if (i > 7 && (ahc->features & AHC_TWIN) != 0) {
5427                         channel = 'B';
5428                         our_id = ahc->our_id_b;
5429                         target_id = i % 8;
5430                 }
5431                 tinfo = ahc_fetch_transinfo(ahc, channel, our_id,
5432                                             target_id, &tstate);
5433                 /* Default to async narrow across the board */
5434                 memset(tinfo, 0, sizeof(*tinfo));
5435                 if (ahc->flags & AHC_USEDEFAULTS) {
5436                         if ((ahc->features & AHC_WIDE) != 0)
5437                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5438
5439                         /*
5440                          * These will be truncated when we determine the
5441                          * connection type we have with the target.
5442                          */
5443                         tinfo->user.period = ahc_syncrates->period;
5444                         tinfo->user.offset = MAX_OFFSET;
5445                 } else {
5446                         u_int scsirate;
5447                         uint16_t mask;
5448
5449                         /* Take the settings leftover in scratch RAM. */
5450                         scsirate = ahc_inb(ahc, TARG_SCSIRATE + i);
5451                         mask = (0x01 << i);
5452                         if ((ahc->features & AHC_ULTRA2) != 0) {
5453                                 u_int offset;
5454                                 u_int maxsync;
5455
5456                                 if ((scsirate & SOFS) == 0x0F) {
5457                                         /*
5458                                          * Haven't negotiated yet,
5459                                          * so the format is different.
5460                                          */
5461                                         scsirate = (scsirate & SXFR) >> 4
5462                                                  | (ultraenb & mask)
5463                                                   ? 0x08 : 0x0
5464                                                  | (scsirate & WIDEXFER);
5465                                         offset = MAX_OFFSET_ULTRA2;
5466                                 } else
5467                                         offset = ahc_inb(ahc, TARG_OFFSET + i);
5468                                 if ((scsirate & ~WIDEXFER) == 0 && offset != 0)
5469                                         /* Set to the lowest sync rate, 5MHz */
5470                                         scsirate |= 0x1c;
5471                                 maxsync = AHC_SYNCRATE_ULTRA2;
5472                                 if ((ahc->features & AHC_DT) != 0)
5473                                         maxsync = AHC_SYNCRATE_DT;
5474                                 tinfo->user.period =
5475                                     ahc_find_period(ahc, scsirate, maxsync);
5476                                 if (offset == 0)
5477                                         tinfo->user.period = 0;
5478                                 else
5479                                         tinfo->user.offset = MAX_OFFSET;
5480                                 if ((scsirate & SXFR_ULTRA2) <= 8/*10MHz*/
5481                                  && (ahc->features & AHC_DT) != 0)
5482                                         tinfo->user.ppr_options =
5483                                             MSG_EXT_PPR_DT_REQ;
5484                         } else if ((scsirate & SOFS) != 0) {
5485                                 if ((scsirate & SXFR) == 0x40
5486                                  && (ultraenb & mask) != 0) {
5487                                         /* Treat 10MHz as a non-ultra speed */
5488                                         scsirate &= ~SXFR;
5489                                         ultraenb &= ~mask;
5490                                 }
5491                                 tinfo->user.period =
5492                                     ahc_find_period(ahc, scsirate,
5493                                                     (ultraenb & mask)
5494                                                    ? AHC_SYNCRATE_ULTRA
5495                                                    : AHC_SYNCRATE_FAST);
5496                                 if (tinfo->user.period != 0)
5497                                         tinfo->user.offset = MAX_OFFSET;
5498                         }
5499                         if (tinfo->user.period == 0)
5500                                 tinfo->user.offset = 0;
5501                         if ((scsirate & WIDEXFER) != 0
5502                          && (ahc->features & AHC_WIDE) != 0)
5503                                 tinfo->user.width = MSG_EXT_WDTR_BUS_16_BIT;
5504                         tinfo->user.protocol_version = 4;
5505                         if ((ahc->features & AHC_DT) != 0)
5506                                 tinfo->user.transport_version = 3;
5507                         else
5508                                 tinfo->user.transport_version = 2;
5509                         tinfo->goal.protocol_version = 2;
5510                         tinfo->goal.transport_version = 2;
5511                         tinfo->curr.protocol_version = 2;
5512                         tinfo->curr.transport_version = 2;
5513                 }
5514                 tstate->ultraenb = 0;
5515         }
5516         ahc->user_discenable = discenable;
5517         ahc->user_tagenable = tagenable;
5518
5519         return (ahc->bus_chip_init(ahc));
5520 }
5521
5522 void
5523 ahc_intr_enable(struct ahc_softc *ahc, int enable)
5524 {
5525         u_int hcntrl;
5526
5527         hcntrl = ahc_inb(ahc, HCNTRL);
5528         hcntrl &= ~INTEN;
5529         ahc->pause &= ~INTEN;
5530         ahc->unpause &= ~INTEN;
5531         if (enable) {
5532                 hcntrl |= INTEN;
5533                 ahc->pause |= INTEN;
5534                 ahc->unpause |= INTEN;
5535         }
5536         ahc_outb(ahc, HCNTRL, hcntrl);
5537 }
5538
5539 /*
5540  * Ensure that the card is paused in a location
5541  * outside of all critical sections and that all
5542  * pending work is completed prior to returning.
5543  * This routine should only be called from outside
5544  * an interrupt context.
5545  */
5546 void
5547 ahc_pause_and_flushwork(struct ahc_softc *ahc)
5548 {
5549         int intstat;
5550         int maxloops;
5551         int paused;
5552
5553         maxloops = 1000;
5554         ahc->flags |= AHC_ALL_INTERRUPTS;
5555         paused = FALSE;
5556         do {
5557                 if (paused) {
5558                         ahc_unpause(ahc);
5559                         /*
5560                          * Give the sequencer some time to service
5561                          * any active selections.
5562                          */
5563                         ahc_delay(500);
5564                 }
5565                 ahc_intr(ahc);
5566                 ahc_pause(ahc);
5567                 paused = TRUE;
5568                 ahc_outb(ahc, SCSISEQ, ahc_inb(ahc, SCSISEQ) & ~ENSELO);
5569                 intstat = ahc_inb(ahc, INTSTAT);
5570                 if ((intstat & INT_PEND) == 0) {
5571                         ahc_clear_critical_section(ahc);
5572                         intstat = ahc_inb(ahc, INTSTAT);
5573                 }
5574         } while (--maxloops
5575               && (intstat != 0xFF || (ahc->features & AHC_REMOVABLE) == 0)
5576               && ((intstat & INT_PEND) != 0
5577                || (ahc_inb(ahc, SSTAT0) & (SELDO|SELINGO)) != 0));
5578         if (maxloops == 0) {
5579                 printf("Infinite interrupt loop, INTSTAT = %x",
5580                        ahc_inb(ahc, INTSTAT));
5581         }
5582         ahc_platform_flushwork(ahc);
5583         ahc->flags &= ~AHC_ALL_INTERRUPTS;
5584 }
5585
5586 #ifdef CONFIG_PM
5587 int
5588 ahc_suspend(struct ahc_softc *ahc)
5589 {
5590
5591         ahc_pause_and_flushwork(ahc);
5592
5593         if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
5594                 ahc_unpause(ahc);
5595                 return (EBUSY);
5596         }
5597
5598 #ifdef AHC_TARGET_MODE
5599         /*
5600          * XXX What about ATIOs that have not yet been serviced?
5601          * Perhaps we should just refuse to be suspended if we
5602          * are acting in a target role.
5603          */
5604         if (ahc->pending_device != NULL) {
5605                 ahc_unpause(ahc);
5606                 return (EBUSY);
5607         }
5608 #endif
5609         ahc_shutdown(ahc);
5610         return (0);
5611 }
5612
5613 int
5614 ahc_resume(struct ahc_softc *ahc)
5615 {
5616
5617         ahc_reset(ahc, /*reinit*/TRUE);
5618         ahc_intr_enable(ahc, TRUE);
5619         ahc_restart(ahc);
5620         return (0);
5621 }
5622 #endif
5623 /************************** Busy Target Table *********************************/
5624 /*
5625  * Return the untagged transaction id for a given target/channel lun.
5626  * Optionally, clear the entry.
5627  */
5628 u_int
5629 ahc_index_busy_tcl(struct ahc_softc *ahc, u_int tcl)
5630 {
5631         u_int scbid;
5632         u_int target_offset;
5633
5634         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5635                 u_int saved_scbptr;
5636
5637                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5638                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5639                 scbid = ahc_inb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl));
5640                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5641         } else {
5642                 target_offset = TCL_TARGET_OFFSET(tcl);
5643                 scbid = ahc_inb(ahc, BUSY_TARGETS + target_offset);
5644         }
5645
5646         return (scbid);
5647 }
5648
5649 void
5650 ahc_unbusy_tcl(struct ahc_softc *ahc, u_int tcl)
5651 {
5652         u_int target_offset;
5653
5654         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5655                 u_int saved_scbptr;
5656
5657                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5658                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5659                 ahc_outb(ahc, SCB_64_BTT+TCL_TARGET_OFFSET(tcl), SCB_LIST_NULL);
5660                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5661         } else {
5662                 target_offset = TCL_TARGET_OFFSET(tcl);
5663                 ahc_outb(ahc, BUSY_TARGETS + target_offset, SCB_LIST_NULL);
5664         }
5665 }
5666
5667 void
5668 ahc_busy_tcl(struct ahc_softc *ahc, u_int tcl, u_int scbid)
5669 {
5670         u_int target_offset;
5671
5672         if ((ahc->flags & AHC_SCB_BTT) != 0) {
5673                 u_int saved_scbptr;
5674
5675                 saved_scbptr = ahc_inb(ahc, SCBPTR);
5676                 ahc_outb(ahc, SCBPTR, TCL_LUN(tcl));
5677                 ahc_outb(ahc, SCB_64_BTT + TCL_TARGET_OFFSET(tcl), scbid);
5678                 ahc_outb(ahc, SCBPTR, saved_scbptr);
5679         } else {
5680                 target_offset = TCL_TARGET_OFFSET(tcl);
5681                 ahc_outb(ahc, BUSY_TARGETS + target_offset, scbid);
5682         }
5683 }
5684
5685 /************************** SCB and SCB queue management **********************/
5686 int
5687 ahc_match_scb(struct ahc_softc *ahc, struct scb *scb, int target,
5688               char channel, int lun, u_int tag, role_t role)
5689 {
5690         int targ = SCB_GET_TARGET(ahc, scb);
5691         char chan = SCB_GET_CHANNEL(ahc, scb);
5692         int slun = SCB_GET_LUN(scb);
5693         int match;
5694
5695         match = ((chan == channel) || (channel == ALL_CHANNELS));
5696         if (match != 0)
5697                 match = ((targ == target) || (target == CAM_TARGET_WILDCARD));
5698         if (match != 0)
5699                 match = ((lun == slun) || (lun == CAM_LUN_WILDCARD));
5700         if (match != 0) {
5701 #ifdef AHC_TARGET_MODE
5702                 int group;
5703
5704                 group = XPT_FC_GROUP(scb->io_ctx->ccb_h.func_code);
5705                 if (role == ROLE_INITIATOR) {
5706                         match = (group != XPT_FC_GROUP_TMODE)
5707                               && ((tag == scb->hscb->tag)
5708                                || (tag == SCB_LIST_NULL));
5709                 } else if (role == ROLE_TARGET) {
5710                         match = (group == XPT_FC_GROUP_TMODE)
5711                               && ((tag == scb->io_ctx->csio.tag_id)
5712                                || (tag == SCB_LIST_NULL));
5713                 }
5714 #else /* !AHC_TARGET_MODE */
5715                 match = ((tag == scb->hscb->tag) || (tag == SCB_LIST_NULL));
5716 #endif /* AHC_TARGET_MODE */
5717         }
5718
5719         return match;
5720 }
5721
5722 void
5723 ahc_freeze_devq(struct ahc_softc *ahc, struct scb *scb)
5724 {
5725         int     target;
5726         char    channel;
5727         int     lun;
5728
5729         target = SCB_GET_TARGET(ahc, scb);
5730         lun = SCB_GET_LUN(scb);
5731         channel = SCB_GET_CHANNEL(ahc, scb);
5732
5733         ahc_search_qinfifo(ahc, target, channel, lun,
5734                            /*tag*/SCB_LIST_NULL, ROLE_UNKNOWN,
5735                            CAM_REQUEUE_REQ, SEARCH_COMPLETE);
5736
5737         ahc_platform_freeze_devq(ahc, scb);
5738 }
5739
5740 void
5741 ahc_qinfifo_requeue_tail(struct ahc_softc *ahc, struct scb *scb)
5742 {
5743         struct scb *prev_scb;
5744
5745         prev_scb = NULL;
5746         if (ahc_qinfifo_count(ahc) != 0) {
5747                 u_int prev_tag;
5748                 uint8_t prev_pos;
5749
5750                 prev_pos = ahc->qinfifonext - 1;
5751                 prev_tag = ahc->qinfifo[prev_pos];
5752                 prev_scb = ahc_lookup_scb(ahc, prev_tag);
5753         }
5754         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5755         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5756                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5757         } else {
5758                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5759         }
5760 }
5761
5762 static void
5763 ahc_qinfifo_requeue(struct ahc_softc *ahc, struct scb *prev_scb,
5764                     struct scb *scb)
5765 {
5766         if (prev_scb == NULL) {
5767                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5768         } else {
5769                 prev_scb->hscb->next = scb->hscb->tag;
5770                 ahc_sync_scb(ahc, prev_scb,
5771                              BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5772         }
5773         ahc->qinfifo[ahc->qinfifonext++] = scb->hscb->tag;
5774         scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5775         ahc_sync_scb(ahc, scb, BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE);
5776 }
5777
5778 static int
5779 ahc_qinfifo_count(struct ahc_softc *ahc)
5780 {
5781         uint8_t qinpos;
5782         uint8_t diff;
5783
5784         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5785                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
5786                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
5787         } else
5788                 qinpos = ahc_inb(ahc, QINPOS);
5789         diff = ahc->qinfifonext - qinpos;
5790         return (diff);
5791 }
5792
5793 int
5794 ahc_search_qinfifo(struct ahc_softc *ahc, int target, char channel,
5795                    int lun, u_int tag, role_t role, uint32_t status,
5796                    ahc_search_action action)
5797 {
5798         struct  scb *scb;
5799         struct  scb *prev_scb;
5800         uint8_t qinstart;
5801         uint8_t qinpos;
5802         uint8_t qintail;
5803         uint8_t next;
5804         uint8_t prev;
5805         uint8_t curscbptr;
5806         int     found;
5807         int     have_qregs;
5808
5809         qintail = ahc->qinfifonext;
5810         have_qregs = (ahc->features & AHC_QUEUE_REGS) != 0;
5811         if (have_qregs) {
5812                 qinstart = ahc_inb(ahc, SNSCB_QOFF);
5813                 ahc_outb(ahc, SNSCB_QOFF, qinstart);
5814         } else
5815                 qinstart = ahc_inb(ahc, QINPOS);
5816         qinpos = qinstart;
5817         found = 0;
5818         prev_scb = NULL;
5819
5820         if (action == SEARCH_COMPLETE) {
5821                 /*
5822                  * Don't attempt to run any queued untagged transactions
5823                  * until we are done with the abort process.
5824                  */
5825                 ahc_freeze_untagged_queues(ahc);
5826         }
5827
5828         /*
5829          * Start with an empty queue.  Entries that are not chosen
5830          * for removal will be re-added to the queue as we go.
5831          */
5832         ahc->qinfifonext = qinpos;
5833         ahc_outb(ahc, NEXT_QUEUED_SCB, ahc->next_queued_scb->hscb->tag);
5834
5835         while (qinpos != qintail) {
5836                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinpos]);
5837                 if (scb == NULL) {
5838                         printf("qinpos = %d, SCB index = %d\n",
5839                                 qinpos, ahc->qinfifo[qinpos]);
5840                         panic("Loop 1\n");
5841                 }
5842
5843                 if (ahc_match_scb(ahc, scb, target, channel, lun, tag, role)) {
5844                         /*
5845                          * We found an scb that needs to be acted on.
5846                          */
5847                         found++;
5848                         switch (action) {
5849                         case SEARCH_COMPLETE:
5850                         {
5851                                 cam_status ostat;
5852                                 cam_status cstat;
5853
5854                                 ostat = ahc_get_transaction_status(scb);
5855                                 if (ostat == CAM_REQ_INPROG)
5856                                         ahc_set_transaction_status(scb, status);
5857                                 cstat = ahc_get_transaction_status(scb);
5858                                 if (cstat != CAM_REQ_CMP)
5859                                         ahc_freeze_scb(scb);
5860                                 if ((scb->flags & SCB_ACTIVE) == 0)
5861                                         printf("Inactive SCB in qinfifo\n");
5862                                 ahc_done(ahc, scb);
5863
5864                                 /* FALLTHROUGH */
5865                         }
5866                         case SEARCH_REMOVE:
5867                                 break;
5868                         case SEARCH_COUNT:
5869                                 ahc_qinfifo_requeue(ahc, prev_scb, scb);
5870                                 prev_scb = scb;
5871                                 break;
5872                         }
5873                 } else {
5874                         ahc_qinfifo_requeue(ahc, prev_scb, scb);
5875                         prev_scb = scb;
5876                 }
5877                 qinpos++;
5878         }
5879
5880         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
5881                 ahc_outb(ahc, HNSCB_QOFF, ahc->qinfifonext);
5882         } else {
5883                 ahc_outb(ahc, KERNEL_QINPOS, ahc->qinfifonext);
5884         }
5885
5886         if (action != SEARCH_COUNT
5887          && (found != 0)
5888          && (qinstart != ahc->qinfifonext)) {
5889                 /*
5890                  * The sequencer may be in the process of dmaing
5891                  * down the SCB at the beginning of the queue.
5892                  * This could be problematic if either the first,
5893                  * or the second SCB is removed from the queue
5894                  * (the first SCB includes a pointer to the "next"
5895                  * SCB to dma). If we have removed any entries, swap
5896                  * the first element in the queue with the next HSCB
5897                  * so the sequencer will notice that NEXT_QUEUED_SCB
5898                  * has changed during its dma attempt and will retry
5899                  * the DMA.
5900                  */
5901                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qinstart]);
5902
5903                 if (scb == NULL) {
5904                         printf("found = %d, qinstart = %d, qinfifionext = %d\n",
5905                                 found, qinstart, ahc->qinfifonext);
5906                         panic("First/Second Qinfifo fixup\n");
5907                 }
5908                 /*
5909                  * ahc_swap_with_next_hscb forces our next pointer to
5910                  * point to the reserved SCB for future commands.  Save
5911                  * and restore our original next pointer to maintain
5912                  * queue integrity.
5913                  */
5914                 next = scb->hscb->next;
5915                 ahc->scb_data->scbindex[scb->hscb->tag] = NULL;
5916                 ahc_swap_with_next_hscb(ahc, scb);
5917                 scb->hscb->next = next;
5918                 ahc->qinfifo[qinstart] = scb->hscb->tag;
5919
5920                 /* Tell the card about the new head of the qinfifo. */
5921                 ahc_outb(ahc, NEXT_QUEUED_SCB, scb->hscb->tag);
5922
5923                 /* Fixup the tail "next" pointer. */
5924                 qintail = ahc->qinfifonext - 1;
5925                 scb = ahc_lookup_scb(ahc, ahc->qinfifo[qintail]);
5926                 scb->hscb->next = ahc->next_queued_scb->hscb->tag;
5927         }
5928
5929         /*
5930          * Search waiting for selection list.
5931          */
5932         curscbptr = ahc_inb(ahc, SCBPTR);
5933         next = ahc_inb(ahc, WAITING_SCBH);  /* Start at head of list. */
5934         prev = SCB_LIST_NULL;
5935
5936         while (next != SCB_LIST_NULL) {
5937                 uint8_t scb_index;
5938
5939                 ahc_outb(ahc, SCBPTR, next);
5940                 scb_index = ahc_inb(ahc, SCB_TAG);
5941                 if (scb_index >= ahc->scb_data->numscbs) {
5942                         printf("Waiting List inconsistency. "
5943                                "SCB index == %d, yet numscbs == %d.",
5944                                scb_index, ahc->scb_data->numscbs);
5945                         ahc_dump_card_state(ahc);
5946                         panic("for safety");
5947                 }
5948                 scb = ahc_lookup_scb(ahc, scb_index);
5949                 if (scb == NULL) {
5950                         printf("scb_index = %d, next = %d\n",
5951                                 scb_index, next);
5952                         panic("Waiting List traversal\n");
5953                 }
5954                 if (ahc_match_scb(ahc, scb, target, channel,
5955                                   lun, SCB_LIST_NULL, role)) {
5956                         /*
5957                          * We found an scb that needs to be acted on.
5958                          */
5959                         found++;
5960                         switch (action) {
5961                         case SEARCH_COMPLETE:
5962                         {
5963                                 cam_status ostat;
5964                                 cam_status cstat;
5965
5966                                 ostat = ahc_get_transaction_status(scb);
5967                                 if (ostat == CAM_REQ_INPROG)
5968                                         ahc_set_transaction_status(scb,
5969                                                                    status);
5970                                 cstat = ahc_get_transaction_status(scb);
5971                                 if (cstat != CAM_REQ_CMP)
5972                                         ahc_freeze_scb(scb);
5973                                 if ((scb->flags & SCB_ACTIVE) == 0)
5974                                         printf("Inactive SCB in Waiting List\n");
5975                                 ahc_done(ahc, scb);
5976                                 /* FALLTHROUGH */
5977                         }
5978                         case SEARCH_REMOVE:
5979                                 next = ahc_rem_wscb(ahc, next, prev);
5980                                 break;
5981                         case SEARCH_COUNT:
5982                                 prev = next;
5983                                 next = ahc_inb(ahc, SCB_NEXT);
5984                                 break;
5985                         }
5986                 } else {
5987
5988                         prev = next;
5989                         next = ahc_inb(ahc, SCB_NEXT);
5990                 }
5991         }
5992         ahc_outb(ahc, SCBPTR, curscbptr);
5993
5994         found += ahc_search_untagged_queues(ahc, /*ahc_io_ctx_t*/NULL, target,
5995                                             channel, lun, status, action);
5996
5997         if (action == SEARCH_COMPLETE)
5998                 ahc_release_untagged_queues(ahc);
5999         return (found);
6000 }
6001
6002 int
6003 ahc_search_untagged_queues(struct ahc_softc *ahc, ahc_io_ctx_t ctx,
6004                            int target, char channel, int lun, uint32_t status,
6005                            ahc_search_action action)
6006 {
6007         struct  scb *scb;
6008         int     maxtarget;
6009         int     found;
6010         int     i;
6011
6012         if (action == SEARCH_COMPLETE) {
6013                 /*
6014                  * Don't attempt to run any queued untagged transactions
6015                  * until we are done with the abort process.
6016                  */
6017                 ahc_freeze_untagged_queues(ahc);
6018         }
6019
6020         found = 0;
6021         i = 0;
6022         if ((ahc->flags & AHC_SCB_BTT) == 0) {
6023
6024                 maxtarget = 16;
6025                 if (target != CAM_TARGET_WILDCARD) {
6026
6027                         i = target;
6028                         if (channel == 'B')
6029                                 i += 8;
6030                         maxtarget = i + 1;
6031                 }
6032         } else {
6033                 maxtarget = 0;
6034         }
6035
6036         for (; i < maxtarget; i++) {
6037                 struct scb_tailq *untagged_q;
6038                 struct scb *next_scb;
6039
6040                 untagged_q = &(ahc->untagged_queues[i]);
6041                 next_scb = TAILQ_FIRST(untagged_q);
6042                 while (next_scb != NULL) {
6043
6044                         scb = next_scb;
6045                         next_scb = TAILQ_NEXT(scb, links.tqe);
6046
6047                         /*
6048                          * The head of the list may be the currently
6049                          * active untagged command for a device.
6050                          * We're only searching for commands that
6051                          * have not been started.  A transaction
6052                          * marked active but still in the qinfifo
6053                          * is removed by the qinfifo scanning code
6054                          * above.
6055                          */
6056                         if ((scb->flags & SCB_ACTIVE) != 0)
6057                                 continue;
6058
6059                         if (ahc_match_scb(ahc, scb, target, channel, lun,
6060                                           SCB_LIST_NULL, ROLE_INITIATOR) == 0
6061                          || (ctx != NULL && ctx != scb->io_ctx))
6062                                 continue;
6063
6064                         /*
6065                          * We found an scb that needs to be acted on.
6066                          */
6067                         found++;
6068                         switch (action) {
6069                         case SEARCH_COMPLETE:
6070                         {
6071                                 cam_status ostat;
6072                                 cam_status cstat;
6073
6074                                 ostat = ahc_get_transaction_status(scb);
6075                                 if (ostat == CAM_REQ_INPROG)
6076                                         ahc_set_transaction_status(scb, status);
6077                                 cstat = ahc_get_transaction_status(scb);
6078                                 if (cstat != CAM_REQ_CMP)
6079                                         ahc_freeze_scb(scb);
6080                                 if ((scb->flags & SCB_ACTIVE) == 0)
6081                                         printf("Inactive SCB in untaggedQ\n");
6082                                 ahc_done(ahc, scb);
6083                                 break;
6084                         }
6085                         case SEARCH_REMOVE:
6086                                 scb->flags &= ~SCB_UNTAGGEDQ;
6087                                 TAILQ_REMOVE(untagged_q, scb, links.tqe);
6088                                 break;
6089                         case SEARCH_COUNT:
6090                                 break;
6091                         }
6092                 }
6093         }
6094
6095         if (action == SEARCH_COMPLETE)
6096                 ahc_release_untagged_queues(ahc);
6097         return (found);
6098 }
6099
6100 int
6101 ahc_search_disc_list(struct ahc_softc *ahc, int target, char channel,
6102                      int lun, u_int tag, int stop_on_first, int remove,
6103                      int save_state)
6104 {
6105         struct  scb *scbp;
6106         u_int   next;
6107         u_int   prev;
6108         u_int   count;
6109         u_int   active_scb;
6110
6111         count = 0;
6112         next = ahc_inb(ahc, DISCONNECTED_SCBH);
6113         prev = SCB_LIST_NULL;
6114
6115         if (save_state) {
6116                 /* restore this when we're done */
6117                 active_scb = ahc_inb(ahc, SCBPTR);
6118         } else
6119                 /* Silence compiler */
6120                 active_scb = SCB_LIST_NULL;
6121
6122         while (next != SCB_LIST_NULL) {
6123                 u_int scb_index;
6124
6125                 ahc_outb(ahc, SCBPTR, next);
6126                 scb_index = ahc_inb(ahc, SCB_TAG);
6127                 if (scb_index >= ahc->scb_data->numscbs) {
6128                         printf("Disconnected List inconsistency. "
6129                                "SCB index == %d, yet numscbs == %d.",
6130                                scb_index, ahc->scb_data->numscbs);
6131                         ahc_dump_card_state(ahc);
6132                         panic("for safety");
6133                 }
6134
6135                 if (next == prev) {
6136                         panic("Disconnected List Loop. "
6137                               "cur SCBPTR == %x, prev SCBPTR == %x.",
6138                               next, prev);
6139                 }
6140                 scbp = ahc_lookup_scb(ahc, scb_index);
6141                 if (ahc_match_scb(ahc, scbp, target, channel, lun,
6142                                   tag, ROLE_INITIATOR)) {
6143                         count++;
6144                         if (remove) {
6145                                 next =
6146                                     ahc_rem_scb_from_disc_list(ahc, prev, next);
6147                         } else {
6148                                 prev = next;
6149                                 next = ahc_inb(ahc, SCB_NEXT);
6150                         }
6151                         if (stop_on_first)
6152                                 break;
6153                 } else {
6154                         prev = next;
6155                         next = ahc_inb(ahc, SCB_NEXT);
6156                 }
6157         }
6158         if (save_state)
6159                 ahc_outb(ahc, SCBPTR, active_scb);
6160         return (count);
6161 }
6162
6163 /*
6164  * Remove an SCB from the on chip list of disconnected transactions.
6165  * This is empty/unused if we are not performing SCB paging.
6166  */
6167 static u_int
6168 ahc_rem_scb_from_disc_list(struct ahc_softc *ahc, u_int prev, u_int scbptr)
6169 {
6170         u_int next;
6171
6172         ahc_outb(ahc, SCBPTR, scbptr);
6173         next = ahc_inb(ahc, SCB_NEXT);
6174
6175         ahc_outb(ahc, SCB_CONTROL, 0);
6176
6177         ahc_add_curscb_to_free_list(ahc);
6178
6179         if (prev != SCB_LIST_NULL) {
6180                 ahc_outb(ahc, SCBPTR, prev);
6181                 ahc_outb(ahc, SCB_NEXT, next);
6182         } else
6183                 ahc_outb(ahc, DISCONNECTED_SCBH, next);
6184
6185         return (next);
6186 }
6187
6188 /*
6189  * Add the SCB as selected by SCBPTR onto the on chip list of
6190  * free hardware SCBs.  This list is empty/unused if we are not
6191  * performing SCB paging.
6192  */
6193 static void
6194 ahc_add_curscb_to_free_list(struct ahc_softc *ahc)
6195 {
6196         /*
6197          * Invalidate the tag so that our abort
6198          * routines don't think it's active.
6199          */
6200         ahc_outb(ahc, SCB_TAG, SCB_LIST_NULL);
6201
6202         if ((ahc->flags & AHC_PAGESCBS) != 0) {
6203                 ahc_outb(ahc, SCB_NEXT, ahc_inb(ahc, FREE_SCBH));
6204                 ahc_outb(ahc, FREE_SCBH, ahc_inb(ahc, SCBPTR));
6205         }
6206 }
6207
6208 /*
6209  * Manipulate the waiting for selection list and return the
6210  * scb that follows the one that we remove.
6211  */
6212 static u_int
6213 ahc_rem_wscb(struct ahc_softc *ahc, u_int scbpos, u_int prev)
6214 {
6215         u_int curscb, next;
6216
6217         /*
6218          * Select the SCB we want to abort and
6219          * pull the next pointer out of it.
6220          */
6221         curscb = ahc_inb(ahc, SCBPTR);
6222         ahc_outb(ahc, SCBPTR, scbpos);
6223         next = ahc_inb(ahc, SCB_NEXT);
6224
6225         /* Clear the necessary fields */
6226         ahc_outb(ahc, SCB_CONTROL, 0);
6227
6228         ahc_add_curscb_to_free_list(ahc);
6229
6230         /* update the waiting list */
6231         if (prev == SCB_LIST_NULL) {
6232                 /* First in the list */
6233                 ahc_outb(ahc, WAITING_SCBH, next);
6234
6235                 /*
6236                  * Ensure we aren't attempting to perform
6237                  * selection for this entry.
6238                  */
6239                 ahc_outb(ahc, SCSISEQ, (ahc_inb(ahc, SCSISEQ) & ~ENSELO));
6240         } else {
6241                 /*
6242                  * Select the scb that pointed to us
6243                  * and update its next pointer.
6244                  */
6245                 ahc_outb(ahc, SCBPTR, prev);
6246                 ahc_outb(ahc, SCB_NEXT, next);
6247         }
6248
6249         /*
6250          * Point us back at the original scb position.
6251          */
6252         ahc_outb(ahc, SCBPTR, curscb);
6253         return next;
6254 }
6255
6256 /******************************** Error Handling ******************************/
6257 /*
6258  * Abort all SCBs that match the given description (target/channel/lun/tag),
6259  * setting their status to the passed in status if the status has not already
6260  * been modified from CAM_REQ_INPROG.  This routine assumes that the sequencer
6261  * is paused before it is called.
6262  */
6263 int
6264 ahc_abort_scbs(struct ahc_softc *ahc, int target, char channel,
6265                int lun, u_int tag, role_t role, uint32_t status)
6266 {
6267         struct  scb *scbp;
6268         struct  scb *scbp_next;
6269         u_int   active_scb;
6270         int     i, j;
6271         int     maxtarget;
6272         int     minlun;
6273         int     maxlun;
6274
6275         int     found;
6276
6277         /*
6278          * Don't attempt to run any queued untagged transactions
6279          * until we are done with the abort process.
6280          */
6281         ahc_freeze_untagged_queues(ahc);
6282
6283         /* restore this when we're done */
6284         active_scb = ahc_inb(ahc, SCBPTR);
6285
6286         found = ahc_search_qinfifo(ahc, target, channel, lun, SCB_LIST_NULL,
6287                                    role, CAM_REQUEUE_REQ, SEARCH_COMPLETE);
6288
6289         /*
6290          * Clean out the busy target table for any untagged commands.
6291          */
6292         i = 0;
6293         maxtarget = 16;
6294         if (target != CAM_TARGET_WILDCARD) {
6295                 i = target;
6296                 if (channel == 'B')
6297                         i += 8;
6298                 maxtarget = i + 1;
6299         }
6300
6301         if (lun == CAM_LUN_WILDCARD) {
6302
6303                 /*
6304                  * Unless we are using an SCB based
6305                  * busy targets table, there is only
6306                  * one table entry for all luns of
6307                  * a target.
6308                  */
6309                 minlun = 0;
6310                 maxlun = 1;
6311                 if ((ahc->flags & AHC_SCB_BTT) != 0)
6312                         maxlun = AHC_NUM_LUNS;
6313         } else {
6314                 minlun = lun;
6315                 maxlun = lun + 1;
6316         }
6317
6318         if (role != ROLE_TARGET) {
6319                 for (;i < maxtarget; i++) {
6320                         for (j = minlun;j < maxlun; j++) {
6321                                 u_int scbid;
6322                                 u_int tcl;
6323
6324                                 tcl = BUILD_TCL(i << 4, j);
6325                                 scbid = ahc_index_busy_tcl(ahc, tcl);
6326                                 scbp = ahc_lookup_scb(ahc, scbid);
6327                                 if (scbp == NULL
6328                                  || ahc_match_scb(ahc, scbp, target, channel,
6329                                                   lun, tag, role) == 0)
6330                                         continue;
6331                                 ahc_unbusy_tcl(ahc, BUILD_TCL(i << 4, j));
6332                         }
6333                 }
6334
6335                 /*
6336                  * Go through the disconnected list and remove any entries we
6337                  * have queued for completion, 0'ing their control byte too.
6338                  * We save the active SCB and restore it ourselves, so there
6339                  * is no reason for this search to restore it too.
6340                  */
6341                 ahc_search_disc_list(ahc, target, channel, lun, tag,
6342                                      /*stop_on_first*/FALSE, /*remove*/TRUE,
6343                                      /*save_state*/FALSE);
6344         }
6345
6346         /*
6347          * Go through the hardware SCB array looking for commands that
6348          * were active but not on any list.  In some cases, these remnants
6349          * might not still have mappings in the scbindex array (e.g. unexpected
6350          * bus free with the same scb queued for an abort).  Don't hold this
6351          * against them.
6352          */
6353         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
6354                 u_int scbid;
6355
6356                 ahc_outb(ahc, SCBPTR, i);
6357                 scbid = ahc_inb(ahc, SCB_TAG);
6358                 scbp = ahc_lookup_scb(ahc, scbid);
6359                 if ((scbp == NULL && scbid != SCB_LIST_NULL)
6360                  || (scbp != NULL
6361                   && ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)))
6362                         ahc_add_curscb_to_free_list(ahc);
6363         }
6364
6365         /*
6366          * Go through the pending CCB list and look for
6367          * commands for this target that are still active.
6368          * These are other tagged commands that were
6369          * disconnected when the reset occurred.
6370          */
6371         scbp_next = LIST_FIRST(&ahc->pending_scbs);
6372         while (scbp_next != NULL) {
6373                 scbp = scbp_next;
6374                 scbp_next = LIST_NEXT(scbp, pending_links);
6375                 if (ahc_match_scb(ahc, scbp, target, channel, lun, tag, role)) {
6376                         cam_status ostat;
6377
6378                         ostat = ahc_get_transaction_status(scbp);
6379                         if (ostat == CAM_REQ_INPROG)
6380                                 ahc_set_transaction_status(scbp, status);
6381                         if (ahc_get_transaction_status(scbp) != CAM_REQ_CMP)
6382                                 ahc_freeze_scb(scbp);
6383                         if ((scbp->flags & SCB_ACTIVE) == 0)
6384                                 printf("Inactive SCB on pending list\n");
6385                         ahc_done(ahc, scbp);
6386                         found++;
6387                 }
6388         }
6389         ahc_outb(ahc, SCBPTR, active_scb);
6390         ahc_platform_abort_scbs(ahc, target, channel, lun, tag, role, status);
6391         ahc_release_untagged_queues(ahc);
6392         return found;
6393 }
6394
6395 static void
6396 ahc_reset_current_bus(struct ahc_softc *ahc)
6397 {
6398         uint8_t scsiseq;
6399
6400         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) & ~ENSCSIRST);
6401         scsiseq = ahc_inb(ahc, SCSISEQ);
6402         ahc_outb(ahc, SCSISEQ, scsiseq | SCSIRSTO);
6403         ahc_flush_device_writes(ahc);
6404         ahc_delay(AHC_BUSRESET_DELAY);
6405         /* Turn off the bus reset */
6406         ahc_outb(ahc, SCSISEQ, scsiseq & ~SCSIRSTO);
6407
6408         ahc_clear_intstat(ahc);
6409
6410         /* Re-enable reset interrupts */
6411         ahc_outb(ahc, SIMODE1, ahc_inb(ahc, SIMODE1) | ENSCSIRST);
6412 }
6413
6414 int
6415 ahc_reset_channel(struct ahc_softc *ahc, char channel, int initiate_reset)
6416 {
6417         struct  ahc_devinfo devinfo;
6418         u_int   initiator, target, max_scsiid;
6419         u_int   sblkctl;
6420         u_int   scsiseq;
6421         u_int   simode1;
6422         int     found;
6423         int     restart_needed;
6424         char    cur_channel;
6425
6426         ahc->pending_device = NULL;
6427
6428         ahc_compile_devinfo(&devinfo,
6429                             CAM_TARGET_WILDCARD,
6430                             CAM_TARGET_WILDCARD,
6431                             CAM_LUN_WILDCARD,
6432                             channel, ROLE_UNKNOWN);
6433         ahc_pause(ahc);
6434
6435         /* Make sure the sequencer is in a safe location. */
6436         ahc_clear_critical_section(ahc);
6437
6438         /*
6439          * Run our command complete fifos to ensure that we perform
6440          * completion processing on any commands that 'completed'
6441          * before the reset occurred.
6442          */
6443         ahc_run_qoutfifo(ahc);
6444 #ifdef AHC_TARGET_MODE
6445         /*
6446          * XXX - In Twin mode, the tqinfifo may have commands
6447          *       for an unaffected channel in it.  However, if
6448          *       we have run out of ATIO resources to drain that
6449          *       queue, we may not get them all out here.  Further,
6450          *       the blocked transactions for the reset channel
6451          *       should just be killed off, irrespecitve of whether
6452          *       we are blocked on ATIO resources.  Write a routine
6453          *       to compact the tqinfifo appropriately.
6454          */
6455         if ((ahc->flags & AHC_TARGETROLE) != 0) {
6456                 ahc_run_tqinfifo(ahc, /*paused*/TRUE);
6457         }
6458 #endif
6459
6460         /*
6461          * Reset the bus if we are initiating this reset
6462          */
6463         sblkctl = ahc_inb(ahc, SBLKCTL);
6464         cur_channel = 'A';
6465         if ((ahc->features & AHC_TWIN) != 0
6466          && ((sblkctl & SELBUSB) != 0))
6467             cur_channel = 'B';
6468         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
6469         if (cur_channel != channel) {
6470                 /* Case 1: Command for another bus is active
6471                  * Stealthily reset the other bus without
6472                  * upsetting the current bus.
6473                  */
6474                 ahc_outb(ahc, SBLKCTL, sblkctl ^ SELBUSB);
6475                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6476 #ifdef AHC_TARGET_MODE
6477                 /*
6478                  * Bus resets clear ENSELI, so we cannot
6479                  * defer re-enabling bus reset interrupts
6480                  * if we are in target mode.
6481                  */
6482                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6483                         simode1 |= ENSCSIRST;
6484 #endif
6485                 ahc_outb(ahc, SIMODE1, simode1);
6486                 if (initiate_reset)
6487                         ahc_reset_current_bus(ahc);
6488                 ahc_clear_intstat(ahc);
6489                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6490                 ahc_outb(ahc, SBLKCTL, sblkctl);
6491                 restart_needed = FALSE;
6492         } else {
6493                 /* Case 2: A command from this bus is active or we're idle */
6494                 simode1 = ahc_inb(ahc, SIMODE1) & ~(ENBUSFREE|ENSCSIRST);
6495 #ifdef AHC_TARGET_MODE
6496                 /*
6497                  * Bus resets clear ENSELI, so we cannot
6498                  * defer re-enabling bus reset interrupts
6499                  * if we are in target mode.
6500                  */
6501                 if ((ahc->flags & AHC_TARGETROLE) != 0)
6502                         simode1 |= ENSCSIRST;
6503 #endif
6504                 ahc_outb(ahc, SIMODE1, simode1);
6505                 if (initiate_reset)
6506                         ahc_reset_current_bus(ahc);
6507                 ahc_clear_intstat(ahc);
6508                 ahc_outb(ahc, SCSISEQ, scsiseq & (ENSELI|ENRSELI|ENAUTOATNP));
6509                 restart_needed = TRUE;
6510         }
6511
6512         /*
6513          * Clean up all the state information for the
6514          * pending transactions on this bus.
6515          */
6516         found = ahc_abort_scbs(ahc, CAM_TARGET_WILDCARD, channel,
6517                                CAM_LUN_WILDCARD, SCB_LIST_NULL,
6518                                ROLE_UNKNOWN, CAM_SCSI_BUS_RESET);
6519
6520         max_scsiid = (ahc->features & AHC_WIDE) ? 15 : 7;
6521
6522 #ifdef AHC_TARGET_MODE
6523         /*
6524          * Send an immediate notify ccb to all target more peripheral
6525          * drivers affected by this action.
6526          */
6527         for (target = 0; target <= max_scsiid; target++) {
6528                 struct ahc_tmode_tstate* tstate;
6529                 u_int lun;
6530
6531                 tstate = ahc->enabled_targets[target];
6532                 if (tstate == NULL)
6533                         continue;
6534                 for (lun = 0; lun < AHC_NUM_LUNS; lun++) {
6535                         struct ahc_tmode_lstate* lstate;
6536
6537                         lstate = tstate->enabled_luns[lun];
6538                         if (lstate == NULL)
6539                                 continue;
6540
6541                         ahc_queue_lstate_event(ahc, lstate, CAM_TARGET_WILDCARD,
6542                                                EVENT_TYPE_BUS_RESET, /*arg*/0);
6543                         ahc_send_lstate_events(ahc, lstate);
6544                 }
6545         }
6546 #endif
6547         /* Notify the XPT that a bus reset occurred */
6548         ahc_send_async(ahc, devinfo.channel, CAM_TARGET_WILDCARD,
6549                        CAM_LUN_WILDCARD, AC_BUS_RESET);
6550
6551         /*
6552          * Revert to async/narrow transfers until we renegotiate.
6553          */
6554         for (target = 0; target <= max_scsiid; target++) {
6555
6556                 if (ahc->enabled_targets[target] == NULL)
6557                         continue;
6558                 for (initiator = 0; initiator <= max_scsiid; initiator++) {
6559                         struct ahc_devinfo devinfo;
6560
6561                         ahc_compile_devinfo(&devinfo, target, initiator,
6562                                             CAM_LUN_WILDCARD,
6563                                             channel, ROLE_UNKNOWN);
6564                         ahc_set_width(ahc, &devinfo, MSG_EXT_WDTR_BUS_8_BIT,
6565                                       AHC_TRANS_CUR, /*paused*/TRUE);
6566                         ahc_set_syncrate(ahc, &devinfo, /*syncrate*/NULL,
6567                                          /*period*/0, /*offset*/0,
6568                                          /*ppr_options*/0, AHC_TRANS_CUR,
6569                                          /*paused*/TRUE);
6570                 }
6571         }
6572
6573         if (restart_needed)
6574                 ahc_restart(ahc);
6575         else
6576                 ahc_unpause(ahc);
6577         return found;
6578 }
6579
6580
6581 /***************************** Residual Processing ****************************/
6582 /*
6583  * Calculate the residual for a just completed SCB.
6584  */
6585 void
6586 ahc_calc_residual(struct ahc_softc *ahc, struct scb *scb)
6587 {
6588         struct hardware_scb *hscb;
6589         struct status_pkt *spkt;
6590         uint32_t sgptr;
6591         uint32_t resid_sgptr;
6592         uint32_t resid;
6593
6594         /*
6595          * 5 cases.
6596          * 1) No residual.
6597          *    SG_RESID_VALID clear in sgptr.
6598          * 2) Transferless command
6599          * 3) Never performed any transfers.
6600          *    sgptr has SG_FULL_RESID set.
6601          * 4) No residual but target did not
6602          *    save data pointers after the
6603          *    last transfer, so sgptr was
6604          *    never updated.
6605          * 5) We have a partial residual.
6606          *    Use residual_sgptr to determine
6607          *    where we are.
6608          */
6609
6610         hscb = scb->hscb;
6611         sgptr = ahc_le32toh(hscb->sgptr);
6612         if ((sgptr & SG_RESID_VALID) == 0)
6613                 /* Case 1 */
6614                 return;
6615         sgptr &= ~SG_RESID_VALID;
6616
6617         if ((sgptr & SG_LIST_NULL) != 0)
6618                 /* Case 2 */
6619                 return;
6620
6621         spkt = &hscb->shared_data.status;
6622         resid_sgptr = ahc_le32toh(spkt->residual_sg_ptr);
6623         if ((sgptr & SG_FULL_RESID) != 0) {
6624                 /* Case 3 */
6625                 resid = ahc_get_transfer_length(scb);
6626         } else if ((resid_sgptr & SG_LIST_NULL) != 0) {
6627                 /* Case 4 */
6628                 return;
6629         } else if ((resid_sgptr & ~SG_PTR_MASK) != 0) {
6630                 panic("Bogus resid sgptr value 0x%x\n", resid_sgptr);
6631         } else {
6632                 struct ahc_dma_seg *sg;
6633
6634                 /*
6635                  * Remainder of the SG where the transfer
6636                  * stopped.
6637                  */
6638                 resid = ahc_le32toh(spkt->residual_datacnt) & AHC_SG_LEN_MASK;
6639                 sg = ahc_sg_bus_to_virt(scb, resid_sgptr & SG_PTR_MASK);
6640
6641                 /* The residual sg_ptr always points to the next sg */
6642                 sg--;
6643
6644                 /*
6645                  * Add up the contents of all residual
6646                  * SG segments that are after the SG where
6647                  * the transfer stopped.
6648                  */
6649                 while ((ahc_le32toh(sg->len) & AHC_DMA_LAST_SEG) == 0) {
6650                         sg++;
6651                         resid += ahc_le32toh(sg->len) & AHC_SG_LEN_MASK;
6652                 }
6653         }
6654         if ((scb->flags & SCB_SENSE) == 0)
6655                 ahc_set_residual(scb, resid);
6656         else
6657                 ahc_set_sense_residual(scb, resid);
6658
6659 #ifdef AHC_DEBUG
6660         if ((ahc_debug & AHC_SHOW_MISC) != 0) {
6661                 ahc_print_path(ahc, scb);
6662                 printf("Handled %sResidual of %d bytes\n",
6663                        (scb->flags & SCB_SENSE) ? "Sense " : "", resid);
6664         }
6665 #endif
6666 }
6667
6668 /******************************* Target Mode **********************************/
6669 #ifdef AHC_TARGET_MODE
6670 /*
6671  * Add a target mode event to this lun's queue
6672  */
6673 static void
6674 ahc_queue_lstate_event(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate,
6675                        u_int initiator_id, u_int event_type, u_int event_arg)
6676 {
6677         struct ahc_tmode_event *event;
6678         int pending;
6679
6680         xpt_freeze_devq(lstate->path, /*count*/1);
6681         if (lstate->event_w_idx >= lstate->event_r_idx)
6682                 pending = lstate->event_w_idx - lstate->event_r_idx;
6683         else
6684                 pending = AHC_TMODE_EVENT_BUFFER_SIZE + 1
6685                         - (lstate->event_r_idx - lstate->event_w_idx);
6686
6687         if (event_type == EVENT_TYPE_BUS_RESET
6688          || event_type == MSG_BUS_DEV_RESET) {
6689                 /*
6690                  * Any earlier events are irrelevant, so reset our buffer.
6691                  * This has the effect of allowing us to deal with reset
6692                  * floods (an external device holding down the reset line)
6693                  * without losing the event that is really interesting.
6694                  */
6695                 lstate->event_r_idx = 0;
6696                 lstate->event_w_idx = 0;
6697                 xpt_release_devq(lstate->path, pending, /*runqueue*/FALSE);
6698         }
6699
6700         if (pending == AHC_TMODE_EVENT_BUFFER_SIZE) {
6701                 xpt_print_path(lstate->path);
6702                 printf("immediate event %x:%x lost\n",
6703                        lstate->event_buffer[lstate->event_r_idx].event_type,
6704                        lstate->event_buffer[lstate->event_r_idx].event_arg);
6705                 lstate->event_r_idx++;
6706                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6707                         lstate->event_r_idx = 0;
6708                 xpt_release_devq(lstate->path, /*count*/1, /*runqueue*/FALSE);
6709         }
6710
6711         event = &lstate->event_buffer[lstate->event_w_idx];
6712         event->initiator_id = initiator_id;
6713         event->event_type = event_type;
6714         event->event_arg = event_arg;
6715         lstate->event_w_idx++;
6716         if (lstate->event_w_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6717                 lstate->event_w_idx = 0;
6718 }
6719
6720 /*
6721  * Send any target mode events queued up waiting
6722  * for immediate notify resources.
6723  */
6724 void
6725 ahc_send_lstate_events(struct ahc_softc *ahc, struct ahc_tmode_lstate *lstate)
6726 {
6727         struct ccb_hdr *ccbh;
6728         struct ccb_immed_notify *inot;
6729
6730         while (lstate->event_r_idx != lstate->event_w_idx
6731             && (ccbh = SLIST_FIRST(&lstate->immed_notifies)) != NULL) {
6732                 struct ahc_tmode_event *event;
6733
6734                 event = &lstate->event_buffer[lstate->event_r_idx];
6735                 SLIST_REMOVE_HEAD(&lstate->immed_notifies, sim_links.sle);
6736                 inot = (struct ccb_immed_notify *)ccbh;
6737                 switch (event->event_type) {
6738                 case EVENT_TYPE_BUS_RESET:
6739                         ccbh->status = CAM_SCSI_BUS_RESET|CAM_DEV_QFRZN;
6740                         break;
6741                 default:
6742                         ccbh->status = CAM_MESSAGE_RECV|CAM_DEV_QFRZN;
6743                         inot->message_args[0] = event->event_type;
6744                         inot->message_args[1] = event->event_arg;
6745                         break;
6746                 }
6747                 inot->initiator_id = event->initiator_id;
6748                 inot->sense_len = 0;
6749                 xpt_done((union ccb *)inot);
6750                 lstate->event_r_idx++;
6751                 if (lstate->event_r_idx == AHC_TMODE_EVENT_BUFFER_SIZE)
6752                         lstate->event_r_idx = 0;
6753         }
6754 }
6755 #endif
6756
6757 /******************** Sequencer Program Patching/Download *********************/
6758
6759 #ifdef AHC_DUMP_SEQ
6760 void
6761 ahc_dumpseq(struct ahc_softc* ahc)
6762 {
6763         int i;
6764
6765         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6766         ahc_outb(ahc, SEQADDR0, 0);
6767         ahc_outb(ahc, SEQADDR1, 0);
6768         for (i = 0; i < ahc->instruction_ram_size; i++) {
6769                 uint8_t ins_bytes[4];
6770
6771                 ahc_insb(ahc, SEQRAM, ins_bytes, 4);
6772                 printf("0x%08x\n", ins_bytes[0] << 24
6773                                  | ins_bytes[1] << 16
6774                                  | ins_bytes[2] << 8
6775                                  | ins_bytes[3]);
6776         }
6777 }
6778 #endif
6779
6780 static int
6781 ahc_loadseq(struct ahc_softc *ahc)
6782 {
6783         struct  cs cs_table[num_critical_sections];
6784         u_int   begin_set[num_critical_sections];
6785         u_int   end_set[num_critical_sections];
6786         struct  patch *cur_patch;
6787         u_int   cs_count;
6788         u_int   cur_cs;
6789         u_int   i;
6790         u_int   skip_addr;
6791         u_int   sg_prefetch_cnt;
6792         int     downloaded;
6793         uint8_t download_consts[7];
6794
6795         /*
6796          * Start out with 0 critical sections
6797          * that apply to this firmware load.
6798          */
6799         cs_count = 0;
6800         cur_cs = 0;
6801         memset(begin_set, 0, sizeof(begin_set));
6802         memset(end_set, 0, sizeof(end_set));
6803
6804         /* Setup downloadable constant table */
6805         download_consts[QOUTFIFO_OFFSET] = 0;
6806         if (ahc->targetcmds != NULL)
6807                 download_consts[QOUTFIFO_OFFSET] += 32;
6808         download_consts[QINFIFO_OFFSET] = download_consts[QOUTFIFO_OFFSET] + 1;
6809         download_consts[CACHESIZE_MASK] = ahc->pci_cachesize - 1;
6810         download_consts[INVERTED_CACHESIZE_MASK] = ~(ahc->pci_cachesize - 1);
6811         sg_prefetch_cnt = ahc->pci_cachesize;
6812         if (sg_prefetch_cnt < (2 * sizeof(struct ahc_dma_seg)))
6813                 sg_prefetch_cnt = 2 * sizeof(struct ahc_dma_seg);
6814         download_consts[SG_PREFETCH_CNT] = sg_prefetch_cnt;
6815         download_consts[SG_PREFETCH_ALIGN_MASK] = ~(sg_prefetch_cnt - 1);
6816         download_consts[SG_PREFETCH_ADDR_MASK] = (sg_prefetch_cnt - 1);
6817
6818         cur_patch = patches;
6819         downloaded = 0;
6820         skip_addr = 0;
6821         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE|LOADRAM);
6822         ahc_outb(ahc, SEQADDR0, 0);
6823         ahc_outb(ahc, SEQADDR1, 0);
6824
6825         for (i = 0; i < sizeof(seqprog)/4; i++) {
6826                 if (ahc_check_patch(ahc, &cur_patch, i, &skip_addr) == 0) {
6827                         /*
6828                          * Don't download this instruction as it
6829                          * is in a patch that was removed.
6830                          */
6831                         continue;
6832                 }
6833
6834                 if (downloaded == ahc->instruction_ram_size) {
6835                         /*
6836                          * We're about to exceed the instruction
6837                          * storage capacity for this chip.  Fail
6838                          * the load.
6839                          */
6840                         printf("\n%s: Program too large for instruction memory "
6841                                "size of %d!\n", ahc_name(ahc),
6842                                ahc->instruction_ram_size);
6843                         return (ENOMEM);
6844                 }
6845
6846                 /*
6847                  * Move through the CS table until we find a CS
6848                  * that might apply to this instruction.
6849                  */
6850                 for (; cur_cs < num_critical_sections; cur_cs++) {
6851                         if (critical_sections[cur_cs].end <= i) {
6852                                 if (begin_set[cs_count] == TRUE
6853                                  && end_set[cs_count] == FALSE) {
6854                                         cs_table[cs_count].end = downloaded;
6855                                         end_set[cs_count] = TRUE;
6856                                         cs_count++;
6857                                 }
6858                                 continue;
6859                         }
6860                         if (critical_sections[cur_cs].begin <= i
6861                          && begin_set[cs_count] == FALSE) {
6862                                 cs_table[cs_count].begin = downloaded;
6863                                 begin_set[cs_count] = TRUE;
6864                         }
6865                         break;
6866                 }
6867                 ahc_download_instr(ahc, i, download_consts);
6868                 downloaded++;
6869         }
6870
6871         ahc->num_critical_sections = cs_count;
6872         if (cs_count != 0) {
6873
6874                 cs_count *= sizeof(struct cs);
6875                 ahc->critical_sections = malloc(cs_count, M_DEVBUF, M_NOWAIT);
6876                 if (ahc->critical_sections == NULL)
6877                         panic("ahc_loadseq: Could not malloc");
6878                 memcpy(ahc->critical_sections, cs_table, cs_count);
6879         }
6880         ahc_outb(ahc, SEQCTL, PERRORDIS|FAILDIS|FASTMODE);
6881
6882         if (bootverbose) {
6883                 printf(" %d instructions downloaded\n", downloaded);
6884                 printf("%s: Features 0x%x, Bugs 0x%x, Flags 0x%x\n",
6885                        ahc_name(ahc), ahc->features, ahc->bugs, ahc->flags);
6886         }
6887         return (0);
6888 }
6889
6890 static int
6891 ahc_check_patch(struct ahc_softc *ahc, struct patch **start_patch,
6892                 u_int start_instr, u_int *skip_addr)
6893 {
6894         struct  patch *cur_patch;
6895         struct  patch *last_patch;
6896         u_int   num_patches;
6897
6898         num_patches = ARRAY_SIZE(patches);
6899         last_patch = &patches[num_patches];
6900         cur_patch = *start_patch;
6901
6902         while (cur_patch < last_patch && start_instr == cur_patch->begin) {
6903
6904                 if (cur_patch->patch_func(ahc) == 0) {
6905
6906                         /* Start rejecting code */
6907                         *skip_addr = start_instr + cur_patch->skip_instr;
6908                         cur_patch += cur_patch->skip_patch;
6909                 } else {
6910                         /* Accepted this patch.  Advance to the next
6911                          * one and wait for our intruction pointer to
6912                          * hit this point.
6913                          */
6914                         cur_patch++;
6915                 }
6916         }
6917
6918         *start_patch = cur_patch;
6919         if (start_instr < *skip_addr)
6920                 /* Still skipping */
6921                 return (0);
6922
6923         return (1);
6924 }
6925
6926 static void
6927 ahc_download_instr(struct ahc_softc *ahc, u_int instrptr, uint8_t *dconsts)
6928 {
6929         union   ins_formats instr;
6930         struct  ins_format1 *fmt1_ins;
6931         struct  ins_format3 *fmt3_ins;
6932         u_int   opcode;
6933
6934         /*
6935          * The firmware is always compiled into a little endian format.
6936          */
6937         instr.integer = ahc_le32toh(*(uint32_t*)&seqprog[instrptr * 4]);
6938
6939         fmt1_ins = &instr.format1;
6940         fmt3_ins = NULL;
6941
6942         /* Pull the opcode */
6943         opcode = instr.format1.opcode;
6944         switch (opcode) {
6945         case AIC_OP_JMP:
6946         case AIC_OP_JC:
6947         case AIC_OP_JNC:
6948         case AIC_OP_CALL:
6949         case AIC_OP_JNE:
6950         case AIC_OP_JNZ:
6951         case AIC_OP_JE:
6952         case AIC_OP_JZ:
6953         {
6954                 struct patch *cur_patch;
6955                 int address_offset;
6956                 u_int address;
6957                 u_int skip_addr;
6958                 u_int i;
6959
6960                 fmt3_ins = &instr.format3;
6961                 address_offset = 0;
6962                 address = fmt3_ins->address;
6963                 cur_patch = patches;
6964                 skip_addr = 0;
6965
6966                 for (i = 0; i < address;) {
6967
6968                         ahc_check_patch(ahc, &cur_patch, i, &skip_addr);
6969
6970                         if (skip_addr > i) {
6971                                 int end_addr;
6972
6973                                 end_addr = min(address, skip_addr);
6974                                 address_offset += end_addr - i;
6975                                 i = skip_addr;
6976                         } else {
6977                                 i++;
6978                         }
6979                 }
6980                 address -= address_offset;
6981                 fmt3_ins->address = address;
6982                 /* FALLTHROUGH */
6983         }
6984         case AIC_OP_OR:
6985         case AIC_OP_AND:
6986         case AIC_OP_XOR:
6987         case AIC_OP_ADD:
6988         case AIC_OP_ADC:
6989         case AIC_OP_BMOV:
6990                 if (fmt1_ins->parity != 0) {
6991                         fmt1_ins->immediate = dconsts[fmt1_ins->immediate];
6992                 }
6993                 fmt1_ins->parity = 0;
6994                 if ((ahc->features & AHC_CMD_CHAN) == 0
6995                  && opcode == AIC_OP_BMOV) {
6996                         /*
6997                          * Block move was added at the same time
6998                          * as the command channel.  Verify that
6999                          * this is only a move of a single element
7000                          * and convert the BMOV to a MOV
7001                          * (AND with an immediate of FF).
7002                          */
7003                         if (fmt1_ins->immediate != 1)
7004                                 panic("%s: BMOV not supported\n",
7005                                       ahc_name(ahc));
7006                         fmt1_ins->opcode = AIC_OP_AND;
7007                         fmt1_ins->immediate = 0xff;
7008                 }
7009                 /* FALLTHROUGH */
7010         case AIC_OP_ROL:
7011                 if ((ahc->features & AHC_ULTRA2) != 0) {
7012                         int i, count;
7013
7014                         /* Calculate odd parity for the instruction */
7015                         for (i = 0, count = 0; i < 31; i++) {
7016                                 uint32_t mask;
7017
7018                                 mask = 0x01 << i;
7019                                 if ((instr.integer & mask) != 0)
7020                                         count++;
7021                         }
7022                         if ((count & 0x01) == 0)
7023                                 instr.format1.parity = 1;
7024                 } else {
7025                         /* Compress the instruction for older sequencers */
7026                         if (fmt3_ins != NULL) {
7027                                 instr.integer =
7028                                         fmt3_ins->immediate
7029                                       | (fmt3_ins->source << 8)
7030                                       | (fmt3_ins->address << 16)
7031                                       | (fmt3_ins->opcode << 25);
7032                         } else {
7033                                 instr.integer =
7034                                         fmt1_ins->immediate
7035                                       | (fmt1_ins->source << 8)
7036                                       | (fmt1_ins->destination << 16)
7037                                       | (fmt1_ins->ret << 24)
7038                                       | (fmt1_ins->opcode << 25);
7039                         }
7040                 }
7041                 /* The sequencer is a little endian cpu */
7042                 instr.integer = ahc_htole32(instr.integer);
7043                 ahc_outsb(ahc, SEQRAM, instr.bytes, 4);
7044                 break;
7045         default:
7046                 panic("Unknown opcode encountered in seq program");
7047                 break;
7048         }
7049 }
7050
7051 int
7052 ahc_print_register(ahc_reg_parse_entry_t *table, u_int num_entries,
7053                    const char *name, u_int address, u_int value,
7054                    u_int *cur_column, u_int wrap_point)
7055 {
7056         int     printed;
7057         u_int   printed_mask;
7058
7059         if (cur_column != NULL && *cur_column >= wrap_point) {
7060                 printf("\n");
7061                 *cur_column = 0;
7062         }
7063         printed = printf("%s[0x%x]", name, value);
7064         if (table == NULL) {
7065                 printed += printf(" ");
7066                 *cur_column += printed;
7067                 return (printed);
7068         }
7069         printed_mask = 0;
7070         while (printed_mask != 0xFF) {
7071                 int entry;
7072
7073                 for (entry = 0; entry < num_entries; entry++) {
7074                         if (((value & table[entry].mask)
7075                           != table[entry].value)
7076                          || ((printed_mask & table[entry].mask)
7077                           == table[entry].mask))
7078                                 continue;
7079
7080                         printed += printf("%s%s",
7081                                           printed_mask == 0 ? ":(" : "|",
7082                                           table[entry].name);
7083                         printed_mask |= table[entry].mask;
7084
7085                         break;
7086                 }
7087                 if (entry >= num_entries)
7088                         break;
7089         }
7090         if (printed_mask != 0)
7091                 printed += printf(") ");
7092         else
7093                 printed += printf(" ");
7094         if (cur_column != NULL)
7095                 *cur_column += printed;
7096         return (printed);
7097 }
7098
7099 void
7100 ahc_dump_card_state(struct ahc_softc *ahc)
7101 {
7102         struct  scb *scb;
7103         struct  scb_tailq *untagged_q;
7104         u_int   cur_col;
7105         int     paused;
7106         int     target;
7107         int     maxtarget;
7108         int     i;
7109         uint8_t last_phase;
7110         uint8_t qinpos;
7111         uint8_t qintail;
7112         uint8_t qoutpos;
7113         uint8_t scb_index;
7114         uint8_t saved_scbptr;
7115
7116         if (ahc_is_paused(ahc)) {
7117                 paused = 1;
7118         } else {
7119                 paused = 0;
7120                 ahc_pause(ahc);
7121         }
7122
7123         saved_scbptr = ahc_inb(ahc, SCBPTR);
7124         last_phase = ahc_inb(ahc, LASTPHASE);
7125         printf(">>>>>>>>>>>>>>>>>> Dump Card State Begins <<<<<<<<<<<<<<<<<\n"
7126                "%s: Dumping Card State %s, at SEQADDR 0x%x\n",
7127                ahc_name(ahc), ahc_lookup_phase_entry(last_phase)->phasemsg,
7128                ahc_inb(ahc, SEQADDR0) | (ahc_inb(ahc, SEQADDR1) << 8));
7129         if (paused)
7130                 printf("Card was paused\n");
7131         printf("ACCUM = 0x%x, SINDEX = 0x%x, DINDEX = 0x%x, ARG_2 = 0x%x\n",
7132                ahc_inb(ahc, ACCUM), ahc_inb(ahc, SINDEX), ahc_inb(ahc, DINDEX),
7133                ahc_inb(ahc, ARG_2));
7134         printf("HCNT = 0x%x SCBPTR = 0x%x\n", ahc_inb(ahc, HCNT),
7135                ahc_inb(ahc, SCBPTR));
7136         cur_col = 0;
7137         if ((ahc->features & AHC_DT) != 0)
7138                 ahc_scsiphase_print(ahc_inb(ahc, SCSIPHASE), &cur_col, 50);
7139         ahc_scsisigi_print(ahc_inb(ahc, SCSISIGI), &cur_col, 50);
7140         ahc_error_print(ahc_inb(ahc, ERROR), &cur_col, 50);
7141         ahc_scsibusl_print(ahc_inb(ahc, SCSIBUSL), &cur_col, 50);
7142         ahc_lastphase_print(ahc_inb(ahc, LASTPHASE), &cur_col, 50);
7143         ahc_scsiseq_print(ahc_inb(ahc, SCSISEQ), &cur_col, 50);
7144         ahc_sblkctl_print(ahc_inb(ahc, SBLKCTL), &cur_col, 50);
7145         ahc_scsirate_print(ahc_inb(ahc, SCSIRATE), &cur_col, 50);
7146         ahc_seqctl_print(ahc_inb(ahc, SEQCTL), &cur_col, 50);
7147         ahc_seq_flags_print(ahc_inb(ahc, SEQ_FLAGS), &cur_col, 50);
7148         ahc_sstat0_print(ahc_inb(ahc, SSTAT0), &cur_col, 50);
7149         ahc_sstat1_print(ahc_inb(ahc, SSTAT1), &cur_col, 50);
7150         ahc_sstat2_print(ahc_inb(ahc, SSTAT2), &cur_col, 50);
7151         ahc_sstat3_print(ahc_inb(ahc, SSTAT3), &cur_col, 50);
7152         ahc_simode0_print(ahc_inb(ahc, SIMODE0), &cur_col, 50);
7153         ahc_simode1_print(ahc_inb(ahc, SIMODE1), &cur_col, 50);
7154         ahc_sxfrctl0_print(ahc_inb(ahc, SXFRCTL0), &cur_col, 50);
7155         ahc_dfcntrl_print(ahc_inb(ahc, DFCNTRL), &cur_col, 50);
7156         ahc_dfstatus_print(ahc_inb(ahc, DFSTATUS), &cur_col, 50);
7157         if (cur_col != 0)
7158                 printf("\n");
7159         printf("STACK:");
7160         for (i = 0; i < STACK_SIZE; i++)
7161                printf(" 0x%x", ahc_inb(ahc, STACK)|(ahc_inb(ahc, STACK) << 8));
7162         printf("\nSCB count = %d\n", ahc->scb_data->numscbs);
7163         printf("Kernel NEXTQSCB = %d\n", ahc->next_queued_scb->hscb->tag);
7164         printf("Card NEXTQSCB = %d\n", ahc_inb(ahc, NEXT_QUEUED_SCB));
7165         /* QINFIFO */
7166         printf("QINFIFO entries: ");
7167         if ((ahc->features & AHC_QUEUE_REGS) != 0) {
7168                 qinpos = ahc_inb(ahc, SNSCB_QOFF);
7169                 ahc_outb(ahc, SNSCB_QOFF, qinpos);
7170         } else
7171                 qinpos = ahc_inb(ahc, QINPOS);
7172         qintail = ahc->qinfifonext;
7173         while (qinpos != qintail) {
7174                 printf("%d ", ahc->qinfifo[qinpos]);
7175                 qinpos++;
7176         }
7177         printf("\n");
7178
7179         printf("Waiting Queue entries: ");
7180         scb_index = ahc_inb(ahc, WAITING_SCBH);
7181         i = 0;
7182         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7183                 ahc_outb(ahc, SCBPTR, scb_index);
7184                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7185                 scb_index = ahc_inb(ahc, SCB_NEXT);
7186         }
7187         printf("\n");
7188
7189         printf("Disconnected Queue entries: ");
7190         scb_index = ahc_inb(ahc, DISCONNECTED_SCBH);
7191         i = 0;
7192         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7193                 ahc_outb(ahc, SCBPTR, scb_index);
7194                 printf("%d:%d ", scb_index, ahc_inb(ahc, SCB_TAG));
7195                 scb_index = ahc_inb(ahc, SCB_NEXT);
7196         }
7197         printf("\n");
7198
7199         ahc_sync_qoutfifo(ahc, BUS_DMASYNC_POSTREAD);
7200         printf("QOUTFIFO entries: ");
7201         qoutpos = ahc->qoutfifonext;
7202         i = 0;
7203         while (ahc->qoutfifo[qoutpos] != SCB_LIST_NULL && i++ < 256) {
7204                 printf("%d ", ahc->qoutfifo[qoutpos]);
7205                 qoutpos++;
7206         }
7207         printf("\n");
7208
7209         printf("Sequencer Free SCB List: ");
7210         scb_index = ahc_inb(ahc, FREE_SCBH);
7211         i = 0;
7212         while (scb_index != SCB_LIST_NULL && i++ < 256) {
7213                 ahc_outb(ahc, SCBPTR, scb_index);
7214                 printf("%d ", scb_index);
7215                 scb_index = ahc_inb(ahc, SCB_NEXT);
7216         }
7217         printf("\n");
7218
7219         printf("Sequencer SCB Info: ");
7220         for (i = 0; i < ahc->scb_data->maxhscbs; i++) {
7221                 ahc_outb(ahc, SCBPTR, i);
7222                 cur_col = printf("\n%3d ", i);
7223
7224                 ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL), &cur_col, 60);
7225                 ahc_scb_scsiid_print(ahc_inb(ahc, SCB_SCSIID), &cur_col, 60);
7226                 ahc_scb_lun_print(ahc_inb(ahc, SCB_LUN), &cur_col, 60);
7227                 ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7228         }
7229         printf("\n");
7230
7231         printf("Pending list: ");
7232         i = 0;
7233         LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7234                 if (i++ > 256)
7235                         break;
7236                 cur_col = printf("\n%3d ", scb->hscb->tag);
7237                 ahc_scb_control_print(scb->hscb->control, &cur_col, 60);
7238                 ahc_scb_scsiid_print(scb->hscb->scsiid, &cur_col, 60);
7239                 ahc_scb_lun_print(scb->hscb->lun, &cur_col, 60);
7240                 if ((ahc->flags & AHC_PAGESCBS) == 0) {
7241                         ahc_outb(ahc, SCBPTR, scb->hscb->tag);
7242                         printf("(");
7243                         ahc_scb_control_print(ahc_inb(ahc, SCB_CONTROL),
7244                                               &cur_col, 60);
7245                         ahc_scb_tag_print(ahc_inb(ahc, SCB_TAG), &cur_col, 60);
7246                         printf(")");
7247                 }
7248         }
7249         printf("\n");
7250
7251         printf("Kernel Free SCB list: ");
7252         i = 0;
7253         SLIST_FOREACH(scb, &ahc->scb_data->free_scbs, links.sle) {
7254                 if (i++ > 256)
7255                         break;
7256                 printf("%d ", scb->hscb->tag);
7257         }
7258         printf("\n");
7259
7260         maxtarget = (ahc->features & (AHC_WIDE|AHC_TWIN)) ? 15 : 7;
7261         for (target = 0; target <= maxtarget; target++) {
7262                 untagged_q = &ahc->untagged_queues[target];
7263                 if (TAILQ_FIRST(untagged_q) == NULL)
7264                         continue;
7265                 printf("Untagged Q(%d): ", target);
7266                 i = 0;
7267                 TAILQ_FOREACH(scb, untagged_q, links.tqe) {
7268                         if (i++ > 256)
7269                                 break;
7270                         printf("%d ", scb->hscb->tag);
7271                 }
7272                 printf("\n");
7273         }
7274
7275         ahc_platform_dump_card_state(ahc);
7276         printf("\n<<<<<<<<<<<<<<<<< Dump Card State Ends >>>>>>>>>>>>>>>>>>\n");
7277         ahc_outb(ahc, SCBPTR, saved_scbptr);
7278         if (paused == 0)
7279                 ahc_unpause(ahc);
7280 }
7281
7282 /************************* Target Mode ****************************************/
7283 #ifdef AHC_TARGET_MODE
7284 cam_status
7285 ahc_find_tmode_devs(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb,
7286                     struct ahc_tmode_tstate **tstate,
7287                     struct ahc_tmode_lstate **lstate,
7288                     int notfound_failure)
7289 {
7290
7291         if ((ahc->features & AHC_TARGETMODE) == 0)
7292                 return (CAM_REQ_INVALID);
7293
7294         /*
7295          * Handle the 'black hole' device that sucks up
7296          * requests to unattached luns on enabled targets.
7297          */
7298         if (ccb->ccb_h.target_id == CAM_TARGET_WILDCARD
7299          && ccb->ccb_h.target_lun == CAM_LUN_WILDCARD) {
7300                 *tstate = NULL;
7301                 *lstate = ahc->black_hole;
7302         } else {
7303                 u_int max_id;
7304
7305                 max_id = (ahc->features & AHC_WIDE) ? 16 : 8;
7306                 if (ccb->ccb_h.target_id >= max_id)
7307                         return (CAM_TID_INVALID);
7308
7309                 if (ccb->ccb_h.target_lun >= AHC_NUM_LUNS)
7310                         return (CAM_LUN_INVALID);
7311
7312                 *tstate = ahc->enabled_targets[ccb->ccb_h.target_id];
7313                 *lstate = NULL;
7314                 if (*tstate != NULL)
7315                         *lstate =
7316                             (*tstate)->enabled_luns[ccb->ccb_h.target_lun];
7317         }
7318
7319         if (notfound_failure != 0 && *lstate == NULL)
7320                 return (CAM_PATH_INVALID);
7321
7322         return (CAM_REQ_CMP);
7323 }
7324
7325 void
7326 ahc_handle_en_lun(struct ahc_softc *ahc, struct cam_sim *sim, union ccb *ccb)
7327 {
7328         struct     ahc_tmode_tstate *tstate;
7329         struct     ahc_tmode_lstate *lstate;
7330         struct     ccb_en_lun *cel;
7331         cam_status status;
7332         u_long     s;
7333         u_int      target;
7334         u_int      lun;
7335         u_int      target_mask;
7336         u_int      our_id;
7337         int        error;
7338         char       channel;
7339
7340         status = ahc_find_tmode_devs(ahc, sim, ccb, &tstate, &lstate,
7341                                      /*notfound_failure*/FALSE);
7342
7343         if (status != CAM_REQ_CMP) {
7344                 ccb->ccb_h.status = status;
7345                 return;
7346         }
7347
7348         if (cam_sim_bus(sim) == 0)
7349                 our_id = ahc->our_id;
7350         else
7351                 our_id = ahc->our_id_b;
7352
7353         if (ccb->ccb_h.target_id != our_id) {
7354                 /*
7355                  * our_id represents our initiator ID, or
7356                  * the ID of the first target to have an
7357                  * enabled lun in target mode.  There are
7358                  * two cases that may preclude enabling a
7359                  * target id other than our_id.
7360                  *
7361                  *   o our_id is for an active initiator role.
7362                  *     Since the hardware does not support
7363                  *     reselections to the initiator role at
7364                  *     anything other than our_id, and our_id
7365                  *     is used by the hardware to indicate the
7366                  *     ID to use for both select-out and
7367                  *     reselect-out operations, the only target
7368                  *     ID we can support in this mode is our_id.
7369                  *
7370                  *   o The MULTARGID feature is not available and
7371                  *     a previous target mode ID has been enabled.
7372                  */
7373                 if ((ahc->features & AHC_MULTIROLE) != 0) {
7374
7375                         if ((ahc->features & AHC_MULTI_TID) != 0
7376                          && (ahc->flags & AHC_INITIATORROLE) != 0) {
7377                                 /*
7378                                  * Only allow additional targets if
7379                                  * the initiator role is disabled.
7380                                  * The hardware cannot handle a re-select-in
7381                                  * on the initiator id during a re-select-out
7382                                  * on a different target id.
7383                                  */
7384                                 status = CAM_TID_INVALID;
7385                         } else if ((ahc->flags & AHC_INITIATORROLE) != 0
7386                                 || ahc->enabled_luns > 0) {
7387                                 /*
7388                                  * Only allow our target id to change
7389                                  * if the initiator role is not configured
7390                                  * and there are no enabled luns which
7391                                  * are attached to the currently registered
7392                                  * scsi id.
7393                                  */
7394                                 status = CAM_TID_INVALID;
7395                         }
7396                 } else if ((ahc->features & AHC_MULTI_TID) == 0
7397                         && ahc->enabled_luns > 0) {
7398
7399                         status = CAM_TID_INVALID;
7400                 }
7401         }
7402
7403         if (status != CAM_REQ_CMP) {
7404                 ccb->ccb_h.status = status;
7405                 return;
7406         }
7407
7408         /*
7409          * We now have an id that is valid.
7410          * If we aren't in target mode, switch modes.
7411          */
7412         if ((ahc->flags & AHC_TARGETROLE) == 0
7413          && ccb->ccb_h.target_id != CAM_TARGET_WILDCARD) {
7414                 u_long   s;
7415                 ahc_flag saved_flags;
7416
7417                 printf("Configuring Target Mode\n");
7418                 ahc_lock(ahc, &s);
7419                 if (LIST_FIRST(&ahc->pending_scbs) != NULL) {
7420                         ccb->ccb_h.status = CAM_BUSY;
7421                         ahc_unlock(ahc, &s);
7422                         return;
7423                 }
7424                 saved_flags = ahc->flags;
7425                 ahc->flags |= AHC_TARGETROLE;
7426                 if ((ahc->features & AHC_MULTIROLE) == 0)
7427                         ahc->flags &= ~AHC_INITIATORROLE;
7428                 ahc_pause(ahc);
7429                 error = ahc_loadseq(ahc);
7430                 if (error != 0) {
7431                         /*
7432                          * Restore original configuration and notify
7433                          * the caller that we cannot support target mode.
7434                          * Since the adapter started out in this
7435                          * configuration, the firmware load will succeed,
7436                          * so there is no point in checking ahc_loadseq's
7437                          * return value.
7438                          */
7439                         ahc->flags = saved_flags;
7440                         (void)ahc_loadseq(ahc);
7441                         ahc_restart(ahc);
7442                         ahc_unlock(ahc, &s);
7443                         ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
7444                         return;
7445                 }
7446                 ahc_restart(ahc);
7447                 ahc_unlock(ahc, &s);
7448         }
7449         cel = &ccb->cel;
7450         target = ccb->ccb_h.target_id;
7451         lun = ccb->ccb_h.target_lun;
7452         channel = SIM_CHANNEL(ahc, sim);
7453         target_mask = 0x01 << target;
7454         if (channel == 'B')
7455                 target_mask <<= 8;
7456
7457         if (cel->enable != 0) {
7458                 u_int scsiseq;
7459
7460                 /* Are we already enabled?? */
7461                 if (lstate != NULL) {
7462                         xpt_print_path(ccb->ccb_h.path);
7463                         printf("Lun already enabled\n");
7464                         ccb->ccb_h.status = CAM_LUN_ALRDY_ENA;
7465                         return;
7466                 }
7467
7468                 if (cel->grp6_len != 0
7469                  || cel->grp7_len != 0) {
7470                         /*
7471                          * Don't (yet?) support vendor
7472                          * specific commands.
7473                          */
7474                         ccb->ccb_h.status = CAM_REQ_INVALID;
7475                         printf("Non-zero Group Codes\n");
7476                         return;
7477                 }
7478
7479                 /*
7480                  * Seems to be okay.
7481                  * Setup our data structures.
7482                  */
7483                 if (target != CAM_TARGET_WILDCARD && tstate == NULL) {
7484                         tstate = ahc_alloc_tstate(ahc, target, channel);
7485                         if (tstate == NULL) {
7486                                 xpt_print_path(ccb->ccb_h.path);
7487                                 printf("Couldn't allocate tstate\n");
7488                                 ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7489                                 return;
7490                         }
7491                 }
7492                 lstate = malloc(sizeof(*lstate), M_DEVBUF, M_NOWAIT);
7493                 if (lstate == NULL) {
7494                         xpt_print_path(ccb->ccb_h.path);
7495                         printf("Couldn't allocate lstate\n");
7496                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7497                         return;
7498                 }
7499                 memset(lstate, 0, sizeof(*lstate));
7500                 status = xpt_create_path(&lstate->path, /*periph*/NULL,
7501                                          xpt_path_path_id(ccb->ccb_h.path),
7502                                          xpt_path_target_id(ccb->ccb_h.path),
7503                                          xpt_path_lun_id(ccb->ccb_h.path));
7504                 if (status != CAM_REQ_CMP) {
7505                         free(lstate, M_DEVBUF);
7506                         xpt_print_path(ccb->ccb_h.path);
7507                         printf("Couldn't allocate path\n");
7508                         ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
7509                         return;
7510                 }
7511                 SLIST_INIT(&lstate->accept_tios);
7512                 SLIST_INIT(&lstate->immed_notifies);
7513                 ahc_lock(ahc, &s);
7514                 ahc_pause(ahc);
7515                 if (target != CAM_TARGET_WILDCARD) {
7516                         tstate->enabled_luns[lun] = lstate;
7517                         ahc->enabled_luns++;
7518
7519                         if ((ahc->features & AHC_MULTI_TID) != 0) {
7520                                 u_int targid_mask;
7521
7522                                 targid_mask = ahc_inb(ahc, TARGID)
7523                                             | (ahc_inb(ahc, TARGID + 1) << 8);
7524
7525                                 targid_mask |= target_mask;
7526                                 ahc_outb(ahc, TARGID, targid_mask);
7527                                 ahc_outb(ahc, TARGID+1, (targid_mask >> 8));
7528
7529                                 ahc_update_scsiid(ahc, targid_mask);
7530                         } else {
7531                                 u_int our_id;
7532                                 char  channel;
7533
7534                                 channel = SIM_CHANNEL(ahc, sim);
7535                                 our_id = SIM_SCSI_ID(ahc, sim);
7536
7537                                 /*
7538                                  * This can only happen if selections
7539                                  * are not enabled
7540                                  */
7541                                 if (target != our_id) {
7542                                         u_int sblkctl;
7543                                         char  cur_channel;
7544                                         int   swap;
7545
7546                                         sblkctl = ahc_inb(ahc, SBLKCTL);
7547                                         cur_channel = (sblkctl & SELBUSB)
7548                                                     ? 'B' : 'A';
7549                                         if ((ahc->features & AHC_TWIN) == 0)
7550                                                 cur_channel = 'A';
7551                                         swap = cur_channel != channel;
7552                                         if (channel == 'A')
7553                                                 ahc->our_id = target;
7554                                         else
7555                                                 ahc->our_id_b = target;
7556
7557                                         if (swap)
7558                                                 ahc_outb(ahc, SBLKCTL,
7559                                                          sblkctl ^ SELBUSB);
7560
7561                                         ahc_outb(ahc, SCSIID, target);
7562
7563                                         if (swap)
7564                                                 ahc_outb(ahc, SBLKCTL, sblkctl);
7565                                 }
7566                         }
7567                 } else
7568                         ahc->black_hole = lstate;
7569                 /* Allow select-in operations */
7570                 if (ahc->black_hole != NULL && ahc->enabled_luns > 0) {
7571                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7572                         scsiseq |= ENSELI;
7573                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7574                         scsiseq = ahc_inb(ahc, SCSISEQ);
7575                         scsiseq |= ENSELI;
7576                         ahc_outb(ahc, SCSISEQ, scsiseq);
7577                 }
7578                 ahc_unpause(ahc);
7579                 ahc_unlock(ahc, &s);
7580                 ccb->ccb_h.status = CAM_REQ_CMP;
7581                 xpt_print_path(ccb->ccb_h.path);
7582                 printf("Lun now enabled for target mode\n");
7583         } else {
7584                 struct scb *scb;
7585                 int i, empty;
7586
7587                 if (lstate == NULL) {
7588                         ccb->ccb_h.status = CAM_LUN_INVALID;
7589                         return;
7590                 }
7591
7592                 ahc_lock(ahc, &s);
7593
7594                 ccb->ccb_h.status = CAM_REQ_CMP;
7595                 LIST_FOREACH(scb, &ahc->pending_scbs, pending_links) {
7596                         struct ccb_hdr *ccbh;
7597
7598                         ccbh = &scb->io_ctx->ccb_h;
7599                         if (ccbh->func_code == XPT_CONT_TARGET_IO
7600                          && !xpt_path_comp(ccbh->path, ccb->ccb_h.path)){
7601                                 printf("CTIO pending\n");
7602                                 ccb->ccb_h.status = CAM_REQ_INVALID;
7603                                 ahc_unlock(ahc, &s);
7604                                 return;
7605                         }
7606                 }
7607
7608                 if (SLIST_FIRST(&lstate->accept_tios) != NULL) {
7609                         printf("ATIOs pending\n");
7610                         ccb->ccb_h.status = CAM_REQ_INVALID;
7611                 }
7612
7613                 if (SLIST_FIRST(&lstate->immed_notifies) != NULL) {
7614                         printf("INOTs pending\n");
7615                         ccb->ccb_h.status = CAM_REQ_INVALID;
7616                 }
7617
7618                 if (ccb->ccb_h.status != CAM_REQ_CMP) {
7619                         ahc_unlock(ahc, &s);
7620                         return;
7621                 }
7622
7623                 xpt_print_path(ccb->ccb_h.path);
7624                 printf("Target mode disabled\n");
7625                 xpt_free_path(lstate->path);
7626                 free(lstate, M_DEVBUF);
7627
7628                 ahc_pause(ahc);
7629                 /* Can we clean up the target too? */
7630                 if (target != CAM_TARGET_WILDCARD) {
7631                         tstate->enabled_luns[lun] = NULL;
7632                         ahc->enabled_luns--;
7633                         for (empty = 1, i = 0; i < 8; i++)
7634                                 if (tstate->enabled_luns[i] != NULL) {
7635                                         empty = 0;
7636                                         break;
7637                                 }
7638
7639                         if (empty) {
7640                                 ahc_free_tstate(ahc, target, channel,
7641                                                 /*force*/FALSE);
7642                                 if (ahc->features & AHC_MULTI_TID) {
7643                                         u_int targid_mask;
7644
7645                                         targid_mask = ahc_inb(ahc, TARGID)
7646                                                     | (ahc_inb(ahc, TARGID + 1)
7647                                                        << 8);
7648
7649                                         targid_mask &= ~target_mask;
7650                                         ahc_outb(ahc, TARGID, targid_mask);
7651                                         ahc_outb(ahc, TARGID+1,
7652                                                  (targid_mask >> 8));
7653                                         ahc_update_scsiid(ahc, targid_mask);
7654                                 }
7655                         }
7656                 } else {
7657
7658                         ahc->black_hole = NULL;
7659
7660                         /*
7661                          * We can't allow selections without
7662                          * our black hole device.
7663                          */
7664                         empty = TRUE;
7665                 }
7666                 if (ahc->enabled_luns == 0) {
7667                         /* Disallow select-in */
7668                         u_int scsiseq;
7669
7670                         scsiseq = ahc_inb(ahc, SCSISEQ_TEMPLATE);
7671                         scsiseq &= ~ENSELI;
7672                         ahc_outb(ahc, SCSISEQ_TEMPLATE, scsiseq);
7673                         scsiseq = ahc_inb(ahc, SCSISEQ);
7674                         scsiseq &= ~ENSELI;
7675                         ahc_outb(ahc, SCSISEQ, scsiseq);
7676
7677                         if ((ahc->features & AHC_MULTIROLE) == 0) {
7678                                 printf("Configuring Initiator Mode\n");
7679                                 ahc->flags &= ~AHC_TARGETROLE;
7680                                 ahc->flags |= AHC_INITIATORROLE;
7681                                 /*
7682                                  * Returning to a configuration that
7683                                  * fit previously will always succeed.
7684                                  */
7685                                 (void)ahc_loadseq(ahc);
7686                                 ahc_restart(ahc);
7687                                 /*
7688                                  * Unpaused.  The extra unpause
7689                                  * that follows is harmless.
7690                                  */
7691                         }
7692                 }
7693                 ahc_unpause(ahc);
7694                 ahc_unlock(ahc, &s);
7695         }
7696 }
7697
7698 static void
7699 ahc_update_scsiid(struct ahc_softc *ahc, u_int targid_mask)
7700 {
7701         u_int scsiid_mask;
7702         u_int scsiid;
7703
7704         if ((ahc->features & AHC_MULTI_TID) == 0)
7705                 panic("ahc_update_scsiid called on non-multitid unit\n");
7706
7707         /*
7708          * Since we will rely on the TARGID mask
7709          * for selection enables, ensure that OID
7710          * in SCSIID is not set to some other ID
7711          * that we don't want to allow selections on.
7712          */
7713         if ((ahc->features & AHC_ULTRA2) != 0)
7714                 scsiid = ahc_inb(ahc, SCSIID_ULTRA2);
7715         else
7716                 scsiid = ahc_inb(ahc, SCSIID);
7717         scsiid_mask = 0x1 << (scsiid & OID);
7718         if ((targid_mask & scsiid_mask) == 0) {
7719                 u_int our_id;
7720
7721                 /* ffs counts from 1 */
7722                 our_id = ffs(targid_mask);
7723                 if (our_id == 0)
7724                         our_id = ahc->our_id;
7725                 else
7726                         our_id--;
7727                 scsiid &= TID;
7728                 scsiid |= our_id;
7729         }
7730         if ((ahc->features & AHC_ULTRA2) != 0)
7731                 ahc_outb(ahc, SCSIID_ULTRA2, scsiid);
7732         else
7733                 ahc_outb(ahc, SCSIID, scsiid);
7734 }
7735
7736 void
7737 ahc_run_tqinfifo(struct ahc_softc *ahc, int paused)
7738 {
7739         struct target_cmd *cmd;
7740
7741         /*
7742          * If the card supports auto-access pause,
7743          * we can access the card directly regardless
7744          * of whether it is paused or not.
7745          */
7746         if ((ahc->features & AHC_AUTOPAUSE) != 0)
7747                 paused = TRUE;
7748
7749         ahc_sync_tqinfifo(ahc, BUS_DMASYNC_POSTREAD);
7750         while ((cmd = &ahc->targetcmds[ahc->tqinfifonext])->cmd_valid != 0) {
7751
7752                 /*
7753                  * Only advance through the queue if we
7754                  * have the resources to process the command.
7755                  */
7756                 if (ahc_handle_target_cmd(ahc, cmd) != 0)
7757                         break;
7758
7759                 cmd->cmd_valid = 0;
7760                 ahc_dmamap_sync(ahc, ahc->shared_data_dmat,
7761                                 ahc->shared_data_dmamap,
7762                                 ahc_targetcmd_offset(ahc, ahc->tqinfifonext),
7763                                 sizeof(struct target_cmd),
7764                                 BUS_DMASYNC_PREREAD);
7765                 ahc->tqinfifonext++;
7766
7767                 /*
7768                  * Lazily update our position in the target mode incoming
7769                  * command queue as seen by the sequencer.
7770                  */
7771                 if ((ahc->tqinfifonext & (HOST_TQINPOS - 1)) == 1) {
7772                         if ((ahc->features & AHC_HS_MAILBOX) != 0) {
7773                                 u_int hs_mailbox;
7774
7775                                 hs_mailbox = ahc_inb(ahc, HS_MAILBOX);
7776                                 hs_mailbox &= ~HOST_TQINPOS;
7777                                 hs_mailbox |= ahc->tqinfifonext & HOST_TQINPOS;
7778                                 ahc_outb(ahc, HS_MAILBOX, hs_mailbox);
7779                         } else {
7780                                 if (!paused)
7781                                         ahc_pause(ahc);
7782                                 ahc_outb(ahc, KERNEL_TQINPOS,
7783                                          ahc->tqinfifonext & HOST_TQINPOS);
7784                                 if (!paused)
7785                                         ahc_unpause(ahc);
7786                         }
7787                 }
7788         }
7789 }
7790
7791 static int
7792 ahc_handle_target_cmd(struct ahc_softc *ahc, struct target_cmd *cmd)
7793 {
7794         struct    ahc_tmode_tstate *tstate;
7795         struct    ahc_tmode_lstate *lstate;
7796         struct    ccb_accept_tio *atio;
7797         uint8_t *byte;
7798         int       initiator;
7799         int       target;
7800         int       lun;
7801
7802         initiator = SCSIID_TARGET(ahc, cmd->scsiid);
7803         target = SCSIID_OUR_ID(cmd->scsiid);
7804         lun    = (cmd->identify & MSG_IDENTIFY_LUNMASK);
7805
7806         byte = cmd->bytes;
7807         tstate = ahc->enabled_targets[target];
7808         lstate = NULL;
7809         if (tstate != NULL)
7810                 lstate = tstate->enabled_luns[lun];
7811
7812         /*
7813          * Commands for disabled luns go to the black hole driver.
7814          */
7815         if (lstate == NULL)
7816                 lstate = ahc->black_hole;
7817
7818         atio = (struct ccb_accept_tio*)SLIST_FIRST(&lstate->accept_tios);
7819         if (atio == NULL) {
7820                 ahc->flags |= AHC_TQINFIFO_BLOCKED;
7821                 /*
7822                  * Wait for more ATIOs from the peripheral driver for this lun.
7823                  */
7824                 if (bootverbose)
7825                         printf("%s: ATIOs exhausted\n", ahc_name(ahc));
7826                 return (1);
7827         } else
7828                 ahc->flags &= ~AHC_TQINFIFO_BLOCKED;
7829 #if 0
7830         printf("Incoming command from %d for %d:%d%s\n",
7831                initiator, target, lun,
7832                lstate == ahc->black_hole ? "(Black Holed)" : "");
7833 #endif
7834         SLIST_REMOVE_HEAD(&lstate->accept_tios, sim_links.sle);
7835
7836         if (lstate == ahc->black_hole) {
7837                 /* Fill in the wildcards */
7838                 atio->ccb_h.target_id = target;
7839                 atio->ccb_h.target_lun = lun;
7840         }
7841
7842         /*
7843          * Package it up and send it off to
7844          * whomever has this lun enabled.
7845          */
7846         atio->sense_len = 0;
7847         atio->init_id = initiator;
7848         if (byte[0] != 0xFF) {
7849                 /* Tag was included */
7850                 atio->tag_action = *byte++;
7851                 atio->tag_id = *byte++;
7852                 atio->ccb_h.flags = CAM_TAG_ACTION_VALID;
7853         } else {
7854                 atio->ccb_h.flags = 0;
7855         }
7856         byte++;
7857
7858         /* Okay.  Now determine the cdb size based on the command code */
7859         switch (*byte >> CMD_GROUP_CODE_SHIFT) {
7860         case 0:
7861                 atio->cdb_len = 6;
7862                 break;
7863         case 1:
7864         case 2:
7865                 atio->cdb_len = 10;
7866                 break;
7867         case 4:
7868                 atio->cdb_len = 16;
7869                 break;
7870         case 5:
7871                 atio->cdb_len = 12;
7872                 break;
7873         case 3:
7874         default:
7875                 /* Only copy the opcode. */
7876                 atio->cdb_len = 1;
7877                 printf("Reserved or VU command code type encountered\n");
7878                 break;
7879         }
7880
7881         memcpy(atio->cdb_io.cdb_bytes, byte, atio->cdb_len);
7882
7883         atio->ccb_h.status |= CAM_CDB_RECVD;
7884
7885         if ((cmd->identify & MSG_IDENTIFY_DISCFLAG) == 0) {
7886                 /*
7887                  * We weren't allowed to disconnect.
7888                  * We're hanging on the bus until a
7889                  * continue target I/O comes in response
7890                  * to this accept tio.
7891                  */
7892 #if 0
7893                 printf("Received Immediate Command %d:%d:%d - %p\n",
7894                        initiator, target, lun, ahc->pending_device);
7895 #endif
7896                 ahc->pending_device = lstate;
7897                 ahc_freeze_ccb((union ccb *)atio);
7898                 atio->ccb_h.flags |= CAM_DIS_DISCONNECT;
7899         }
7900         xpt_done((union ccb*)atio);
7901         return (0);
7902 }
7903
7904 #endif