#include <linux/usb.h>
#include <linux/timer.h>
+#include <linux/kernel.h>
#include "../core/hcd.h"
/* Code sharing between pci-quirks and xhci hcd */
* xHCI register interface.
* This corresponds to the eXtensible Host Controller Interface (xHCI)
* Revision 0.95 specification
- *
- * Registers should always be accessed with double word or quad word accesses.
- *
- * Some xHCI implementations may support 64-bit address pointers. Registers
- * with 64-bit address pointers should be written to with dword accesses by
- * writing the low dword first (ptr[0]), then the high dword (ptr[1]) second.
- * xHCI implementations that do not support 64-bit address pointers will ignore
- * the high dword, and write order is irrelevant.
*/
/**
u32 db_off;
u32 run_regs_off;
/* Reserved up to (CAPLENGTH - 0x1C) */
-} __attribute__ ((packed));
+};
/* hc_capbase bitmasks */
/* bits 7:0 - how long is the Capabilities register */
#define HCS_ERST_MAX(p) (((p) >> 4) & 0xf)
/* bit 26 Scratchpad restore - for save/restore HW state - not used yet */
/* bits 27:31 number of Scratchpad buffers SW must allocate for the HW */
+#define HCS_MAX_SCRATCHPAD(p) (((p) >> 27) & 0x1f)
/* HCSPARAMS3 - hcs_params3 - bitmasks */
/* bits 0:7, Max U1 to U0 latency for the roothub ports */
u32 reserved1;
u32 reserved2;
u32 dev_notification;
- u32 cmd_ring[2];
+ u64 cmd_ring;
/* rsvd: offset 0x20-2F */
u32 reserved3[4];
- u32 dcbaa_ptr[2];
+ u64 dcbaa_ptr;
u32 config_reg;
/* rsvd: offset 0x3C-3FF */
u32 reserved4[241];
u32 reserved5;
/* registers for ports 2-255 */
u32 reserved6[NUM_PORT_REGS*254];
-} __attribute__ ((packed));
+};
/* USBCMD - USB command - command bitmasks */
/* start/stop HC execution - do not write unless HC is halted*/
#define CMD_RING_RUNNING (1 << 3)
/* bits 4:5 reserved and should be preserved */
/* Command Ring pointer - bit mask for the lower 32 bits. */
-#define CMD_RING_ADDR_MASK (0xffffffc0)
+#define CMD_RING_RSVD_BITS (0x3f)
/* CONFIG - Configure Register - config_reg bitmasks */
/* bits 0:7 - maximum number of device slots enabled (NumSlotsEn) */
/**
- * struct intr_reg - Interrupt Register Set
+ * struct xhci_intr_reg - Interrupt Register Set
* @irq_pending: IMAN - Interrupt Management Register. Used to enable
* interrupts and check for pending interrupts.
* @irq_control: IMOD - Interrupt Moderation Register.
* position of the Enqueue Pointer." The HCD (Linux) processes those events and
* updates the dequeue pointer.
*/
-struct intr_reg {
+struct xhci_intr_reg {
u32 irq_pending;
u32 irq_control;
u32 erst_size;
u32 rsvd;
- u32 erst_base[2];
- u32 erst_dequeue[2];
-} __attribute__ ((packed));
+ u64 erst_base;
+ u64 erst_dequeue;
+};
/* irq_pending bitmasks */
#define ER_IRQ_PENDING(p) ((p) & 0x1)
* or larger accesses"
*/
struct xhci_run_regs {
- u32 microframe_index;
- u32 rsvd[7];
- struct intr_reg ir_set[128];
-} __attribute__ ((packed));
+ u32 microframe_index;
+ u32 rsvd[7];
+ struct xhci_intr_reg ir_set[128];
+};
/**
* struct doorbell_array
*/
struct xhci_doorbell_array {
u32 doorbell[256];
-} __attribute__ ((packed));
+};
#define DB_TARGET_MASK 0xFFFFFF00
#define DB_STREAM_ID_MASK 0x0000FFFF
/**
+ * struct xhci_container_ctx
+ * @type: Type of context. Used to calculated offsets to contained contexts.
+ * @size: Size of the context data
+ * @bytes: The raw context data given to HW
+ * @dma: dma address of the bytes
+ *
+ * Represents either a Device or Input context. Holds a pointer to the raw
+ * memory used for the context (bytes) and dma address of it (dma).
+ */
+struct xhci_container_ctx {
+ unsigned type;
+#define XHCI_CTX_TYPE_DEVICE 0x1
+#define XHCI_CTX_TYPE_INPUT 0x2
+
+ int size;
+
+ u8 *bytes;
+ dma_addr_t dma;
+};
+
+/**
* struct xhci_slot_ctx
* @dev_info: Route string, device speed, hub info, and last valid endpoint
* @dev_info2: Max exit latency for device number, root hub port number
u32 dev_state;
/* offset 0x10 to 0x1f reserved for HC internal use */
u32 reserved[4];
-} __attribute__ ((packed));
+};
/* dev_info bitmasks */
/* Route String - 0:19 */
/* bits 8:26 reserved */
/* Slot state */
#define SLOT_STATE (0x1f << 27)
+#define GET_SLOT_STATE(p) (((p) & (0x1f << 27)) >> 27)
/**
struct xhci_ep_ctx {
u32 ep_info;
u32 ep_info2;
- u32 deq[2];
+ u64 deq;
u32 tx_info;
/* offset 0x14 - 0x1f reserved for HC internal use */
u32 reserved[3];
-} __attribute__ ((packed));
+};
/* ep_info bitmasks */
/*
/* bit 15 is Linear Stream Array */
/* Interval - period between requests to an endpoint - 125u increments. */
#define EP_INTERVAL(p) ((p & 0xff) << 16)
+#define EP_INTERVAL_TO_UFRAMES(p) (1 << (((p) >> 16) & 0xff))
/* ep_info2 bitmasks */
/*
*/
#define FORCE_EVENT (0x1)
#define ERROR_COUNT(p) (((p) & 0x3) << 1)
+#define CTX_TO_EP_TYPE(p) (((p) >> 3) & 0x7)
#define EP_TYPE(p) ((p) << 3)
#define ISOC_OUT_EP 1
#define BULK_OUT_EP 2
/* bit 7 is Host Initiate Disable - for disabling stream selection */
#define MAX_BURST(p) (((p)&0xff) << 8)
#define MAX_PACKET(p) (((p)&0xffff) << 16)
+#define MAX_PACKET_MASK (0xffff << 16)
+#define MAX_PACKET_DECODED(p) (((p) >> 16) & 0xffff)
/**
- * struct xhci_device_control
- * Input/Output context; see section 6.2.5.
+ * struct xhci_input_control_context
+ * Input control context; see section 6.2.5.
*
* @drop_context: set the bit of the endpoint context you want to disable
* @add_context: set the bit of the endpoint context you want to enable
*/
-struct xhci_device_control {
+struct xhci_input_control_ctx {
u32 drop_flags;
u32 add_flags;
- u32 rsvd[6];
- struct xhci_slot_ctx slot;
- struct xhci_ep_ctx ep[31];
-} __attribute__ ((packed));
+ u32 rsvd2[6];
+};
/* drop context bitmasks */
#define DROP_EP(x) (0x1 << x)
/* add context bitmasks */
#define ADD_EP(x) (0x1 << x)
+struct xhci_virt_ep {
+ struct xhci_ring *ring;
+ /* Temporary storage in case the configure endpoint command fails and we
+ * have to restore the device state to the previous state
+ */
+ struct xhci_ring *new_ring;
+ unsigned int ep_state;
+#define SET_DEQ_PENDING (1 << 0)
+#define EP_HALTED (1 << 1)
+ /* ---- Related to URB cancellation ---- */
+ struct list_head cancelled_td_list;
+ unsigned int cancels_pending;
+ /* The TRB that was last reported in a stopped endpoint ring */
+ union xhci_trb *stopped_trb;
+ struct xhci_td *stopped_td;
+};
struct xhci_virt_device {
/*
* track of input and output contexts separately because
* these commands might fail and we don't trust the hardware.
*/
- struct xhci_device_control *out_ctx;
- dma_addr_t out_ctx_dma;
+ struct xhci_container_ctx *out_ctx;
/* Used for addressing devices and configuration changes */
- struct xhci_device_control *in_ctx;
- dma_addr_t in_ctx_dma;
- /* FIXME when stream support is added */
- struct xhci_ring *ep_rings[31];
- /* Temporary storage in case the configure endpoint command fails and we
- * have to restore the device state to the previous state
- */
- struct xhci_ring *new_ep_rings[31];
+ struct xhci_container_ctx *in_ctx;
+ struct xhci_virt_ep eps[31];
struct completion cmd_completion;
/* Status of the last command issued for this device */
u32 cmd_status;
*/
struct xhci_device_context_array {
/* 64-bit device addresses; we only write 32-bit addresses */
- u32 dev_context_ptrs[2*MAX_HC_SLOTS];
+ u64 dev_context_ptrs[MAX_HC_SLOTS];
/* private xHCD pointers */
dma_addr_t dma;
-} __attribute__ ((packed));
+};
/* TODO: write function to set the 64-bit device DMA address */
/*
* TODO: change this to be dynamically sized at HC mem init time since the HC
struct xhci_stream_ctx {
/* 64-bit stream ring address, cycle state, and stream type */
- u32 stream_ring[2];
+ u64 stream_ring;
/* offset 0x14 - 0x1f reserved for HC internal use */
u32 reserved[2];
-} __attribute__ ((packed));
+};
struct xhci_transfer_event {
/* 64-bit buffer address, or immediate data */
- u32 buffer[2];
+ u64 buffer;
u32 transfer_len;
/* This field is interpreted differently based on the type of TRB */
u32 flags;
-} __attribute__ ((packed));
+};
/** Transfer Event bit fields **/
#define TRB_TO_EP_ID(p) (((p) >> 16) & 0x1f)
struct xhci_link_trb {
/* 64-bit segment pointer*/
- u32 segment_ptr[2];
+ u64 segment_ptr;
u32 intr_target;
u32 control;
-} __attribute__ ((packed));
+};
/* control bitfields */
#define LINK_TOGGLE (0x1<<1)
/* Command completion event TRB */
struct xhci_event_cmd {
/* Pointer to command TRB, or the value passed by the event data trb */
- u32 cmd_trb[2];
+ u64 cmd_trb;
u32 status;
u32 flags;
-} __attribute__ ((packed));
+};
/* flags bitmasks */
/* bits 16:23 are the virtual function ID */
#define TRB_TO_SLOT_ID(p) (((p) & (0xff<<24)) >> 24)
#define SLOT_ID_FOR_TRB(p) (((p) & 0xff) << 24)
+/* Stop Endpoint TRB - ep_index to endpoint ID for this TRB */
+#define TRB_TO_EP_INDEX(p) ((((p) & (0x1f << 16)) >> 16) - 1)
+#define EP_ID_FOR_TRB(p) ((((p) + 1) & 0x1f) << 16)
+
+
/* Port Status Change Event TRB fields */
/* Port ID - bits 31:24 */
#define GET_PORT_ID(p) (((p) & (0xff << 24)) >> 24)
struct xhci_generic_trb {
u32 field[4];
-} __attribute__ ((packed));
+};
union xhci_trb {
struct xhci_link_trb link;
#define TRB_CONFIG_EP 12
/* Evaluate Context Command */
#define TRB_EVAL_CONTEXT 13
-/* Reset Transfer Ring Command */
-#define TRB_RESET_RING 14
+/* Reset Endpoint Command */
+#define TRB_RESET_EP 14
/* Stop Transfer Ring Command */
#define TRB_STOP_RING 15
/* Set Transfer Ring Dequeue Pointer Command */
#define TRB_MAX_BUFF_SHIFT 16
#define TRB_MAX_BUFF_SIZE (1 << TRB_MAX_BUFF_SHIFT)
+struct xhci_segment {
+ union xhci_trb *trbs;
+ /* private to HCD */
+ struct xhci_segment *next;
+ dma_addr_t dma;
+};
+
struct xhci_td {
struct list_head td_list;
+ struct list_head cancelled_td_list;
struct urb *urb;
+ struct xhci_segment *start_seg;
+ union xhci_trb *first_trb;
union xhci_trb *last_trb;
};
-struct xhci_segment {
- union xhci_trb *trbs;
- /* private to HCD */
- struct xhci_segment *next;
- dma_addr_t dma;
-} __attribute__ ((packed));
+struct xhci_dequeue_state {
+ struct xhci_segment *new_deq_seg;
+ union xhci_trb *new_deq_ptr;
+ int new_cycle_state;
+};
struct xhci_ring {
struct xhci_segment *first_seg;
struct xhci_erst_entry {
/* 64-bit event ring segment address */
- u32 seg_addr[2];
+ u64 seg_addr;
u32 seg_size;
/* Set to zero */
u32 rsvd;
-} __attribute__ ((packed));
+};
struct xhci_erst {
struct xhci_erst_entry *entries;
unsigned int erst_size;
};
+struct xhci_scratchpad {
+ u64 *sp_array;
+ dma_addr_t sp_dma;
+ void **sp_buffers;
+ dma_addr_t *sp_dma_buffers;
+};
+
/*
* Each segment table entry is 4*32bits long. 1K seems like an ok size:
* (1K bytes * 8bytes/bit) / (4*32 bits) = 64 segment entries in the table,
struct xhci_run_regs __iomem *run_regs;
struct xhci_doorbell_array __iomem *dba;
/* Our HCD's current interrupter register set */
- struct intr_reg __iomem *ir_set;
+ struct xhci_intr_reg __iomem *ir_set;
/* Cached register copies of read-only HC data */
__u32 hcs_params1;
struct xhci_ring *cmd_ring;
struct xhci_ring *event_ring;
struct xhci_erst erst;
+ /* Scratchpad */
+ struct xhci_scratchpad *scratchpad;
+
/* slot enabling and address device helpers */
struct completion addr_dev;
int slot_id;
int noops_submitted;
int noops_handled;
int error_bitmask;
+ unsigned int quirks;
+#define XHCI_LINK_TRB_QUIRK (1 << 0)
+#define XHCI_RESET_EP_QUIRK (1 << 1)
};
/* For testing purposes */
{
return readl(regs);
}
-static inline void xhci_writel(const struct xhci_hcd *xhci,
+static inline void xhci_writel(struct xhci_hcd *xhci,
const unsigned int val, __u32 __iomem *regs)
{
- if (!in_interrupt())
- xhci_dbg(xhci, "`MEM_WRITE_DWORD(3'b000, 32'h%0x, 32'h%0x, 4'hf);\n",
- (unsigned int) regs, val);
+ xhci_dbg(xhci,
+ "`MEM_WRITE_DWORD(3'b000, 32'h%p, 32'h%0x, 4'hf);\n",
+ regs, val);
writel(val, regs);
}
+/*
+ * Registers should always be accessed with double word or quad word accesses.
+ *
+ * Some xHCI implementations may support 64-bit address pointers. Registers
+ * with 64-bit address pointers should be written to with dword accesses by
+ * writing the low dword first (ptr[0]), then the high dword (ptr[1]) second.
+ * xHCI implementations that do not support 64-bit address pointers will ignore
+ * the high dword, and write order is irrelevant.
+ */
+static inline u64 xhci_read_64(const struct xhci_hcd *xhci,
+ __u64 __iomem *regs)
+{
+ __u32 __iomem *ptr = (__u32 __iomem *) regs;
+ u64 val_lo = readl(ptr);
+ u64 val_hi = readl(ptr + 1);
+ return val_lo + (val_hi << 32);
+}
+static inline void xhci_write_64(struct xhci_hcd *xhci,
+ const u64 val, __u64 __iomem *regs)
+{
+ __u32 __iomem *ptr = (__u32 __iomem *) regs;
+ u32 val_lo = lower_32_bits(val);
+ u32 val_hi = upper_32_bits(val);
+
+ xhci_dbg(xhci,
+ "`MEM_WRITE_DWORD(3'b000, 64'h%p, 64'h%0lx, 4'hf);\n",
+ regs, (long unsigned int) val);
+ writel(val_lo, ptr);
+ writel(val_hi, ptr + 1);
+}
+
+static inline int xhci_link_trb_quirk(struct xhci_hcd *xhci)
+{
+ u32 temp = xhci_readl(xhci, &xhci->cap_regs->hc_capbase);
+ return ((HC_VERSION(temp) == 0x95) &&
+ (xhci->quirks & XHCI_LINK_TRB_QUIRK));
+}
+
/* xHCI debugging */
-void xhci_print_ir_set(struct xhci_hcd *xhci, struct intr_reg *ir_set, int set_num);
+void xhci_print_ir_set(struct xhci_hcd *xhci, struct xhci_intr_reg *ir_set, int set_num);
void xhci_print_registers(struct xhci_hcd *xhci);
void xhci_dbg_regs(struct xhci_hcd *xhci);
void xhci_print_run_regs(struct xhci_hcd *xhci);
void xhci_dbg_erst(struct xhci_hcd *xhci, struct xhci_erst *erst);
void xhci_dbg_cmd_ptrs(struct xhci_hcd *xhci);
void xhci_dbg_ring_ptrs(struct xhci_hcd *xhci, struct xhci_ring *ring);
-void xhci_dbg_ctx(struct xhci_hcd *xhci, struct xhci_device_control *ctx, dma_addr_t dma, unsigned int last_ep);
+void xhci_dbg_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx, unsigned int last_ep);
-/* xHCI memory managment */
+/* xHCI memory management */
void xhci_mem_cleanup(struct xhci_hcd *xhci);
int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags);
void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id);
int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev);
unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc);
unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc);
+unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index);
+unsigned int xhci_last_valid_endpoint(u32 added_ctxs);
void xhci_endpoint_zero(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, struct usb_host_endpoint *ep);
-int xhci_endpoint_init(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, struct usb_device *udev, struct usb_host_endpoint *ep);
+void xhci_endpoint_copy(struct xhci_hcd *xhci,
+ struct xhci_virt_device *vdev, unsigned int ep_index);
+void xhci_slot_copy(struct xhci_hcd *xhci, struct xhci_virt_device *vdev);
+int xhci_endpoint_init(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev,
+ struct usb_device *udev, struct usb_host_endpoint *ep,
+ gfp_t mem_flags);
void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring);
#ifdef CONFIG_PCI
int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status);
int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *ep);
int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev, struct usb_host_endpoint *ep);
+void xhci_endpoint_reset(struct usb_hcd *hcd, struct usb_host_endpoint *ep);
int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev);
void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev);
/* xHCI ring, segment, TRB, and TD functions */
-dma_addr_t trb_virt_to_dma(struct xhci_segment *seg, union xhci_trb *trb);
-void ring_cmd_db(struct xhci_hcd *xhci);
-void *setup_one_noop(struct xhci_hcd *xhci);
-void handle_event(struct xhci_hcd *xhci);
-void set_hc_event_deq(struct xhci_hcd *xhci);
-int queue_slot_control(struct xhci_hcd *xhci, u32 trb_type, u32 slot_id);
-int queue_address_device(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr, u32 slot_id);
-int queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb, int slot_id, unsigned int ep_index);
-int queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb, int slot_id, unsigned int ep_index);
-int queue_configure_endpoint(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr, u32 slot_id);
+dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg, union xhci_trb *trb);
+void xhci_ring_cmd_db(struct xhci_hcd *xhci);
+void *xhci_setup_one_noop(struct xhci_hcd *xhci);
+void xhci_handle_event(struct xhci_hcd *xhci);
+void xhci_set_hc_event_deq(struct xhci_hcd *xhci);
+int xhci_queue_slot_control(struct xhci_hcd *xhci, u32 trb_type, u32 slot_id);
+int xhci_queue_address_device(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr,
+ u32 slot_id);
+int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, int slot_id,
+ unsigned int ep_index);
+int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
+ int slot_id, unsigned int ep_index);
+int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
+ int slot_id, unsigned int ep_index);
+int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
+ int slot_id, unsigned int ep_index);
+int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr,
+ u32 slot_id);
+int xhci_queue_evaluate_context(struct xhci_hcd *xhci, dma_addr_t in_ctx_ptr,
+ u32 slot_id);
+int xhci_queue_reset_ep(struct xhci_hcd *xhci, int slot_id,
+ unsigned int ep_index);
+void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
+ unsigned int slot_id, unsigned int ep_index,
+ struct xhci_td *cur_td, struct xhci_dequeue_state *state);
+void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
+ unsigned int slot_id, unsigned int ep_index,
+ struct xhci_dequeue_state *deq_state);
+void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci,
+ struct usb_device *udev, unsigned int ep_index);
+void xhci_queue_config_ep_quirk(struct xhci_hcd *xhci,
+ unsigned int slot_id, unsigned int ep_index,
+ struct xhci_dequeue_state *deq_state);
/* xHCI roothub code */
int xhci_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue, u16 wIndex,
char *buf, u16 wLength);
int xhci_hub_status_data(struct usb_hcd *hcd, char *buf);
+/* xHCI contexts */
+struct xhci_input_control_ctx *xhci_get_input_control_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx);
+struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx);
+struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx, unsigned int ep_index);
+
#endif /* __LINUX_XHCI_HCD_H */