G.1: xHCI bulk-endpoint stall recovery (per F.14), built + verified

Full BOT-spec stall recovery per FABRIC-3.md F.14: new STALL_ERROR handling,
Reset Endpoint + Set TR Dequeue Pointer commands, CLEAR_FEATURE(ENDPOINT_HALT),
escalating to Bulk-Only Mass Storage Reset, capped retries
(XHCI_BOT_STALL_MAX_RECOVERIES=2) mirroring bot_tur_retries, clean terminal
failure via xhci_stall_fail().

Purely additive recovery path off the non-success transfer-event branch; the
normal path is unchanged. Builds clean on amd64/aarch64/riscv64. QEMU amd64
boot regression passes: zero stalls, BOT attach (READ CAPACITY10 -> READ10 ->
home-blocks) completes, normal-path xHCI trace identical to baseline. Live
stall injection is not provable under qemu-xhci; deferred to v2.5.0 hardware.

FABRIC-3.md G.1 documented; ROADMAP release-versioning policy folded in.
This commit is contained in:
Robert Allan James
2026-08-29 00:58:59 -04:00
parent 5689c397fc
commit 49a3faa331
7 changed files with 730 additions and 5 deletions
+371 -1
View File
@@ -130,6 +130,10 @@ int xhci_cmd_disable_slot(xhci_dev_t *dev, uint32_t slot_id);
int xhci_cmd_address_device(xhci_dev_t *dev, uint32_t slot_id,
uint32_t port_id, uint32_t speed);
int xhci_cmd_configure_endpoint(xhci_dev_t *dev, uint32_t slot_id);
int xhci_cmd_reset_endpoint(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id);
int xhci_cmd_set_tr_dequeue_pointer(xhci_dev_t *dev, uint32_t slot_id,
uint32_t ep_id, uint64_t new_dequeue,
uint32_t dcs);
int xhci_bot_send_read10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba,
uint16_t num_blocks, uint32_t block_size);
int xhci_bot_send_write10(xhci_dev_t *dev, uint32_t slot_id, uint32_t lba,
@@ -667,6 +671,51 @@ int xhci_cmd_configure_endpoint(xhci_dev_t *dev, uint32_t slot_id)
return 0;
}
/* G.1 / §F.14 stall recovery: Reset Endpoint command. ep_id is a full
* bEndpointAddress (bit 7 = direction) of the stalled bulk endpoint -- the
* xHC is told which endpoint to un-halt via its Endpoint ID (the DCI) in
* parameter[31:24], exactly the field layout Reset Endpoint uses (xHCI 1.2
* spec table 6-88): parameter[31:24] = Endpoint ID, control[31:24] = Slot
* ID. Mirrors the existing xhci_cmd_disable_slot() shape (submit, ring
* doorbell 0, don't wait) -- this touches no doorbell but [0]. */
int xhci_cmd_reset_endpoint(xhci_dev_t *dev, uint32_t slot_id, uint32_t ep_id)
{
if (!dev || !dev->cmd_ring) return -1;
uint32_t dci = XHCI_EP_ADDR_TO_DCI(ep_id);
if (dci == 0) return -1; /* EP0 (DCI 1) is the only non-bulk case -- refuse */
xhci_submit_command(dev, (uint64_t)dci << 24, 0,
XHCI_TRB_TYPE_RESET_ENDPOINT_CMD, slot_id << 24);
console_println("xhci: reset endpoint command submitted");
return 0;
}
/* G.1 / §F.14 stall recovery: Set TR Dequeue Pointer command. new_dequeue
* is a pointer into the stalled endpoint's Transfer Ring (the current
* producer slot, past any failed TRBs) and dcs is the ring's current Cycle
* State -- both written into parameter exactly as the spec's Set TR
* Dequeue Pointer (xHCI 1.2 table 6-91) defines: parameter[63:4] = the
* dequeue pointer, parameter[0] = DCS, control[31:24] = Slot ID. */
int xhci_cmd_set_tr_dequeue_pointer(xhci_dev_t *dev, uint32_t slot_id,
uint32_t ep_id, uint64_t new_dequeue,
uint32_t dcs)
{
if (!dev || !dev->cmd_ring) return -1;
/* ep_id names the endpoint whose ring new_dequeue points into. The Set
* TR Dequeue Pointer TRB itself carries no Endpoint ID field -- the
* ring the controller should update is implied by the dequeue pointer
* + the controller already having Reset this endpoint (G.1's sequence
* runs Reset Endpoint before this) -- so ep_id is a documentation /
* call-site-clarity parameter, kept for symmetry with its sibling.
* Either way the controller knows which ring because RESET ENDPOINT
* names it first. */
(void)ep_id;
uint64_t param = (new_dequeue & ~0xFULL) | (uint64_t)(dcs ? 1u : 0u);
xhci_submit_command(dev, param, 0,
XHCI_TRB_TYPE_SET_TR_DEQUEUE_POINTER_CMD, slot_id << 24);
console_println("xhci: set TR dequeue pointer command submitted");
return 0;
}
/* Enqueue one Normal TRB to the bulk OUT Transfer Ring -- same fixed-
* ring-plus-Link-TRB wraparound pattern as xhci_ep0_enqueue_trb(),
* operating on bulk_out_ring/bulk_out_ring_enq/bulk_out_ring_cycle
@@ -1156,6 +1205,68 @@ int xhci_ep0_set_configuration(xhci_dev_t *dev, uint32_t slot_id, uint8_t config
return 0;
}
/* G.1 / §F.14 stall recovery: CLEAR_FEATURE(ENDPOINT_HALT) on a specific
* endpoint. Reuses the existing no-data control-transfer machinery
* (xhci_ep0_control_write_nodata(), the same Setup+Status shape as
* SET_CONFIGURATION) -- just a different request payload: bRequest =
* USB_REQ_CLEAR_FEATURE, wValue = USB_FEATURE_ENDPOINT_HALT, wIndex = the
* endpoint's own bEndpointAddress (standard, host-to-device, endpoint-
* recipient -- bmRequestType inferred by the helper as 0x00). This is the
* USB-level clear that clears the device's halt condition and resets its
* data toggle after the two xHCI command steps (Reset Endpoint, Set TR
* Dequeue Pointer) have un-halted the xHC side. */
int xhci_ep0_clear_endpoint_halt(xhci_dev_t *dev, uint32_t slot_id, uint8_t ep_addr)
{
if (!dev || !dev->ep0_ring) return -1;
dev->transfer_purpose = XHCI_XFER_CLEAR_HALT;
xhci_ep0_control_write_nodata(dev, USB_REQ_CLEAR_FEATURE,
USB_FEATURE_ENDPOINT_HALT, ep_addr);
dev->pending_transfer_slot_id = slot_id;
dev->doorbell[slot_id] = XHCI_DB_TARGET(1);
console_println("xhci: clear endpoint halt submitted");
return 0;
}
/* G.1 / §F.14 stall recovery escalation: Bulk-Only Transport Mass Storage
* Reset. bmRequestType = 0x21 (class, interface recipient), bRequest =
* 0xFF, no data stage -- also built on xhci_ep0_control_write_nodata(), but
* that helper hardcodes bmRequestType = 0x00 (standard, host-to-device,
* device recipient). A BOT Mass Storage Reset is class + interface
* recipient, so it can't reuse the helper directly; the request bytes are
* stamped into the Setup packet by hand here (matching the same explicit-
* layout, no-padding discipline usb_setup_packet_t is documented for). */
int xhci_ep0_bot_mass_storage_reset(xhci_dev_t *dev, uint32_t slot_id)
{
if (!dev || !dev->ep0_ring) return -1;
usb_setup_packet_t setup = {
.bmRequestType = (USB_REQ_TYPE_CLASS << 5) | USB_RECIP_INTERFACE,
.bRequest = USB_BOT_MASS_STORAGE_RESET,
.wValue = 0,
.wIndex = 0,
.wLength = 0
};
uint64_t setup_bits;
memcpy(&setup_bits, &setup, sizeof(setup_bits));
xhci_ep0_enqueue_trb(dev, setup_bits, 8u,
(XHCI_TRB_TYPE_SETUP_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) |
XHCI_TRB_CONTROL_IDT |
(XHCI_SETUP_TRT_NO_DATA << XHCI_TRB_CONTROL_TRT_SHIFT));
xhci_ep0_enqueue_trb(dev, 0, 0,
(XHCI_TRB_TYPE_STATUS_STAGE << XHCI_TRB_CONTROL_TYPE_SHIFT) |
XHCI_TRB_CONTROL_IOC |
XHCI_TRB_CONTROL_DIR_IN);
dev->transfer_purpose = XHCI_XFER_BOT_RESET;
dev->pending_transfer_slot_id = slot_id;
dev->doorbell[slot_id] = XHCI_DB_TARGET(1);
console_println("xhci: BOT mass storage reset submitted");
return 0;
}
/* -------------------------------------------------------------------------
* Milestone 2d: Event Ring servicing, polled from sk_repl_idle().
*
@@ -1248,6 +1359,122 @@ static void xhci_scan_ports_for_already_connected(xhci_dev_t *dev)
}
}
/* G.1 / §F.14: clean terminal failure of the BOT command currently in
* flight, used by every stall-recovery bail-out path. Explicitly sets
* bot_last_status = BOT_STATUS_FAILED and bot_cmd_kind = BOT_CMD_NONE -- a
* clean signal to a synchronous caller blocked in
* xhci_bot_wait_for_idle() (which would otherwise just time out, leaving
* state ambiguous), rather than relying on the outer timeout the way the
* pre-G.1 un-recovered stall did. */
static void xhci_stall_fail(xhci_dev_t *dev)
{
console_println("xhci: stall recovery exhausted -- failing command cleanly");
dev->bot_last_status = BOT_STATUS_FAILED;
dev->bot_cmd_kind = BOT_CMD_NONE;
dev->stall_retry_action = XHCI_NEXT_ACTION_NONE;
dev->bot_reset_clear_remaining = 0;
}
/* G.1 / §F.14: is `purpose` a bulk (BOT) transfer rather than an EP0
* control transfer? Stall recovery is defined only for the bulk endpoints
* (CBW_SENT and BOT_DATA_OUT run on bulk_out; BOT_DATA_IN and CSW_RECEIVED
* on bulk_in) -- an EP0 control transfer that stalls during enumeration is
* a different, out-of-scope case this driver deliberately does not attempt
* to recover. */
static int xhci_bulk_purpose_stalled(xhci_dev_t *dev, uint32_t purpose)
{
(void)dev;
switch (purpose) {
case XHCI_XFER_CBW_SENT:
case XHCI_XFER_BOT_DATA_IN:
case XHCI_XFER_BOT_DATA_OUT:
case XHCI_XFER_CSW_RECEIVED:
return 1;
default:
return 0;
}
}
/* G.1 / §F.14: start stall recovery for a bulk transfer that completed
* with XHCI_COMPLETION_CODE_STALL_ERROR in xhci_poll_events()'s transfer-
* event handler. Identifies the stalled endpoint from `purpose` (which
* bulk ring it ran on) and the command-stage to retry once recovery
* completes, then either (a) kicks off the basic xHCI Reset Endpoint /
* Set TR Dequeue Pointer / CLEAR_FEATURE(ENDPOINT_HALT) recovery, or on a
* repeated stall (b) escalates to a full BOT Mass Storage Reset. Recovery
* is bounded by bot_stall_recoveries vs XHCI_BOT_STALL_MAX_RECOVERIES;
* exhausting it bails out clean per the helper above. Called with purpose
* already read (it is cleared by the caller immediately after this call). */
static void xhci_handle_bulk_stall(xhci_dev_t *dev, uint32_t slot_id,
uint32_t purpose)
{
switch (purpose) {
case XHCI_XFER_CBW_SENT:
dev->stall_ep_addr = dev->bulk_out_ep_addr;
/* Re-send the whole SCSI command from its CBW -- which takes
* the existing TUR/READ/WRITE/CAPACITY entry points, picked
* by what's in flight. */
switch (dev->bot_cmd_kind) {
case BOT_CMD_WRITE10:
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_WRITE10;
break;
case BOT_CMD_READ_CAPACITY10:
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_READ_CAPACITY10;
break;
case BOT_CMD_TEST_UNIT_READY:
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_TUR;
break;
default: /* BOT_CMD_READ10 (and anything unexpected) */
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_SEND_READ10;
break;
}
break;
case XHCI_XFER_BOT_DATA_OUT:
dev->stall_ep_addr = dev->bulk_out_ep_addr;
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_DATA_OUT;
break;
case XHCI_XFER_BOT_DATA_IN:
dev->stall_ep_addr = dev->bulk_in_ep_addr;
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_DATA_IN;
break;
case XHCI_XFER_CSW_RECEIVED:
dev->stall_ep_addr = dev->bulk_in_ep_addr;
dev->stall_retry_action = XHCI_NEXT_ACTION_BOT_CSW_RECEIVE;
break;
default:
/* Unreachable -- xhci_bulk_purpose_stalled() gates entry. */
return;
}
dev->stall_dci = XHCI_EP_ADDR_TO_DCI(dev->stall_ep_addr);
if (dev->bot_stall_recoveries >= XHCI_BOT_STALL_MAX_RECOVERIES) {
xhci_stall_fail(dev);
return;
}
dev->bot_stall_recoveries++;
console_println("xhci: bulk endpoint STALL -- starting recovery");
if (dev->bot_stall_recoveries == 1) {
/* First recovery: the basic xHCI-level Reset Endpoint sequence
* (commands are submitted synchronously -- they ring doorbell 0,
* which is safe from inside event processing; only transfer-ring
* doorbells are deferred). */
dev->connect_state = XHCI_CONN_AWAIT_RESET_ENDPOINT;
if (xhci_cmd_reset_endpoint(dev, slot_id, dev->stall_ep_addr) != 0) {
console_println("xhci: reset endpoint submit failed -- stalling out clean");
dev->connect_state = XHCI_CONN_IDLE;
xhci_stall_fail(dev);
}
} else {
/* Repeated stall: escalate to a full BOT Mass Storage Reset + clear
* halt on both endpoints -- deferred, since BOT Mass Storage Reset
* is a control transfer that rings doorbell[slot] (must not happen
* inside event processing). */
dev->next_action = XHCI_NEXT_ACTION_BOT_RESET;
dev->next_action_slot_id = slot_id;
}
}
void xhci_poll_events(void)
{
xhci_dev_t *dev = g_xhci_dev;
@@ -1393,6 +1620,61 @@ void xhci_poll_events(void)
console_println("xhci: configure endpoint failed");
}
dev->connect_state = XHCI_CONN_IDLE;
} else if (dev->connect_state == XHCI_CONN_AWAIT_RESET_ENDPOINT) {
/* G.1 / §F.14 stall recovery, command step 1 of 2: the
* Reset Endpoint command has completed. On success,
* the xHC endpoint is back in the Stopped state and we
* immediately issue step 2, Set TR Dequeue Pointer, to
* reposition the stalled Transfer Ring's dequeue past
* the failed TRB (at the ring's current producer slot,
* with the producer's cycle as DCS) so a freshly
* enqueued retry TRB is consumed cleanly. stall_dci
* tells which ring. */
if (code == XHCI_COMPLETION_CODE_SUCCESS) {
console_println("xhci: reset endpoint succeeded");
uintptr_t new_dequeue;
uint32_t dcs;
if (dev->stall_ep_addr & USB_EP_ADDR_DIR_MASK) {
new_dequeue = (uintptr_t)&dev->bulk_in_ring[dev->bulk_in_ring_enq];
dcs = dev->bulk_in_ring_cycle;
} else {
new_dequeue = (uintptr_t)&dev->bulk_out_ring[dev->bulk_out_ring_enq];
dcs = dev->bulk_out_ring_cycle;
}
dev->connect_state = XHCI_CONN_AWAIT_SET_TR_DEQUEUE;
if (xhci_cmd_set_tr_dequeue_pointer(dev, slot_id,
dev->stall_ep_addr,
new_dequeue, dcs) != 0) {
console_println("xhci: set tr dequeue pointer submit failed");
dev->connect_state = XHCI_CONN_IDLE;
/* Recovery couldn't even be issued -- clean
* terminal failure so the synchronous waiter
* doesn't spin forever on a wedged command. */
xhci_stall_fail(dev);
}
} else {
console_println("xhci: reset endpoint failed -- stalling out clean");
dev->connect_state = XHCI_CONN_IDLE;
xhci_stall_fail(dev);
}
} else if (dev->connect_state == XHCI_CONN_AWAIT_SET_TR_DEQUEUE) {
/* G.1 / §F.14 stall recovery, command step 2 of 2: Set
* TR Dequeue Pointer has completed. The xHC side is
* now fully recovered; the device side still needs its
* halt cleared via control transfer, so chain into the
* deferred CLEAR_FEATURE(ENDPOINT_HALT). */
if (code == XHCI_COMPLETION_CODE_SUCCESS) {
console_println("xhci: set TR dequeue pointer succeeded");
} else {
console_println("xhci: set TR dequeue pointer failed -- stalling out clean");
xhci_stall_fail(dev);
}
dev->connect_state = XHCI_CONN_IDLE;
if (dev->bot_last_status != BOT_STATUS_FAILED) {
dev->bot_reset_clear_remaining = 1;
dev->next_action = XHCI_NEXT_ACTION_CLEAR_HALT;
dev->next_action_slot_id = slot_id;
}
} else {
console_println("xhci: command completion event");
}
@@ -1407,7 +1689,19 @@ void xhci_poll_events(void)
dev->transfer_purpose = XHCI_XFER_NONE;
if (code != XHCI_COMPLETION_CODE_SUCCESS) {
console_println("xhci: control transfer failed");
/* G.1 / §F.14: distinguish a true STALL (the one
* recoverable xHCI transfer completion) from every
* other failure. A stall on a bulk endpoint enters
* recovery; a plain failure (or a stall on an EP0
* control transfer, which this driver only meets
* during enumeration and does not attempt to
* recover) keeps today's behavior -- log and bail. */
if (code == XHCI_COMPLETION_CODE_STALL_ERROR &&
xhci_bulk_purpose_stalled(dev, purpose)) {
xhci_handle_bulk_stall(dev, xfer_slot_id, purpose);
} else {
console_println("xhci: control transfer failed");
}
break;
}
@@ -1686,6 +1980,68 @@ void xhci_poll_events(void)
}
break;
}
case XHCI_XFER_CLEAR_HALT: {
/* G.1 / §F.14: CLEAR_FEATURE(ENDPOINT_HALT)
* completed. This is the device-side clear; the
* xHC side was already recovered by Reset
* Endpoint + Set TR Dequeue Pointer. On success
* this may be one of two CLEAR_FEATUREs owed in
* a BOT-reset escalation (bot_reset_clear_remaining
* counts how many remain; >1 means "also clear
* the other bulk endpoint"). When all clears are
* done, the recovery is complete and the stalled
* command stage is retried (stall_retry_action).
* A CLEAR_FEATURE that itself fails or stalls
* is the "step 3 itself stalls" escalation case
* from §F.14 -- escalate to a full BOT Mass
* Storage Reset. */
if (code != XHCI_COMPLETION_CODE_SUCCESS) {
console_println("xhci: clear endpoint halt failed -- escalating to BOT reset");
dev->next_action = XHCI_NEXT_ACTION_BOT_RESET;
dev->next_action_slot_id = xfer_slot_id;
break;
}
console_println("xhci: clear endpoint halt succeeded");
if (dev->bot_reset_clear_remaining > 1) {
dev->bot_reset_clear_remaining--;
/* Flip to the other bulk endpoint for the
* second CLEAR_FEATURE owed in a BOT-reset
* escalation. */
dev->stall_ep_addr = (dev->stall_ep_addr & USB_EP_ADDR_DIR_MASK)
? dev->bulk_out_ep_addr
: dev->bulk_in_ep_addr;
dev->next_action = XHCI_NEXT_ACTION_CLEAR_HALT;
dev->next_action_slot_id = xfer_slot_id;
} else {
dev->bot_reset_clear_remaining = 0;
/* Recovery done -- resume the rest of the
* command chain at the stalled stage. */
if (dev->stall_retry_action != XHCI_NEXT_ACTION_NONE) {
dev->next_action = dev->stall_retry_action;
dev->next_action_slot_id = xfer_slot_id;
}
}
break;
}
case XHCI_XFER_BOT_RESET: {
/* G.1 / §F.14: BOT Mass Storage Reset completed.
* Per BOT spec 5.3.4's full procedure this is
* followed by CLEAR_FEATURE(ENDPOINT_HALT) on
* *both* bulk endpoints before the original
* command is retried; stage that as two
* chained CLEAR_HALT transfers. */
if (code != XHCI_COMPLETION_CODE_SUCCESS) {
console_println("xhci: BOT reset failed -- stalling out clean");
xhci_stall_fail(dev);
break;
}
console_println("xhci: BOT reset succeeded -- clearing both endpoints");
dev->bot_reset_clear_remaining = 2;
dev->stall_ep_addr = dev->bulk_in_ep_addr;
dev->next_action = XHCI_NEXT_ACTION_CLEAR_HALT;
dev->next_action_slot_id = xfer_slot_id;
break;
}
default:
console_println("xhci: transfer event");
break;
@@ -1796,5 +2152,19 @@ void xhci_poll_events(void)
dev->bot_write10_block_size) != 0) {
console_println("xhci: deferred WRITE10 setup failed");
}
} else if (dev->next_action == XHCI_NEXT_ACTION_CLEAR_HALT) {
uint32_t next_slot_id = dev->next_action_slot_id;
dev->next_action = XHCI_NEXT_ACTION_NONE;
if (xhci_ep0_clear_endpoint_halt(dev, next_slot_id, dev->stall_ep_addr) != 0) {
console_println("xhci: deferred clear endpoint halt setup failed");
xhci_stall_fail(dev);
}
} else if (dev->next_action == XHCI_NEXT_ACTION_BOT_RESET) {
uint32_t next_slot_id = dev->next_action_slot_id;
dev->next_action = XHCI_NEXT_ACTION_NONE;
if (xhci_ep0_bot_mass_storage_reset(dev, next_slot_id) != 0) {
console_println("xhci: deferred BOT mass storage reset setup failed");
xhci_stall_fail(dev);
}
}
}