Files
seaweedfs/weed/storage/blockvol/iscsi/session.go
T
Ping QiuandClaude Opus 4.6 bd73a81e00 fix: handle pipelined SCSI commands during Data-Out collection
The Linux kernel iSCSI initiator pipelines multiple SCSI commands on
the same TCP connection (command queuing). When a write needs R2T for
data beyond the immediate portion, collectDataOut may read a pipelined
SCSI command instead of the expected Data-Out PDU.

Fix: queue non-Data-Out PDUs received during collectDataOut into a
pending buffer. The main dispatch loop drains pending PDUs before
reading from the connection. This correctly handles interleaved
commands during multi-PDU write transfers.

Bug found during WSL2 smoke test: mkfs.ext4 hangs at "Writing
superblocks" because inode table zeroing sends large writes that
exceed FirstBurstLength, triggering R2T while the kernel has already
queued the next command.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-28 10:08:32 -08:00

441 lines
10 KiB
Go

package iscsi
import (
"errors"
"fmt"
"io"
"log"
"net"
"sync"
"sync/atomic"
)
var (
ErrSessionClosed = errors.New("iscsi: session closed")
ErrCmdSNOutOfWindow = errors.New("iscsi: CmdSN out of window")
)
// SessionState tracks the lifecycle of an iSCSI session.
type SessionState int
const (
SessionLogin SessionState = iota // login phase
SessionLoggedIn // full feature phase
SessionLogout // logout requested
SessionClosed // terminated
)
// Session manages a single iSCSI session (one initiator connection).
type Session struct {
mu sync.Mutex
state SessionState
conn net.Conn
scsi *SCSIHandler
config TargetConfig
resolver TargetResolver
devices DeviceLookup
// Sequence numbers
expCmdSN atomic.Uint32 // expected CmdSN from initiator
maxCmdSN atomic.Uint32 // max CmdSN we allow
statSN uint32 // target status sequence number
// Login state
negotiator *LoginNegotiator
loginDone bool
// Data sequencing
dataInWriter *DataInWriter
// PDU queue for commands received during Data-Out collection.
// The initiator pipelines commands; we may read a SCSI Command
// while waiting for Data-Out PDUs.
pending []*PDU
// Shutdown
closed atomic.Bool
closeErr error
// Logging
logger *log.Logger
}
// NewSession creates a new iSCSI session on the given connection.
func NewSession(conn net.Conn, config TargetConfig, resolver TargetResolver, devices DeviceLookup, logger *log.Logger) *Session {
if logger == nil {
logger = log.Default()
}
s := &Session{
state: SessionLogin,
conn: conn,
config: config,
resolver: resolver,
devices: devices,
negotiator: NewLoginNegotiator(config),
logger: logger,
}
s.expCmdSN.Store(1)
s.maxCmdSN.Store(32) // window of 32 commands
return s
}
// HandleConnection processes PDUs until the connection is closed or an error occurs.
func (s *Session) HandleConnection() error {
defer s.close()
for !s.closed.Load() {
pdu, err := s.nextPDU()
if err != nil {
if s.closed.Load() {
return nil
}
if errors.Is(err, io.EOF) || errors.Is(err, net.ErrClosed) {
return nil
}
return fmt.Errorf("read PDU: %w", err)
}
if err := s.dispatch(pdu); err != nil {
if s.closed.Load() {
return nil
}
return fmt.Errorf("dispatch %s: %w", OpcodeName(pdu.Opcode()), err)
}
}
return nil
}
// nextPDU returns the next PDU to process, draining the pending queue
// (populated during Data-Out collection) before reading from the connection.
func (s *Session) nextPDU() (*PDU, error) {
if len(s.pending) > 0 {
pdu := s.pending[0]
s.pending = s.pending[1:]
return pdu, nil
}
return ReadPDU(s.conn)
}
// Close terminates the session.
func (s *Session) Close() error {
s.closed.Store(true)
return s.conn.Close()
}
func (s *Session) close() {
s.closed.Store(true)
s.mu.Lock()
s.state = SessionClosed
s.mu.Unlock()
s.conn.Close()
}
func (s *Session) dispatch(pdu *PDU) error {
op := pdu.Opcode()
switch op {
case OpLoginReq:
return s.handleLogin(pdu)
case OpTextReq:
return s.handleText(pdu)
case OpSCSICmd:
return s.handleSCSICmd(pdu)
case OpSCSIDataOut:
// Handled inline during write command processing
return nil
case OpNOPOut:
return s.handleNOPOut(pdu)
case OpLogoutReq:
return s.handleLogout(pdu)
case OpSCSITaskMgmt:
return s.handleTaskMgmt(pdu)
default:
s.logger.Printf("unhandled opcode: %s", OpcodeName(op))
return s.sendReject(pdu, 0x04) // command not supported
}
}
func (s *Session) handleLogin(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
resp := s.negotiator.HandleLoginPDU(pdu, s.resolver)
// Set sequence numbers
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
if err := WritePDU(s.conn, resp); err != nil {
return err
}
if s.negotiator.Done() {
s.loginDone = true
s.state = SessionLoggedIn
result := s.negotiator.Result()
s.dataInWriter = NewDataInWriter(uint32(result.MaxRecvDataSegLen))
// Bind SCSI handler to the device for the target the initiator logged into
if s.devices != nil && result.TargetName != "" {
dev := s.devices.LookupDevice(result.TargetName)
s.scsi = NewSCSIHandler(dev)
}
s.logger.Printf("login complete: initiator=%s target=%s session=%s",
result.InitiatorName, result.TargetName, result.SessionType)
}
return nil
}
func (s *Session) handleText(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
// Gather discovery targets from resolver if it supports listing
var targets []DiscoveryTarget
if lister, ok := s.resolver.(TargetLister); ok {
targets = lister.ListTargets()
}
resp := HandleTextRequest(pdu, targets)
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
return WritePDU(s.conn, resp)
}
func (s *Session) handleSCSICmd(pdu *PDU) error {
if !s.loginDone {
return s.sendReject(pdu, 0x0b) // protocol error
}
cdb := pdu.CDB()
itt := pdu.InitiatorTaskTag()
flags := pdu.OpSpecific1()
// Advance CmdSN
if !pdu.Immediate() {
s.advanceCmdSN()
}
isWrite := flags&FlagW != 0
isRead := flags&FlagR != 0
expectedLen := pdu.ExpectedDataTransferLength()
// Handle write commands — collect data
var dataOut []byte
if isWrite && expectedLen > 0 {
collector := NewDataOutCollector(expectedLen)
// Immediate data
if len(pdu.DataSegment) > 0 {
if err := collector.AddImmediateData(pdu.DataSegment); err != nil {
return s.sendCheckCondition(itt, SenseIllegalRequest, ASCInvalidFieldInCDB, ASCQLuk)
}
}
// If more data needed, send R2T and collect Data-Out PDUs
if !collector.Done() {
if err := s.collectDataOut(collector, itt); err != nil {
return err
}
}
dataOut = collector.Data()
}
// Execute SCSI command
result := s.scsi.HandleCommand(cdb, dataOut)
// Send response
s.mu.Lock()
expCmdSN := s.expCmdSN.Load()
maxCmdSN := s.maxCmdSN.Load()
s.mu.Unlock()
if isRead && result.Status == SCSIStatusGood && len(result.Data) > 0 {
// Send Data-In PDUs
s.mu.Lock()
_, err := s.dataInWriter.WriteDataIn(s.conn, result.Data, itt, expCmdSN, maxCmdSN, &s.statSN)
s.mu.Unlock()
return err
}
// Send SCSI Response
s.mu.Lock()
err := SendSCSIResponse(s.conn, result, itt, &s.statSN, expCmdSN, maxCmdSN)
s.mu.Unlock()
return err
}
func (s *Session) collectDataOut(collector *DataOutCollector, itt uint32) error {
var r2tSN uint32
ttt := itt // use ITT as TTT for simplicity
for !collector.Done() {
// Send R2T
s.mu.Lock()
r2t := BuildR2T(itt, ttt, r2tSN, s.totalReceived(collector), collector.Remaining(),
s.statSN, s.expCmdSN.Load(), s.maxCmdSN.Load())
s.mu.Unlock()
if err := WritePDU(s.conn, r2t); err != nil {
return err
}
r2tSN++
// Read Data-Out PDUs until F-bit.
// The initiator may pipeline other commands; queue them for later.
for {
doPDU, err := ReadPDU(s.conn)
if err != nil {
return err
}
if doPDU.Opcode() != OpSCSIDataOut {
// Not our Data-Out — queue for later dispatch
s.pending = append(s.pending, doPDU)
continue
}
if err := collector.AddDataOut(doPDU); err != nil {
return err
}
if doPDU.OpSpecific1()&FlagF != 0 {
break
}
}
}
return nil
}
func (s *Session) totalReceived(c *DataOutCollector) uint32 {
return c.expectedLen - c.Remaining()
}
func (s *Session) handleNOPOut(pdu *PDU) error {
resp := &PDU{}
resp.SetOpcode(OpNOPIn)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.SetTargetTransferTag(0xFFFFFFFF)
s.mu.Lock()
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
// Echo back data if present
if len(pdu.DataSegment) > 0 {
resp.DataSegment = pdu.DataSegment
}
return WritePDU(s.conn, resp)
}
func (s *Session) handleLogout(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
s.state = SessionLogout
resp := &PDU{}
resp.SetOpcode(OpLogoutResp)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.BHS[2] = 0x00 // response: connection/session closed successfully
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
if err := WritePDU(s.conn, resp); err != nil {
return err
}
// Signal HandleConnection to exit
s.closed.Store(true)
s.conn.Close()
return nil
}
func (s *Session) handleTaskMgmt(pdu *PDU) error {
s.mu.Lock()
defer s.mu.Unlock()
// Simplified: always respond with "function complete"
resp := &PDU{}
resp.SetOpcode(OpSCSITaskResp)
resp.SetOpSpecific1(FlagF)
resp.SetInitiatorTaskTag(pdu.InitiatorTaskTag())
resp.BHS[2] = 0x00 // function complete
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
return WritePDU(s.conn, resp)
}
func (s *Session) advanceCmdSN() {
s.expCmdSN.Add(1)
s.maxCmdSN.Add(1)
}
func (s *Session) sendReject(origPDU *PDU, reason uint8) error {
resp := &PDU{}
resp.SetOpcode(OpReject)
resp.SetOpSpecific1(FlagF)
resp.BHS[2] = reason
resp.SetInitiatorTaskTag(0xFFFFFFFF)
s.mu.Lock()
resp.SetStatSN(s.statSN)
s.statSN++
resp.SetExpCmdSN(s.expCmdSN.Load())
resp.SetMaxCmdSN(s.maxCmdSN.Load())
s.mu.Unlock()
// Include the rejected BHS in the data segment
resp.DataSegment = origPDU.BHS[:]
return WritePDU(s.conn, resp)
}
func (s *Session) sendCheckCondition(itt uint32, senseKey, asc, ascq uint8) error {
result := SCSIResult{
Status: SCSIStatusCheckCond,
SenseKey: senseKey,
SenseASC: asc,
SenseASCQ: ascq,
}
s.mu.Lock()
defer s.mu.Unlock()
return SendSCSIResponse(s.conn, result, itt, &s.statSN, s.expCmdSN.Load(), s.maxCmdSN.Load())
}
// TargetLister is an optional interface that TargetResolver can implement
// to support SendTargets discovery.
type TargetLister interface {
ListTargets() []DiscoveryTarget
}
// DeviceLookup resolves a target IQN to a BlockDevice.
// Used after login to bind the session to the correct volume.
type DeviceLookup interface {
LookupDevice(iqn string) BlockDevice
}
// State returns the current session state.
func (s *Session) State() SessionState {
s.mu.Lock()
defer s.mu.Unlock()
return s.state
}