FreeBSD bNXT Ethernet driver source code reading record 2:

bnxt Summary of functions related to interrupts in (2022/7/25)

It mainly involves the following 5 A function , Let's see what they have done .

    DEVMETHOD(ifdi_intr_enable, bnxt_intr_enable),
    DEVMETHOD(ifdi_tx_queue_intr_enable, bnxt_tx_queue_intr_enable),
    DEVMETHOD(ifdi_rx_queue_intr_enable, bnxt_rx_queue_intr_enable),
    DEVMETHOD(ifdi_intr_disable, bnxt_disable_intr),
    DEVMETHOD(ifdi_msix_intr_assign, bnxt_msix_intr_assign),

bnxt_intr_enable：

/* Enable all interrupts */
static void
bnxt_intr_enable(if_ctx_t ctx)
{
    
    struct bnxt_softc *softc = iflib_get_softc(ctx);
    int i;
    bnxt_do_enable_intr(&softc->def_cp_ring);
    for (i = 0; i < softc->nrxqsets; i++)
        bnxt_do_enable_intr(&softc->rx_cp_rings[i]);
    return;
}

bnxt_tx_queue_intr_enable：

/* Enable interrupt for a single queue */
static int
bnxt_tx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
{
    
    struct bnxt_softc *softc = iflib_get_softc(ctx);
    bnxt_do_enable_intr(&softc->tx_cp_rings[qid]);
    return 0;
}

bnxt_rx_queue_intr_enable：

static int
bnxt_rx_queue_intr_enable(if_ctx_t ctx, uint16_t qid)
{
    
    struct bnxt_softc *softc = iflib_get_softc(ctx);
    bnxt_do_enable_intr(&softc->rx_cp_rings[qid]);
    return 0;
}

bnxt_disable_intr：

/* Disable all interrupts */
static void
bnxt_disable_intr(if_ctx_t ctx)
{
    
    struct bnxt_softc *softc = iflib_get_softc(ctx);
    int i;
    /* * NOTE: These TX interrupts should never get enabled, so don't * update the index */
    for (i = 0; i < softc->ntxqsets; i++)
        bnxt_do_disable_intr(&softc->tx_cp_rings[i]);
    for (i = 0; i < softc->nrxqsets; i++)
        bnxt_do_disable_intr(&softc->rx_cp_rings[i]);
    return;
}

uppermost 4 All functions involve ,bnxt_do_disable_intr Functions and bnxt_do_enable_intr function ,
bnxt_do_enable_intr：

static void inline
bnxt_do_enable_intr(struct bnxt_cp_ring *cpr)
{
    
    if (cpr->ring.phys_id != (uint16_t)HWRM_NA_SIGNATURE) {
    
        /* First time enabling, do not set index */
        if (cpr->cons == UINT32_MAX)
            BNXT_CP_ENABLE_DB(&cpr->ring);
        else
            BNXT_CP_IDX_ENABLE_DB(&cpr->ring, cpr->cons);
    }
}

bnxt_do_disable_intr：

static void inline
bnxt_do_disable_intr(struct bnxt_cp_ring *cpr)
{
    
    if (cpr->ring.phys_id != (uint16_t)HWRM_NA_SIGNATURE)
        BNXT_CP_DISABLE_DB(&cpr->ring);
}

and BNXT_CP_ENABLE_DB and BNXT_CP_DISABLE_DB,BNXT_CP_IDX_ENABLE_DB It's all macros .

#define BNXT_CP_DISABLE_DB(ring) do {
       \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_MASK)); \ } while (0)

#define BNXT_CP_ENABLE_DB(ring) do {
       \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL)); \ } while (0)

#define BNXT_CP_IDX_ENABLE_DB(ring, cons) do {
       \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_IDX_VALID | \ (cons))); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ } while (0)

#define BNXT_CP_IDX_DISABLE_DB(ring, cons) do {
       \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, 4, \ BUS_SPACE_BARRIER_WRITE); \ bus_space_write_4((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, (ring)->doorbell, \ htole32(CMPL_DOORBELL_KEY_CMPL | CMPL_DOORBELL_IDX_VALID | \ CMPL_DOORBELL_MASK | (cons))); \ bus_space_barrier((ring)->softc->doorbell_bar.tag, \ (ring)->softc->doorbell_bar.handle, 0, \ (ring)->softc->doorbell_bar.size, BUS_SPACE_BARRIER_WRITE); \ } while (0)

therefore , Above 4 In a function , In fact, they all write doorbell Register operation , The key is to understand the differences between these macros , We go on to analyze ：

First of all BNXT_CP_DISABLE_DB：

 It has CMPL_DOORBELL_MASK position , This bit means mask interrupt ,
 Because this macro is called in the interrupt function , So there are interruptions in the follow-up , I'll shield them first , Finish the current work first ..

BNXT_CP_ENABLE_DB：

 This is just the opposite of the one above , Can receive interrupt , But does this mean ？ It's not easy to understand , Just write a tape CMPL_DOORBELL_KEY_CMPL The bit of is enabled ？ You can let the subsequent interruption come ？
 But from another perspective , This write operation , It has no belt CMPL_DOORBELL_MASK This , In fact, it is equivalent to that there may be an interrupt trigger , Right .

These two are interesting , Because it brings cons,,, Analyze where they will be called .
BNXT_CP_IDX_ENABLE_DB：

 Insert a code chip here 

BNXT_CP_IDX_DISABLE_DB：

 Insert a code chip here 

bnxt_msix_intr_assign：
This function is actually equivalent to applying for interrupt number and setting interrupt callback function .
The callback functions involved are bnxt_handle_def_cp and bnxt_handle_rx_cp.

static int
bnxt_msix_intr_assign(if_ctx_t ctx, int msix)
{
    
    struct bnxt_softc *softc = iflib_get_softc(ctx);
    int rc;
    int i;
    char irq_name[16];
    //def_cp
    rc = iflib_irq_alloc_generic(ctx, &softc->def_cp_ring.irq, softc->def_cp_ring.ring.id + 1, IFLIB_INTR_ADMIN,
                bnxt_handle_def_cp, softc, 0, "def_cp");
    if (rc) {
    
        device_printf(iflib_get_dev(ctx),  "Failed to register default completion ring handler\n");
        return rc;
    }
    //rx
    for (i = 0; i<softc->scctx->isc_nrxqsets; i++) {
    
        snprintf(irq_name, sizeof(irq_name), "rxq%d", i);
        rc = iflib_irq_alloc_generic(ctx, &softc->rx_cp_rings[i].irq, softc->rx_cp_rings[i].ring.id + 1, IFLIB_INTR_RX,
            bnxt_handle_rx_cp, &softc->rx_cp_rings[i], i, irq_name);
        if (rc) {
    
            device_printf(iflib_get_dev(ctx), "Failed to register RX completion ring handler\n");
            i--;
            goto fail;
        }
    }
    //tx
    for (i=0; i<softc->scctx->isc_ntxqsets; i++)
        iflib_softirq_alloc_generic(ctx, NULL, IFLIB_INTR_TX, NULL, i, "tx_cp");

    return rc;
fail:
    for (; i>=0; i--)
        iflib_irq_free(ctx, &softc->rx_cp_rings[i].irq);
    iflib_irq_free(ctx, &softc->def_cp_ring.irq);
    return rc;
}

bnxt_handle_def_cp：

// Interrupt handling callback function 
static int
bnxt_handle_def_cp(void *arg)
{
    
    struct bnxt_softc *softc = arg;

    BNXT_CP_DISABLE_DB(&softc->def_cp_ring.ring);
    GROUPTASK_ENQUEUE(&softc->def_cp_task); // This will be analyzed later 
    return FILTER_HANDLED;
}

bnxt_handle_rx_cp：

// Interrupt handling callback function 
static int
bnxt_handle_rx_cp(void *arg)
{
    
    struct bnxt_cp_ring *cpr = arg;

    /* Disable further interrupts for this queue */
    BNXT_CP_DISABLE_DB(&cpr->ring);
    return FILTER_SCHEDULE_THREAD;
}

From this point of view, what is done in these two callback functions is also called BNXT_CP_DISABLE_DB Write doorbell The register of .

bnxt_ring Structure and bnxt_cp_ring Analysis of the meaning of each member variable of the structure (2022/7/25)

bnxt_ring ：

struct bnxt_ring {
    
    uint64_t        paddr;
    vm_offset_t     doorbell;
    caddr_t         vaddr;
    struct bnxt_softc   *softc;
    uint32_t        ring_size;  /* Must be a power of two */
    uint16_t        id;     /* Logical ID */
    uint16_t        phys_id;
    struct bnxt_full_tpa_start *tpa_start;
};

Analyze one by one , It feels like this piece , Almost understand the logic of the whole contract .
First of all softc member , Needless to say , This points to us bnxt The structure of this driver .
paddr and vaddr：
Its two assigned values are in the corresponding alloc In the function ：

softc->tx_cp_rings[i].ring.vaddr = vaddrs[i * ntxqs];
softc->tx_cp_rings[i].ring.paddr = paddrs[i * ntxqs];

 among ,vaddrs and paddrs,ntxqs It's all in iflib Parameters passed when calling this function over there .
 among paddr Value is used later hwrm_ring_alloc( This command makes some basic preparations for ring allocation and operation ) Will be filled in ,
 In fact, it is a value , This value points to the first address of the page table of the ring ( It can be understood that ), In fact, I applied for a large memory address ,paddr Point to the beginning of this large memory address .
 This block of memory address is understood in terms of bytes , It is divided into one by one BD Structure , The type of this structure will be discussed later . This is a physical address , and 
vaddrs It's a virtual address , Why tell the hardware this address ?? I'm not sure , Later, when I look in depth, I'm recording , But I'm sure , It must be to establish a connection between hardware and host memory , Right .

doorbell：

 This is doorbell The address of the register , In fact, my question here is how many such registers , Only through the chip manual ...
 Back , For example, when you want to contract , I will be the tx ring dependent doorbell Write related idx That's all right. .

 The place of assignment is here , In a cycle .
softc->tx_rings[i].id = (softc->scctx->isc_nrxqsets * 2) + 1 + i;
softc->tx_rings[i].doorbell = softc->tx_rings[i].id * 0x80; //linux Over there and here, we also have to multiply by 0x80

 In fact, make a guess here , It's just one. ring There should be a special related doorbell The register of .

ring_size：
I think this value corresponds to this ring There are several in it BD The structure of this .

 The place of assignment ：
softc->tx_rings[i].ring_size = softc->scctx->isc_ntxd[1];

id：
ring The logic of id, Probably used to mark the memory of the host malloc Apply for one ring Mark when constructing .

 Initial value ：
softc->tx_rings[i].id = (softc->scctx->isc_nrxqsets * 2) + 1 + i;
softc->tx_rings[i].doorbell = softc->tx_rings[i].id * 0x80;

 Where to use , Fill here in hwrm_alloc_ring Inside 
// The logical ring number of the ring to be assigned . This value determines where the ring will be updated in the doorbell area 
 req.logical_id = htole16(ring->id);

// Associated with it tx ring Of cmpl ring Of id,, these id The function of is mainly to identify the corresponding ring Of ,
// And actually this ring It is decided by hardware , That is to say ring->phys_id
req.cmpl_ring_id = htole16(cmpl_ring_id);

// Hardware reply 
ring->phys_id = le16toh(resp->ring_id);
 in other words , I give the hardware one id Number is 1, The hardware reply may also be 2,, Mainly hardware .

phys_id：
This phys_id I don't think it's of much use , From the whole code , Its initial value is -1, However, it will call hwrm_alloc_ring yes , You will get a new value

ring->phys_id = le16toh(resp->ring_id);

Use this value to make some judgments when doing some operations later , It mainly means whether there is a call hwrm_alloc_ring Series of functions , If you don't call , I won't let you do these operations ..

tpa_start：
Let's not analyze this for the time being .

bnxt_cp_ring ：

struct bnxt_cp_ring {
    
    struct bnxt_ring    ring;
    struct if_irq       irq;
    uint32_t        cons;
    bool            v_bit;      /* Value of valid bit */
    struct ctx_hw_stats *stats;
    uint32_t        stats_ctx_id;
    uint32_t        last_idx;   /* Used by RX rings only * set to the last read pidx */
};

The members of this structure are also analyzed one by one .
ring：
Don't need to say more ,cmpl ring Also a ring Well , So it's a little similar to inheritance , hold ring The features of have been inherited .
irq：
Related to interruption . The function of this is related to the top analysis .
cons and v_bit The function of is not carried out , It has been analyzed earlier .
stats and stats_ctx_id This is related to statistics ... I don't care about .
last_idx See the notes are and RX ring of , Let's not analyze .

bnxt_hwrm_ring_alloc Call situation analysis (2022/7/25)

It's not hard to see. , A call to 6 This function , in other words , In the driver, I hope to get from the hardware 6 A unique ring.
according to type It can be seen that , Yes 3 Yes L2_CMPL(softc->rx_cp_rings[i].ring,softc->def_cp_ring.ring,softc->tx_cp_rings[i].ring),
2 Yes RX(softc->rx_rings[i], softc->ag_rings[i]),1 Yes TX(softc->tx_rings[i]).