Linux DMA: Using the DMAengine for scatter-gather transactions
<p>I try to use the DMAengine API from a custom kernel driver to perform a <em>scatter-gather</em> operation. I have a contiguous memory region as source and I want to copy its data in several distributed buffers through a <strong>scatterlist</strong> structure. The DMA controller is the PL330 one that supports the DMAengine API (see <a href="http://lxr.free-electrons.com/source/drivers/dma/pl330.c" rel="nofollow" title="PL330 driver">PL330 DMA controller</a>).</p>

<p>My test code is the following:</p>

<p>In my driver header file (<code>test_driver.h</code>):</p>

<pre><code>#ifndef __TEST_DRIVER_H__
#define __TEST_DRIVER_H__

#include &lt;linux/platform_device.h&gt;
#include &lt;linux/device.h&gt;

#include &lt;linux/scatterlist.h&gt;
#include &lt;linux/dma-mapping.h&gt;
#include &lt;linux/dmaengine.h&gt;
#include &lt;linux/of_dma.h&gt;

#define SG_ENTRIES 3
#define BUF_SIZE 16
#define DEV_BUF 0x10000000

struct dma_block {
    void * data;
    int size;
};

struct dma_private_info {

    struct sg_table sgt;

    struct dma_block * blocks;
    int nblocks;

    int dma_started;

    struct dma_chan * dma_chan;
    struct dma_slave_config dma_config;
    struct dma_async_tx_descriptor * dma_desc;
    dma_cookie_t cookie;
};

struct test_platform_device {
    struct platform_device * pdev;

    struct dma_private_info dma_priv;
};

#define _get_devp(tdev) (&amp;((tdev)-&gt;pdev-&gt;dev))
#define _get_dmapip(tdev) (&amp;((tdev)-&gt;dma_priv))

int dma_stop(struct test_platform_device * tdev);
int dma_start(struct test_platform_device * tdev);
int dma_start_block(struct test_platform_device * tdev);
int dma_init(struct test_platform_device * tdev);
int dma_exit(struct test_platform_device * tdev);

#endif
</code></pre>

<p>In my source that contains the dma functions (<code>dma_functions.c</code>):</p>

<pre><code>#include &lt;linux/slab.h&gt;

#include "test_driver.h"

#define BARE_RAM_BASE 0x10000000
#define BARE_RAM_SIZE 0x10000000

struct ram_bare {
    uint32_t * __iomem map;

    uint32_t base;
    uint32_t size;
};

static void dma_sg_check(struct test_platform_device * tdev)
{
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);
    uint32_t * buf;
    unsigned int bufsize;
    int nwords;
    int nbytes_word = sizeof(uint32_t);
    int nblocks;
    struct ram_bare ramb;
    uint32_t * p;
    int i;
    int j;

    ramb.map = ioremap(BARE_RAM_BASE,BARE_RAM_SIZE);
    ramb.base = BARE_RAM_BASE;
    ramb.size = BARE_RAM_SIZE;

    dev_info(dev,"nblocks: %d \n",dma_priv-&gt;nblocks);

    p = ramb.map;

    nblocks = dma_priv-&gt;nblocks;

    for( i = 0 ; i &lt; nblocks ; i++ ) {

        buf = (uint32_t *) dma_priv-&gt;blocks[i].data;
        bufsize = dma_priv-&gt;blocks[i].size;
        nwords = dma_priv-&gt;blocks[i].size/nbytes_word;

        dev_info(dev,"block[%d],size %d: ",i,bufsize);

        for ( j = 0 ; j &lt;  nwords; j++, p++) {
            dev_info(dev,"DMA: 0x%x, RAM: 0x%x",buf[j],ioread32(p));
        }
    }

    iounmap(ramb.map);
}

static int dma_sg_exit(struct test_platform_device * tdev)
{
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    int ret = 0;
    int i;

    for( i = 0 ; i &lt; dma_priv-&gt;nblocks ; i++ ) {
        kfree(dma_priv-&gt;blocks[i].data);
    }

    kfree(dma_priv-&gt;blocks);

    sg_free_table(&amp;(dma_priv-&gt;sgt));

    return ret;
}

int dma_stop(struct test_platform_device * tdev)
{
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);
    int ret = 0;

    dma_unmap_sg(dev,dma_priv-&gt;sgt.sgl,\
        dma_priv-&gt;sgt.nents, DMA_FROM_DEVICE);

    dma_sg_exit(tdev);

    dma_priv-&gt;dma_started = 0;

    return ret;
}

static void dma_callback(void * param)
{
    enum dma_status dma_stat;
    struct test_platform_device * tdev = (struct test_platform_device *) param;
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);

    dev_info(dev,"Checking the DMA state....\n");

    dma_stat = dma_async_is_tx_complete(dma_priv-&gt;dma_chan,\
        dma_priv-&gt;cookie, NULL, NULL);

    if(dma_stat == DMA_COMPLETE) {
        dev_info(dev,"DMA complete! \n");
        dma_sg_check(tdev);
        dma_stop(tdev);
    } else if (unlikely(dma_stat == DMA_ERROR)) {
        dev_info(dev,"DMA error! \n");
        dma_stop(tdev);
    }
}

static void dma_busy_loop(struct test_platform_device * tdev)
{
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);

    enum dma_status status;
    int status_change = -1;

    do {
        status = dma_async_is_tx_complete(dma_priv-&gt;dma_chan, dma_priv-&gt;cookie, NULL, NULL);

        switch(status) {
        case DMA_COMPLETE:
            if(status_change != 0)
                dev_info(dev,"DMA status: COMPLETE\n");
            status_change = 0;
            break;
        case DMA_PAUSED:
            if (status_change != 1)
                dev_info(dev,"DMA status: PAUSED\n");
            status_change = 1;
            break;
        case DMA_IN_PROGRESS:
            if(status_change != 2)
                dev_info(dev,"DMA status: IN PROGRESS\n");
            status_change = 2;
            break;
        case DMA_ERROR:
            if (status_change != 3)
                dev_info(dev,"DMA status: ERROR\n");
            status_change = 3;
            break;
        default:
            dev_info(dev,"DMA status: UNKNOWN\n");
            status_change = -1;
            break;
        }
    } while(status != DMA_COMPLETE);

    dev_info(dev,"DMA transaction completed! \n");
}

static int dma_sg_init(struct test_platform_device * tdev)
{

    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct scatterlist *sg;
    int ret = 0;
    int i;

    ret = sg_alloc_table(&amp;(dma_priv-&gt;sgt), SG_ENTRIES, GFP_ATOMIC);
    if(ret)
        goto out_mem2;

    dma_priv-&gt;nblocks = SG_ENTRIES;
    dma_priv-&gt;blocks = (struct dma_block *) kmalloc(dma_priv-&gt;nblocks\
        *sizeof(struct dma_block), GFP_ATOMIC);
    if(dma_priv-&gt;blocks == NULL) 
         goto out_mem1;


    for( i = 0 ; i &lt; dma_priv-&gt;nblocks ; i++ ) {
        dma_priv-&gt;blocks[i].size = BUF_SIZE;
        dma_priv-&gt;blocks[i].data = kmalloc(dma_priv-&gt;blocks[i].size, GFP_ATOMIC);
        if(dma_priv-&gt;blocks[i].data == NULL)
            goto out_mem3;
    }

    for_each_sg(dma_priv-&gt;sgt.sgl, sg, dma_priv-&gt;sgt.nents, i)
        sg_set_buf(sg,dma_priv-&gt;blocks[i].data,dma_priv-&gt;blocks[i].size);

    return ret;

out_mem3:
    i--;

    while(i &gt;= 0)
        kfree(dma_priv-&gt;blocks[i].data);

    kfree(dma_priv-&gt;blocks);

out_mem2:
    sg_free_table(&amp;(dma_priv-&gt;sgt));

out_mem1:
    ret = -ENOMEM;  

    return ret;

}

static int _dma_start(struct test_platform_device * tdev,int block)
{
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);
    int ret = 0;
    int sglen;

    /* Step 1: Allocate and initialize the SG list */
    dma_sg_init(tdev);

    /* Step 2: Map the SG list */
    sglen = dma_map_sg(dev,dma_priv-&gt;sgt.sgl,\
        dma_priv-&gt;sgt.nents, DMA_FROM_DEVICE);
    if(! sglen)
        goto out2;

    /* Step 3: Configure the DMA */
    (dma_priv-&gt;dma_config).direction = DMA_DEV_TO_MEM;
    (dma_priv-&gt;dma_config).src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
    (dma_priv-&gt;dma_config).src_maxburst = 1;
    (dma_priv-&gt;dma_config).src_addr = (dma_addr_t) DEV_BUF;

    dmaengine_slave_config(dma_priv-&gt;dma_chan, \
        &amp;(dma_priv-&gt;dma_config));

    /* Step 4: Prepare the SG descriptor */
    dma_priv-&gt;dma_desc = dmaengine_prep_slave_sg(dma_priv-&gt;dma_chan, \
        dma_priv-&gt;sgt.sgl, dma_priv-&gt;sgt.nents, DMA_DEV_TO_MEM, \
        DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
    if (dma_priv-&gt;dma_desc == NULL) {
        dev_err(dev,"DMA could not assign a descriptor! \n");
        goto out1;
    }

    /* Step 5: Set the callback method */
    (dma_priv-&gt;dma_desc)-&gt;callback = dma_callback;
    (dma_priv-&gt;dma_desc)-&gt;callback_param = (void *) tdev;

    /* Step 6: Put the DMA descriptor in the queue */
    dma_priv-&gt;cookie = dmaengine_submit(dma_priv-&gt;dma_desc);

    /* Step 7: Fires the DMA transaction */
    dma_async_issue_pending(dma_priv-&gt;dma_chan);

    dma_priv-&gt;dma_started = 1;

    if(block)
        dma_busy_loop(tdev);

    return ret;

out1:
    dma_stop(tdev);
out2:
    ret = -1;

    return ret;
}

int dma_start(struct test_platform_device * tdev) {
    return _dma_start(tdev,0);
}

int dma_start_block(struct test_platform_device * tdev) {
    return _dma_start(tdev,1);
}

int dma_init(struct test_platform_device * tdev)
{
    int ret = 0;
    struct dma_private_info * dma_priv = _get_dmapip(tdev);
    struct device * dev = _get_devp(tdev);

    dma_priv-&gt;dma_chan = dma_request_slave_channel(dev, \
        "dma_chan0");
    if (dma_priv-&gt;dma_chan == NULL) {
        dev_err(dev,"DMA channel busy! \n");
        ret = -1;
    }

    dma_priv-&gt;dma_started = 0;

    return ret;
}

int dma_exit(struct test_platform_device * tdev)
{
    int ret = 0;
    struct dma_private_info * dma_priv = _get_dmapip(tdev);

    if(dma_priv-&gt;dma_started) {
        dmaengine_terminate_all(dma_priv-&gt;dma_chan);
        dma_stop(tdev);
        dma_priv-&gt;dma_started = 0;
    }

    if(dma_priv-&gt;dma_chan != NULL)
        dma_release_channel(dma_priv-&gt;dma_chan);

    return ret;
}
</code></pre>

<p>In my driver source file (<code>test_driver.c</code>):</p>

<pre><code>#include &lt;linux/kernel.h&gt;
#include &lt;linux/module.h&gt;
#include &lt;linux/init.h&gt;
#include &lt;linux/version.h&gt;
#include &lt;linux/device.h&gt;
#include &lt;linux/platform_device.h&gt;
#include &lt;linux/of_device.h&gt;
#include &lt;linux/of_address.h&gt;
#include &lt;linux/of_irq.h&gt;
#include &lt;linux/interrupt.h&gt;

#include "test_driver.h"

static int dma_block=0;
module_param_named(dma_block, dma_block, int, 0444);

static struct test_platform_device tdev;

static struct of_device_id test_of_match[] = {
  { .compatible = "custom,test-driver-1.0", },
  {}
};

static int test_probe(struct platform_device *op)
{
    int ret = 0;
    struct device * dev = &amp;(op-&gt;dev);

    const struct of_device_id *match = of_match_device(test_of_match, &amp;op-&gt;dev);

    if (!match)
        return -EINVAL;

    tdev.pdev = op;

    dma_init(&amp;tdev);

    if(dma_block)
        ret = dma_start_block(&amp;tdev);
    else
        ret = dma_start(&amp;tdev);

    if(ret) {
        dev_err(dev,"Error to start DMA transaction! \n");
    } else {
        dev_info(dev,"DMA OK! \n");
    }

    return ret;
}

static int test_remove(struct platform_device *op)
{       
    dma_exit(&amp;tdev);

    return 0;
}

static struct platform_driver test_platform_driver = {
  .probe = test_probe,
  .remove = test_remove,
  .driver = {
    .name = "test-driver",
    .owner = THIS_MODULE,
    .of_match_table = test_of_match,
  },
};

static int test_init(void)
{
    platform_driver_register(&amp;test_platform_driver);
    return 0;
}

static void test_exit(void)
{
    platform_driver_unregister(&amp;test_platform_driver);
}

module_init(test_init);
module_exit(test_exit);

MODULE_AUTHOR("klyone");
MODULE_DESCRIPTION("DMA SG test module");
MODULE_LICENSE("GPL");
</code></pre>

<p>However, the DMA never calls my callback function and I do not have any idea why it happens. Maybe, I am misunderstanding something...</p>

<p>Could anyone help me?</p>

<p>Thanks in advance.</p>