pcitool/driver/umem.c

/**
 *
 * @file umem.c
 * @brief This file contains the functions handling user space memory.
 * @author Guillermo Marcus
 * @date 2009-04-05
 *
 */
#include <linux/version.h>
#include <linux/string.h>
#include <linux/types.h>
#include <linux/list.h>
#include <linux/interrupt.h>
#include <linux/pci.h>
#include <linux/cdev.h>
#include <linux/wait.h>
#include <linux/mm.h>
#include <linux/pagemap.h>
#include <linux/sched.h>
#include <linux/vmalloc.h>


#include "base.h"

/**
 *
 * Reserve a new scatter/gather list and map it from memory to PCI bus addresses.
 *
 */
int pcidriver_umem_sgmap(pcidriver_privdata_t *privdata, umem_handle_t *umem_handle)
{
    int i, res, nr_pages;
    struct page **pages;
    struct scatterlist *sg = NULL;
    pcidriver_umem_entry_t *umem_entry;
    unsigned int nents;
    unsigned long count,offset,length;

    /*
     * We do some checks first. Then, the following is necessary to create a
     * Scatter/Gather list from a user memory area:
     *  - Determine the number of pages
     *  - Get the pages for the memory area
     * 	- Lock them.
     *  - Create a scatter/gather list of the pages
     *  - Map the list from memory to PCI bus addresses
     *
     * Then, we:
     *  - Create an entry on the umem list of the device, to cache the mapping.
     *  - Create a sysfs attribute that gives easy access to the SG list
     */

    /* zero-size?? */
    if (umem_handle->size == 0)
        return -EINVAL;

    /* Direction is better ignoring during mapping. */
    /* We assume bidirectional buffers always, except when sync'ing */

    /* calculate the number of pages */
    nr_pages = ((umem_handle->vma & ~PAGE_MASK) + umem_handle->size + ~PAGE_MASK) >> PAGE_SHIFT;

    mod_info_dbg("nr_pages computed: %u\n", nr_pages);

    /* Allocate space for the page information */
    /* This can be very big, so we use vmalloc */
    if ((pages = vmalloc(nr_pages * sizeof(*pages))) == NULL)
        return -ENOMEM;

    mod_info_dbg("allocated space for the pages.\n");

    /* Allocate space for the scatterlist */
    /* We do not know how many entries will be, but the maximum is nr_pages. */
    /* This can be very big, so we use vmalloc */
    if ((sg = vmalloc(nr_pages * sizeof(*sg))) == NULL)
        goto umem_sgmap_pages;

    sg_init_table(sg, nr_pages);

    mod_info_dbg("allocated space for the SG list.\n");

    /* Get the page information */
#if LINUX_VERSION_CODE >= KERNEL_VERSION(5,8,0)
    /*As of Kernel 5.8.0, the mmap_sem member of the MM struct has been
     * renamed to mmap_lock
     * See:
     * https://github.com/torvalds/linux/commit/da1c55f1b272f4bd54671d459b39ea7b54944ef9
     */
    down_read(&current->mm->mmap_lock);
    res = get_user_pages_compat(umem_handle->vma, nr_pages, pages);
    up_read(&current->mm->mmap_lock);
#else
    down_read(&current->mm->mmap_sem);
    res = get_user_pages_compat(umem_handle->vma, nr_pages, pages);
    up_read(&current->mm->mmap_sem);
#endif

    /* Error, not all pages mapped */
    if (res < (int)nr_pages) {
        mod_info("Could not map all user pages (%d of %d)\n", res, nr_pages);
        /* If only some pages could be mapped, we release those. If a real
         * error occured, we set nr_pages to 0 */
        nr_pages = (res > 0 ? res : 0);
        goto umem_sgmap_unmap;
    }

    mod_info_dbg("Got the pages (%d).\n", res);

    /* Lock the pages, then populate the SG list with the pages */
    /* page0 is different */
    if ( !PageReserved(pages[0]) )
        lock_page(pages[0]);

    offset = (umem_handle->vma & ~PAGE_MASK);
    length = (umem_handle->size > (PAGE_SIZE-offset) ? (PAGE_SIZE-offset) : umem_handle->size);

    sg_set_page(&sg[0], pages[0], length, offset);

    count = umem_handle->size - length;
    for(i=1; i<nr_pages; i++) {
        /* Lock page first */
        if ( !PageReserved(pages[i]) )
            lock_page(pages[i]);

        /* Populate the list */
        sg_set_page(&sg[i], pages[i], ((count > PAGE_SIZE) ? PAGE_SIZE : count), 0);
        count -= sg[i].length;
    }

    /* Use the page list to populate the SG list */
    /* SG entries may be merged, res is the number of used entries */
    /* We have originally nr_pages entries in the sg list */
    if ((nents = pci_map_sg(privdata->pdev, sg, nr_pages, PCI_DMA_BIDIRECTIONAL)) == 0)
        goto umem_sgmap_unmap;

    mod_info_dbg("Mapped SG list (%d entries).\n", nents);

    /* Add an entry to the umem_list of the device, and update the handle with the id */
    /* Allocate space for the new umem entry */
    if ((umem_entry = kmalloc(sizeof(*umem_entry), GFP_KERNEL)) == NULL)
        goto umem_sgmap_entry;

    /* Fill entry to be added to the umem list */
    umem_entry->id = atomic_inc_return(&privdata->umem_count) - 1;
    umem_entry->nr_pages = nr_pages;	/* Will be needed when unmapping */
    umem_entry->pages = pages;
    umem_entry->nents = nents;
    umem_entry->sg = sg;

    if (pcidriver_sysfs_initialize_umem(privdata, umem_entry->id, &(umem_entry->sysfs_attr)) != 0)
        goto umem_sgmap_name_fail;

    /* Add entry to the umem list */
    spin_lock( &(privdata->umemlist_lock) );
    list_add_tail( &(umem_entry->list), &(privdata->umem_list) );
    spin_unlock( &(privdata->umemlist_lock) );

    /* Update the Handle with the Handle ID of the entry */
    umem_handle->handle_id = umem_entry->id;

    return 0;

umem_sgmap_name_fail:
    kfree(umem_entry);
umem_sgmap_entry:
    pci_unmap_sg( privdata->pdev, sg, nr_pages, PCI_DMA_BIDIRECTIONAL );
umem_sgmap_unmap:
    /* release pages */
    if (nr_pages > 0) {
        for(i=0; i<nr_pages; i++) {
            if (PageLocked(pages[i]))
                unlock_page(pages[i]);
            if (!PageReserved(pages[i]))
                set_page_dirty(pages[i]);
            put_page(pages[i]);
        }
    }
    vfree(sg);
umem_sgmap_pages:
    vfree(pages);
    return -ENOMEM;

}

/**
 *
 * Unmap a scatter/gather list
 *
 */
int pcidriver_umem_sgunmap(pcidriver_privdata_t *privdata, pcidriver_umem_entry_t *umem_entry)
{
    int i;
    pcidriver_sysfs_remove(privdata, &(umem_entry->sysfs_attr));

    /* Unmap user memory */
    pci_unmap_sg( privdata->pdev, umem_entry->sg, umem_entry->nr_pages, PCI_DMA_BIDIRECTIONAL );

    /* Release the pages */
    if (umem_entry->nr_pages > 0) {
        for(i=0; i<(umem_entry->nr_pages); i++) {
            /* Mark pages as Dirty and unlock it */
            if ( !PageReserved( umem_entry->pages[i] )) {
                SetPageDirty( umem_entry->pages[i] );
                unlock_page(umem_entry->pages[i]);
            }
            /* and release it from the cache */
            put_page( umem_entry->pages[i] );
        }
    }

    /* Remove the umem list entry */
    spin_lock( &(privdata->umemlist_lock) );
    list_del( &(umem_entry->list) );
    spin_unlock( &(privdata->umemlist_lock) );

    /* Release SG list and page list memory */
    /* These two are in the vm area of the kernel */
    vfree(umem_entry->pages);
    vfree(umem_entry->sg);

    /* Release umem_entry memory */
    kfree(umem_entry);

    return 0;
}

/**
 *
 * Unmap all scatter/gather lists.
 *
 */
int pcidriver_umem_sgunmap_all(pcidriver_privdata_t *privdata)
{
    struct list_head *ptr, *next;
    pcidriver_umem_entry_t *umem_entry;

    /* iterate safely over the entries and delete them */
    list_for_each_safe( ptr, next, &(privdata->umem_list) ) {
        umem_entry = list_entry(ptr, pcidriver_umem_entry_t, list );
        pcidriver_umem_sgunmap( privdata, umem_entry ); 		/* spin lock inside! */
    }

    return 0;
}

/**
 *
 * Copies the scatter/gather list from kernelspace to userspace.
 *
 */
int pcidriver_umem_sgget(pcidriver_privdata_t *privdata, umem_sglist_t *umem_sglist)
{
    int i;
    pcidriver_umem_entry_t *umem_entry;
    struct scatterlist *sg;
    int idx = 0;
    dma_addr_t cur_addr;
    unsigned int cur_size;

    /* Find the associated umem_entry for this buffer */
    umem_entry = pcidriver_umem_find_entry_id( privdata, umem_sglist->handle_id );
    if (umem_entry == NULL)
        return -EINVAL;					/* umem_handle is not valid */

    /* Check if passed SG list is enough */
    if (umem_sglist->nents < umem_entry->nents)
        return -EINVAL;					/* sg has not enough entries */

    /* Copy the SG list to the user format */
    if (umem_sglist->type == PCIDRIVER_SG_MERGED) {
        for_each_sg(umem_entry->sg, sg, umem_entry->nents, i ) {
            if (i==0) {
                umem_sglist->sg[0].addr = sg_dma_address( sg );
                umem_sglist->sg[0].size = sg_dma_len( sg );
                idx = 0;
            }
            else {
                cur_addr = sg_dma_address( sg );
                cur_size = sg_dma_len( sg );

                /* Check if entry fits after current entry */
                if (cur_addr == (umem_sglist->sg[idx].addr + umem_sglist->sg[idx].size)) {
                    umem_sglist->sg[idx].size += cur_size;
                    continue;
                }

                /* Skip if the entry is zero-length (yes, it can happen.... at the end of the list) */
                if (cur_size == 0)
                    continue;

                /* None of the above, add new entry */
                idx++;
                umem_sglist->sg[idx].addr = cur_addr;
                umem_sglist->sg[idx].size = cur_size;
            }
        }
        /* Set the used size of the SG list */
        umem_sglist->nents = idx+1;
    } else {
        for_each_sg(umem_entry->sg, sg, umem_entry->nents, i ) {
            mod_info("entry: %d\n",i);
            umem_sglist->sg[i].addr = sg_dma_address( sg );
            umem_sglist->sg[i].size = sg_dma_len( sg );
        }

        /* Set the used size of the SG list */
        /* Check if the last one is zero-length */
        if ( umem_sglist->sg[ umem_entry->nents - 1].size == 0)
            umem_sglist->nents = umem_entry->nents -1;
        else
            umem_sglist->nents = umem_entry->nents;
    }

    return 0;
}

/**
 *
 * Sync user space memory from/to device
 *
 */
int pcidriver_umem_sync( pcidriver_privdata_t *privdata, umem_handle_t *umem_handle )
{
    pcidriver_umem_entry_t *umem_entry;

    /* Find the associated umem_entry for this buffer */
    umem_entry = pcidriver_umem_find_entry_id( privdata, umem_handle->handle_id );
    if (umem_entry == NULL)
        return -EINVAL;					/* umem_handle is not valid */

    switch (umem_handle->dir) {
    case PCIDRIVER_DMA_TODEVICE:
        pci_dma_sync_sg_for_device( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_TODEVICE );
        break;
    case PCIDRIVER_DMA_FROMDEVICE:
        pci_dma_sync_sg_for_cpu( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_FROMDEVICE );
        break;
    case PCIDRIVER_DMA_BIDIRECTIONAL:
        pci_dma_sync_sg_for_device( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_BIDIRECTIONAL );
        pci_dma_sync_sg_for_cpu( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_BIDIRECTIONAL );
        break;
    default:
        return -EINVAL;				/* wrong direction parameter */
    }

    return 0;
}

/*
 *
 * Get the pcidriver_umem_entry_t structure for the given id.
 *
 * @param id ID of the umem entry to search for
 *
 */
pcidriver_umem_entry_t *pcidriver_umem_find_entry_id(pcidriver_privdata_t *privdata, int id)
{
    struct list_head *ptr;
    pcidriver_umem_entry_t *entry;

    spin_lock(&(privdata->umemlist_lock));
    list_for_each(ptr, &(privdata->umem_list)) {
        entry = list_entry(ptr, pcidriver_umem_entry_t, list );

        if (entry->id == id) {
            spin_unlock( &(privdata->umemlist_lock) );
            return entry;
        }
    }

    spin_unlock(&(privdata->umemlist_lock));
    return NULL;
}