Properly perform synchronization of DMA buffers

dma/nwl_engine.c

@@ -289,10 +289,12 @@ int dma_nwl_stream_read(pcilib_dma_context_t *vctx, pcilib_dma_engine_t dma, uin
 	}
 #endif /*  NWL_FIX_EOP_FOR_BIG_PACKETS */
 	
-	//sync
+	pcilib_sync_kernel_memory(ctx->pcilib, info->pages, PCILIB_KMEM_SYNC_FROMDEVICE);
         void *buf = pcilib_kmem_get_block_ua(ctx->pcilib, info->pages, bufnum);
 	ret = cb(cbattr, eop?PCILIB_DMA_FLAG_EOP:0, bufsize, buf);
+	pcilib_sync_kernel_memory(ctx->pcilib, info->pages, PCILIB_KMEM_SYNC_TODEVICE);
 	dma_nwl_return_buffer(ctx, info);
+
 	
 	res += bufsize;
 

dma/nwl_engine_buffers.h

@@ -95,20 +95,22 @@ static int dma_nwl_allocate_engine_buffers(nwl_dma_t *ctx, pcilib_nwl_engine_des
     uint64_t buf_pa;
     pcilib_kmem_reuse_state_t reuse_ring, reuse_pages;
     pcilib_kmem_flags_t flags;
+    pcilib_kmem_type_t type;
 
     char *base = info->base_addr;
     
     if (info->pages) return 0;
     
 	// Or bidirectional specified by 0x0|addr, or read 0x0|addr and write 0x80|addr
+    type = (info->desc.direction == PCILIB_DMA_TO_DEVICE)?PCILIB_KMEM_TYPE_DMA_S2C_PAGE:PCILIB_KMEM_TYPE_DMA_C2S_PAGE;
     sub_use = info->desc.addr|((info->desc.direction == PCILIB_DMA_TO_DEVICE)?0x80:0x00);
     flags = PCILIB_KMEM_FLAG_REUSE|PCILIB_KMEM_FLAG_EXCLUSIVE|PCILIB_KMEM_FLAG_HARDWARE|(info->preserve?PCILIB_KMEM_FLAG_PERSISTENT:0);
     
     pcilib_kmem_handle_t *ring = pcilib_alloc_kernel_memory(ctx->pcilib, PCILIB_KMEM_TYPE_CONSISTENT, 1, PCILIB_NWL_DMA_PAGES * PCILIB_NWL_DMA_DESCRIPTOR_SIZE, PCILIB_NWL_ALIGNMENT, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_RING, sub_use), flags);
-    pcilib_kmem_handle_t *pages = pcilib_alloc_kernel_memory(ctx->pcilib, PCILIB_KMEM_TYPE_PAGE, PCILIB_NWL_DMA_PAGES, 0, 0, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, sub_use), flags);
+    pcilib_kmem_handle_t *pages = pcilib_alloc_kernel_memory(ctx->pcilib, type, PCILIB_NWL_DMA_PAGES, 0, 0, PCILIB_KMEM_USE(PCILIB_KMEM_USE_DMA_PAGES, sub_use), flags);
 
-    if ((ring)&&(pages)) err = dma_nwl_sync_buffers(ctx, info, pages);
-    else err = PCILIB_ERROR_FAILED;
+//    if ((ring)&&(pages)) err = dma_nwl_sync_buffers(ctx, info, pages);
+//    else err = PCILIB_ERROR_FAILED;
 
     if (err) {
 	if (pages) pcilib_free_kernel_memory(ctx->pcilib, pages, 0);

driver/common.h

@@ -11,6 +11,8 @@
 /* This list keeps references to the allocated kernel buffers */
 typedef struct {
 	int id;
+	enum dma_data_direction direction;
+	
 	struct list_head list;
 	dma_addr_t dma_handle;
 	unsigned long cpua;

driver/kmem.c

@@ -46,19 +46,19 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 		    kmem_handle->align = kmem_entry->align;
 		} else {
 		    if (kmem_handle->type != kmem_entry->type) {
-		    	mod_info("Invalid type of reusable kmem_entry\n");
+		    	mod_info("Invalid type of reusable kmem_entry, currently: %lu, but requested: %lu\n", kmem_entry->type, kmem_handle->type);
 			return -EINVAL;
 		    }
 
-		    if (kmem_handle->type == PCILIB_KMEM_TYPE_PAGE) {
+		    if ((kmem_handle->type&PCILIB_KMEM_TYPE_MASK) == PCILIB_KMEM_TYPE_PAGE) {
 			    kmem_handle->size = kmem_entry->size;
 		    } else if (kmem_handle->size != kmem_entry->size) {
-			mod_info("Invalid size of reusable kmem_entry\n");
+			mod_info("Invalid size of reusable kmem_entry, currently: %lu, but requested: %lu\n", kmem_entry->size, kmem_handle->size);
 			return -EINVAL;
 		    }
 		    
 		    if (kmem_handle->align != kmem_entry->align) {
-			mod_info("Invalid alignment of reusable kmem_entry\n");
+			mod_info("Invalid alignment of reusable kmem_entry, currently: %lu, but requested: %lu\n", kmem_entry->align, kmem_handle->align);
 			return -EINVAL;
 		    }
 
@@ -112,6 +112,7 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 	kmem_entry->item = kmem_handle->item;
 	kmem_entry->type = kmem_handle->type;
 	kmem_entry->align = kmem_handle->align;
+	kmem_entry->direction = PCI_DMA_NONE;
 
 	/* Initialize sysfs if possible */
 	if (pcidriver_sysfs_initialize_kmem(privdata, kmem_entry->id, &(kmem_entry->sysfs_attr)) != 0)
@@ -124,7 +125,7 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 	 * CPU address is used for the mmap (internal to the driver), and
 	 * PCI address is the address passed to the DMA Controller in the device.
 	 */
-	switch (kmem_entry->type) {
+	switch (kmem_entry->type&PCILIB_KMEM_TYPE_MASK) {
 	 case PCILIB_KMEM_TYPE_CONSISTENT:
 	    retptr = pci_alloc_consistent( privdata->pdev, kmem_handle->size, &(kmem_entry->dma_handle) );
 	    break;
@@ -132,13 +133,32 @@ int pcidriver_kmem_alloc(pcidriver_privdata_t *privdata, kmem_handle_t *kmem_han
 	    retptr = (void*)__get_free_pages(GFP_KERNEL, get_order(PAGE_SIZE));
 	    kmem_entry->dma_handle = 0;
 	    kmem_handle->size = PAGE_SIZE;
-
-//    	    kmem_entry->dma_handle = pci_map_single(privdata->pdev, retptr, PAGE_SIZE,  PCI_DMA_FROMDEVICE);
-//	    printk("%llx %lx\n", kmem_entry->dma_handle, retptr);
+	    
+	    if (retptr) {
+	        if (kmem_entry->type == PCILIB_KMEM_TYPE_DMA_S2C_PAGE) {
+		    kmem_entry->direction = PCI_DMA_TODEVICE;
+    		    kmem_entry->dma_handle = pci_map_single(privdata->pdev, retptr, PAGE_SIZE, PCI_DMA_TODEVICE);
+		    if (pci_dma_mapping_error(privdata->pdev, kmem_entry->dma_handle)) {
+			free_page((unsigned long)retptr);
+			goto kmem_alloc_mem_fail;
+		    }
+		} else if (kmem_entry->type == PCILIB_KMEM_TYPE_DMA_C2S_PAGE) {
+		    kmem_entry->direction = PCI_DMA_FROMDEVICE;
+    		    kmem_entry->dma_handle = pci_map_single(privdata->pdev, retptr, PAGE_SIZE, PCI_DMA_FROMDEVICE);
+		    if (pci_dma_mapping_error(privdata->pdev, kmem_entry->dma_handle)) {
+			free_page((unsigned long)retptr);
+			goto kmem_alloc_mem_fail;
+		    
+		    }
+		}
+	    }
+	    
 	    break;
 	 default:
 	    goto kmem_alloc_mem_fail;
 	}
+	
+	
 	if (retptr == NULL)
 		goto kmem_alloc_mem_fail;
 
@@ -316,42 +336,34 @@ int pcidriver_kmem_sync( pcidriver_privdata_t *privdata, kmem_sync_t *kmem_sync
 	if ((kmem_entry = pcidriver_kmem_find_entry(privdata, &(kmem_sync->handle))) == NULL)
 		return -EINVAL;					/* kmem_handle is not valid */
 
+	if (kmem_entry->direction == PCI_DMA_NONE)
+		return -EINVAL;
 
-	if (!kmem_entry->dma_handle) {
-	    mod_info_dbg("Instead of synchronization, we are mapping kmem_entry with id: %d\n", kmem_entry->id);
-	    if (kmem_sync->dir == PCIDRIVER_DMA_TODEVICE) 
-		kmem_entry->dma_handle = pci_map_single(privdata->pdev, (void*)kmem_entry->cpua, kmem_entry->size,  PCI_DMA_TODEVICE);
-	    else
-		kmem_entry->dma_handle = pci_map_single(privdata->pdev, (void*)kmem_entry->cpua, kmem_entry->size,  PCI_DMA_FROMDEVICE);
-
-	    kmem_sync->handle.pa = kmem_entry->dma_handle;
-	}
-	
 #if LINUX_VERSION_CODE >= KERNEL_VERSION(2,6,11)
 	switch (kmem_sync->dir) {
-		case PCIDRIVER_DMA_TODEVICE:
-			pci_dma_sync_single_for_device( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_TODEVICE );
+		case PCILIB_KMEM_SYNC_TODEVICE:
+			pci_dma_sync_single_for_device( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
-		case PCIDRIVER_DMA_FROMDEVICE:
-			pci_dma_sync_single_for_cpu( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_FROMDEVICE );
+		case PCILIB_KMEM_SYNC_FROMDEVICE:
+			pci_dma_sync_single_for_cpu( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
-		case PCIDRIVER_DMA_BIDIRECTIONAL:
-			pci_dma_sync_single_for_device( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_BIDIRECTIONAL );
-			pci_dma_sync_single_for_cpu( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_BIDIRECTIONAL );
+		case PCILIB_KMEM_SYNC_BIDIRECTIONAL:
+			pci_dma_sync_single_for_device( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
+			pci_dma_sync_single_for_cpu( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
 		default:
 			return -EINVAL;				/* wrong direction parameter */
 	}
 #else
 	switch (kmem_sync->dir) {
-		case PCIDRIVER_DMA_TODEVICE:
-			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_TODEVICE );
+		case PCILIB_KMEM_SYNC_TODEVICE:
+			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
-		case PCIDRIVER_DMA_FROMDEVICE:
-			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_FROMDEVICE );
+		case PCILIB_KMEM_SYNC_FROMDEVICE:
+			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
-		case PCIDRIVER_DMA_BIDIRECTIONAL:
-			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_BIDIRECTIONAL );
+		case PCILIB_KMEM_SYNC_BIDIRECTIONAL:
+			pci_dma_sync_single( privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, kmem_entry->direction );
 			break;
 		default:
 			return -EINVAL;				/* wrong direction parameter */
@@ -400,12 +412,18 @@ int pcidriver_kmem_free_entry(pcidriver_privdata_t *privdata, pcidriver_kmem_ent
 #endif
 
 	/* Release DMA memory */
-	switch (kmem_entry->type) {
+	switch (kmem_entry->type&PCILIB_KMEM_TYPE_MASK) {
 	 case PCILIB_KMEM_TYPE_CONSISTENT:
 	    pci_free_consistent( privdata->pdev, kmem_entry->size, (void *)(kmem_entry->cpua), kmem_entry->dma_handle );
 	    break;
 	 case PCILIB_KMEM_TYPE_PAGE:
-	    if (kmem_entry->dma_handle) pci_unmap_single(privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_TODEVICE);
+	    if (kmem_entry->dma_handle) {
+		if (kmem_entry->type == PCILIB_KMEM_TYPE_DMA_S2C_PAGE) {
+		    pci_unmap_single(privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_TODEVICE);
+		} else if (kmem_entry->type == PCILIB_KMEM_TYPE_DMA_C2S_PAGE) {
+		    pci_unmap_single(privdata->pdev, kmem_entry->dma_handle, kmem_entry->size, PCI_DMA_FROMDEVICE);
+		}
+	    }
 	    free_page((unsigned long)kmem_entry->cpua);
 	    break;
 	}

driver/sysfs.c

@@ -102,9 +102,9 @@ static SYSFS_GET_FUNCTION(pcidriver_show_kmem_entry)
 	pcidriver_kmem_entry_t *entry = pcidriver_kmem_find_entry_id(privdata, id);
 	if (entry)
 	    if (entry->size >= 16)
-		return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lu\nuse: 0x%lx\nitem: %lu\nsize: %lu\nrefs: %lu\nhw ref: %i\nmode: 0x%lx\n data: %8x %8x %8x %8x\n", id, entry->type, entry->use, entry->item, entry->size, entry->refs&KMEM_REF_COUNT, (entry->refs&KMEM_REF_HW)?1:0, entry->mode, *(u32*)(entry->cpua), *(u32*)(entry->cpua + 4),  *(u32*)(entry->cpua + 8), *(u32*)(entry->cpua + 12));
+		return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lx\nuse: 0x%lx\nitem: %lu\nsize: %lu\nrefs: %lu\nhw ref: %i\nmode: 0x%lx\ndata: %8x %8x %8x %8x\n", id, entry->type, entry->use, entry->item, entry->size, entry->refs&KMEM_REF_COUNT, (entry->refs&KMEM_REF_HW)?1:0, entry->mode, *(u32*)(entry->cpua), *(u32*)(entry->cpua + 4),  *(u32*)(entry->cpua + 8), *(u32*)(entry->cpua + 12));
 	    else
-		return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lu\nuse: 0x%lx\nitem: %lu\nsize: %lu\nrefs: %lu\nhw ref: %i\nmode: 0x%lx\n", id, entry->type, entry->use, entry->item, entry->size, entry->refs&KMEM_REF_COUNT, (entry->refs&KMEM_REF_HW)?1:0, entry->mode);
+		return snprintf(buf, PAGE_SIZE, "buffer: %d\ntype: %lx\nuse: 0x%lx\nitem: %lu\nsize: %lu\nrefs: %lu\nhw ref: %i\nmode: 0x%lx\n", id, entry->type, entry->use, entry->item, entry->size, entry->refs&KMEM_REF_COUNT, (entry->refs&KMEM_REF_HW)?1:0, entry->mode);
 	else
 	    return snprintf(buf, PAGE_SIZE, "I am in the kmem_entry show function for buffer %d\n", id);
 #else

pcilib.h

@@ -161,8 +161,8 @@ typedef enum {
 } pcilib_irq_type_t;
 
 typedef enum {
-    PCILIB_DMA_FROM_DEVICE = 1,
-    PCILIB_DMA_TO_DEVICE = 2,
+    PCILIB_DMA_TO_DEVICE = 1,
+    PCILIB_DMA_FROM_DEVICE = 2,
     PCILIB_DMA_BIDIRECTIONAL = 3
 } pcilib_dma_direction_t;
 

pcilib_types.h

@@ -9,9 +9,13 @@
 #define KMEM_MODE_PERSISTENT	0x20000000	/**< Persistent mode instructs kmem_free to preserve buffer in memory */
 #define KMEM_MODE_COUNT		0x0FFFFFFF	/**< Mask of reuse counter (alloc/free) */
 
+#define PCILIB_KMEM_TYPE_MASK	0xFFFF0000
+
 typedef enum {
-    PCILIB_KMEM_TYPE_CONSISTENT = 0,
-    PCILIB_KMEM_TYPE_PAGE,
+    PCILIB_KMEM_TYPE_CONSISTENT = 0x00000,
+    PCILIB_KMEM_TYPE_PAGE = 0x10000,
+    PCILIB_KMEM_TYPE_DMA_S2C_PAGE = 0x10001,
+    PCILIB_KMEM_TYPE_DMA_C2S_PAGE = 0x10002
 } pcilib_kmem_type_t;
 
 typedef enum {
@@ -21,6 +25,7 @@ typedef enum {
 } pcilib_kmem_use_t;
 
 typedef enum {
+    PCILIB_KMEM_SYNC_BIDIRECTIONAL = 0,
     PCILIB_KMEM_SYNC_TODEVICE = 1,
     PCILIB_KMEM_SYNC_FROMDEVICE = 2
 } pcilib_kmem_sync_direction_t;