Functional DMA transfer FPGA->GPU

include/common.h

@@ -8,6 +8,7 @@
 #define ASSERT_SUCCESS 1
 #define GPU_NAME_LENGTH 30
 #define PAGE_SIZE 4096
+#define GPU_PAGE 65536
 
 void assert_cuda(cudaError_t err_id); /* for runtime api*/
 void assert_cu(CUresult res_id); /* for driver api */

src/main.cu

@@ -24,13 +24,7 @@ int main()
     assert_cu( cuInit(0) );
     gdr_t g = gdr_open();
 
-    /* Try some stuff... */
-    printf("Using binary data to feed FPGA...\n");
-    char* data=(char*)calloc(4096,sizeof(*data));
-    memset(data,0xBB,4096);
-    init_to_send(data);
-    system("/home/mathiasb/sources/gpuFirstComm/launch.sh");
-
+    
     /* Manage NVIDIA GPU */
     printf("\nInitialisation of the GPU\n");
     CUdevice GPU;
@@ -42,35 +36,47 @@ int main()
     /* Check context */
     assert_cu( cuCtxGetDevice(&GPU) );
     CUcontext cuCtx;
-    assert_cu( cuCtxCreate(&cuCtx,CU_CTX_MAP_HOST,GPU) );
-    assert_cu( cuCtxGetDevice(&GPU) );
+    assert_cu( cuCtxCreate(&cuCtx,CU_CTX_MAP_HOST|CU_CTX_SCHED_AUTO,GPU) );
+    assert_cu( cuCtxSetCurrent(cuCtx) );
+
+    
+    /* Try some stuff... */
+    printf("Using binary data to feed FPGA...\n");
+    char* data=(char*)calloc(4096,sizeof(*data));
+    memset(data,0xCC,4096);
+    init_to_send(data);
+    system("/home/mathiasb/sources/gpuFirstComm/launch.sh");
 
+    
     /* Allocate memory on the device, pin and map */
     uint8_t flagValueToSet = 1;
+    
     printf("\nMemory mapping with the GPU for pages\n");
     CUdeviceptr gpuPagePtr;
-    assert_cu( cuMemAlloc(&gpuPagePtr,3*PAGE_SIZE) );
+    assert_cu( cuMemAlloc(&gpuPagePtr,PAGE_SIZE) );
     assert_cu( cuPointerSetAttribute(&flagValueToSet,CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,gpuPagePtr) );
     gdr_mh_t GPUMemHandlePage;
-    assert_gdr( gdr_pin_buffer(g,gpuPagePtr,3*PAGE_SIZE,0,0,&GPUMemHandlePage) );
+    assert_gdr( gdr_pin_buffer(g,gpuPagePtr,PAGE_SIZE,0,0,&GPUMemHandlePage) );
     void* gpuPageVa;
-    assert_gdr( gdr_map(g,GPUMemHandlePage,&gpuPageVa,3*PAGE_SIZE) );
+    assert_gdr( gdr_map(g,GPUMemHandlePage,&gpuPageVa,PAGE_SIZE) );
     gdr_info_t pageInfo;
     assert_gdr( gdr_get_info(g,GPUMemHandlePage,&pageInfo) );
-    printf("Bus ptr = %lx\nVA = 0x%lx\nSize = %lu\n",pageInfo.bus_addr,pageInfo.va,pageInfo.mapped_size);
+    printf("Bus ptr = %lx\nVA = %lx\nSize = %lu\n",pageInfo.bus_addr,pageInfo.va,pageInfo.mapped_size);
+    
+
     printf("Memory mapping with the GPU for descriptors\n");
     CUdeviceptr gpuDescPtr;
-    assert_cu( cuMemAlloc(&gpuDescPtr,PAGE_SIZE) );
+    assert_cu( cuMemAlloc(&gpuDescPtr,GPU_PAGE) );       
     assert_cu( cuPointerSetAttribute(&flagValueToSet,CU_POINTER_ATTRIBUTE_SYNC_MEMOPS,gpuDescPtr) );
     gdr_mh_t GPUMemHandleDesc;
-    assert_gdr( gdr_pin_buffer(g,gpuDescPtr,PAGE_SIZE,0,0,&GPUMemHandleDesc) );
+    assert_gdr( gdr_pin_buffer(g,gpuDescPtr,GPU_PAGE,0,0,&GPUMemHandleDesc) );
     void* gpuDescVa;
-    assert_gdr( gdr_map(g,GPUMemHandleDesc,&gpuDescVa,PAGE_SIZE) );
-    
-    printf("All set\n");
+    assert_gdr( gdr_map(g,GPUMemHandleDesc,&gpuDescVa,GPU_PAGE) );
     gdr_info_t descInfo;
     assert_gdr( gdr_get_info(g,GPUMemHandleDesc,&descInfo) );
-    printf("Bus ptr = %lx\nVA = 0x%lx\nSize = %lu\n",descInfo.bus_addr,descInfo.va,descInfo.mapped_size);
+    printf("Bus ptr = %lx\nVA = %lx\nSize = %lu\n",descInfo.bus_addr,descInfo.va,descInfo.mapped_size);
+    
+    printf("All set\n");
     
     /* PCI */
     printf("\nSetting up the PCI\n");
@@ -92,6 +98,7 @@ int main()
     CUdeviceptr dBAR;
     assert_cu( cuMemHostRegister((void*)pciVa,128,CU_MEMHOSTREGISTER_IOMEMORY) );
     assert_cu( cuMemHostGetDevicePointer(&dBAR,(void*)pciVa, 0) );
+
     
     /* Config PCI for Pages*/
     pcilib_kmem_handle_t* pciHandlePage;
@@ -103,7 +110,7 @@ int main()
     }
     volatile void* pciMemPtrPage;
     uintptr_t pciBusPage;
-    pciMemPtrPage = (uint32_t*) pcilib_kmem_get_block_ua(pciCtx,pciHandlePage,0);
+    pciMemPtrPage = (uint64_t*) pcilib_kmem_get_block_ua(pciCtx,pciHandlePage,0);
     if( pciMemPtrPage == NULL )
     {
 	printf("Cannot get PCI pointer to kernel memory\n");
@@ -118,7 +125,7 @@ int main()
 
     /* Config PCI for Desc */
     pcilib_kmem_handle_t* pciHandleDesc;
-    pciHandleDesc = pcilib_alloc_kernel_memory(pciCtx,PCILIB_KMEM_TYPE_CONSISTENT, 1, 128, 4096, KMEM_USE_RING, KMEM_DEFAULT_FLAGS);
+    pciHandleDesc = pcilib_alloc_kernel_memory(pciCtx,PCILIB_KMEM_TYPE_CONSISTENT, 1, 128, 4096,KMEM_USE_RING, KMEM_DEFAULT_FLAGS);
     if( pciHandleDesc == NULL )
     {
 	printf("Cannot allocate  PCI kernel memory\n");
@@ -126,7 +133,7 @@ int main()
     }
     volatile void* pciMemPtrDesc;
     uintptr_t pciBusDesc;
-    pciMemPtrDesc = (uint32_t*) pcilib_kmem_get_block_ua(pciCtx,pciHandleDesc,0);
+    pciMemPtrDesc = (uint64_t*) pcilib_kmem_get_block_ua(pciCtx,pciHandleDesc,0);
     if( pciMemPtrDesc == NULL )
     {
 	printf("Cannot get PCI pointer to kernel memory\n");
@@ -138,6 +145,8 @@ int main()
 	printf("Cannot get PCI Bus address on kernel memory\n");
 	exit( EXIT_FAILURE );
     }
+    printf("pciMemPtrPage = %lx\npciMemPtrDesc = %lx\n",(uint64_t)pciMemPtrPage,(uint64_t)pciMemPtrDesc);
+
     
     /* FPGA */
     printf("\nWorking on the FPGA\n");
@@ -147,29 +156,36 @@ int main()
     usleep(100000);
     WR32_sleep(REG_NUM_PACKETS_PER_DESCRIPTOR,16);
     WR32_sleep(REG_PACKET_LENGTH,64);
+    WR32_sleep(REG_TIMER_THRESHOLD, 0x1);
     WR32_sleep(REG_UPDATE_THRESHOLD, 0x1);
-    /* WR64_sleep(REG_UPDATE_COUNTER,descInfo.bus_addr); */
+/*    WR64_sleep(REG_UPDATE_COUNTER,pciBusDesc);*/
     WR64_sleep(REG_UPDATE_ADDRESS,pciBusPage+DESCRIPTOR_OFFSET);
     WR32_sleep(REG_CONTROL,CONTROL_ENABLE_READ|CONTROL_SOURCE_RX_FIFO);
-    WR64_sleep(REG_DESCRIPTOR_ADDRESS,descInfo.bus_addr);
+    /* WR64_sleep(REG_DESCRIPTOR_ADDRESS,descInfo.bus_addr); */
     WR32_sleep(REG_DMA,1);
     WR32_sleep(REG_INTERCONNECT, 0x262);
-    /* WR32_sleep(REG_COUNTER,1); */
+    WR32_sleep(REG_COUNTER,1);
     usleep(100000);
 
-    WR64_sleep(REG_DESCRIPTOR_ADDRESS,descInfo.bus_addr);
-    printf("pciVa = %x\npciMemPtrPage = %lx\npciMemPtrDesc = %lx\n",pciVa,*(uint64_t*)pciMemPtrPage,*(uint64_t*)pciMemPtrDesc);
+    WR64_sleep(REG_DESCRIPTOR_ADDRESS,pciBusDesc);
+    /* usleep(100000); */
+    printf("pageInfo.bus_addr = %lx\ndescInfo.bus_addr = %lx\npciBusPage = %lx\npciBusDesc = %lx\n",pageInfo.bus_addr,descInfo.bus_addr,pciBusPage,pciBusDesc);
+    printf("gpuDescVa = %lx\ngpuPageVa = %lx\n",(uint64_t)gpuDescVa,(uint64_t)gpuPageVa);
+    printf("pciVa = %x\npciMemPtrPage = %lx\npciMemPtrDesc = %lx\n",pciVa,(uint64_t)pciMemPtrPage,(uint64_t)pciMemPtrDesc);
 
-    
+    /* assert_cu( cuMemcpyHtoD(gpuDescPtr,(const void*)data,4096) ); */
+    /* uint64_t* trackAddr = (uint64_t*); */
+    /* printf("Data were written at %lx\n",*trackAddr); */
     char* getBack=(char*)calloc(4096,sizeof(*getBack));
-    memcpy(getBack,(const void*)gpuDescVa,4096);
+    memcpy(getBack,(const void*)pciMemPtrDesc,4096);
     int i;
     for(i=0;i<4096;i++)
     {
     	printf("%hhx",getBack[i]);
     }
     printf("\n");
-    
+    free(getBack);
+
     
     /* Close everything */
     printf("\nClosing the connections\n");
@@ -180,17 +196,18 @@ int main()
     WR32(REG_RESET_DMA, 0);
     usleep (100000);
 
-    pcilib_free_kernel_memory(pciCtx,pciHandleDesc,(pcilib_kmem_flags_t)(PCILIB_KMEM_FLAG_PERSISTENT|PCILIB_KMEM_FLAG_HARDWARE));
-    pcilib_free_kernel_memory(pciCtx,pciHandlePage,(pcilib_kmem_flags_t)(PCILIB_KMEM_FLAG_PERSISTENT|PCILIB_KMEM_FLAG_HARDWARE));
+    pcilib_free_kernel_memory(pciCtx,pciHandleDesc,PCILIB_KMEM_FLAG_FORCE);
+    pcilib_free_kernel_memory(pciCtx,pciHandlePage,PCILIB_KMEM_FLAG_FORCE);
     assert_cu( cuMemHostUnregister((void*) pciVa) );
     pcilib_close(pciCtx);
-    assert_gdr( gdr_unmap(g,GPUMemHandlePage,gpuPageVa,3*PAGE_SIZE) );
+    assert_gdr( gdr_unmap(g,GPUMemHandlePage,gpuPageVa,PAGE_SIZE) );
     assert_gdr( gdr_unpin_buffer(g,GPUMemHandlePage) );
-    assert_gdr( gdr_unmap(g,GPUMemHandleDesc,gpuDescVa,3*PAGE_SIZE) );
+    assert_gdr( gdr_unmap(g,GPUMemHandleDesc,gpuDescVa,GPU_PAGE) );
     assert_gdr( gdr_unpin_buffer(g,GPUMemHandleDesc) );
     assert_gdr( gdr_close(g) );
     assert_cu( cuMemFree(gpuPagePtr) );
     assert_cu( cuMemFree(gpuDescPtr) );
+    assert_cu( cuCtxDestroy(cuCtx) );
     
     printf("All Cleared\n");