umem.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368
  1. /**
  2. *
  3. * @file umem.c
  4. * @brief This file contains the functions handling user space memory.
  5. * @author Guillermo Marcus
  6. * @date 2009-04-05
  7. *
  8. */
  9. #include <linux/version.h>
  10. #include <linux/string.h>
  11. #include <linux/types.h>
  12. #include <linux/list.h>
  13. #include <linux/interrupt.h>
  14. #include <linux/pci.h>
  15. #include <linux/cdev.h>
  16. #include <linux/wait.h>
  17. #include <linux/mm.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/sched.h>
  20. #include "base.h"
  21. /**
  22. *
  23. * Reserve a new scatter/gather list and map it from memory to PCI bus addresses.
  24. *
  25. */
  26. int pcidriver_umem_sgmap(pcidriver_privdata_t *privdata, umem_handle_t *umem_handle)
  27. {
  28. int i, res, nr_pages;
  29. struct page **pages;
  30. struct scatterlist *sg = NULL;
  31. pcidriver_umem_entry_t *umem_entry;
  32. unsigned int nents;
  33. unsigned long count,offset,length;
  34. /*
  35. * We do some checks first. Then, the following is necessary to create a
  36. * Scatter/Gather list from a user memory area:
  37. * - Determine the number of pages
  38. * - Get the pages for the memory area
  39. * - Lock them.
  40. * - Create a scatter/gather list of the pages
  41. * - Map the list from memory to PCI bus addresses
  42. *
  43. * Then, we:
  44. * - Create an entry on the umem list of the device, to cache the mapping.
  45. * - Create a sysfs attribute that gives easy access to the SG list
  46. */
  47. /* zero-size?? */
  48. if (umem_handle->size == 0)
  49. return -EINVAL;
  50. /* Direction is better ignoring during mapping. */
  51. /* We assume bidirectional buffers always, except when sync'ing */
  52. /* calculate the number of pages */
  53. nr_pages = ((umem_handle->vma & ~PAGE_MASK) + umem_handle->size + ~PAGE_MASK) >> PAGE_SHIFT;
  54. mod_info_dbg("nr_pages computed: %u\n", nr_pages);
  55. /* Allocate space for the page information */
  56. /* This can be very big, so we use vmalloc */
  57. if ((pages = vmalloc(nr_pages * sizeof(*pages))) == NULL)
  58. return -ENOMEM;
  59. mod_info_dbg("allocated space for the pages.\n");
  60. /* Allocate space for the scatterlist */
  61. /* We do not know how many entries will be, but the maximum is nr_pages. */
  62. /* This can be very big, so we use vmalloc */
  63. if ((sg = vmalloc(nr_pages * sizeof(*sg))) == NULL)
  64. goto umem_sgmap_pages;
  65. sg_init_table(sg, nr_pages);
  66. mod_info_dbg("allocated space for the SG list.\n");
  67. /* Get the page information */
  68. down_read(&current->mm->mmap_sem);
  69. res = get_user_pages(
  70. current,
  71. current->mm,
  72. umem_handle->vma,
  73. nr_pages,
  74. 1,
  75. 0, /* do not force, FIXME: shall I? */
  76. pages,
  77. NULL );
  78. up_read(&current->mm->mmap_sem);
  79. /* Error, not all pages mapped */
  80. if (res < (int)nr_pages) {
  81. mod_info("Could not map all user pages (%d of %d)\n", res, nr_pages);
  82. /* If only some pages could be mapped, we release those. If a real
  83. * error occured, we set nr_pages to 0 */
  84. nr_pages = (res > 0 ? res : 0);
  85. goto umem_sgmap_unmap;
  86. }
  87. mod_info_dbg("Got the pages (%d).\n", res);
  88. /* Lock the pages, then populate the SG list with the pages */
  89. /* page0 is different */
  90. if ( !PageReserved(pages[0]) )
  91. __set_page_locked(pages[0]);
  92. offset = (umem_handle->vma & ~PAGE_MASK);
  93. length = (umem_handle->size > (PAGE_SIZE-offset) ? (PAGE_SIZE-offset) : umem_handle->size);
  94. sg_set_page(&sg[0], pages[0], length, offset);
  95. count = umem_handle->size - length;
  96. for(i=1; i<nr_pages; i++) {
  97. /* Lock page first */
  98. if ( !PageReserved(pages[i]) )
  99. __set_page_locked(pages[i]);
  100. /* Populate the list */
  101. sg_set_page(&sg[i], pages[i], ((count > PAGE_SIZE) ? PAGE_SIZE : count), 0);
  102. count -= sg[i].length;
  103. }
  104. /* Use the page list to populate the SG list */
  105. /* SG entries may be merged, res is the number of used entries */
  106. /* We have originally nr_pages entries in the sg list */
  107. if ((nents = pci_map_sg(privdata->pdev, sg, nr_pages, PCI_DMA_BIDIRECTIONAL)) == 0)
  108. goto umem_sgmap_unmap;
  109. mod_info_dbg("Mapped SG list (%d entries).\n", nents);
  110. /* Add an entry to the umem_list of the device, and update the handle with the id */
  111. /* Allocate space for the new umem entry */
  112. if ((umem_entry = kmalloc(sizeof(*umem_entry), GFP_KERNEL)) == NULL)
  113. goto umem_sgmap_entry;
  114. /* Fill entry to be added to the umem list */
  115. umem_entry->id = atomic_inc_return(&privdata->umem_count) - 1;
  116. umem_entry->nr_pages = nr_pages; /* Will be needed when unmapping */
  117. umem_entry->pages = pages;
  118. umem_entry->nents = nents;
  119. umem_entry->sg = sg;
  120. if (pcidriver_sysfs_initialize_umem(privdata, umem_entry->id, &(umem_entry->sysfs_attr)) != 0)
  121. goto umem_sgmap_name_fail;
  122. /* Add entry to the umem list */
  123. spin_lock( &(privdata->umemlist_lock) );
  124. list_add_tail( &(umem_entry->list), &(privdata->umem_list) );
  125. spin_unlock( &(privdata->umemlist_lock) );
  126. /* Update the Handle with the Handle ID of the entry */
  127. umem_handle->handle_id = umem_entry->id;
  128. return 0;
  129. umem_sgmap_name_fail:
  130. kfree(umem_entry);
  131. umem_sgmap_entry:
  132. pci_unmap_sg( privdata->pdev, sg, nr_pages, PCI_DMA_BIDIRECTIONAL );
  133. umem_sgmap_unmap:
  134. /* release pages */
  135. if (nr_pages > 0) {
  136. for(i=0; i<nr_pages; i++) {
  137. if (PageLocked(pages[i]))
  138. __clear_page_locked(pages[i]);
  139. if (!PageReserved(pages[i]))
  140. set_page_dirty(pages[i]);
  141. page_cache_release(pages[i]);
  142. }
  143. }
  144. vfree(sg);
  145. umem_sgmap_pages:
  146. vfree(pages);
  147. return -ENOMEM;
  148. }
  149. /**
  150. *
  151. * Unmap a scatter/gather list
  152. *
  153. */
  154. int pcidriver_umem_sgunmap(pcidriver_privdata_t *privdata, pcidriver_umem_entry_t *umem_entry)
  155. {
  156. int i;
  157. pcidriver_sysfs_remove(privdata, &(umem_entry->sysfs_attr));
  158. /* Unmap user memory */
  159. pci_unmap_sg( privdata->pdev, umem_entry->sg, umem_entry->nr_pages, PCI_DMA_BIDIRECTIONAL );
  160. /* Release the pages */
  161. if (umem_entry->nr_pages > 0) {
  162. for(i=0; i<(umem_entry->nr_pages); i++) {
  163. /* Mark pages as Dirty and unlock it */
  164. if ( !PageReserved( umem_entry->pages[i] )) {
  165. SetPageDirty( umem_entry->pages[i] );
  166. __clear_page_locked(umem_entry->pages[i]);
  167. }
  168. /* and release it from the cache */
  169. page_cache_release( umem_entry->pages[i] );
  170. }
  171. }
  172. /* Remove the umem list entry */
  173. spin_lock( &(privdata->umemlist_lock) );
  174. list_del( &(umem_entry->list) );
  175. spin_unlock( &(privdata->umemlist_lock) );
  176. /* Release SG list and page list memory */
  177. /* These two are in the vm area of the kernel */
  178. vfree(umem_entry->pages);
  179. vfree(umem_entry->sg);
  180. /* Release umem_entry memory */
  181. kfree(umem_entry);
  182. return 0;
  183. }
  184. /**
  185. *
  186. * Unmap all scatter/gather lists.
  187. *
  188. */
  189. int pcidriver_umem_sgunmap_all(pcidriver_privdata_t *privdata)
  190. {
  191. struct list_head *ptr, *next;
  192. pcidriver_umem_entry_t *umem_entry;
  193. /* iterate safely over the entries and delete them */
  194. list_for_each_safe( ptr, next, &(privdata->umem_list) ) {
  195. umem_entry = list_entry(ptr, pcidriver_umem_entry_t, list );
  196. pcidriver_umem_sgunmap( privdata, umem_entry ); /* spin lock inside! */
  197. }
  198. return 0;
  199. }
  200. /**
  201. *
  202. * Copies the scatter/gather list from kernelspace to userspace.
  203. *
  204. */
  205. int pcidriver_umem_sgget(pcidriver_privdata_t *privdata, umem_sglist_t *umem_sglist)
  206. {
  207. int i;
  208. pcidriver_umem_entry_t *umem_entry;
  209. struct scatterlist *sg;
  210. int idx = 0;
  211. dma_addr_t cur_addr;
  212. unsigned int cur_size;
  213. /* Find the associated umem_entry for this buffer */
  214. umem_entry = pcidriver_umem_find_entry_id( privdata, umem_sglist->handle_id );
  215. if (umem_entry == NULL)
  216. return -EINVAL; /* umem_handle is not valid */
  217. /* Check if passed SG list is enough */
  218. if (umem_sglist->nents < umem_entry->nents)
  219. return -EINVAL; /* sg has not enough entries */
  220. /* Copy the SG list to the user format */
  221. if (umem_sglist->type == PCIDRIVER_SG_MERGED) {
  222. for_each_sg(umem_entry->sg, sg, umem_entry->nents, i ) {
  223. if (i==0) {
  224. umem_sglist->sg[0].addr = sg_dma_address( sg );
  225. umem_sglist->sg[0].size = sg_dma_len( sg );
  226. idx = 0;
  227. }
  228. else {
  229. cur_addr = sg_dma_address( sg );
  230. cur_size = sg_dma_len( sg );
  231. /* Check if entry fits after current entry */
  232. if (cur_addr == (umem_sglist->sg[idx].addr + umem_sglist->sg[idx].size)) {
  233. umem_sglist->sg[idx].size += cur_size;
  234. continue;
  235. }
  236. /* Skip if the entry is zero-length (yes, it can happen.... at the end of the list) */
  237. if (cur_size == 0)
  238. continue;
  239. /* None of the above, add new entry */
  240. idx++;
  241. umem_sglist->sg[idx].addr = cur_addr;
  242. umem_sglist->sg[idx].size = cur_size;
  243. }
  244. }
  245. /* Set the used size of the SG list */
  246. umem_sglist->nents = idx+1;
  247. } else {
  248. for_each_sg(umem_entry->sg, sg, umem_entry->nents, i ) {
  249. mod_info("entry: %d\n",i);
  250. umem_sglist->sg[i].addr = sg_dma_address( sg );
  251. umem_sglist->sg[i].size = sg_dma_len( sg );
  252. }
  253. /* Set the used size of the SG list */
  254. /* Check if the last one is zero-length */
  255. if ( umem_sglist->sg[ umem_entry->nents - 1].size == 0)
  256. umem_sglist->nents = umem_entry->nents -1;
  257. else
  258. umem_sglist->nents = umem_entry->nents;
  259. }
  260. return 0;
  261. }
  262. /**
  263. *
  264. * Sync user space memory from/to device
  265. *
  266. */
  267. int pcidriver_umem_sync( pcidriver_privdata_t *privdata, umem_handle_t *umem_handle )
  268. {
  269. pcidriver_umem_entry_t *umem_entry;
  270. /* Find the associated umem_entry for this buffer */
  271. umem_entry = pcidriver_umem_find_entry_id( privdata, umem_handle->handle_id );
  272. if (umem_entry == NULL)
  273. return -EINVAL; /* umem_handle is not valid */
  274. switch (umem_handle->dir) {
  275. case PCIDRIVER_DMA_TODEVICE:
  276. pci_dma_sync_sg_for_device( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_TODEVICE );
  277. break;
  278. case PCIDRIVER_DMA_FROMDEVICE:
  279. pci_dma_sync_sg_for_cpu( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_FROMDEVICE );
  280. break;
  281. case PCIDRIVER_DMA_BIDIRECTIONAL:
  282. pci_dma_sync_sg_for_device( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_BIDIRECTIONAL );
  283. pci_dma_sync_sg_for_cpu( privdata->pdev, umem_entry->sg, umem_entry->nents, PCI_DMA_BIDIRECTIONAL );
  284. break;
  285. default:
  286. return -EINVAL; /* wrong direction parameter */
  287. }
  288. return 0;
  289. }
  290. /*
  291. *
  292. * Get the pcidriver_umem_entry_t structure for the given id.
  293. *
  294. * @param id ID of the umem entry to search for
  295. *
  296. */
  297. pcidriver_umem_entry_t *pcidriver_umem_find_entry_id(pcidriver_privdata_t *privdata, int id)
  298. {
  299. struct list_head *ptr;
  300. pcidriver_umem_entry_t *entry;
  301. spin_lock(&(privdata->umemlist_lock));
  302. list_for_each(ptr, &(privdata->umem_list)) {
  303. entry = list_entry(ptr, pcidriver_umem_entry_t, list );
  304. if (entry->id == id) {
  305. spin_unlock( &(privdata->umemlist_lock) );
  306. return entry;
  307. }
  308. }
  309. spin_unlock(&(privdata->umemlist_lock));
  310. return NULL;
  311. }