123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410 |
- #include <stdio.h>
- #include <string.h>
- #include <strings.h>
- #include <stdlib.h>
- #include <stdint.h>
- #include <stdarg.h>
- #include <fcntl.h>
- #include <unistd.h>
- #include <sys/ioctl.h>
- #include <sys/mman.h>
- #include <arpa/inet.h>
- #include <errno.h>
- #include <assert.h>
- #include "pcilib.h"
- #include "pci.h"
- #include "kmem.h"
- #include "error.h"
- int pcilib_clean_kernel_memory(pcilib_t *ctx, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
- kmem_handle_t kh = {0};
- kh.use = use;
- kh.flags = flags|PCILIB_KMEM_FLAG_MASS;
- return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
- }
- static int pcilib_free_kernel_buffer(pcilib_t *ctx, pcilib_kmem_list_t *kbuf, size_t i, pcilib_kmem_flags_t flags) {
- kmem_handle_t kh = {0};
- if (kbuf->buf.blocks[i].ua) munmap((void*)kbuf->buf.blocks[i].ua, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset);
- kh.handle_id = kbuf->buf.blocks[i].handle_id;
- kh.pa = kbuf->buf.blocks[i].pa;
- kh.flags = flags;
-
- return ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
- }
- static void pcilib_cancel_kernel_memory(pcilib_t *ctx, pcilib_kmem_list_t *kbuf, pcilib_kmem_flags_t flags, int last_flags) {
- int ret;
-
- if (!kbuf->buf.n_blocks) return;
- // consistency error during processing of last block, special treatment could be needed
- if (last_flags) {
- pcilib_kmem_flags_t failed_flags = flags;
-
- if (last_flags&KMEM_FLAG_REUSED_PERSISTENT) failed_flags&=~PCILIB_KMEM_FLAG_PERSISTENT;
- if (last_flags&KMEM_FLAG_REUSED_HW) failed_flags&=~PCILIB_KMEM_FLAG_HARDWARE;
-
- if (failed_flags != flags) {
- ret = pcilib_free_kernel_buffer(ctx, kbuf, --kbuf->buf.n_blocks, failed_flags);
- if (ret) pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
- }
- }
- pcilib_free_kernel_memory(ctx, kbuf, flags);
- }
- pcilib_kmem_handle_t *pcilib_alloc_kernel_memory(pcilib_t *ctx, pcilib_kmem_type_t type, size_t nmemb, size_t size, size_t alignment, pcilib_kmem_use_t use, pcilib_kmem_flags_t flags) {
- int err = 0;
- char error[256];
-
- int ret;
- size_t i, allocated = nmemb;
- void *addr;
-
- pcilib_tristate_t reused = PCILIB_TRISTATE_NO;
- int persistent = -1;
- int hardware = -1;
- kmem_handle_t kh = {0};
-
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)malloc(sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
- if (!kbuf) {
- pcilib_error("Memory allocation has failed");
- return NULL;
- }
-
- memset(kbuf, 0, sizeof(pcilib_kmem_list_t) + nmemb * sizeof(pcilib_kmem_addr_t));
- err = pcilib_lock_global(ctx);
- if (err) {
- pcilib_error("Error (%i) acquiring mmap lock", err);
- return NULL;
- }
- ret = ioctl( ctx->handle, PCIDRIVER_IOC_MMAP_MODE, PCIDRIVER_MMAP_KMEM );
- if (ret) {
- pcilib_unlock_global(ctx);
- pcilib_error("PCIDRIVER_IOC_MMAP_MODE ioctl have failed");
- return NULL;
- }
- kh.type = type;
- kh.size = size;
- kh.align = alignment;
- kh.use = use;
- if ((type&PCILIB_KMEM_TYPE_MASK) == PCILIB_KMEM_TYPE_REGION) {
- kh.align = 0;
- } else if ((type&PCILIB_KMEM_TYPE_MASK) != PCILIB_KMEM_TYPE_PAGE) {
- kh.size += alignment;
- }
- for ( i = 0; (i < nmemb)||(flags&PCILIB_KMEM_FLAG_MASS); i++) {
- kh.item = i;
- kh.flags = flags;
- if (i >= nmemb)
- kh.flags |= KMEM_FLAG_TRY;
- if ((type&PCILIB_KMEM_TYPE_MASK) == PCILIB_KMEM_TYPE_REGION) {
- kh.pa = alignment + i * size;
- }
-
- ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_ALLOC, &kh);
- if (ret) {
- kbuf->buf.n_blocks = i;
- if ((i < nmemb)||(errno != ENOENT)) {
- err = PCILIB_ERROR_FAILED;
- if (errno == EINVAL) {
- if (kh.type != type)
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), we have requested type %u but buffer is of type %lu", use, i, type, kh.type);
- else if (kh.size != size)
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), we have requested size %lu but buffer is of size %lu", use, i, size, kh.size);
- else if (kh.align != alignment)
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), we have requested alignment %lu but buffer is of alignment %lu", use, i, size, kh.size);
- else if ((kh.flags&KMEM_FLAG_EXCLUSIVE) != (flags&KMEM_FLAG_EXCLUSIVE))
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), we have requested size %s but buffer is of size %s", use, i, ((flags&KMEM_FLAG_EXCLUSIVE)?"exclusive":"non-exclusive"), ((kh.flags&KMEM_FLAG_EXCLUSIVE)?"exclusive":"non exclusive"));
- else
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), unknown consistency error", use, i);
- } else if (errno == EBUSY) {
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), reuse counter of kmem_entry is overflown", use, i);
- } else if (errno == ENOMEM) {
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), memory allocation (%zu bytes) failed", use, i, size);
- } else {
- sprintf(error, "Driver prevents us from re-using buffer (use 0x%x, block: %zu), PCIDRIVER_IOC_KMEM_ALLOC ioctl have failed with errno %i", use, i, errno);
- }
- }
- break;
- }
-
- if (i >= allocated) {
- void *kbuf_new = realloc(kbuf, sizeof(pcilib_kmem_list_t) + 2 * allocated * sizeof(pcilib_kmem_addr_t));
- if (!kbuf_new) {
- kbuf->buf.n_blocks = i;
- err = PCILIB_ERROR_MEMORY;
- sprintf(error, "Failed to allocate extra %zu bytes of user-space memory for kmem structures", allocated * sizeof(pcilib_kmem_addr_t));
- break;
- }
- memset(kbuf_new + sizeof(pcilib_kmem_list_t) + allocated * sizeof(pcilib_kmem_addr_t) , 0, allocated * sizeof(pcilib_kmem_addr_t));
- kbuf = kbuf_new;
- allocated *= 2;
- }
-
- kbuf->buf.blocks[i].handle_id = kh.handle_id;
- kbuf->buf.blocks[i].pa = kh.pa;
- kbuf->buf.blocks[i].ba = kh.ba;
- kbuf->buf.blocks[i].size = kh.size;
-
- if (!i) reused = (kh.flags&KMEM_FLAG_REUSED)?PCILIB_TRISTATE_YES:PCILIB_TRISTATE_NO;
- if (kh.flags&KMEM_FLAG_REUSED) {
- if (!i) reused = PCILIB_TRISTATE_YES;
- else if (!reused) reused = PCILIB_TRISTATE_PARTIAL;
-
- if (persistent) {
- if (persistent < 0) {
- /*if (((flags&PCILIB_KMEM_FLAG_PERSISTENT) == 0)&&(kh.flags&KMEM_FLAG_REUSED_PERSISTENT)) err = PCILIB_ERROR_INVALID_STATE;
- else*/ persistent = (kh.flags&KMEM_FLAG_REUSED_PERSISTENT)?1:0;
- } else if ((kh.flags&KMEM_FLAG_REUSED_PERSISTENT) == 0) err = PCILIB_ERROR_INVALID_STATE;
- } else if (kh.flags&KMEM_FLAG_REUSED_PERSISTENT) err = PCILIB_ERROR_INVALID_STATE;
- if (err) {
- kbuf->buf.n_blocks = i + 1;
- sprintf(error, "Mistmatch in persistent modes of the re-used kmem blocks. Current buffer (use 0x%x, block: %zu) is %s, but prior ones %s",
- use, i, ((kh.flags&KMEM_FLAG_REUSED_PERSISTENT)?"persistent":"not persistent"), (persistent?"are":"are not"));
- break;
- }
- if (hardware) {
- if (hardware < 0) {
- /*if (((flags&PCILIB_KMEM_FLAG_HARDWARE) == 0)&&(kh.flags&KMEM_FLAG_REUSED_HW)) err = PCILIB_ERROR_INVALID_STATE;
- else*/ hardware = (kh.flags&KMEM_FLAG_REUSED_HW)?1:0;
- } else if ((kh.flags&KMEM_FLAG_REUSED_HW) == 0) err = PCILIB_ERROR_INVALID_STATE;
- } else if (kh.flags&KMEM_FLAG_REUSED_HW) err = PCILIB_ERROR_INVALID_STATE;
- if (err) {
- kbuf->buf.n_blocks = i + 1;
- sprintf(error, "Mistmatch in hardware modes of the re-used kmem blocks. Current buffer (use 0x%x, block: %zu) is %s, but prior ones %s",
- use, i, ((kh.flags&KMEM_FLAG_REUSED_HW)?"hardware-locked":"not hardware-locked"), (hardware?"are":"are not"));
- break;
- }
- } else {
- if (!i) reused = PCILIB_TRISTATE_NO;
- else if (reused) reused = PCILIB_TRISTATE_PARTIAL;
- if ((persistent > 0)&&((flags&PCILIB_KMEM_FLAG_PERSISTENT) == 0)) {
- err = PCILIB_ERROR_INVALID_STATE;
- sprintf(error, "Expecting to re-use persistent blocks, but buffer (use 0x%x, block: %zu) is not", use, i);
- }
- else if ((hardware > 0)&&((flags&PCILIB_KMEM_FLAG_HARDWARE) == 0)) {
- err = PCILIB_ERROR_INVALID_STATE;
- sprintf(error, "Expecting to re-use hardware-locked blocks, but buffer (use 0x%x, block: %zu) is not", use, i);
- }
- if (err) {
- kbuf->buf.n_blocks = i + 1;
- break;
- }
- }
-
- if ((kh.align)&&((kh.type&PCILIB_KMEM_TYPE_MASK) != PCILIB_KMEM_TYPE_PAGE)) {
- // Physical or bus address here?
- if (kh.ba) {
- if (kh.ba % kh.align) kbuf->buf.blocks[i].alignment_offset = kh.align - kh.ba % kh.align;
- } else {
- if (kh.pa % kh.align) kbuf->buf.blocks[i].alignment_offset = kh.align - kh.pa % kh.align;
- }
- kbuf->buf.blocks[i].size -= kh.align;
- }
- addr = mmap( 0, kbuf->buf.blocks[i].size + kbuf->buf.blocks[i].alignment_offset, PROT_WRITE | PROT_READ, MAP_SHARED, ctx->handle, 0 );
- if ((!addr)||(addr == MAP_FAILED)) {
- kbuf->buf.n_blocks = i + 1;
- err = PCILIB_ERROR_FAILED;
- sprintf(error, "Driver prevents us from mmaping buffer (use 0x%x, block: %zu), mmap have failed with errno %i", use, i, errno);
- break;
- }
- kbuf->buf.blocks[i].ua = addr;
- kbuf->buf.blocks[i].mmap_offset = kh.pa & ctx->page_mask;
- }
- if (err) kbuf->buf.n_blocks = i + 1;
- else kbuf->buf.n_blocks = i;
- // Check if there are more unpicked buffers
- if ((!err)&&((flags&PCILIB_KMEM_FLAG_MASS) == 0)&&(reused == PCILIB_TRISTATE_YES)&&((type&PCILIB_KMEM_TYPE_MASK) != PCILIB_KMEM_TYPE_REGION)) {
- kh.item = kbuf->buf.n_blocks;
- kh.flags = KMEM_FLAG_REUSE|KMEM_FLAG_TRY;
- ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_ALLOC, &kh);
- if (!ret) {
- kh.flags = KMEM_FLAG_REUSE;
- ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_FREE, &kh);
- reused = PCILIB_TRISTATE_PARTIAL;
- } else if (errno != ENOENT) {
- reused = PCILIB_TRISTATE_PARTIAL;
- }
- }
- pcilib_unlock_global(ctx);
- //This is possible in the case of error (nothing is allocated yet) or if buffers are not reused
- if (persistent < 0) persistent = 0;
- if (hardware < 0) hardware = 0;
- if (err) {
- // do not clean if we have reused (even partially) persistent/hardware-locked buffers
- if (((persistent)||(hardware))&&(reused != PCILIB_TRISTATE_NO)) {
- pcilib_cancel_kernel_memory(ctx, kbuf, KMEM_FLAG_REUSE, 0);
- } else {
- pcilib_kmem_flags_t free_flags = 0;
- if (flags&PCILIB_KMEM_FLAG_PERSISTENT) {
- free_flags |= PCILIB_KMEM_FLAG_PERSISTENT;
- }
- if (flags&PCILIB_KMEM_FLAG_HARDWARE) {
- free_flags |= PCILIB_KMEM_FLAG_HARDWARE;
- }
- // err indicates consistensy error. The last ioctl have succeeded and we need to clean it in a special way
- pcilib_cancel_kernel_memory(ctx, kbuf, free_flags, (err == PCILIB_ERROR_INVALID_STATE)?kh.flags:0);
- }
- pcilib_warning("Error %i: %s", err, error);
- return NULL;
- }
-
- if (nmemb == 1) {
- memcpy(&kbuf->buf.addr, &kbuf->buf.blocks[0], sizeof(pcilib_kmem_addr_t));
- }
- kbuf->buf.type = type;
- kbuf->buf.use = use;
- kbuf->buf.reused = reused|(persistent?PCILIB_KMEM_REUSE_PERSISTENT:0)|(hardware?PCILIB_KMEM_REUSE_HARDWARE:0);
- kbuf->prev = NULL;
- kbuf->next = ctx->kmem_list;
- if (ctx->kmem_list) ctx->kmem_list->prev = kbuf;
- ctx->kmem_list = kbuf;
- return (pcilib_kmem_handle_t*)kbuf;
- }
- void pcilib_free_kernel_memory(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_flags_t flags) {
- int ret, err = 0;
- int i;
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- // if linked in to the list
- if (kbuf->next) kbuf->next->prev = kbuf->prev;
- if (kbuf->prev) kbuf->prev->next = kbuf->next;
- else if (ctx->kmem_list == kbuf) ctx->kmem_list = kbuf->next;
- for (i = 0; i < kbuf->buf.n_blocks; i++) {
- ret = pcilib_free_kernel_buffer(ctx, kbuf, i, flags);
- if ((ret)&&(!err)) err = ret;
- }
-
- free(kbuf);
-
- if (err) {
- pcilib_error("PCIDRIVER_IOC_KMEM_FREE ioctl have failed");
- }
- }
- /*
- int pcilib_kmem_sync(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_sync_direction_t dir) {
- int i;
- int ret;
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
-
- for (i = 0; i < kbuf->buf.n_blocks; i++) {
- ret = pcilib_kmem_sync_block(ctx, k, dir, i);
- if (ret) {
- pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
- return PCILIB_ERROR_FAILED;
- }
- }
-
- return 0;
- }
- */
- int pcilib_kmem_sync_block(pcilib_t *ctx, pcilib_kmem_handle_t *k, pcilib_kmem_sync_direction_t dir, size_t block) {
- int ret;
- kmem_sync_t ks;
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- switch (kbuf->buf.type) {
- case PCILIB_KMEM_TYPE_DMA_S2C_PAGE:
- case PCILIB_KMEM_TYPE_DMA_C2S_PAGE:
- case PCILIB_KMEM_TYPE_REGION_S2C:
- case PCILIB_KMEM_TYPE_REGION_C2S:
- ks.dir = dir;
- ks.handle.handle_id = kbuf->buf.blocks[block].handle_id;
- ks.handle.pa = kbuf->buf.blocks[block].pa;
-
- ret = ioctl(ctx->handle, PCIDRIVER_IOC_KMEM_SYNC, &ks);
- if (ret) {
- pcilib_error("PCIDRIVER_IOC_KMEM_SYNC ioctl have failed");
- return PCILIB_ERROR_FAILED;
- }
- break;
- default:
- ;
- }
- return 0;
- }
- volatile void *pcilib_kmem_get_ua(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.addr.ua + kbuf->buf.addr.alignment_offset + kbuf->buf.addr.mmap_offset;
- }
- uintptr_t pcilib_kmem_get_pa(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.addr.pa + kbuf->buf.addr.alignment_offset;
- }
- uintptr_t pcilib_kmem_get_ba(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- if (kbuf->buf.addr.ba)
- return kbuf->buf.addr.ba + kbuf->buf.addr.alignment_offset;
- return 0;
- }
- volatile void *pcilib_kmem_get_block_ua(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.blocks[block].ua + kbuf->buf.blocks[block].alignment_offset + kbuf->buf.blocks[block].mmap_offset;
- }
- uintptr_t pcilib_kmem_get_block_pa(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.blocks[block].pa + kbuf->buf.blocks[block].alignment_offset;
- }
- uintptr_t pcilib_kmem_get_block_ba(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- if (kbuf->buf.blocks[block].ba)
- return kbuf->buf.blocks[block].ba + kbuf->buf.blocks[block].alignment_offset;
- return 0;
- }
- size_t pcilib_kmem_get_block_size(pcilib_t *ctx, pcilib_kmem_handle_t *k, size_t block) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.blocks[block].size;
- }
- pcilib_kmem_reuse_state_t pcilib_kmem_is_reused(pcilib_t *ctx, pcilib_kmem_handle_t *k) {
- pcilib_kmem_list_t *kbuf = (pcilib_kmem_list_t*)k;
- return kbuf->buf.reused;
- }
|