123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209 |
- #define _PCILIB_DMA_IPE_C
- #define _BSD_SOURCE
- #define _DEFAULT_SOURCE
- #define _POSIX_C_SOURCE 200112L
- #include <stdio.h>
- #include <stdlib.h>
- #include <string.h>
- #include <unistd.h>
- #include <sched.h>
- #include <sys/time.h>
- #include <arpa/inet.h>
- #include "pci.h"
- #include "pcilib.h"
- #include "error.h"
- #include "tools.h"
- #include "debug.h"
- #include "ipe.h"
- #include "ipe_private.h"
- typedef struct {
- size_t size;
- size_t pos;
- pcilib_dma_flags_t flags;
- } dma_ipe_skim_callback_context_t;
- static int dma_ipe_skim_callback(void *arg, pcilib_dma_flags_t flags, size_t bufsize, void *buf) {
- dma_ipe_skim_callback_context_t *ctx = (dma_ipe_skim_callback_context_t*)arg;
- ctx->pos += bufsize;
- if (flags & PCILIB_DMA_FLAG_EOP) {
- if ((ctx->pos < ctx->size)&&(ctx->flags&PCILIB_DMA_FLAG_MULTIPACKET)) {
- if (ctx->flags&PCILIB_DMA_FLAG_WAIT) return PCILIB_STREAMING_WAIT;
- else return PCILIB_STREAMING_CONTINUE;
- }
- return PCILIB_STREAMING_STOP;
- }
- return PCILIB_STREAMING_REQ_FRAGMENT;
- }
- int dma_ipe_skim_dma_custom(pcilib_t *ctx, pcilib_dma_engine_t dma, uintptr_t addr, size_t size, pcilib_dma_flags_t flags, pcilib_timeout_t timeout, void *buf, size_t *read_bytes) {
- int err;
- dma_ipe_skim_callback_context_t opts = {
- size, 0, flags
- };
- err = pcilib_stream_dma(ctx, dma, addr, size, flags, timeout, dma_ipe_skim_callback, &opts);
- if (read_bytes) *read_bytes = opts.pos;
- return err;
- }
- double dma_ipe_benchmark(pcilib_dma_context_t *vctx, pcilib_dma_engine_addr_t dma, uintptr_t addr, size_t size, size_t iterations, pcilib_dma_direction_t direction) {
- int err = 0;
- ipe_dma_t *ctx = (ipe_dma_t*)vctx;
- int iter;
- size_t us = 0;
- struct timeval start, cur;
-
- void *buf;
- size_t bytes, rbytes;
- int (*read_dma)(pcilib_t *ctx, pcilib_dma_engine_t dma, uintptr_t addr, size_t size, pcilib_dma_flags_t flags, pcilib_timeout_t timeout, void *buf, size_t *read_bytes);
- if ((direction == PCILIB_DMA_TO_DEVICE)||(direction == PCILIB_DMA_BIDIRECTIONAL)) return -1.;
- if ((dma != PCILIB_DMA_ENGINE_INVALID)&&(dma > 1)) return -1.;
- err = dma_ipe_start(vctx, 0, PCILIB_DMA_FLAGS_DEFAULT);
- if (err) return err;
- if (size%ctx->page_size) size = (1 + size / ctx->page_size) * ctx->page_size;
- if (getenv("PCILIB_BENCHMARK_HARDWARE"))
- read_dma = dma_ipe_skim_dma_custom;
- else
- read_dma = pcilib_read_dma_custom;
- // There is no significant difference and we can remove this when testing phase is over.
- // DS: With large number of buffers this is quite slow due to skimming of initially written buffers
- if (getenv("PCILIB_BENCHMARK_STREAMING")) {
- size_t dma_buffer_space;
- pcilib_dma_engine_status_t dma_status;
-
- if (read_dma == pcilib_read_dma_custom)
- pcilib_info_once("Benchmarking the DMA streaming (with memcpy)");
- else
- pcilib_info_once("Benchmarking the DMA streaming (without memcpy)");
- // Starting DMA
- WR(IPEDMA_REG_CONTROL, 0x1);
- gettimeofday(&start, NULL);
- pcilib_calc_deadline(&start, ctx->dma_timeout * IPEDMA_DMA_PAGES);
- #ifdef IPEDMA_BUG_LAST_READ
- dma_buffer_space = (IPEDMA_DMA_PAGES - 2) * ctx->page_size;
- #else /* IPEDMA_BUG_LAST_READ */
- dma_buffer_space = (IPEDMA_DMA_PAGES - 1) * ctx->page_size;
- #endif /* IPEDMA_BUG_LAST_READ */
- // Allocate memory and prepare data
- err = posix_memalign(&buf, 4096, size + dma_buffer_space);
- if ((err)||(!buf)) return -1;
- // Wait all DMA buffers are filled
- memset(&dma_status, 0, sizeof(dma_status));
- do {
- usleep(10 * IPEDMA_NODATA_SLEEP);
- err = dma_ipe_get_status(vctx, dma, &dma_status, 0, NULL);
- } while ((!err)&&(dma_status.written_bytes < dma_buffer_space)&&(pcilib_calc_time_to_deadline(&start) > 0));
- if (err) {
- pcilib_error("Error (%i) getting dma status", err);
- return -1;
- } else if (dma_status.written_bytes < dma_buffer_space) {
- pcilib_error("Timeout while waiting DMA engine to feel the buffer space completely, only %zu bytes of %zu written", dma_status.written_bytes, dma_buffer_space);
- return -1;
- }
- gettimeofday(&start, NULL);
- for (iter = 0; iter < iterations; iter++) {
- for (bytes = 0; bytes < (size + dma_buffer_space); bytes += rbytes) {
- err = read_dma(ctx->dmactx.pcilib, 0, addr, size + dma_buffer_space - bytes, PCILIB_DMA_FLAG_MULTIPACKET, ctx->dma_timeout, buf + bytes, &rbytes);
- if (err) {
- pcilib_error("Can't read data from DMA, error %i", err);
- return -1;
- }
- }
- dma_buffer_space = 0;
- }
- gettimeofday(&cur, NULL);
- us += ((cur.tv_sec - start.tv_sec)*1000000 + (cur.tv_usec - start.tv_usec));
- // Stopping DMA
- WR(IPEDMA_REG_CONTROL, 0x0);
- usleep(IPEDMA_RESET_DELAY);
-
- pcilib_skip_dma(ctx->dmactx.pcilib, 0);
- } else {
- if (read_dma == dma_ipe_skim_dma_custom)
- pcilib_info_once("Benchmarking the DMA hardware (without memcpy)");
- WR(IPEDMA_REG_CONTROL, 0x0);
- usleep(IPEDMA_RESET_DELAY);
- err = pcilib_skip_dma(ctx->dmactx.pcilib, 0);
- if (err) {
- pcilib_error("Can't start benchmark, devices continuously writes unexpected data using DMA engine");
- return -1;
- }
- // Allocate memory and prepare data
- err = posix_memalign(&buf, 4096, size);
- if ((err)||(!buf)) return -1;
- for (iter = 0; iter <= iterations; iter++) {
- gettimeofday(&start, NULL);
- // Starting DMA
- WR(IPEDMA_REG_CONTROL, 0x1);
-
- for (bytes = 0; bytes < size; bytes += rbytes) {
- err = read_dma(ctx->dmactx.pcilib, 0, addr, size - bytes, PCILIB_DMA_FLAG_MULTIPACKET, ctx->dma_timeout, buf + bytes, &rbytes);
- if (err) {
- pcilib_error("Can't read data from DMA (iteration: %zu, offset: %zu), error %i", iter, bytes, err);
- return -1;
- }
- }
- gettimeofday(&cur, NULL);
- // Stopping DMA
- WR(IPEDMA_REG_CONTROL, 0x0);
- usleep(IPEDMA_RESET_DELAY);
- if (err) break;
-
- // Heating up during the first iteration
- if (iter)
- us += ((cur.tv_sec - start.tv_sec)*1000000 + (cur.tv_usec - start.tv_usec));
- pcilib_info("Iteration %-4i latency: %lu", iter, ((cur.tv_sec - start.tv_sec)*1000000 + (cur.tv_usec - start.tv_usec)));
- err = pcilib_skip_dma(ctx->dmactx.pcilib, 0);
- if (err) {
- pcilib_error("Can't start iteration, devices continuously writes unexpected data using DMA engine");
- break;
- }
-
- usleep(ctx->dma_timeout);
- }
- }
- free(buf);
- return err?-1:((1. * size * iterations * 1000000) / (1024. * 1024. * us));
- }
|