pciutils/ddrio.c

#define _XOPEN_SOURCE   500

#include <stdio.h>
#include <stdlib.h>
#include <stdbool.h>
#include <string.h>
#include <unistd.h>
#include <getopt.h>
#include <time.h>
#include <assert.h>
#include <pcilib.h>
#include <pcilib/bar.h>
#include <pcilib/kmem.h>

#include "hf-interface.h"

/* this should actually come from the distributed pcitool sources */
#include "pciDriver.h"


#ifndef NDEBUG
#define WR32(addr, value) *(uint32_t *) (bar + (addr)) = (value);\
        printf ("WR32N %4x  <- %x\n", (addr), (value));
#define WR32_sleep(addr, value) *(uint32_t *) (bar + (addr)) = (value); usleep (1000);\
        printf ("WR32S %4x  <- %x\n", (addr), (value));
#define WR64(addr, value) *(uint64_t *) (bar + (addr)) = (value);\
        printf ("WR64N %4x  <- %x\n", (addr), (value));
#define WR64_sleep(addr, value) *(uint64_t *) (bar + (addr)) = (value); usleep (1000);\
        printf ("WR64S %4x  <- %x\n", (addr), (value));
#else
#define WR32(addr, value) *(uint32_t *) (bar + (addr)) = (value);
#define WR32_sleep(addr, value) *(uint32_t *) (bar + (addr)) = (value); usleep (1000);
#define WR64(addr, value) *(uint64_t *) (bar + (addr)) = (value);
#define WR64_sleep(addr, value) *(uint64_t *) (bar + (addr)) = (value); usleep (1000);
#endif

#define RD32(addr) (*(uint32_t *) (bar + (addr)))
#define RD64(addr) (*(uint64_t *) (bar + (addr)))

#define KMEM_USE_RING           PCILIB_KMEM_USE(PCILIB_KMEM_USE_USER, 1)
#define KMEM_DEFAULT_FLAGS      PCILIB_KMEM_FLAG_HARDWARE | \
                                PCILIB_KMEM_FLAG_PERSISTENT | \
                                PCILIB_KMEM_FLAG_EXCLUSIVE

typedef struct {
    const char *input;
    const char *output;
    bool keep;
    bool verbose;
    size_t size;
    enum {
        COPY_UINT64 = 0,
        COPY_MEMCPY,
    } copy;
    int number;
} Options;


static void
usage (void)
{
    printf ("Usage: ddr [OPTION] [FILE]\n"
            "Options:\n"
            "  -h, --help       Show this help message and exit\n"
            "  -v, --verbose    Be more verbose\n"
            "  -k, --keep-data  Keep data and don't reset DMA\n"
            "  -o, --output     Output filename\n"
            "  -i, --input      Input filename\n"
            "  -s, --size       Size of data (leave if reading supplied input)\n"
            "  -n, --number     Number of times data should be read\n"
            "  -c, --copy       Copy method, either `memcpy' or `64'\n");
}

static void
parse_options (int argc, char *const *argv, Options *opts)
{
    enum {
        OPT_HELP = 'h',
        OPT_VERBOSE = 'v',
        OPT_KEEP = 'k',
        OPT_INPUT = 'i',
        OPT_OUTPUT = 'o',
        OPT_SIZE = 's',
        OPT_COPY = 'c',
        OPT_NUMBER = 'n',
    };

    static struct option long_options[] = {
        { "help", no_argument, 0, OPT_HELP },
        { "verbose", no_argument, 0, OPT_VERBOSE },
        { "keep", no_argument, 0, OPT_KEEP },
        { "input", required_argument, 0, OPT_INPUT },
        { "output", required_argument, 0, OPT_OUTPUT },
        { "size", required_argument, 0, OPT_SIZE },
        { "copy", required_argument, 0, OPT_COPY },
        { "number", 0, 0, OPT_NUMBER },
        { 0, 0, 0, 0 },
    };

    int ret;
    int index;

    if (argc == 1) {
        usage ();
        exit (0);
    }

    memset (opts, 0, sizeof (Options));
    opts->number = 1;

    while ((ret = getopt_long (argc, argv, "i:o:s:c:n:khv", long_options, &index)) != -1) {
        switch (ret) {
            case OPT_HELP:
                usage ();
                exit (0);
            case OPT_VERBOSE:
                opts->verbose = true;
                break;
            case OPT_KEEP:
                opts->keep = true;
                break;
            case OPT_INPUT:
                opts->input = optarg;
                break;
            case OPT_OUTPUT:
                opts->output = optarg;
                break;
            case OPT_SIZE:
                opts->size = (size_t) atol (optarg);
                break;
            case OPT_COPY:
                if (!strcmp (optarg, "memcpy"))
                    opts->copy = COPY_MEMCPY;
                else if (!strcmp (optarg, "64"))
                    opts->copy = COPY_UINT64;
                break;
            case OPT_NUMBER:
                opts->number = atoi (optarg);
                break;
            default:
                break;
        }
    }
}

static double
elapsed_seconds (struct timespec *start, struct timespec *end)
{
    return (end->tv_sec + end->tv_nsec / 1000000000.0) - (start->tv_sec + start->tv_nsec / 1000000000.0);
}

static void
reset_dma (pcilib_t *pci, volatile void *bar)
{
    uint32_t value;

    WR32_sleep (HF_REG_BASE, HF_BASE_RESET);
    WR32_sleep (HF_REG_BASE, 0);
    value = RD32 (HF_REG_BASE);
    assert (value == 0x14021700 || value == 0x14031700);
}

static void
reset_ddr_data (pcilib_t *pci, volatile void *bar, Options *opts)
{
    reset_dma (pci, bar);

    /* reset DDR and FIFOs */
    WR32_sleep (HF_REG_CONTROL,
                HF_CONTROL_RESET |
                HF_CONTROL_SOFT_RESET |
                HF_CONTROL_SOURCE_RX_FIFO);

    /* write only in DDR mode, disable read */
    WR32_sleep (HF_REG_CONTROL,
                HF_CONTROL_SOURCE_RX_FIFO);
}

static void
copy_data (volatile void *bar, char *data, size_t size, Options *opts)
{
    if (opts->verbose)
        printf ("Writing %zu bytes\n", size);

    switch (opts->copy) {
        case COPY_MEMCPY:
            memcpy (bar + 0x9400, data, size);
            break;
        case COPY_UINT64:
            {
                uint64_t *src = (uint64_t *) data;
                uint64_t *dst = (uint64_t *) (bar + 0x9400);
                int remaining;

                for (size_t i = 0; i < size / 8; i++)
                    dst[0] = src[i];

                remaining = size % 8;

                for (int i = 0; i < remaining; i++)
                    ((uint8_t *) bar + 0x9400)[0] = data[size - 1 - i];
            }
            break;
        default:
            break;
    }
}

static void
write_to_ddr (pcilib_t *pci, volatile void *bar, Options *opts)
{
    FILE *fp;
    char *data;
    size_t size;
    size_t read_size;

    if ((fp = fopen (opts->input, "rb")) == NULL) {
        fprintf (stderr, "Could not open `%s'\n", opts->input);
        return;
    }

    if (opts->size == 0) {
        fseek (fp, 0, SEEK_END);
        size = (size_t) ftell (fp);
        rewind (fp);
        opts->size = size;
    }
    else
        size = opts->size;

    data = malloc (size);
    read_size = fread (data, 1, size, fp);

    if (read_size != size) {
        fprintf (stderr, "Could only read %zu/%zu bytes, expect inconsistencies\n",
                 read_size, size);
    }

    copy_data (bar, data, size, opts);

    if (size >= 4096 && size % 4096) {
        size_t remaining = 4096 - size % 4096;

        memset (data, 0, remaining);
        copy_data (bar, data, remaining, opts);
    }

    free (data);
    fclose (fp);
}

static void
read_from_ddr (pcilib_t *pci, volatile void *bar, Options *opts)
{
    pcilib_kmem_handle_t *kmem_data;
    pcilib_kmem_handle_t *kmem_desc;
    uintptr_t bus_addr_data;
    uintptr_t bus_addr_desc;
    volatile uint32_t *data;
    volatile uint32_t *desc;
    uint32_t hardware_ptr;
    size_t size;
    FILE *fp = NULL;
    int started = 0;
    const size_t mem_size = 4096;
    const int flag_index = 2;
    struct timespec start;
    struct timespec end;

    if (opts->size == 0) {
        fprintf (stderr, "Read size is zero, not going to read\n");
        return;
    }

    if (opts->output) {
        if ((fp = fopen (opts->output, "wb")) == NULL) {
            fprintf (stderr, "Could not open `%s'\n", opts->output);
            return;
        }
    }

    kmem_data = pcilib_alloc_kernel_memory (pci, PCILIB_KMEM_TYPE_CONSISTENT, 1, mem_size, mem_size, PCILIB_KMEM_USE_DMA_PAGES, KMEM_DEFAULT_FLAGS);
    kmem_desc = pcilib_alloc_kernel_memory (pci, PCILIB_KMEM_TYPE_CONSISTENT, 1, 128, 4096, KMEM_USE_RING, KMEM_DEFAULT_FLAGS);
    data = (uint32_t *) pcilib_kmem_get_block_ua (pci, kmem_data, 0);
    desc = (uint32_t *) pcilib_kmem_get_block_ua (pci, kmem_desc, 0);
    bus_addr_data = pcilib_kmem_get_block_ba (pci, kmem_data, 0);
    bus_addr_desc = pcilib_kmem_get_block_ba (pci, kmem_desc, 0);

    /* DMA config */
    WR32_sleep (HF_REG_NUM_PACKETS, 0x20);
    WR32_sleep (HF_REG_PACKET_LENGTH, 0x20);
    WR32_sleep (HF_REG_UPDATE_THRESHOLD, 1);
    WR64_sleep (HF_REG_UPDATE_ADDRESS, bus_addr_desc);
    WR32_sleep (HF_REG_TIMER_THRESHOLD, 0x20000);

    if (opts->verbose)
        printf ("Reading %i times %zu B\n", opts->number, opts->size);

    clock_gettime (CLOCK_MONOTONIC, &start);

    for (int i = 0; i < opts->number; i++) {
        size = opts->size;
        hardware_ptr = 0;

        /* enable multi-read from DDR */
        WR32_sleep (HF_REG_CONTROL,
                    HF_CONTROL_ENABLE_READ |
                    HF_CONTROL_ENABLE_MULTI_READ |
                    HF_CONTROL_SOURCE_RX_FIFO);

        WR32_sleep (HF_REG_CONTROL,
                    HF_CONTROL_ENABLE_READ |
                    HF_CONTROL_SOURCE_RX_FIFO);

        desc[flag_index] = 0;

        while (size >= mem_size) {
            desc[flag_index] = 0;

            /* set write addr */
            WR64 (HF_REG_DESCRIPTOR_ADDRESS, bus_addr_data);

            /* start DMA */
            if (!started) {
                WR32_sleep (HF_REG_DMA, HF_DMA_START);
                started = 1;
            }

            do {
                hardware_ptr = desc[flag_index];
            }
            while (hardware_ptr != (bus_addr_data & 0xFFFFFFFF));

            if (fp != NULL)
                fwrite (data, 1, mem_size, fp);

            size -= mem_size;
        }

        memset (data, 0xf0, mem_size);

        if (size > 0) {
            desc[flag_index] = 0;

            if (opts->verbose)
                printf ("Read remaining %zu bytes\n", size);

            WR64_sleep (HF_REG_DESCRIPTOR_ADDRESS, bus_addr_data);

            /* start DMA */
            if (!started) {
                WR32_sleep (HF_REG_DMA, HF_DMA_START);
                started = 1;
            }

            do {
                hardware_ptr = desc[flag_index];
            }
            while (hardware_ptr != (bus_addr_data & 0xFFFFFFFF));

            if (fp != NULL)
                fwrite (data, 1, size, fp);
        }

        if (opts->verbose)
            printf ("Descriptor: update_pattern=%x empty=%x last_address=%lx\n", desc[0], 1 & desc[1], desc[2] | (((uint64_t) desc[3]) << 32));
    }

    clock_gettime (CLOCK_MONOTONIC, &end);

    WR32_sleep (HF_REG_DMA, HF_DMA_STOP);

    if (fp)
        fclose (fp);

    pcilib_free_kernel_memory (pci, kmem_data, KMEM_DEFAULT_FLAGS);
    pcilib_free_kernel_memory (pci, kmem_desc, KMEM_DEFAULT_FLAGS);

    if (opts->verbose) {
        double time;

        time = elapsed_seconds (&start, &end);
        printf ("Read in %f s equivalent to %3.5f MB/s\n",
                time, opts->size * opts->number / 1024. / 1024. / time);
    }
}

int
main (int argc, char const* argv[])
{
    static const char *DEVICE = "/dev/fpga0";

    Options opts;
    pcilib_t *pci;
    volatile void *bar;

    parse_options (argc, (char *const *) argv, &opts);

    pci = pcilib_open (DEVICE, "pci");

    if (pci == NULL) {
        fprintf (stderr, "Could not open `%s'", DEVICE);
        return 1;
    }

    pcilib_map_bar (pci, PCILIB_BAR0);
    bar = pcilib_resolve_bar_address (pci, PCILIB_BAR0, 0);

    /* reset dbg tx */
    WR32_sleep (HF_REG_DEBUG_REQUESTER_RESET, 1);
    WR32_sleep (HF_REG_DEBUG_REQUESTER_RESET, 0);
    WR32_sleep (HF_REG_DEBUG_COMPLETER_RESET, 1);
    WR32_sleep (HF_REG_DEBUG_COMPLETER_RESET, 0);

    /* configure interconnect */
    WR32_sleep (HF_REG_INTERCONNECT,
                HF_INTERCONNECT_MASTER_DMA |
                HF_INTERCONNECT_DDR_TO_DMA |
                HF_INTERCONNECT_DDR_FROM_64);

    if (opts.input != NULL) {
        if (!opts.keep) {
            if (opts.verbose)
                printf ("Reset DDR and data\n");

            reset_ddr_data (pci, bar, &opts);
        }

        write_to_ddr (pci, bar, &opts);
    }

    if (opts.input == NULL)
        read_from_ddr (pci, bar, &opts);

    pcilib_unmap_bar (pci, PCILIB_BAR0, (void *) bar);
    pcilib_close (pci);
}