Test application for resolving user-space BAR addresses

.bzrignore

@@ -0,0 +1,12 @@
+.dev.o.cmd
+.mod.o.cmd
+.sysfs.o.cmd
+.test.mod.o.cmd
+.test.o.cmd
+app
+modules.order
+Module.symvers
+test.ko
+test.mod.c
+.test.ko.cmd
+.tmp_versions

Makefile

@@ -0,0 +1,38 @@
+obj-m := test.o
+test-objs := mod.o dev.o sysfs.o
+
+KERNELDIR ?= /lib/modules/$(shell uname -r)/build
+INSTALLDIR ?= /lib/modules/$(shell uname -r)/kernel/extra
+PWD := $(shell pwd)
+
+EXTRA_CFLAGS += -I$(M)/..
+
+default: app
+	@KERNEL_GCC_VERSION=`cat /proc/version | head -n1 | cut -d " " -f 7` ;\
+	GCC_VERSION=`$(CC) --version | head -n 1 | tr ' ' '\n' | grep -e "[0-9]\+\.[0-9]" | tail -n 1` ;\
+	if [ $$KERNEL_GCC_VERSION != $$GCC_VERSION ]; then \
+	    echo "Kernel is compiled with gcc $$KERNEL_GCC_VERSION, but you are now using $$GCC_VERSION" ;\
+	    GCC_MAJOR=`echo $$KERNEL_GCC_VERSION | cut -d "." -f 1-2` ;\
+	    newCC=gcc-$$GCC_MAJOR ;\
+	    CC=`which $$newCC 2>/dev/null` ;\
+	    if [ $$? -ne 0 ]; then \
+		echo "No compiler of $$GCC_MAJOR series is installed" ;\
+		exit 1 ;\
+	    fi ;\
+	    GCC_VERSION=`$$CC --version | head -n 1 | tr ' ' '\n' | grep -e "[0-9]\+\.[0-9]" | tail -n 1` ;\
+	    if [ $$KERNEL_GCC_VERSION != $$GCC_VERSION -a -z "$$RELAXED_GCC_CHECK" ]; then \
+		echo "The $$GCC_VERSION of $$GCC_MAJOR series is installed" ;\
+		exit 1 ;\
+	    fi ;\
+	    echo "Setting CC to $$newCC" ;\
+	else \
+	    CC=$(CC) ;\
+	fi ;\
+	$(MAKE) $(CFLAGS) -C $(KERNELDIR) M=$(PWD) CC=$$CC modules
+
+app: app.c
+	gcc app.c -o app -lpcilib
+
+
+clean:
+	rm -rf *.o *.ko *.mod.c .*.o.cmd .*.o.tmp .*.ko.cmd  .*.o *.symvers modules.order .tmp_versions

app.c

@@ -0,0 +1,68 @@
+#define _BSD_SOURCE
+#define _DEFAULT_SOURCE
+#define _POSIX_C_SOURCE 199309L
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <stdarg.h>
+#include <time.h>
+#include <sched.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/time.h>
+#include <sys/stat.h>
+
+#include <pcilib.h>
+#include "pcilib/kmem.h"
+
+#define DEVICE "/dev/fpga0"
+
+#define WR(addr, value) { *(uint32_t*)(bar + addr) = value; }
+#define RD(addr, value) { value = *(uint32_t*)(bar + addr); }
+
+int main(int argc, char *argv[]) {
+    uint32_t i, j;
+    pcilib_t *pci;
+    pcilib_kmem_handle_t *kbuf;
+
+    void* volatile bar;
+    
+    pci = pcilib_open(DEVICE, PCILIB_MODEL_DETECT);
+    if (!pci) {
+	printf("pcilib_open\n");
+	exit(1);
+    }
+
+    kbuf = pcilib_alloc_kernel_memory(pci, PCILIB_KMEM_TYPE_PAGE, 1, 4096, 4096, 0, 0);
+    volatile uint32_t *ua = pcilib_kmem_get_ua(pci, kbuf);
+    uintptr_t pa = pcilib_kmem_get_pa(pci, kbuf);
+
+    printf("User: %p\n", ua);
+    printf("HW  : 0x%lx\n", pa);
+
+    int fd = open("/sys/class/test/test0/test_request",  O_RDWR);
+    if (fd >= 0) {
+	size_t bytes;
+	char res[64];
+
+	sprintf(res, "%p", ua);
+	write(fd, res, strlen(res));
+	lseek(fd, SEEK_SET, 0);
+	bytes = read(fd, res, sizeof(res) - 1); 
+
+	printf("Res : ");
+	if (bytes < 0) {
+	    puts("Error");
+	} else {
+	    res[bytes] = 0;
+	    puts(res);
+	}
+
+	close(fd);
+    }
+
+    pcilib_free_kernel_memory(pci, kbuf, 0);
+    
+    pcilib_close(pci);
+}

debug.h

@@ -0,0 +1,19 @@
+#ifndef _TEST_DEBUG_H
+#define _TEST_DEBUG_H
+
+#ifdef DEBUG
+ #define mod_info( args... ) \
+    do { printk( KERN_INFO "%s - %s : ", MODNAME , __FUNCTION__ );\
+    printk( args ); } while(0)
+ #define mod_info_dbg( args... ) \
+    do { printk( KERN_INFO "%s - %s : ", MODNAME , __FUNCTION__ );\
+    printk( args ); } while(0)
+#else
+ #define mod_info( args... ) \
+    do { printk( KERN_INFO "%s: ", MODNAME );\
+    printk( args ); } while(0)
+ #define mod_info_dbg( args... ) 
+#endif
+
+
+#endif /* _TEST_DEBUG_H */

dev.c

@@ -0,0 +1,53 @@
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+
+#include "mod.h"
+#include "dev.h"
+#include "debug.h"
+
+#define sysfs_attr(name) do { \
+	if (device_create_file(dev->dev, &dev_attr_##name) != 0) \
+				goto probe_device_create_fail; \
+    } while (0)
+
+
+int test_device_open(struct inode *inode, struct file *filp)
+{
+    test_dev_t *test;
+
+    test = container_of( inode->i_cdev, test_dev_t, cdev);
+    filp->private_data = test;
+    
+    mod_info("open\n");
+
+    
+    return 0;
+}
+
+int test_device_release(struct inode *inode, struct file *filp)
+{
+    test_dev_t *test = (test_dev_t*)filp->private_data;
+
+    mod_info("close\n");
+
+    return 0;
+}
+
+
+static struct file_operations test_fops = {
+    .owner = THIS_MODULE,
+//	.unlocked_ioctl = pcidriver_ioctl,
+//	.mmap = pcidriver_mmap,
+    .open = test_device_open,
+    .release = test_device_release,
+};
+
+const struct file_operations *test_get_fops(void)
+{
+    return &test_fops;
+}

dev.h

@@ -0,0 +1,17 @@
+#ifndef _TEST_DEV_H
+#define _TEST_DEV_H
+
+#include <linux/fs.h>
+
+typedef struct {
+    dev_t devno;
+    struct device *dev;					/**< Class device */
+
+    unsigned long addr;
+
+    struct cdev cdev;
+} test_dev_t;
+
+const struct file_operations *test_get_fops(void);
+
+#endif /* _TEST_BASE_H */

mod.c

@@ -0,0 +1,168 @@
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+
+
+#include "mod.h"
+#include "dev.h"
+#include "debug.h"
+#include "sysfs.h"
+
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Suren A. Chilingaryan <csa@suren.me>");
+MODULE_DESCRIPTION("Testing module");
+MODULE_VERSION("0.0.1");
+
+int test_minors = TEST_MINORS;
+
+module_param(test_minors, int, S_IRUGO);
+
+
+static dev_t test_devno;			/**< major number */
+static test_dev_t **test_devs = NULL;	/**< per-device context */
+static spinlock_t test_devs_lock;	/**< lock protecting creation/destruction of devices */
+static struct class *test_class;		/**< device class */
+
+static void test_module_destroy_cdev(test_dev_t *test)
+{
+    if (test->dev) {
+	test_sysfs_free(test);
+	device_destroy(test_class, test->devno);
+    }
+    cdev_del(&test->cdev);
+    kfree(test);
+}
+
+static int test_module_setup_cdev(void)
+{
+    int i;
+    int err = 0;
+    dev_t devno;
+    test_dev_t *test;
+
+    test = kmalloc(sizeof(test_dev_t), GFP_KERNEL);
+    if (!test) {
+	mod_info("Couldn't allocate memory. Device is not created.\n");
+	return -ENOMEM;
+    }
+
+    cdev_init(&test->cdev, test_get_fops());
+    test->cdev.owner = THIS_MODULE;
+    test->cdev.ops = test_get_fops();
+
+    spin_lock(&test_devs_lock);
+    for (i = 0; i < test_minors; i++) {
+	if (!test_devs[i])
+	    break;
+    }
+
+    if (i == test_minors) {
+	mod_info("No free minor numbers left");
+	err = -EBUSY;
+	goto fail;
+    }
+
+    devno = MKDEV(MAJOR(test_devno), MINOR(test_devno) + i);
+    test->devno = devno;
+
+    err = cdev_add(&test->cdev, devno, 1);
+    if (err) {
+	mod_info("Error %d adding device kmm%d", err, i);
+	goto fail;
+    }
+
+    test->dev = device_create(test_class, NULL, devno, test, TEST_NODE_FMT, MINOR(test_devno) + i);
+    if (!test->dev) {
+	mod_info("Error creating /dev/%s%d\n", TEST_NODE, MINOR(test_devno) + i);
+	goto fail;
+    }
+    dev_set_drvdata(test->dev, test);
+
+    test_devs[i] = test;
+    spin_unlock(&test_devs_lock);
+
+    err = test_sysfs_init(test);
+    if (err) goto fail_sysfs;
+
+    mod_info("Device /dev/%s%d added\n", TEST_NODE, MINOR(test_devno) + i);
+
+    return 0;
+
+fail:
+    spin_unlock(&test_devs_lock);
+fail_sysfs:
+    test_module_destroy_cdev(test);
+    return err;
+}
+
+
+static void test_module_cleanup(void)
+{
+    int i;
+
+    if (test_devs) {
+	for (i = 0; i < test_minors; i++) {
+	    if (test_devs[i])
+		test_module_destroy_cdev(test_devs[i]);
+	}
+	kfree(test_devs);
+    }
+
+
+    if (test_class)
+    	class_destroy(test_class);
+
+    unregister_chrdev_region(test_devno, test_minors);
+}
+
+static int __init test_module_init(void)
+{
+    int err;
+
+    spin_lock_init(&test_devs_lock);
+
+    if ((err = alloc_chrdev_region(&test_devno, 0, test_minors, TEST_NODE))) {
+	mod_info("Couldn't allocate chrdev region. Module not loaded.\n");
+	goto alloc_chrdev_fail;
+    }
+
+    test_class = class_create(THIS_MODULE, TEST_NODE);
+    if (IS_ERR(test_class)) {
+	mod_info("No sysfs support. Module not loaded.\n");
+	goto fail;
+    }
+
+    test_devs = kmalloc(test_minors * sizeof(test_dev_t*), GFP_KERNEL);
+    if (!test_devs) {
+	mod_info("Couldn't allocate memory. Module not loaded.\n");
+	err = -ENOMEM;
+    }
+    memset(test_devs, 0, test_minors * sizeof(test_dev_t*));
+
+    mod_info("Major %d allocated to node '%s'\n", MAJOR(test_devno), TEST_NODE);
+
+    err = test_module_setup_cdev();
+    if (err) goto fail;
+
+
+    return 0;
+
+fail:
+    test_module_cleanup();
+alloc_chrdev_fail:
+    return err;
+}
+
+
+static void __exit test_module_exit(void)
+{
+    test_module_cleanup();
+}
+
+module_init(test_module_init);
+module_exit(test_module_exit);

mod.h

@@ -0,0 +1,9 @@
+#ifndef _TEST_MODE_H
+#define _TEST_MODE_H
+
+#define MODNAME		"test"
+#define TEST_NODE	"test"
+#define TEST_NODE_FMT	"test%i"
+#define TEST_MINORS 	16
+
+#endif /* _TEST_MODE_H */

rdma.h

@@ -0,0 +1,9 @@
+#ifndef _PCIDRIVER_RDMA_H
+#define _PCIDRIVER_RDMA_H
+
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+
+extern unsigned long pcidriver_resolve_bar(unsigned long address);
+
+#endif /* _PCIDRIVER_RDMA_H */

sysfs.c

@@ -0,0 +1,51 @@
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/device.h>
+#include <linux/types.h>
+#include <linux/cdev.h>
+#include <linux/fs.h>
+#include <linux/slab.h>
+#include <linux/sysfs.h>
+
+#include "mod.h"
+#include "dev.h"
+#include "debug.h"
+
+#include "rdma.h"
+
+static ssize_t test_request_show(struct device *dev, struct device_attribute *attr, char *buf) {
+    test_dev_t *test = dev_get_drvdata(dev);
+
+    return sprintf(buf, "%lx\n", test->addr);
+}
+
+static ssize_t test_request_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t count) {
+    unsigned long ua;
+
+    test_dev_t *test = dev_get_drvdata(dev);
+
+    if (sscanf(buf, "%lx", &ua) != 1) {
+	mod_info("Error processing request\n");
+	return 0;
+    }
+
+    test->addr = pcidriver_resolve_bar(ua);
+
+    return count;
+}
+
+static DEVICE_ATTR(test_request, 0664, test_request_show, test_request_store);
+
+
+
+int test_sysfs_init(test_dev_t *dev) {
+    int err;
+
+    err = device_create_file(dev->dev, &dev_attr_test_request);
+
+    return err;
+}
+
+void test_sysfs_free(test_dev_t *dev) {
+    device_remove_file(dev->dev, &dev_attr_test_request);
+}

sysfs.h

@@ -0,0 +1,7 @@
+#ifndef _TEST_SYSFS_H
+#define _TEST_SYSFS_H
+
+int test_sysfs_init(test_dev_t *dev);
+void test_sysfs_free(test_dev_t *dev);
+
+#endif /* _TEST_SYSFS_H */