updates

cpp/data_recorder.cpp

@@ -59,15 +59,15 @@ DataRecorder::DataRecorder() {
     allocate_memory();
 
     // Prep access to PL registers
-    plfd = open("/dev/wsrpl0", O_RDWR);
+    plfd = open("/dev/drexpl0", O_RDWR);
     if (plfd < 0){
-        printf("Failed to open %s\n", "/dev/wsrpl0");
+        printf("Failed to open %s\n", "/dev/drexpl0");
     }
 
     plmmap = (uint32_t *)mmap(NULL, 0x1000000, PROT_WRITE | PROT_READ, MAP_SHARED, plfd, 0);
 
     if (plmmap < (uint32_t *)0){
-        printf("Failed to mmap %s\n", "/dev/wsrpl0");
+        printf("Failed to mmap %s\n", "/dev/drexpl0");
         close(plfd);
     }
 
@@ -266,7 +266,7 @@ void DataRecorder::set_validate_cnt_data(bool enable, uint32_t pri, uint32_t int
 void DataRecorder::get_data(int ch_ind, int save_to_disk) {
 
     char* file;
-    asprintf(&file, "/dev/wsrdma%d", ch_ind);
+    asprintf(&file, "/dev/drexdma%d", ch_ind);
 
     // Start write to disk thread
     if (save_to_disk) {
@@ -279,27 +279,27 @@ void DataRecorder::get_data(int ch_ind, int save_to_disk) {
         return;
     }
 
-    wsrpcie_ioctl_t wsrpcie_ioctl;
+    drexpcie_ioctl_t drexpcie_ioctl;
 
-    wsrpcie_ioctl.cmd = WSRDMA_SET_NUM_BYTES;
+    drexpcie_ioctl.cmd = DREXDMA_SET_NUM_BYTES;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = BUFFER_SIZE;
+    drexpcie_ioctl.value = BUFFER_SIZE;
-    ioctl(fd, 0, &wsrpcie_ioctl);
+    ioctl(fd, 0, &drexpcie_ioctl);
 
-    wsrpcie_ioctl.cmd = WSRDMA_SET_NUM_BUFS;
+    drexpcie_ioctl.cmd = DREXDMA_SET_NUM_BUFS;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = NUM_BUFFERS;
+    drexpcie_ioctl.value = NUM_BUFFERS;
-    ioctl(fd, 0, &wsrpcie_ioctl);
+    ioctl(fd, 0, &drexpcie_ioctl);
 
-    wsrpcie_ioctl.cmd = WSRDMA_DMA_INIT;
+    drexpcie_ioctl.cmd = DREXDMA_DMA_INIT;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = 0;
+    drexpcie_ioctl.value = 0;
-    ioctl(fd, 0, &wsrpcie_ioctl);
+    ioctl(fd, 0, &drexpcie_ioctl);
 
-    wsrpcie_ioctl.cmd = WSRDMA_DMA_START;
+    drexpcie_ioctl.cmd = DREXDMA_DMA_START;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = 0;
+    drexpcie_ioctl.value = 0;
-    ioctl(fd, 0, &wsrpcie_ioctl);    
+    ioctl(fd, 0, &drexpcie_ioctl);    
 
 
     struct timespec ts_now;
@@ -394,17 +394,17 @@ void DataRecorder::get_data(int ch_ind, int save_to_disk) {
 
     }
 
-    wsrpcie_ioctl.cmd = WSRDMA_DMA_STOP;
+    drexpcie_ioctl.cmd = DREXDMA_DMA_STOP;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = 0;
+    drexpcie_ioctl.value = 0;
 
-    ioctl(fd, 0, &wsrpcie_ioctl);
+    ioctl(fd, 0, &drexpcie_ioctl);
 
-    wsrpcie_ioctl.cmd = WSRDMA_DMA_CLEAR;
+    drexpcie_ioctl.cmd = DREXDMA_DMA_CLEAR;
-    wsrpcie_ioctl.offset = 0;
+    drexpcie_ioctl.offset = 0;
-    wsrpcie_ioctl.value = 0;
+    drexpcie_ioctl.value = 0;
 
-    ioctl(fd, 0, &wsrpcie_ioctl);
+    ioctl(fd, 0, &drexpcie_ioctl);
 
     // Make sure write thread is done before closing file handle
     if (writer[ch_ind].joinable()) {

cpp/data_recorder.h

@@ -16,21 +16,21 @@
 #define TIMING_REG_BASE 0x20000
 #define DATA_GEN_REG_BASE 0x30000
 
-#define WSRDMA_DMA_INIT             0
+#define DREXDMA_DMA_INIT             0
-#define WSRDMA_DMA_CLEAR            1
+#define DREXDMA_DMA_CLEAR            1
-#define WSRDMA_DMA_START            2
+#define DREXDMA_DMA_START            2
-#define WSRDMA_DMA_STOP             3
+#define DREXDMA_DMA_STOP             3
-#define WSRDMA_SET_NUM_BUFS         4
+#define DREXDMA_SET_NUM_BUFS         4
-#define WSRDMA_SET_NUM_BYTES        5
+#define DREXDMA_SET_NUM_BYTES        5
-#define WSRDMA_GET_NUM_BUFS         6
+#define DREXDMA_GET_NUM_BUFS         6
-#define WSRDMA_GET_NUM_BYTES        7
+#define DREXDMA_GET_NUM_BYTES        7
-#define WSRDMA_GET_FREE_BUFS        8
+#define DREXDMA_GET_FREE_BUFS        8
 
 typedef struct {
    unsigned int cmd;
    unsigned int offset;
    unsigned int value;
-} wsrpcie_ioctl_t;
+} drexpcie_ioctl_t;
 
 #define NUM_DMA_CH      4
 #define BUFFER_SIZE     (4 * 1024 * 1024)

cpp/test_data_recorder.cpp

@@ -11,8 +11,13 @@ int main() {
     uint32_t n_pulses = 128;
     float inter_cpi = 100e-6;
     // float pri = 75e-6;
-    float pri = 200e-6;
+    float pri = 100e-6;
     uint32_t n_samples = 16384;
+    int test_time_ms = 10000;
+
+    // Set false to just pull data across the pcie and run the data validator.
+    // Set true to save data to disk
+    bool save_to_disk = false;
 
     float cpi_time = n_pulses * pri + inter_cpi;
     float cpi_num_bytes = n_pulses * n_samples * 16;
@@ -40,13 +45,12 @@ int main() {
     dr.set_validate_cnt_data(true, dr.read_reg(TIMING_REG_BASE + 0x4) + 1, dr.read_reg(TIMING_REG_BASE + 0x10) + 1, n_pulses);
 
     // Start listening for data
-    // dr.start_recording("test.bin", 1);
+    dr.start_recording("/media/hptnvme/test.bin", save_to_disk);
-    dr.start_recording("/media/hptnvme/test.bin", 1);
     sleep(1);
     // Start the timing engine so data starts flowing
     dr.write_reg(TIMING_REG_BASE + 0x0, 0);
     // Wait a while
-    std::this_thread::sleep_for(std::chrono::milliseconds(2000));
+    std::this_thread::sleep_for(std::chrono::milliseconds(test_time_ms));
     dr.write_reg(TIMING_REG_BASE + 0x0, 1);
     sleep(2);
     dr.stop_recording();

pcie_driver/Makefile

@@ -0,0 +1,31 @@
+BINARY     := drexpcie_module
+KERNEL      := /lib/modules/$(shell uname -r)/build
+ARCH        := x86
+C_FLAGS     := -Wall
+KMOD_DIR    := $(shell pwd)
+TARGET_PATH := /lib/modules/$(shell uname -r)/kernel/drivers/char
+
+OBJECTS := \
+	drexpcie.o \
+	drexdma.o \
+	drexchar.o \
+
+ccflags-y += $(C_FLAGS)
+
+obj-m += $(BINARY).o
+
+$(BINARY)-y := $(OBJECTS)
+
+$(BINARY).ko:
+	make -C $(KERNEL) M=$(KMOD_DIR) modules
+
+install:
+	cp $(BINARY).ko $(TARGET_PATH)
+	depmod -a
+
+uninstall:
+	rm $(TARGET_PATH)/$(BINARY).ko
+	depmod -a
+
+clean:
+	make -C $(KERNEL) M=$(KMOD_DIR) clean

pcie_driver/drex.h

@@ -0,0 +1,18 @@
+#ifndef DREX_H_
+#define DREX_H_
+
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+/* This is a "private" data structure */
+/* You can store there any data that should be passed between driver's functions */
+struct drexpcie_driver_priv {
+   struct pci_dev *pdev;
+   unsigned long mem_start;
+   unsigned long mem_len;   
+   u8 __iomem *bar0_mem;
+};
+
+#endif

pcie_driver/drexchar.c

@@ -0,0 +1,292 @@
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+
+#include <linux/iomap.h>
+
+#include "drexchar.h"
+#include "drexdma.h"
+
+#define MAX_DEV 16
+
+static int drexpcie_open(struct inode *inode, struct file *file);
+static int drexpcie_release(struct inode *inode, struct file *file);
+static int drexpcie_mmap(struct file *file, struct vm_area_struct *vma);
+static long drexpcie_ioctl(struct file *file, unsigned int cmd, unsigned long arg);
+static ssize_t drexpcie_read(struct file *file, char __user *buf, size_t count, loff_t *offset);
+static ssize_t drexpcie_write(struct file *file, const char __user *buf, size_t count, loff_t *offset);
+
+static const struct file_operations drexpcie_fops = {
+	.owner      = THIS_MODULE,
+	.open       = drexpcie_open,
+	.release    = drexpcie_release,
+   .mmap       = drexpcie_mmap,   
+	.unlocked_ioctl = drexpcie_ioctl,
+	.read       = drexpcie_read,
+	.write      = drexpcie_write
+};
+
+
+struct drexpcie_device_data {
+	struct device* dev;
+	struct cdev cdev;
+   uint32_t type;
+   uint32_t offset;
+};
+
+
+static int dev_major = 0;
+static struct class *drexpcie_class = NULL;
+static struct drexpcie_device_data drexpcie_dev_data[MAX_DEV];
+static struct drexpcie_driver_priv* driv_access = NULL;
+
+static int drexpcie_uevent(const struct device *dev, struct kobj_uevent_env *env)
+{
+	add_uevent_var(env, "DEVMODE=%#o", 0666);
+
+	return 0;
+}
+
+int drexpcie_chardev_create(struct drexpcie_driver_priv *driv_priv)
+{
+	int err, i;
+	dev_t dev;
+
+   char device_string[64];
+
+   int pl_count = 0;
+   int dma_count = 0;
+
+   driv_access = driv_priv;
+
+   err = alloc_chrdev_region(&dev, 0, MAX_DEV, "drexpcie");
+
+	dev_major = MAJOR(dev);
+
+	// drexpcie_class = class_create(THIS_MODULE, "drexpcie-dev");
+   drexpcie_class = class_create("drexpcie-dev");
+
+	drexpcie_class->dev_uevent = drexpcie_uevent;
+
+	for (i = 0; i < MAX_DEV; i++) {
+      drexpcie_dev_data[i].type = ioread32(driv_access->bar0_mem+(i*4)*4);
+      if (drexpcie_dev_data[i].type == 0x10000) {
+         snprintf(device_string,sizeof(device_string),"drexpl%d",pl_count);  
+         pl_count++;       
+      }
+      else if (drexpcie_dev_data[i].type == 0x20000) {
+         snprintf(device_string,sizeof(device_string),"drexdma%d",dma_count);
+         dma_count++;
+      }
+      else {
+         drexpcie_dev_data[i].type = 0;
+         break;
+      }
+
+      // save the offset of the device area;
+      drexpcie_dev_data[i].offset = ioread32(driv_access->bar0_mem+(i*4+1)*4);
+
+		cdev_init(&drexpcie_dev_data[i].cdev, &drexpcie_fops);
+		drexpcie_dev_data[i].cdev.owner = THIS_MODULE;
+		cdev_add(&drexpcie_dev_data[i].cdev, MKDEV(dev_major, i), 1);
+      
+		drexpcie_dev_data[i].dev = device_create(drexpcie_class, NULL, MKDEV(dev_major, i), NULL, "%s",device_string);
+
+	}
+
+	return 0;
+}
+
+int drexpcie_chardev_destroy(void)
+{
+	int i;
+
+	for (i = 0; i < MAX_DEV; i++) {
+		device_destroy(drexpcie_class, MKDEV(dev_major, i));
+	}
+
+	class_unregister(drexpcie_class);
+	class_destroy(drexpcie_class);
+
+	unregister_chrdev_region(MKDEV(dev_major, 0), MINORMASK);
+
+	return 0;
+}
+
+
+static int drexpcie_open(struct inode *inode, struct file *file)
+{
+	struct drexpcie_device_priv* drexpcie_priv;
+	unsigned int minor = iminor(inode);
+
+	drexpcie_priv = kzalloc(sizeof(struct drexpcie_device_priv), GFP_KERNEL);
+	drexpcie_priv->chnum = minor;
+	drexpcie_priv->driv_priv = driv_access;
+//   drexpcie_priv->dev = &(drexpcie_dev_data[drexpcie_priv->chnum].pdev->dev);
+   drexpcie_priv->type = drexpcie_dev_data[drexpcie_priv->chnum].type;
+   drexpcie_priv->base_mem = driv_access->bar0_mem +  drexpcie_dev_data[drexpcie_priv->chnum].offset;
+
+   drexpcie_priv->num_bufs = 512;
+   drexpcie_priv->num_bytes = 4194304;
+   drexpcie_priv->valid_bufs = 0;
+
+   drexpcie_priv->bufPtr = NULL;
+   drexpcie_priv->dmaPtr = (dma_addr_t *) NULL;    
+
+	file->private_data = drexpcie_priv;
+
+	return 0;
+}
+
+static int drexpcie_release(struct inode *inode, struct file *file)
+{
+	struct drexpcie_device_priv* drexpcie_priv = file->private_data;
+
+	kfree(drexpcie_priv);
+
+	drexpcie_priv = NULL;
+
+	return 0;
+}
+
+static int drexpcie_mmap(struct file *file, struct vm_area_struct *vma)
+{
+   struct drexpcie_device_priv* drexpcie_priv = file->private_data;
+   int    ret;
+
+   // DMA doesn't support mmap
+   if (drexpcie_priv->type == 0x20000) return -EINVAL; 
+
+   vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+    
+   ret = io_remap_pfn_range(vma, vma->vm_start, ((unsigned long) drexpcie_priv->driv_priv->mem_start) >> PAGE_SHIFT, vma->vm_end - vma->vm_start, vma->vm_page_prot);
+   if (ret != 0) {
+      printk(KERN_ERR "Error in calling remap_pfn_range: returned %d\n", ret);
+      return -EAGAIN;
+   }
+    
+	 return 0;
+}
+
+static long drexpcie_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
+{
+	struct drexpcie_device_priv* drexpcie_priv = file->private_data;
+   drexpcie_ioctl_t drexpcie_ioctl;
+
+   // uint32_t temp;
+
+   if (copy_from_user(&drexpcie_ioctl, (drexpcie_ioctl_t *)arg,
+                       sizeof(drexpcie_ioctl_t))) {
+      return -EACCES;
+   } 
+
+   printk("drexpcie[%d]: Received IOCTL cmd=%d\n", drexpcie_priv->chnum,  drexpcie_ioctl.cmd);
+
+   switch (drexpcie_ioctl.cmd) {
+      case DREXDMA_DMA_INIT:
+         return dma_init(drexpcie_priv);
+         break;
+
+      case DREXDMA_DMA_CLEAR:
+         return dma_clear(drexpcie_priv);
+         break;         
+
+      case DREXDMA_DMA_START:
+         return dma_start(drexpcie_priv);
+         break;
+               
+      case DREXDMA_DMA_STOP:
+         return dma_stop(drexpcie_priv);
+         break;
+
+      case DREXDMA_SET_NUM_BUFS:
+         drexpcie_priv->num_bufs = drexpcie_ioctl.value;
+         break;
+
+      case DREXDMA_SET_NUM_BYTES:
+         drexpcie_priv->num_bytes = drexpcie_ioctl.value;
+         break;
+
+      case DREXDMA_GET_NUM_BUFS:
+         drexpcie_ioctl.value = drexpcie_priv->num_bufs;
+         if (copy_to_user((drexpcie_ioctl_t *)arg, &drexpcie_ioctl, sizeof(drexpcie_ioctl_t))) {
+            return -EACCES;
+         }
+         break;
+
+      case DREXDMA_GET_NUM_BYTES:
+         drexpcie_ioctl.value = drexpcie_priv->num_bytes;
+         if (copy_to_user((drexpcie_ioctl_t *)arg, &drexpcie_ioctl, sizeof(drexpcie_ioctl_t))) {
+            return -EACCES;
+         }
+         break;    
+
+      case DREXDMA_GET_FREE_BUFS:
+         drexpcie_ioctl.value = ioread32(drexpcie_priv->base_mem + DREXDMA_OFFSET_STAT + 0x8);
+         if (copy_to_user((drexpcie_ioctl_t *)arg, &drexpcie_ioctl, sizeof(drexpcie_ioctl_t))) {
+            return -EACCES;
+         }
+         break;                 
+
+      default:
+         printk("drexpcie[%d]: error, invalid IOCTL cmd\n", drexpcie_priv->chnum);
+         return -EINVAL;
+    }
+
+	return 0;
+}
+
+static ssize_t drexpcie_read(struct file *file, char __user *buf, size_t count, loff_t *offset)
+{
+   struct drexpcie_device_priv* drexpcie_priv = file->private_data;
+
+   uint32_t temp[3];
+   uint32_t dma_index;
+
+   // read status register to determine is something is in the complete queue
+   temp[0] = ioread32(drexpcie_priv->base_mem + DREXDMA_OFFSET_STAT+0xC);
+
+   // if bit 0 is high, FIFO is empty return 0 count value;
+   if (temp[0] & 0x01) {
+      count = 0;
+   }
+   // else read 72 bit FIFO value;  value is popped after MSW read;
+   else {
+      temp[0] = ioread32(drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO);
+      // temp[1] = ioread32(drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO+4);
+      temp[2] = ioread32(drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO+8);
+      
+      // determine which buffer index;
+      dma_index = temp[0] & 0x000001FF;
+
+      // determine amount of data in buffer;
+      count = (temp[0] & 0xFFFFFE00) >> 9;
+
+      //using dma_index, copy correct buffer to user space 
+      if (copy_to_user(buf, drexpcie_priv->bufPtr[dma_index], count)) {
+         count = 0;
+      }      
+   
+      // create new LSW using index and size
+      temp[0] = (((drexpcie_priv->num_bytes) & 0x7FFFFF) << 9);
+      temp[0] |= (((dma_index) & 0x000001FF));
+
+      // write now unused buffer back to the available queue
+      // If using loop mode dont need to write buffers back in
+      // iowrite32(temp[0], drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO);
+      // iowrite32(temp[1], drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO+4);
+      // iowrite32(temp[2], drexpcie_priv->base_mem + DREXDMA_OFFSET_FIFO+8);     
+
+   }
+
+	return count;
+}
+
+static ssize_t drexpcie_write(struct file *file, const char __user *buf, size_t count, loff_t *offset)
+{
+	return count;
+}
+

pcie_driver/drexchar.h

@@ -0,0 +1,48 @@
+
+#ifndef DREXCHAR_H
+#define DREXCHAR_H
+
+#include "drex.h"
+
+#define DREXDMA_DMA_INIT             0
+#define DREXDMA_DMA_CLEAR            1
+#define DREXDMA_DMA_START            2
+#define DREXDMA_DMA_STOP             3
+#define DREXDMA_SET_NUM_BUFS         4
+#define DREXDMA_SET_NUM_BYTES        5
+#define DREXDMA_GET_NUM_BUFS         6
+#define DREXDMA_GET_NUM_BYTES        7
+#define DREXDMA_GET_FREE_BUFS        8
+
+
+typedef struct {
+   unsigned int cmd;
+   unsigned int offset;
+   unsigned int value;
+} drexpcie_ioctl_t;  
+
+
+struct drexpcie_device_priv {
+	uint8_t chnum;
+	struct drexpcie_driver_priv* driv_priv;
+   
+   uint32_t type;
+   u8 __iomem *base_mem;
+
+   // Parameters used to allocate dma buffers
+   uint32_t num_bufs;
+   uint32_t valid_bufs;
+   uint32_t num_bytes;
+
+   // Physical memory address returned by dma_alloc_coherent
+   dma_addr_t *dmaPtr;
+                    
+   // Virtual memory address returned by dma_alloc_coherent
+   void  **bufPtr;
+
+};
+
+int drexpcie_chardev_create(struct drexpcie_driver_priv *driv_priv);
+int drexpcie_chardev_destroy(void);
+
+#endif

pcie_driver/drexdma.c

@@ -0,0 +1,158 @@
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <linux/uaccess.h>
+#include <linux/fs.h>
+
+#include <linux/dma-mapping.h>
+
+#include "drexdma.h"
+
+
+
+// Initial DMA buffers function
+int init_dma_buffers(struct drexpcie_device_priv *drexpcie_priv)
+{
+   int buffer_index;
+
+   // allocate an array for the buffer pointers (kernel space)
+   drexpcie_priv->bufPtr = kzalloc(drexpcie_priv->num_bufs*sizeof(void *),GFP_KERNEL);
+   if (drexpcie_priv->bufPtr == NULL) {
+      printk("drexpcie[%d]: error, failed to alloc bufPtr memory\n", drexpcie_priv->chnum);
+      return -ENOMEM;
+   }
+
+   // allocate an array for the dma pointers (physical address)
+   drexpcie_priv->dmaPtr = kzalloc(drexpcie_priv->num_bufs*sizeof(dma_addr_t),GFP_KERNEL);
+   if (drexpcie_priv->dmaPtr == NULL) {
+      printk("drexpcie[%d]: error, failed to alloc dmaPtr memory\n", drexpcie_priv->chnum);
+      return -ENOMEM;
+   }
+
+   // loop through the total number of desired buffer and attempt to allocate;
+   // This will populate both the bufPtr and dmaPtr
+   for (buffer_index = 0; buffer_index < drexpcie_priv->num_bufs; buffer_index++) {
+      drexpcie_priv->bufPtr[buffer_index] = dma_alloc_coherent(&(drexpcie_priv->driv_priv->pdev->dev), drexpcie_priv->num_bytes, &drexpcie_priv->dmaPtr[buffer_index], GFP_KERNEL);
+      printk("Allocating Memory: %p, %llX\n",drexpcie_priv->bufPtr[buffer_index],drexpcie_priv->dmaPtr[buffer_index]);
+            
+      if (drexpcie_priv->bufPtr[buffer_index] == NULL) {
+         printk("drexpcie[%d]: warning, DMA mem allocation failed at %d buffers\n", drexpcie_priv->chnum, buffer_index);
+         break;
+      }
+   } 
+
+   drexpcie_priv->valid_bufs = buffer_index;  
+
+   return 0;
+}
+
+// Release (free) DMA buffers function
+void release_dma_buffers(struct drexpcie_device_priv *drexpcie_priv)
+{
+   int buffer_index;
+
+   if (drexpcie_priv->bufPtr) {
+      for (buffer_index=0; buffer_index<drexpcie_priv->valid_bufs; buffer_index++) {
+         if (drexpcie_priv->bufPtr[buffer_index]) {
+            printk("Freeing Memory: %p, %llX\n",drexpcie_priv->bufPtr[buffer_index],drexpcie_priv->dmaPtr[buffer_index]);
+            dma_free_coherent(&(drexpcie_priv->driv_priv->pdev->dev), drexpcie_priv->num_bytes , drexpcie_priv->bufPtr[buffer_index], drexpcie_priv->dmaPtr[buffer_index]);
+            
+            // This is unnecessary, but cleaner
+            drexpcie_priv->bufPtr[buffer_index] = NULL;
+            drexpcie_priv->dmaPtr[buffer_index] = (dma_addr_t) NULL;
+         }
+      }
+      kfree(drexpcie_priv->bufPtr);
+      drexpcie_priv->bufPtr = NULL;
+   }
+
+   if (drexpcie_priv->dmaPtr) {
+      kfree(drexpcie_priv->dmaPtr);
+      drexpcie_priv->dmaPtr = NULL;
+   }
+
+   drexpcie_priv->valid_bufs = 0;
+
+}
+
+// Start the DMA function
+int dma_init(struct drexpcie_device_priv *drexpcie_priv)
+{
+   int i;
+   int ret;
+   uint32_t temp[3];
+   uint64_t temp64;
+
+   // set reset bits
+   iowrite32(0x00000007, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL);
+                      
+   // clears queues if dma_start is called multiple times.
+   release_dma_buffers(drexpcie_priv);
+
+   ret = init_dma_buffers(drexpcie_priv);
+
+   if (ret) {
+      release_dma_buffers(drexpcie_priv);
+      return ret;
+   }
+   
+   // clear reset bits
+   iowrite32(0x00000000, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL);
+       
+   // writes buffer to the available FIFO
+   printk("drexpcie[%d]: writing %d buffers to FIFO\n", drexpcie_priv->chnum, drexpcie_priv->valid_bufs );
+   for (i = 0; i < drexpcie_priv->valid_bufs; i++) {
+      temp64 = drexpcie_priv->dmaPtr[i];
+      temp[0] = (((drexpcie_priv->num_bytes) & 0x7FFFFF) << 9);
+      temp[0] |= (((i) & 0x000001FF));
+      temp[1] = temp64 & 0xFFFFFFFF;
+      temp[2] = (temp64 >> 32) & 0x000000FF;
+
+      printk("drexpcie[%d]: writing %x %x %x to FIFO\n", drexpcie_priv->chnum, temp[2],temp[1],temp[0]);
+      iowrite32(temp[0], drexpcie_priv->base_mem+DREXDMA_OFFSET_FIFO);
+      iowrite32(temp[1], drexpcie_priv->base_mem+DREXDMA_OFFSET_FIFO+4);
+      iowrite32(temp[2], drexpcie_priv->base_mem+DREXDMA_OFFSET_FIFO+8);            
+   }
+   
+   // Enable Loop Mode
+   printk("drexpcie[%d]: enabling loop mode\n", drexpcie_priv->chnum);
+   iowrite32(1, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL+4);
+
+   // Enable DMA Engine
+   // iowrite32(0x00000100, drexpcie_privDREXDMA_OFFSET_CTRL);
+
+   return 0;
+}
+
+int dma_clear(struct drexpcie_device_priv *drexpcie_priv)
+{
+
+   // set reset bits
+   iowrite32(0x00000007, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL);
+      
+   // Disable Loop Mode
+   iowrite32(0, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL+4);
+
+   release_dma_buffers(drexpcie_priv);
+ 
+   return 0;
+}
+
+int dma_start(struct drexpcie_device_priv *drexpcie_priv)
+{
+   
+   // Enable DMA Engine
+   iowrite32(0x00000100, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL);
+ 
+   return 0;
+}
+
+int dma_stop(struct drexpcie_device_priv *drexpcie_priv)
+{
+   
+   // Disable DMA Engine
+   iowrite32(0x00000000, drexpcie_priv->base_mem+DREXDMA_OFFSET_CTRL);
+
+   return 0;
+}

pcie_driver/drexdma.h

@@ -0,0 +1,20 @@
+
+#ifndef DREXDMA_H
+#define DREXDMA_H
+
+#include "drexchar.h"
+
+#define DREXDMA_OFFSET_CTRL 0x00
+#define DREXDMA_OFFSET_STAT 0x10
+#define DREXDMA_OFFSET_FIFO 0x20
+
+int init_dma_buffers(struct drexpcie_device_priv *drexpcie_priv);
+void release_dma_buffers(struct drexpcie_device_priv *drexpcie_priv);
+
+int dma_init(struct drexpcie_device_priv *drexpcie_priv);
+int dma_clear(struct drexpcie_device_priv *drexpcie_priv);
+int dma_start(struct drexpcie_device_priv *drexpcie_priv);
+int dma_stop(struct drexpcie_device_priv *drexpcie_priv);
+
+
+#endif

pcie_driver/drexpcie.c

@@ -0,0 +1,189 @@
+
+#include "drex.h"
+#include "drexchar.h"
+
+#define DRIVER_NAME "drexpcie"
+
+/* This sample driver supports device with VID = 0x010F, and PID = 0x0F0E*/
+static struct pci_device_id drexpcie_driver_id_table[] = {
+   { PCI_DEVICE(0x10EE, 0x903F) },
+   {0,}
+};
+
+MODULE_DEVICE_TABLE(pci, drexpcie_driver_id_table);
+
+static int drexpcie_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
+static void drexpcie_driver_remove(struct pci_dev *pdev);
+
+/* Driver registration structure */
+static struct pci_driver drexpcie_driver = {
+   .name = DRIVER_NAME,
+   .id_table = drexpcie_driver_id_table,
+   .probe = drexpcie_driver_probe,
+   .remove = drexpcie_driver_remove
+};
+
+
+static int __init drexpcie_driver_init(void)
+{
+   /* Register new PCI driver */
+   int result = pci_register_driver(&drexpcie_driver);
+   if (result < 0) {
+      // This print will tell you exactly why it failed
+      pr_err("pci_register_driver failed with error: %d\n", result);
+      return result; 
+   }
+
+   pr_info("Xilinx custom driver registered successfully!\n");   
+
+   return result;
+}
+
+static void __exit drexpcie_driver_exit(void)
+{
+   /* Unregister */
+   pci_unregister_driver(&drexpcie_driver);
+}
+
+void release_device(struct pci_dev *pdev)
+{
+   /* Disable IRQ #42*/
+//    free_irq(42, pdev);
+   /* Free memory region */
+   pci_release_region(pdev, pci_select_bars(pdev, IORESOURCE_MEM));
+   /* And disable device */
+   pci_disable_device(pdev);
+}
+
+/* */
+
+static irqreturn_t irq_handler(int irq, void *cookie)
+{
+   (void) cookie;
+   printk("Handle IRQ #%d\n", irq);
+   return IRQ_HANDLED;
+}
+
+/* Reqest interrupt and setup handler */
+int set_interrupts(struct pci_dev *pdev)
+{
+   /* We want MSI interrupt, 3 lines (just an example) */
+   int ret = pci_alloc_irq_vectors(pdev, 3, 3, PCI_IRQ_MSI);
+
+   if (ret < 0) {
+      return ret;
+   }
+
+   /* Request IRQ #42 */
+   return request_threaded_irq(42, irq_handler, NULL, 0, "TEST IRQ", pdev);
+}
+
+
+
+/* This function is called by the kernel */
+static int drexpcie_driver_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+   int bar, err;
+   u16 vendor, device;
+//    unsigned long mmio_start,mmio_len;
+   struct drexpcie_driver_priv *drv_priv;
+
+   printk("drexpcie probe\n");
+
+   /* Let's read data from the PCI device configuration registers */
+   pci_read_config_word(pdev, PCI_VENDOR_ID, &vendor);
+   pci_read_config_word(pdev, PCI_DEVICE_ID, &device);
+
+   printk(KERN_INFO "Device vid: 0x%X pid: 0x%X\n", vendor, device);
+
+   /* Request IO BAR */
+   bar = pci_select_bars(pdev, IORESOURCE_MEM);
+
+   /* Enable device memory */
+   err = pci_enable_device_mem(pdev);
+
+   if (err) {
+      return err;
+   }
+
+   /* Request memory region for the BAR */
+   err = pci_request_region(pdev, bar, DRIVER_NAME);
+
+   if (err) {
+      pci_disable_device(pdev);
+      return err;
+   }
+   
+   pci_set_master(pdev);
+
+   if (dma_set_mask_and_coherent(&(pdev->dev), DMA_BIT_MASK(40))) {
+      printk("dma_set_mask_and_coherent error\n");
+   }
+
+   /* Allocate memory for the driver private data */
+   drv_priv = kzalloc(sizeof(struct drexpcie_driver_priv), GFP_KERNEL);
+
+   if (!drv_priv) {
+      release_device(pdev);
+      return -ENOMEM;
+   }
+
+   drv_priv->pdev = pdev;
+
+   /* Get start and stop memory offsets */
+   drv_priv->mem_start = pci_resource_start(pdev, 0);
+   drv_priv->mem_len = pci_resource_len(pdev, 0);
+
+   printk(KERN_INFO "Resource Start: 0x%lX length: 0x%lX\n", drv_priv->mem_start, drv_priv->mem_len);
+
+   /* Remap BAR to the local pointer */
+   drv_priv->bar0_mem = ioremap(drv_priv->mem_start, drv_priv->mem_len);
+
+   if (!drv_priv->bar0_mem) {
+      release_device(pdev);
+      return -EIO;
+   }
+
+   drexpcie_chardev_create(drv_priv);
+
+   /* Set driver private data */
+   /* Now we can access mapped "bar0_mem" from the any driver's function */
+   pci_set_drvdata(pdev, drv_priv);
+
+   return 0;
+    
+//    return set_interrupts(pdev);
+}
+
+/* Clean up */
+static void drexpcie_driver_remove(struct pci_dev *pdev)
+{
+   struct drexpcie_driver_priv *drv_priv = pci_get_drvdata(pdev);
+
+   drexpcie_chardev_destroy();
+
+   if (drv_priv) {
+      if (drv_priv->bar0_mem) {
+         iounmap(drv_priv->bar0_mem);
+      }
+
+      pci_free_irq_vectors(pdev);
+
+      kfree(drv_priv);
+      pci_set_drvdata(pdev, NULL); 
+   }
+
+   // release_device(pdev);
+   pci_release_region(pdev, 0); 
+   pci_disable_device(pdev);    
+
+}
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("");
+MODULE_DESCRIPTION("DREX PCIe driver");
+MODULE_VERSION("0.1");
+
+module_init(drexpcie_driver_init);
+module_exit(drexpcie_driver_exit);
+