diff options
| -rw-r--r-- | linux/raptor.c | 1386 |
1 files changed, 695 insertions, 691 deletions
diff --git a/linux/raptor.c b/linux/raptor.c index cb28c44..024096e 100644 --- a/linux/raptor.c +++ b/linux/raptor.c @@ -67,874 +67,878 @@ MODULE_LICENSE("GPL"); struct packet; struct packet { - uint64_t u_addr; - uint16_t length; - struct page* page; - dma_addr_t dma_addr; - struct packet* next; + uint64_t u_addr; + uint16_t length; + struct page *page; + dma_addr_t dma_addr; + struct packet *next; }; struct queue { - struct packet* first; - struct packet* last; - uint64_t count; + struct packet *first; + struct packet *last; + uint64_t count; }; -struct raptor { - dev_t base; - struct class * class; +struct { + dev_t base; + struct class *class; - dev_t dev; - struct device * device; - struct cdev cdev; + dev_t dev; + struct device *device; + struct cdev cdev; - uint64_t *bar; - struct pci_dev *pdev; + uint64_t *bar; + struct pci_dev *pdev; - bool interrupt; + bool interrupt; - struct queue send_queue; - struct queue recv_queue; - struct queue send_done_queue; - struct queue recv_done_queue; - struct queue send_free_queue; - struct queue recv_free_queue; + struct queue send_queue; + struct queue recv_queue; + struct queue send_done_queue; + struct queue recv_done_queue; + struct queue send_free_queue; + struct queue recv_free_queue; - struct packet* send_packets; - struct packet* recv_packets; + struct packet *send_packets; + struct packet *recv_packets; - uint64_t recv_need; + uint64_t recv_need; - spinlock_t lock; - wait_queue_head_t wait; + spinlock_t lock; + wait_queue_head_t wait; - struct task_struct* kthread; + struct task_struct *kthread; - uint64_t int_count; - uint64_t send_busy; - uint64_t recv_busy; -}; - -struct raptor raptor; - -// ------------------------------------------- + uint64_t int_count; + uint64_t send_busy; + uint64_t recv_busy; +} raptor; -static void queue_push(struct queue* queue, struct packet* packet) +static void queue_push(struct queue *queue, struct packet *packet) { - spin_lock_irq(&raptor.lock); - - if (queue->count == 0) { - queue->first = packet; - queue->last = packet; - } - else { - queue->last->next = packet; - queue->last = packet; - } - queue->count++; - - spin_unlock_irq(&raptor.lock); - wake_up(&raptor.wait); + spin_lock_irq(&raptor.lock); + + if (queue->count == 0) { + queue->first = packet; + queue->last = packet; + } else { + queue->last->next = packet; + queue->last = packet; + } + queue->count++; + + spin_unlock_irq(&raptor.lock); + wake_up(&raptor.wait); } -static struct packet* queue_pop(struct queue* queue, bool block) +static struct packet *queue_pop(struct queue *queue, bool block) { - struct packet* packet; - - spin_lock_irq(&raptor.lock); - - if (block) { - if (wait_event_interruptible_lock_irq_timeout( - raptor.wait, queue->count > 0, raptor.lock, HZ) <= 0) { - - spin_unlock_irq(&raptor.lock); - return NULL; - } - } - else { - if (queue->count == 0) { - spin_unlock_irq(&raptor.lock); - return NULL; - } - } - - packet = queue->first; - queue->first = packet->next; - queue->count--; - - spin_unlock_irq(&raptor.lock); - wake_up(&raptor.wait); - return packet; + struct packet *packet; + + spin_lock_irq(&raptor.lock); + + if (block) { + if (wait_event_interruptible_lock_irq_timeout + (raptor.wait, queue->count > 0, raptor.lock, HZ) <= 0) { + + spin_unlock_irq(&raptor.lock); + return NULL; + } + } else { + if (queue->count == 0) { + spin_unlock_irq(&raptor.lock); + return NULL; + } + } + + packet = queue->first; + queue->first = packet->next; + queue->count--; + + spin_unlock_irq(&raptor.lock); + wake_up(&raptor.wait); + return packet; } -// ------------------------------------------- - uint64_t raptor_read(uint64_t offset) { - return raptor.bar[offset]; + return raptor.bar[offset]; } void raptor_write(uint64_t offset, uint64_t data) { - raptor.bar[offset] = data; + raptor.bar[offset] = data; } uint64_t endian64(uint64_t data) { - uint64_t byte0 = (data & 0x00000000000000FF) << 7*8; - uint64_t byte1 = (data & 0x000000000000FF00) << 5*8; - uint64_t byte2 = (data & 0x0000000000FF0000) << 3*8; - uint64_t byte3 = (data & 0x00000000FF000000) << 1*8; - uint64_t byte4 = (data & 0x000000FF00000000) >> 1*8; - uint64_t byte5 = (data & 0x0000FF0000000000) >> 3*8; - uint64_t byte6 = (data & 0x00FF000000000000) >> 5*8; - uint64_t byte7 = (data & 0xFF00000000000000) >> 7*8; - - return byte0 | byte1 | byte2 | byte3 | byte4 | byte5 | byte6 | byte7; + uint64_t byte0 = (data & 0x00000000000000FF) << 7 * 8; + uint64_t byte1 = (data & 0x000000000000FF00) << 5 * 8; + uint64_t byte2 = (data & 0x0000000000FF0000) << 3 * 8; + uint64_t byte3 = (data & 0x00000000FF000000) << 1 * 8; + uint64_t byte4 = (data & 0x000000FF00000000) >> 1 * 8; + uint64_t byte5 = (data & 0x0000FF0000000000) >> 3 * 8; + uint64_t byte6 = (data & 0x00FF000000000000) >> 5 * 8; + uint64_t byte7 = (data & 0xFF00000000000000) >> 7 * 8; + + return byte0 | byte1 | byte2 | byte3 | byte4 | byte5 | byte6 | byte7; } static irqreturn_t raptor_interrupt(int irq, void *dev_id) { - spin_lock(&raptor.lock); - raptor.interrupt = true; - spin_unlock(&raptor.lock); - wake_up(&raptor.wait); + spin_lock(&raptor.lock); + raptor.interrupt = true; + spin_unlock(&raptor.lock); + wake_up(&raptor.wait); - return IRQ_HANDLED; + return IRQ_HANDLED; } -void mdio_raw(unsigned operation, unsigned phy_addr, unsigned dev_addr, unsigned data) +void mdio_raw(unsigned int operation, unsigned int phy_addr, + unsigned int dev_addr, unsigned int data) { - unsigned command = (operation << 26) | (phy_addr << 21) | (dev_addr << 16) | (data & 0xFFFF); + unsigned int command = + (operation << 26) | (phy_addr << 21) | (dev_addr << 16) | (data & + 0xFFFF); - raptor_write(CMD_MDIO, endian64((uint64_t)command)); - msleep(1); + raptor_write(CMD_MDIO, endian64((uint64_t) command)); + msleep(20); } -void mdio_write(unsigned phy_addr, unsigned dev_addr, unsigned data_addr, unsigned data) +void mdio_write(unsigned int phy_addr, unsigned int dev_addr, + unsigned int data_addr, unsigned int data) { - mdio_raw(0, phy_addr, dev_addr, data_addr); - mdio_raw(1, phy_addr, dev_addr, data); + mdio_raw(0, phy_addr, dev_addr, data_addr); + mdio_raw(1, phy_addr, dev_addr, data); } -unsigned mdio_read(unsigned phy_addr, unsigned dev_addr, unsigned data_addr) +unsigned int mdio_read(unsigned int phy_addr, unsigned int dev_addr, + unsigned int data_addr) { - mdio_raw(0, phy_addr, dev_addr, data_addr); - mdio_raw(3, phy_addr, dev_addr, 0); + mdio_raw(0, phy_addr, dev_addr, data_addr); + mdio_raw(3, phy_addr, dev_addr, 0); - return (unsigned)endian64(raptor_read(BAR_MDIO)); + return (unsigned int) endian64(raptor_read(BAR_MDIO)); } -int mdio_i2c_read(uint32_t phy_addr, uint16_t dev_addr, uint16_t word_addr, uint16_t * data) +int mdio_i2c_read(uint32_t phy_addr, uint16_t dev_addr, uint16_t word_addr, + uint16_t *data) { - uint16_t stat; - int i; + uint16_t stat; + int i; - mdio_write(phy_addr, 1, AEL_I2C_CTRL, (dev_addr << 8) | (1 << 8) | word_addr); + mdio_write(phy_addr, 1, AEL_I2C_CTRL, + (dev_addr << 8) | (1 << 8) | word_addr); - for (i = 0; i < 20; i++) { - msleep(2); - stat = mdio_read(phy_addr, 1, AEL_I2C_STAT); + for (i = 0; i < 20; i++) { + msleep(20); + stat = mdio_read(phy_addr, 1, AEL_I2C_STAT); - if ((stat & 3) == 1){ - stat = mdio_read(phy_addr, 1, AEL_I2C_DATA); + if ((stat & 3) == 1) { + stat = mdio_read(phy_addr, 1, AEL_I2C_DATA); - *data = stat >> 8; - return 0; - } - } - return -1; + *data = stat >> 8; + return 0; + } + } + return -1; } void mdio_initialize(uint32_t phy_addr, int mode) { - int size, i; - - // Step 1 - size = sizeof(reset) / sizeof(uint16_t); - - for(i = 0; i < size; i += 2) { - mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, reset[i], reset[i+1]); - } - - msleep(5); - - // Step 2 - if (mode == MODE_SR) { - size = sizeof(sr_edc) / sizeof(uint16_t); - - for(i = 0; i < size; i += 2) { - mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, sr_edc[i], sr_edc[i+1]); - } - } - else if (mode == MODE_TWINAX) { - size = sizeof(twinax_edc) / sizeof(uint16_t); - - for(i = 0; i < size; i += 2) { - mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, twinax_edc[i], twinax_edc[i+1]); - } - } - - // Step 3 - size = sizeof(regs1) / sizeof(uint16_t); - for(i = 0; i < size; i+=2) { - mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, regs1[i], regs1[i+1]); - } - - msleep(5); + int size, i; + + /* Step 1 */ + size = sizeof(reset) / sizeof(uint16_t); + + for (i = 0; i < size; i += 2) { + mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, reset[i], + reset[i + 1]); + } + + msleep(20); + + /* Step 2 */ + if (mode == MODE_SR) { + size = sizeof(sr_edc) / sizeof(uint16_t); + + for (i = 0; i < size; i += 2) { + mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, sr_edc[i], + sr_edc[i + 1]); + } + } else if (mode == MODE_TWINAX) { + size = sizeof(twinax_edc) / sizeof(uint16_t); + + for (i = 0; i < size; i += 2) { + mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, + twinax_edc[i], twinax_edc[i + 1]); + } + } + /* Step 3 */ + size = sizeof(regs1) / sizeof(uint16_t); + for (i = 0; i < size; i += 2) { + mdio_write(phy_addr, PMA_MDIO_DEVICE_ADDRESS, regs1[i], + regs1[i + 1]); + } + + msleep(20); } int phy_configuration(void) { - int port, dev; - uint16_t value = 0; - char port_mode; - - // check if we need initialization - value = mdio_read(2, PMA_MDIO_DEVICE_ADDRESS, AEL_MICRO_CONTROLLER_CTL_ADDRESS); - - //printk(KERN_INFO "raptor: 0xc04a: %04x\n",value); - if (value & 0x8000) { // uC held in reset - printk(KERN_INFO "raptor: Programming the AEL2005 PHY chips...\n"); - - for (port = 0; port < 4; port++) { - if (port == 0) dev = 2; - else if (port == 1) dev = 1; - else if (port == 2) dev = 0; - else dev = 3; - - value = 0; - - // check if we have a 10GBASE-SR cable - mdio_i2c_read(dev, MODULE_DEV_ADDR, 0x3, &value); - - if((value >> 4) == 1) { - port_mode = MODE_SR; - } - else { - port_mode = MODE_TWINAX; - } - - printk(KERN_INFO "raptor: Programming PHY %d...\n", port); - - mdio_initialize(dev, port_mode); - } - - return 0; - } - - return -1; + int port, dev; + uint16_t value = 0; + char port_mode; + + /* check if we need initialization */ + value = + mdio_read(2, PMA_MDIO_DEVICE_ADDRESS, + AEL_MICRO_CONTROLLER_CTL_ADDRESS); + + if (value & 0x8000) { /* uC held in reset */ + for (port = 0; port < 4; port++) { + if (port == 0) + dev = 2; + else if (port == 1) + dev = 1; + else if (port == 2) + dev = 0; + else + dev = 3; + + value = 0; + + /* check if we have a 10GBASE-SR cable */ + mdio_i2c_read(dev, MODULE_DEV_ADDR, 0x3, &value); + + if ((value >> 4) == 1) + port_mode = MODE_SR; + else + port_mode = MODE_TWINAX; + + mdio_initialize(dev, port_mode); + } + + return 0; + } + + return -1; } static void read_debug(void) { - uint64_t status; - uint64_t address; - uint64_t data; - uint64_t cmd; - uint64_t buff; - uint64_t length; - uint64_t cmpl_count; - uint64_t cmpl_data; - uint64_t cmpl_tag; - uint64_t cmpl_state; - uint64_t write_count; - uint64_t write_length; - uint64_t write_addr; - uint64_t write_data; - uint64_t accept_count; - uint64_t done_count; - - status = endian64(raptor_read(0)); - address = endian64(raptor_read(1)); - data = endian64(raptor_read(2)); - cmd = endian64(raptor_read(3)); - buff = endian64(raptor_read(4)); - length = endian64(raptor_read(5)); - cmpl_count = endian64(raptor_read(10)); - cmpl_data = endian64(raptor_read(11)); - cmpl_tag = endian64(raptor_read(12)); - cmpl_state = endian64(raptor_read(13)); - - write_count = endian64(raptor_read(20)); - write_length = endian64(raptor_read(21)); - write_addr = endian64(raptor_read(22)); - write_data = endian64(raptor_read(23)); - accept_count = endian64(raptor_read(24)); - done_count = endian64(raptor_read(25)); - - printk(KERN_INFO "raptor: read_debug:\n"); - printk(KERN_INFO "raptor: status = 0x%llx.\n", status); - printk(KERN_INFO "raptor: address = 0x%llx.\n", address); - printk(KERN_INFO "raptor: data = 0x%llx.\n", data); - printk(KERN_INFO "raptor: cmd = 0x%llx.\n", cmd); - printk(KERN_INFO "raptor: buff = 0x%llx.\n", buff); - printk(KERN_INFO "raptor: length = %llu.\n", length); - printk(KERN_INFO "raptor: cmpl_count = %llu.\n", cmpl_count); - printk(KERN_INFO "raptor: cmpl_data = 0x%llx.\n", cmpl_data); - printk(KERN_INFO "raptor: cmpl_tag = 0x%llx.\n", cmpl_tag); - printk(KERN_INFO "raptor: cmpl_state = 0x%llx.\n", cmpl_state); - printk(KERN_INFO "raptor: write_count = %llu.\n", write_count); - printk(KERN_INFO "raptor: write_length = %llu.\n", write_length); - printk(KERN_INFO "raptor: write_addr = 0x%llx.\n", write_addr); - printk(KERN_INFO "raptor: write_data = 0x%llx.\n", write_data); - printk(KERN_INFO "raptor: accept_count = %llu.\n", accept_count); - printk(KERN_INFO "raptor: done_count = %llu.\n", done_count); + uint64_t status; + uint64_t address; + uint64_t data; + uint64_t cmd; + uint64_t buff; + uint64_t length; + uint64_t cmpl_count; + uint64_t cmpl_data; + uint64_t cmpl_tag; + uint64_t cmpl_state; + uint64_t write_count; + uint64_t write_length; + uint64_t write_addr; + uint64_t write_data; + uint64_t accept_count; + uint64_t done_count; + + status = endian64(raptor_read(0)); + address = endian64(raptor_read(1)); + data = endian64(raptor_read(2)); + cmd = endian64(raptor_read(3)); + buff = endian64(raptor_read(4)); + length = endian64(raptor_read(5)); + cmpl_count = endian64(raptor_read(10)); + cmpl_data = endian64(raptor_read(11)); + cmpl_tag = endian64(raptor_read(12)); + cmpl_state = endian64(raptor_read(13)); + + write_count = endian64(raptor_read(20)); + write_length = endian64(raptor_read(21)); + write_addr = endian64(raptor_read(22)); + write_data = endian64(raptor_read(23)); + accept_count = endian64(raptor_read(24)); + done_count = endian64(raptor_read(25)); + + dev_info(raptor.device, "%s:\n", __func__); + dev_info(raptor.device, " status = 0x%llx.\n", status); + dev_info(raptor.device, " address = 0x%llx.\n", address); + dev_info(raptor.device, " data = 0x%llx.\n", data); + dev_info(raptor.device, " cmd = 0x%llx.\n", cmd); + dev_info(raptor.device, " buff = 0x%llx.\n", buff); + dev_info(raptor.device, " length = %llu.\n", length); + dev_info(raptor.device, " cmpl_count = %llu.\n", cmpl_count); + dev_info(raptor.device, " cmpl_data = 0x%llx.\n", cmpl_data); + dev_info(raptor.device, " cmpl_tag = 0x%llx.\n", cmpl_tag); + dev_info(raptor.device, " cmpl_state = 0x%llx.\n", cmpl_state); + dev_info(raptor.device, " write_count = %llu.\n", write_count); + dev_info(raptor.device, " write_length = %llu.\n", write_length); + dev_info(raptor.device, " write_addr = 0x%llx.\n", write_addr); + dev_info(raptor.device, " write_data = 0x%llx.\n", write_data); + dev_info(raptor.device, " accept_count = %llu.\n", accept_count); + dev_info(raptor.device, " done_count = %llu.\n", done_count); } #define get_send(status, i) ((status >> (i * 4 + 0)) & 0x3) #define get_recv(status, i) ((status >> (i * 4 + 2)) & 0x3) -struct packet* send_packets[BUFF_COUNT]; -struct packet* recv_packets[BUFF_COUNT]; +struct packet *send_packets[BUFF_COUNT]; +struct packet *recv_packets[BUFF_COUNT]; uint8_t send_buff[BUFF_COUNT]; uint8_t recv_buff[BUFF_COUNT]; -static int raptor_kthread(void* data) +static int raptor_kthread(void *data) { - bool interrupt; - uint64_t status = 0; - uint8_t send_new; - uint8_t recv_new; - uint8_t send_old; - uint8_t recv_old; - uint64_t i; - bool send_full = false; - bool recv_full = false; - struct packet* packet; - struct device* dev = &raptor.pdev->dev; - uint64_t offset; - uint64_t rest; - - printk(KERN_INFO "raptor_kthread: started\n"); - - memset(send_packets, 0, sizeof(send_packets)); - memset(recv_packets, 0, sizeof(recv_packets)); - - raptor.interrupt = true; - - while (true) { - spin_lock_irq(&raptor.lock); - - if (wait_event_interruptible_lock_irq( - raptor.wait, - kthread_should_stop() || - raptor.interrupt || - (! send_full && raptor.send_queue.count > 0) || - (! recv_full && raptor.recv_queue.count > 0), - raptor.lock) != 0) { - - printk(KERN_ERR "raptor_kthread: wait_event failed\n"); - spin_unlock_irq(&raptor.lock); - break; - } - - interrupt = raptor.interrupt; - raptor.interrupt = false; - spin_unlock_irq(&raptor.lock); - - if (kthread_should_stop()) - break; - - if (interrupt) { - // read status - // update done_queues - // incr recv_need - status = endian64(raptor_read(0)); - - raptor.int_count++; - - for (i = 0; i < BUFF_COUNT; i++) { - send_new = get_send(status, i); - recv_new = get_recv(status, i); - send_old = send_buff[i]; - recv_old = recv_buff[i]; - - if (send_new != BUFF_EMPTY) - raptor.send_busy++; - if (recv_new != BUFF_EMPTY) - raptor.recv_busy++; - - if ((send_old == BUFF_ADDR || send_old == BUFF_DATA) && - (send_new == BUFF_FULL || send_new == BUFF_EMPTY) && - send_packets[i] != NULL) { - - packet = send_packets[i]; - offset = packet->u_addr & (~PAGE_MASK); - rest = PAGE_SIZE - offset; - - dma_unmap_page(dev, packet->dma_addr, rest, DMA_TO_DEVICE); - queue_push(&raptor.send_done_queue, packet); - send_packets[i] = NULL; - } - if ((recv_old == BUFF_ADDR || recv_old == BUFF_DATA) && - (recv_new == BUFF_FULL || recv_new == BUFF_EMPTY) && - recv_packets[i] != NULL) { - - packet = recv_packets[i]; - offset = packet->u_addr & (~PAGE_MASK); - rest = PAGE_SIZE - offset; - - dma_unmap_page(dev, packet->dma_addr, rest, DMA_FROM_DEVICE); - queue_push(&raptor.recv_done_queue, packet); - recv_packets[i] = NULL; - } - if (recv_old != BUFF_FULL && recv_new == BUFF_FULL) { - spin_lock_irq(&raptor.lock); - raptor.recv_need++; - spin_unlock_irq(&raptor.lock); - wake_up(&raptor.wait); - } - - send_buff[i] = send_new; - recv_buff[i] = recv_new; - } - } - - // handle send_queue - send_full = true; - - for (i = 0; i < BUFF_COUNT; i++) { - if (send_buff[i] == BUFF_EMPTY && send_packets[i] == NULL) { - - packet = queue_pop(&raptor.send_queue, false); - - if (packet == NULL) { - send_full = false; - break; - } - - // map page to dma_addr - // write that to hardware - - offset = packet->u_addr & (~PAGE_MASK); - rest = PAGE_SIZE - offset; - - packet->dma_addr = dma_map_page(dev, packet->page, offset, rest, DMA_TO_DEVICE); - - raptor_write(CMD_SEND | (i << 2) | (packet->length << 7), endian64(packet->dma_addr)); - - send_buff[i] = BUFF_ADDR; - send_packets[i] = packet; - } - } - - // handle recv_queue - recv_full = true; - - for (i = 0; i < BUFF_COUNT; i++) { - if (recv_buff[i] == BUFF_FULL && recv_packets[i] == NULL) { - - packet = queue_pop(&raptor.recv_queue, false); - - if (packet == NULL) { - recv_full = false; - break; - } - - // map page to dma_addr - // write that to hardware - - offset = packet->u_addr & (~PAGE_MASK); - rest = PAGE_SIZE - offset; - - packet->dma_addr = dma_map_page(dev, packet->page, offset, rest, DMA_FROM_DEVICE); - - raptor_write(CMD_RECV | (i << 2), endian64(packet->dma_addr)); - - recv_buff[i] = BUFF_ADDR; - recv_packets[i] = packet; - } - } - } - - printk(KERN_INFO "raptor_kthread: stopped\n"); - return 0; + bool interrupt; + uint64_t status = 0; + uint8_t send_new; + uint8_t recv_new; + uint8_t send_old; + uint8_t recv_old; + uint64_t i; + bool send_full = false; + bool recv_full = false; + struct packet *packet; + struct device *dev = &raptor.pdev->dev; + uint64_t offset; + uint64_t rest; + + memset(send_packets, 0, sizeof(send_packets)); + memset(recv_packets, 0, sizeof(recv_packets)); + + raptor.interrupt = true; + + while (true) { + spin_lock_irq(&raptor.lock); + + if (wait_event_interruptible_lock_irq( + raptor.wait, + kthread_should_stop() || raptor.interrupt || + (!send_full && raptor. send_queue.count > 0) || + (!recv_full && raptor.recv_queue.count > 0), + raptor.lock) != 0) { + dev_err(raptor.device, "wait_event failed\n"); + spin_unlock_irq(&raptor.lock); + break; + } + + interrupt = raptor.interrupt; + raptor.interrupt = false; + spin_unlock_irq(&raptor.lock); + + if (kthread_should_stop()) + break; + + if (interrupt) { + /* + *read status + * update done_queues + * incr recv_need + */ + status = endian64(raptor_read(0)); + + raptor.int_count++; + + for (i = 0; i < BUFF_COUNT; i++) { + send_new = get_send(status, i); + recv_new = get_recv(status, i); + send_old = send_buff[i]; + recv_old = recv_buff[i]; + + if (send_new != BUFF_EMPTY) + raptor.send_busy++; + if (recv_new != BUFF_EMPTY) + raptor.recv_busy++; + + if ((send_old == BUFF_ADDR + || send_old == BUFF_DATA) + && (send_new == BUFF_FULL + || send_new == BUFF_EMPTY) + && send_packets[i] != NULL) { + + packet = send_packets[i]; + offset = packet->u_addr & (~PAGE_MASK); + rest = PAGE_SIZE - offset; + + dma_unmap_page(dev, packet->dma_addr, + rest, DMA_TO_DEVICE); + queue_push(&raptor.send_done_queue, + packet); + send_packets[i] = NULL; + } + if ((recv_old == BUFF_ADDR + || recv_old == BUFF_DATA) + && (recv_new == BUFF_FULL + || recv_new == BUFF_EMPTY) + && recv_packets[i] != NULL) { + + packet = recv_packets[i]; + offset = packet->u_addr & (~PAGE_MASK); + rest = PAGE_SIZE - offset; + + dma_unmap_page(dev, packet->dma_addr, + rest, DMA_FROM_DEVICE); + queue_push(&raptor.recv_done_queue, + packet); + recv_packets[i] = NULL; + } + if (recv_old != BUFF_FULL + && recv_new == BUFF_FULL) { + spin_lock_irq(&raptor.lock); + raptor.recv_need++; + spin_unlock_irq(&raptor.lock); + wake_up(&raptor.wait); + } + + send_buff[i] = send_new; + recv_buff[i] = recv_new; + } + } + /* handle send_queue */ + send_full = true; + + for (i = 0; i < BUFF_COUNT; i++) { + if (send_buff[i] == BUFF_EMPTY + && send_packets[i] == NULL) { + + packet = queue_pop(&raptor.send_queue, false); + + if (packet == NULL) { + send_full = false; + break; + } + /* + * map page to dma_addr + * write that to hardware + */ + + offset = packet->u_addr & (~PAGE_MASK); + rest = PAGE_SIZE - offset; + + packet->dma_addr = + dma_map_page(dev, packet->page, offset, + rest, DMA_TO_DEVICE); + + raptor_write(CMD_SEND | (i << 2) | + (packet->length << 7), + endian64(packet->dma_addr)); + + send_buff[i] = BUFF_ADDR; + send_packets[i] = packet; + } + } + + /* handle recv_queue */ + recv_full = true; + + for (i = 0; i < BUFF_COUNT; i++) { + if (recv_buff[i] == BUFF_FULL + && recv_packets[i] == NULL) { + + packet = queue_pop(&raptor.recv_queue, false); + + if (packet == NULL) { + recv_full = false; + break; + } + /* + * map page to dma_addr + * write that to hardware + */ + + offset = packet->u_addr & (~PAGE_MASK); + rest = PAGE_SIZE - offset; + + packet->dma_addr = + dma_map_page(dev, packet->page, offset, + rest, DMA_FROM_DEVICE); + + raptor_write(CMD_RECV | (i << 2), + endian64(packet->dma_addr)); + + recv_buff[i] = BUFF_ADDR; + recv_packets[i] = packet; + } + } + } + + return 0; } static int pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { - if (pci_enable_device(pdev) != 0) - goto error_no_cleanup; - - if (pci_request_region(pdev, RAPTOR_BAR, "raptor") != 0) - goto error_disable_device; + if (pci_enable_device(pdev) != 0) + goto error_no_cleanup; - if ((raptor.bar = pci_ioremap_bar(pdev, RAPTOR_BAR)) == NULL) - goto error_release_region; + if (pci_request_region(pdev, RAPTOR_BAR, "raptor") != 0) + goto error_disable_device; - if (pci_enable_msi(pdev) < 0) - goto error_iounmap; + raptor.bar = pci_ioremap_bar(pdev, RAPTOR_BAR); + if (raptor.bar == NULL) + goto error_release_region; - if (request_irq(pdev->irq, raptor_interrupt, 0, "raptor", NULL) < 0) - goto error_disable_msi; + if (pci_enable_msi(pdev) < 0) + goto error_iounmap; - if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0) - goto error_free_irq; + if (request_irq(pdev->irq, raptor_interrupt, 0, "raptor", NULL) < 0) + goto error_disable_msi; - if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0) - goto error_free_irq; + if (dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0) + goto error_free_irq; - pci_set_master(pdev); - raptor.pdev = pdev; + if (dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)) < 0) + goto error_free_irq; - phy_configuration(); + pci_set_master(pdev); + raptor.pdev = pdev; - read_debug(); + phy_configuration(); - raptor.kthread = kthread_run(raptor_kthread, NULL, "raptor_kthread"); + read_debug(); - if (IS_ERR(raptor.kthread)) - goto error_clear_master; + raptor.kthread = kthread_run(raptor_kthread, NULL, "raptor_kthread"); - printk(KERN_INFO "raptor: pci_probe succes\n"); + if (IS_ERR(raptor.kthread)) + goto error_clear_master; - return 0; + return 0; -error_clear_master: - pci_clear_master(pdev); -error_free_irq: - free_irq(pdev->irq, NULL); -error_disable_msi: - pci_disable_msi(pdev); -error_iounmap: - iounmap(raptor.bar); -error_release_region: - pci_release_region(pdev, 0); -error_disable_device: - pci_disable_device(pdev); -error_no_cleanup: - raptor.bar = NULL; - raptor.pdev = NULL; - printk(KERN_ALERT "raptor: pci_probe FAILED\n"); - return -1; + error_clear_master: + pci_clear_master(pdev); + error_free_irq: + free_irq(pdev->irq, NULL); + error_disable_msi: + pci_disable_msi(pdev); + error_iounmap: + iounmap(raptor.bar); + error_release_region: + pci_release_region(pdev, 0); + error_disable_device: + pci_disable_device(pdev); + error_no_cleanup: + raptor.bar = NULL; + raptor.pdev = NULL; + dev_alert(raptor.device, "%s FAILED\n", __func__); + return -1; } static void pci_remove(struct pci_dev *pdev) { - printk(KERN_INFO "raptor: pci_remove started...\n"); - - (void)kthread_stop(raptor.kthread); - - pci_clear_master(pdev); - free_irq(pdev->irq, NULL); - pci_disable_msi(pdev); - iounmap(raptor.bar); - pci_release_region(pdev, 0); - pci_disable_device(pdev); - raptor.bar = NULL; - raptor.pdev = NULL; - - printk(KERN_INFO "raptor: pci_remove done\n"); + (void) kthread_stop(raptor.kthread); + + pci_clear_master(pdev); + free_irq(pdev->irq, NULL); + pci_disable_msi(pdev); + iounmap(raptor.bar); + pci_release_region(pdev, 0); + pci_disable_device(pdev); + raptor.bar = NULL; + raptor.pdev = NULL; } -static pci_ers_result_t pci_error_detected(struct pci_dev *pdev, enum pci_channel_state state) +static pci_ers_result_t pci_error_detected(struct pci_dev *pdev, + enum pci_channel_state state) { - printk(KERN_ALERT "raptor: pcie error %d\n", state); - return PCI_ERS_RESULT_RECOVERED; + return PCI_ERS_RESULT_RECOVERED; } -static int raptor_send(uint64_t count, uint64_t* u_addrs) +static int raptor_send(uint64_t count, uint64_t *u_addrs) { - uint64_t i; - uint64_t u_addr; - uint16_t bytes; - uint16_t length; - struct page* page; - struct packet* packet; - - for (i = 0; i < count; i++) { - if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) { - printk(KERN_ERR "raptor_send: copy_from_user u_addr failed\n"); - return i; - } - - if (copy_from_user(&bytes, (void*)u_addr, sizeof(bytes)) != 0) { - printk(KERN_ERR "raptor_send: copy_from_user bytes failed\n"); - return i; - } - - if (get_user_pages_fast(u_addr, 1, false, &page) != 1) { - printk(KERN_ERR "raptor_send: get_user_pages_fast failed\n"); - return i; - } - - packet = queue_pop(&raptor.send_free_queue, true); - - if (packet == NULL) { - printk(KERN_ERR "raptor_send: queue_pop send_free_queue failed\n"); - put_page(page); - return i; - } - - length = (bytes >> 3); - - if (bytes & 0x7) - length += 1; - - packet->u_addr = u_addr; - packet->length = length; - packet->page = page; - - queue_push(&raptor.send_queue, packet); - } - return count; + uint64_t i; + uint64_t u_addr; + uint16_t bytes; + uint16_t length; + struct page *page; + struct packet *packet; + + for (i = 0; i < count; i++) { + if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) { + dev_err(raptor.device, "copy_from_user u_addr failed\n"); + return i; + } + + if (copy_from_user(&bytes, (void *) u_addr, sizeof(bytes)) + != 0) { + dev_err(raptor.device, "copy_from_user bytes failed\n"); + return i; + } + + if (get_user_pages_fast(u_addr, 1, false, &page) != 1) { + dev_err(raptor.device, "get_user_pages_fast failed\n"); + return i; + } + + packet = queue_pop(&raptor.send_free_queue, true); + + if (packet == NULL) { + dev_err(raptor.device, "queue_pop send_free_queue failed\n"); + put_page(page); + return i; + } + + length = (bytes >> 3); + + if (bytes & 0x7) + length += 1; + + packet->u_addr = u_addr; + packet->length = length; + packet->page = page; + + queue_push(&raptor.send_queue, packet); + } + return count; } -static int raptor_recv(uint64_t count, uint64_t* u_addrs) +static int raptor_recv(uint64_t count, uint64_t *u_addrs) { - uint64_t i; - uint64_t u_addr; - struct page* page; - struct packet* packet; - - for (i = 0; i < count; i++) { - if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) { - printk(KERN_ERR "raptor_recv: copy_from_user u_addr failed\n"); - return i; - } - - if (get_user_pages_fast(u_addr, 1, true, &page) != 1) { - printk(KERN_ERR "raptor_recv: get_user_pages_fast failed\n"); - return i; - } - - packet = queue_pop(&raptor.recv_free_queue, true); - - if (packet == NULL) { - printk(KERN_ERR "raptor_recv: queue_pop recv_free_queue failed\n"); - put_page(page); - return i; - } - - packet->u_addr = u_addr; - packet->page = page; - - queue_push(&raptor.recv_queue, packet); - } - return count; + uint64_t i; + uint64_t u_addr; + struct page *page; + struct packet *packet; + + for (i = 0; i < count; i++) { + if (copy_from_user(&u_addr, &u_addrs[i], sizeof(u_addr)) != 0) { + dev_err(raptor.device, "recv copy_from_user u_addr failed\n"); + return i; + } + + if (get_user_pages_fast(u_addr, 1, true, &page) != 1) { + dev_err(raptor.device, "recv get_user_pages_fast failed\n"); + return i; + } + + packet = queue_pop(&raptor.recv_free_queue, true); + + if (packet == NULL) { + dev_err(raptor.device, "recv: queue_pop recv_free_queue failed\n"); + put_page(page); + return i; + } + + packet->u_addr = u_addr; + packet->page = page; + + queue_push(&raptor.recv_queue, packet); + } + return count; } -static int raptor_send_done(uint64_t count, uint64_t* u_addrs) +static int raptor_send_done(uint64_t count, uint64_t *u_addrs) { - uint64_t i; - struct packet* packet; + uint64_t i; + struct packet *packet; - for (i = 0; i < count; i++) { - packet = queue_pop(&raptor.send_done_queue, i == 0); + for (i = 0; i < count; i++) { + packet = queue_pop(&raptor.send_done_queue, i == 0); - if (packet == NULL) - return i; + if (packet == NULL) + return i; - put_page(packet->page); + put_page(packet->page); - if (copy_to_user(&u_addrs[i], &packet->u_addr, sizeof(packet->u_addr)) != 0) { - printk(KERN_ERR "raptor_send_done: copy_to_user u_addr failed\n"); - return i; - } + if (copy_to_user + (&u_addrs[i], &packet->u_addr, + sizeof(packet->u_addr)) != 0) { + dev_err(raptor.device, "done copy_to_user u_addr failed\n"); + return i; + } - queue_push(&raptor.send_free_queue, packet); - } - return count; + queue_push(&raptor.send_free_queue, packet); + } + return count; } -static int raptor_recv_done(uint64_t count, uint64_t* u_addrs) +static int raptor_recv_done(uint64_t count, uint64_t *u_addrs) { - uint64_t i; - struct packet* packet; + uint64_t i; + struct packet *packet; - for (i = 0; i < count; i++) { - packet = queue_pop(&raptor.recv_done_queue, i == 0); + for (i = 0; i < count; i++) { + packet = queue_pop(&raptor.recv_done_queue, i == 0); - if (packet == NULL) - return i; + if (packet == NULL) + return i; - put_page(packet->page); + put_page(packet->page); - if (copy_to_user(&u_addrs[i], &packet->u_addr, sizeof(packet->u_addr)) != 0) { - printk(KERN_ERR "raptor_recv_done: copy_to_user u_addr failed\n"); - return i; - } + if (copy_to_user + (&u_addrs[i], &packet->u_addr, + sizeof(packet->u_addr)) != 0) { + dev_err(raptor.device, + "recv_done: copy_to_user u_addr failed\n"); + return i; + } - queue_push(&raptor.recv_free_queue, packet); - } - return count; + queue_push(&raptor.recv_free_queue, packet); + } + return count; } static int raptor_recv_need(uint64_t count) { - uint64_t ret; + uint64_t ret; - spin_lock_irq(&raptor.lock); + spin_lock_irq(&raptor.lock); - if (wait_event_interruptible_lock_irq_timeout( - raptor.wait, raptor.recv_need > 0, raptor.lock, HZ) <= 0) { + if (wait_event_interruptible_lock_irq_timeout + (raptor.wait, raptor.recv_need > 0, raptor.lock, HZ) <= 0) { - spin_unlock_irq(&raptor.lock); - return -1; - } + spin_unlock_irq(&raptor.lock); + return -1; + } - if (raptor.recv_need > count) { - ret = count; - raptor.recv_need -= count; - } - else { - ret = raptor.recv_need; - raptor.recv_need = 0; - } + if (raptor.recv_need > count) { + ret = count; + raptor.recv_need -= count; + } else { + ret = raptor.recv_need; + raptor.recv_need = 0; + } - spin_unlock_irq(&raptor.lock); - return ret; + spin_unlock_irq(&raptor.lock); + return ret; } -static int raptor_debug(uint64_t* u_array) +static int raptor_debug(uint64_t *u_array) { - if (copy_to_user(&u_array[0], &raptor.int_count, sizeof(uint64_t)) || - copy_to_user(&u_array[1], &raptor.send_busy, sizeof(uint64_t)) || - copy_to_user(&u_array[2], &raptor.recv_busy, sizeof(uint64_t))) { + if (copy_to_user(&u_array[0], &raptor.int_count, sizeof(uint64_t)) || + copy_to_user(&u_array[1], &raptor.send_busy, sizeof(uint64_t)) || + copy_to_user(&u_array[2], &raptor.recv_busy, sizeof(uint64_t))) { - printk(KERN_ERR "raptor_debug: copy_to_user failed\n"); - return -1; - } + dev_err(raptor.device, "%s: copy_to_user failed\n", __func__); + return -1; + } - raptor.int_count = 0; - raptor.send_busy = 0; - raptor.recv_busy = 0; + raptor.int_count = 0; + raptor.send_busy = 0; + raptor.recv_busy = 0; - return 0; + return 0; } - -static long raptor_ioctl(struct file * file, unsigned int cmd, unsigned long arg) +static long raptor_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { - uint64_t type = cmd & 0xFFFF0000; - uint64_t count = cmd & 0x0000FFFF; - - if (raptor.bar == NULL) { - printk(KERN_ERR "raptor: no pcie device\n"); - return -1; - } - - switch (type) { - case IOCTL_SEND: - return raptor_send(count, (uint64_t*)arg); - case IOCTL_RECV: - return raptor_recv(count, (uint64_t*)arg); - case IOCTL_SEND_DONE: - return raptor_send_done(count, (uint64_t*)arg); - case IOCTL_RECV_DONE: - return raptor_recv_done(count, (uint64_t*)arg); - case IOCTL_RECV_NEED: - return raptor_recv_need(count); - case IOCTL_DEBUG: - return raptor_debug((uint64_t*)arg); - default: - printk(KERN_ERR "raptor: unknown ioctl\n"); - return -1; - } + uint64_t type = cmd & 0xFFFF0000; + uint64_t count = cmd & 0x0000FFFF; + + if (raptor.bar == NULL) { + dev_err(raptor.device, "no pcie device\n"); + return -1; + } + + switch (type) { + case IOCTL_SEND: + return raptor_send(count, (uint64_t *) arg); + case IOCTL_RECV: + return raptor_recv(count, (uint64_t *) arg); + case IOCTL_SEND_DONE: + return raptor_send_done(count, (uint64_t *) arg); + case IOCTL_RECV_DONE: + return raptor_recv_done(count, (uint64_t *) arg); + case IOCTL_RECV_NEED: + return raptor_recv_need(count); + case IOCTL_DEBUG: + return raptor_debug((uint64_t *) arg); + default: + dev_err(raptor.device, "unknown ioctl\n"); + return -1; + } } static struct pci_device_id pci_id_table[] = { - {PCI_DEVICE(RAPTOR_VENDOR_ID, RAPTOR_DEVICE_ID)}, - {0} + { + PCI_DEVICE(RAPTOR_VENDOR_ID, RAPTOR_DEVICE_ID)}, { + 0} }; + MODULE_DEVICE_TABLE(pci, pci_id_table); static struct pci_error_handlers pci_err_handlers = { - .error_detected = pci_error_detected + .error_detected = pci_error_detected }; static struct pci_driver pci_driver = { - .name = "raptor", - .id_table = pci_id_table, - .probe = pci_probe, - .remove = pci_remove, - .err_handler = &pci_err_handlers + .name = "raptor", + .id_table = pci_id_table, + .probe = pci_probe, + .remove = pci_remove, + .err_handler = &pci_err_handlers }; -static struct file_operations raptor_fops = { - .unlocked_ioctl = raptor_ioctl, +static const struct file_operations raptor_fops = { + .unlocked_ioctl = raptor_ioctl, }; int init_module(void) { - int i; - - printk(KERN_INFO "raptor: inserting module...\n"); - - memset(&raptor, 0, sizeof(raptor)); + int i; - spin_lock_init(&raptor.lock); - init_waitqueue_head(&raptor.wait); + memset(&raptor, 0, sizeof(raptor)); - raptor.send_packets = kmalloc(PACKET_COUNT * sizeof(struct packet), GFP_KERNEL); - raptor.recv_packets = kmalloc(PACKET_COUNT * sizeof(struct packet), GFP_KERNEL); + spin_lock_init(&raptor.lock); + init_waitqueue_head(&raptor.wait); - if (raptor.send_packets == NULL || raptor.recv_packets == NULL) { - printk(KERN_ERR "raptor: failed to allocate packets\n"); - return -1; - } + raptor.send_packets = + kmalloc_array(PACKET_COUNT, sizeof(struct packet), GFP_KERNEL); + raptor.recv_packets = + kmalloc_array(PACKET_COUNT, sizeof(struct packet), GFP_KERNEL); - for (i = 0; i < PACKET_COUNT; i++) { - queue_push(&raptor.send_free_queue, &raptor.send_packets[i]); - queue_push(&raptor.recv_free_queue, &raptor.recv_packets[i]); - } + if (raptor.send_packets == NULL || raptor.recv_packets == NULL) + return -1; - if (alloc_chrdev_region(&raptor.base, 0, 1, "raptor") < 0) { - printk(KERN_ERR "raptor: alloc_chrdev_region failed\n"); - return -1; - } + for (i = 0; i < PACKET_COUNT; i++) { + queue_push(&raptor.send_free_queue, &raptor.send_packets[i]); + queue_push(&raptor.recv_free_queue, &raptor.recv_packets[i]); + } - raptor.class = class_create(THIS_MODULE, "raptor"); + if (alloc_chrdev_region(&raptor.base, 0, 1, "raptor") < 0) + return -1; - if (IS_ERR(raptor.class)) { - printk(KERN_ERR "raptor: class_create failed\n"); - return -1; - } + raptor.class = class_create(THIS_MODULE, "raptor"); - raptor.dev = MKDEV(MAJOR(raptor.base), 0); + if (IS_ERR(raptor.class)) + return -1; - raptor.device = device_create(raptor.class, NULL, raptor.dev, NULL, "raptor"); + raptor.dev = MKDEV(MAJOR(raptor.base), 0); - if (IS_ERR(raptor.device)) { - printk(KERN_ERR "raptor: device_create failed\n"); - return -1; - } + raptor.device = + device_create(raptor.class, NULL, raptor.dev, NULL, "raptor"); - cdev_init(&raptor.cdev, &raptor_fops); + if (IS_ERR(raptor.device)) + return -1; - if (cdev_add(&raptor.cdev, raptor.dev, 1) < 0) { - printk(KERN_ERR "raptor: cdev_add failed\n"); - return -1; - } + cdev_init(&raptor.cdev, &raptor_fops); - if (pci_register_driver(&pci_driver) != 0) { - printk(KERN_ALERT "raptor: pci driver NOT loaded\n"); - return -1; - } + if (cdev_add(&raptor.cdev, raptor.dev, 1) < 0) { + dev_err(raptor.device, "cdev_add failed\n"); + return -1; + } - printk(KERN_INFO "raptor: module inserted\n"); + if (pci_register_driver(&pci_driver) != 0) { + dev_err(raptor.device, "pci driver NOT loaded\n"); + return -1; + } - return 0; + return 0; } void cleanup_module(void) { - printk(KERN_INFO "raptor: removing module...\n"); - - pci_unregister_driver(&pci_driver); - - cdev_del(&raptor.cdev); - device_destroy(raptor.class, raptor.dev); + pci_unregister_driver(&pci_driver); - class_destroy(raptor.class); - unregister_chrdev_region(raptor.base, 1); + cdev_del(&raptor.cdev); + device_destroy(raptor.class, raptor.dev); - kfree(raptor.send_packets); - kfree(raptor.recv_packets); + class_destroy(raptor.class); + unregister_chrdev_region(raptor.base, 1); - memset(&raptor, 0, sizeof(raptor)); + kfree(raptor.send_packets); + kfree(raptor.recv_packets); - printk(KERN_INFO "raptor: module removed\n"); + memset(&raptor, 0, sizeof(raptor)); } |