#include <asm/div64.h> /* do_div */
#include <asm/timex.h>
-#define VERSION "pktgen v2.69: Packet Generator for packet performance testing.\n"
+#define VERSION "pktgen v2.70: Packet Generator for packet performance testing.\n"
#define IP_NAME_SZ 32
#define MAX_MPLS_LABELS 16 /* This is the max label stack depth */
#define F_FLOW_SEQ (1<<11) /* Sequential flows */
#define F_IPSEC_ON (1<<12) /* ipsec on for flows */
#define F_QUEUE_MAP_RND (1<<13) /* queue map Random */
+#define F_QUEUE_MAP_CPU (1<<14) /* queue map mirrors smp_processor_id() */
/* Thread control flag bits */
#define T_TERMINATE (1<<0)
int cpu;
wait_queue_head_t queue;
+ struct completion start_done;
};
#define REMOVE 1
const char *ifname);
static int pktgen_device_event(struct notifier_block *, unsigned long, void *);
static void pktgen_run_all_threads(void);
+static void pktgen_reset_all_threads(void);
static void pktgen_stop_all_threads_ifs(void);
static int pktgen_stop_device(struct pktgen_dev *pkt_dev);
static void pktgen_stop(struct pktgen_thread *t);
else if (!strcmp(data, "start"))
pktgen_run_all_threads();
+ else if (!strcmp(data, "reset"))
+ pktgen_reset_all_threads();
+
else
printk(KERN_WARNING "pktgen: Unknown command: %s\n", data);
__u64 sa;
__u64 stopped;
__u64 now = getCurUs();
- DECLARE_MAC_BUF(mac);
seq_printf(seq,
"Params: count %llu min_pkt_size: %u max_pkt_size: %u\n",
seq_puts(seq, " src_mac: ");
- seq_printf(seq, "%s ",
- print_mac(mac, is_zero_ether_addr(pkt_dev->src_mac) ?
- pkt_dev->odev->dev_addr : pkt_dev->src_mac));
+ seq_printf(seq, "%pM ",
+ is_zero_ether_addr(pkt_dev->src_mac) ?
+ pkt_dev->odev->dev_addr : pkt_dev->src_mac);
seq_printf(seq, "dst_mac: ");
- seq_printf(seq, "%s\n", print_mac(mac, pkt_dev->dst_mac));
+ seq_printf(seq, "%pM\n", pkt_dev->dst_mac);
seq_printf(seq,
" udp_src_min: %d udp_src_max: %d udp_dst_min: %d udp_dst_max: %d\n",
if (pkt_dev->flags & F_QUEUE_MAP_RND)
seq_printf(seq, "QUEUE_MAP_RND ");
+ if (pkt_dev->flags & F_QUEUE_MAP_CPU)
+ seq_printf(seq, "QUEUE_MAP_CPU ");
+
if (pkt_dev->cflows) {
if (pkt_dev->flags & F_FLOW_SEQ)
seq_printf(seq, "FLOW_SEQ "); /*in sequence flows*/
else if (strcmp(f, "!QUEUE_MAP_RND") == 0)
pkt_dev->flags &= ~F_QUEUE_MAP_RND;
+
+ else if (strcmp(f, "QUEUE_MAP_CPU") == 0)
+ pkt_dev->flags |= F_QUEUE_MAP_CPU;
+
+ else if (strcmp(f, "!QUEUE_MAP_CPU") == 0)
+ pkt_dev->flags &= ~F_QUEUE_MAP_CPU;
#ifdef CONFIG_XFRM
else if (strcmp(f, "IPSEC") == 0)
pkt_dev->flags |= F_IPSEC_ON;
{
struct net_device *dev = ptr;
- if (dev->nd_net != &init_net)
+ if (!net_eq(dev_net(dev), &init_net))
return NOTIFY_DONE;
/* It is OK that we do not hold the group lock right now,
return NOTIFY_DONE;
}
+static struct net_device *pktgen_dev_get_by_name(struct pktgen_dev *pkt_dev, const char *ifname)
+{
+ char b[IFNAMSIZ+5];
+ int i = 0;
+
+ for(i=0; ifname[i] != '@'; i++) {
+ if(i == IFNAMSIZ)
+ break;
+
+ b[i] = ifname[i];
+ }
+ b[i] = 0;
+
+ return dev_get_by_name(&init_net, b);
+}
+
+
/* Associate pktgen_dev with a device. */
static int pktgen_setup_dev(struct pktgen_dev *pkt_dev, const char *ifname)
pkt_dev->odev = NULL;
}
- odev = dev_get_by_name(&init_net, ifname);
+ odev = pktgen_dev_get_by_name(pkt_dev, ifname);
if (!odev) {
printk(KERN_ERR "pktgen: no such netdevice: \"%s\"\n", ifname);
return -ENODEV;
*/
static void pktgen_setup_inject(struct pktgen_dev *pkt_dev)
{
+ int ntxq;
+
if (!pkt_dev->odev) {
printk(KERN_ERR "pktgen: ERROR: pkt_dev->odev == NULL in "
"setup_inject.\n");
return;
}
+ /* make sure that we don't pick a non-existing transmit queue */
+ ntxq = pkt_dev->odev->real_num_tx_queues;
+
+ if (ntxq <= pkt_dev->queue_map_min) {
+ printk(KERN_WARNING "pktgen: WARNING: Requested "
+ "queue_map_min (zero-based) (%d) exceeds valid range "
+ "[0 - %d] for (%d) queues on %s, resetting\n",
+ pkt_dev->queue_map_min, (ntxq ?: 1)- 1, ntxq,
+ pkt_dev->odev->name);
+ pkt_dev->queue_map_min = ntxq - 1;
+ }
+ if (pkt_dev->queue_map_max >= ntxq) {
+ printk(KERN_WARNING "pktgen: WARNING: Requested "
+ "queue_map_max (zero-based) (%d) exceeds valid range "
+ "[0 - %d] for (%d) queues on %s, resetting\n",
+ pkt_dev->queue_map_max, (ntxq ?: 1)- 1, ntxq,
+ pkt_dev->odev->name);
+ pkt_dev->queue_map_max = ntxq - 1;
+ }
+
/* Default to the interface's mac if not explicitly set. */
if (is_zero_ether_addr(pkt_dev->src_mac))
if (pkt_dev->flows[flow].count >= pkt_dev->lflow) {
/* reset time */
pkt_dev->flows[flow].count = 0;
+ pkt_dev->flows[flow].flags = 0;
pkt_dev->curfl += 1;
if (pkt_dev->curfl >= pkt_dev->cflows)
pkt_dev->curfl = 0; /*reset */
}
} else {
flow = random32() % pkt_dev->cflows;
+ pkt_dev->curfl = flow;
- if (pkt_dev->flows[flow].count > pkt_dev->lflow)
+ if (pkt_dev->flows[flow].count > pkt_dev->lflow) {
pkt_dev->flows[flow].count = 0;
+ pkt_dev->flows[flow].flags = 0;
+ }
}
return pkt_dev->curfl;
struct xfrm_state *x = pkt_dev->flows[flow].x;
if (!x) {
/*slow path: we dont already have xfrm_state*/
- x = xfrm_stateonly_find((xfrm_address_t *)&pkt_dev->cur_daddr,
+ x = xfrm_stateonly_find(&init_net,
+ (xfrm_address_t *)&pkt_dev->cur_daddr,
(xfrm_address_t *)&pkt_dev->cur_saddr,
AF_INET,
pkt_dev->ipsmode,
}
}
#endif
+static void set_cur_queue_map(struct pktgen_dev *pkt_dev)
+{
+
+ if (pkt_dev->flags & F_QUEUE_MAP_CPU)
+ pkt_dev->cur_queue_map = smp_processor_id();
+
+ else if (pkt_dev->queue_map_min < pkt_dev->queue_map_max) {
+ __u16 t;
+ if (pkt_dev->flags & F_QUEUE_MAP_RND) {
+ t = random32() %
+ (pkt_dev->queue_map_max -
+ pkt_dev->queue_map_min + 1)
+ + pkt_dev->queue_map_min;
+ } else {
+ t = pkt_dev->cur_queue_map + 1;
+ if (t > pkt_dev->queue_map_max)
+ t = pkt_dev->queue_map_min;
+ }
+ pkt_dev->cur_queue_map = t;
+ }
+ pkt_dev->cur_queue_map = pkt_dev->cur_queue_map % pkt_dev->odev->real_num_tx_queues;
+}
+
/* Increment/randomize headers according to flags and current values
* for IP src/dest, UDP src/dst port, MAC-Addr src/dst
*/
mc = random32() % pkt_dev->src_mac_count;
else {
mc = pkt_dev->cur_src_mac_offset++;
- if (pkt_dev->cur_src_mac_offset >
+ if (pkt_dev->cur_src_mac_offset >=
pkt_dev->src_mac_count)
pkt_dev->cur_src_mac_offset = 0;
}
else {
mc = pkt_dev->cur_dst_mac_offset++;
- if (pkt_dev->cur_dst_mac_offset >
+ if (pkt_dev->cur_dst_mac_offset >=
pkt_dev->dst_mac_count) {
pkt_dev->cur_dst_mac_offset = 0;
}
pkt_dev->cur_pkt_size = t;
}
- if (pkt_dev->queue_map_min < pkt_dev->queue_map_max) {
- __u16 t;
- if (pkt_dev->flags & F_QUEUE_MAP_RND) {
- t = random32() %
- (pkt_dev->queue_map_max - pkt_dev->queue_map_min + 1)
- + pkt_dev->queue_map_min;
- } else {
- t = pkt_dev->cur_queue_map + 1;
- if (t > pkt_dev->queue_map_max)
- t = pkt_dev->queue_map_min;
- }
- pkt_dev->cur_queue_map = t;
- }
+ set_cur_queue_map(pkt_dev);
pkt_dev->flows[flow].count++;
}
if (pkt_dev->cflows) {
/* let go of the SAs if we have them */
int i = 0;
- for (; i < pkt_dev->nflows; i++){
+ for (; i < pkt_dev->cflows; i++) {
struct xfrm_state *x = pkt_dev->flows[i].x;
if (x) {
xfrm_state_put(x);
if (ret < 0) {
printk(KERN_ERR "Error expanding "
"ipsec packet %d\n",ret);
- return 0;
+ goto err;
}
}
if (ret) {
printk(KERN_ERR "Error creating ipsec "
"packet %d\n",ret);
- kfree_skb(skb);
- return 0;
+ goto err;
}
/* restore ll */
eth = (__u8 *) skb_push(skb, ETH_HLEN);
}
}
return 1;
+err:
+ kfree_skb(skb);
+ return 0;
}
#endif
__be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
__be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
__be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
-
+ u16 queue_map;
if (pkt_dev->nr_labels)
protocol = htons(ETH_P_MPLS_UC);
/* Update any of the values, used when we're incrementing various
* fields.
*/
+ queue_map = pkt_dev->cur_queue_map;
mod_cur_headers(pkt_dev);
datalen = (odev->hard_header_len + 16) & ~0xf;
skb->network_header = skb->tail;
skb->transport_header = skb->network_header + sizeof(struct iphdr);
skb_put(skb, sizeof(struct iphdr) + sizeof(struct udphdr));
- skb_set_queue_mapping(skb, pkt_dev->cur_queue_map);
+ skb_set_queue_mapping(skb, queue_map);
iph = ip_hdr(skb);
udph = udp_hdr(skb);
__be16 *vlan_encapsulated_proto = NULL; /* packet type ID field (or len) for VLAN tag */
__be16 *svlan_tci = NULL; /* Encapsulates priority and SVLAN ID */
__be16 *svlan_encapsulated_proto = NULL; /* packet type ID field (or len) for SVLAN tag */
+ u16 queue_map;
if (pkt_dev->nr_labels)
protocol = htons(ETH_P_MPLS_UC);
/* Update any of the values, used when we're incrementing various
* fields.
*/
+ queue_map = pkt_dev->cur_queue_map;
mod_cur_headers(pkt_dev);
skb = alloc_skb(pkt_dev->cur_pkt_size + 64 + 16 +
skb->network_header = skb->tail;
skb->transport_header = skb->network_header + sizeof(struct ipv6hdr);
skb_put(skb, sizeof(struct ipv6hdr) + sizeof(struct udphdr));
- skb_set_queue_mapping(skb, pkt_dev->cur_queue_map);
+ skb_set_queue_mapping(skb, queue_map);
iph = ipv6_hdr(skb);
udph = udp_hdr(skb);
pktgen_wait_all_threads_run();
}
+static void pktgen_reset_all_threads(void)
+{
+ struct pktgen_thread *t;
+
+ pr_debug("pktgen: entering pktgen_reset_all_threads.\n");
+
+ mutex_lock(&pktgen_thread_lock);
+
+ list_for_each_entry(t, &pktgen_threads, th_list)
+ t->control |= (T_REMDEVALL);
+
+ mutex_unlock(&pktgen_thread_lock);
+
+ schedule_timeout_interruptible(msecs_to_jiffies(125)); /* Propagate thread->control */
+
+ pktgen_wait_all_threads_run();
+}
+
static void show_results(struct pktgen_dev *pkt_dev, int nr_frags)
{
__u64 total_us, bps, mbps, pps, idle;
list_for_each_entry(pkt_dev, &t->if_list, list) {
pktgen_stop_device(pkt_dev);
- if (pkt_dev->skb)
- kfree_skb(pkt_dev->skb);
+ kfree_skb(pkt_dev->skb);
pkt_dev->skb = NULL;
}
if (!cur->removal_mark)
continue;
- if (cur->skb)
- kfree_skb(cur->skb);
+ kfree_skb(cur->skb);
cur->skb = NULL;
pktgen_remove_device(t, cur);
list_for_each_safe(q, n, &t->if_list) {
cur = list_entry(q, struct pktgen_dev, list);
- if (cur->skb)
- kfree_skb(cur->skb);
+ kfree_skb(cur->skb);
cur->skb = NULL;
pktgen_remove_device(t, cur);
static __inline__ void pktgen_xmit(struct pktgen_dev *pkt_dev)
{
- struct net_device *odev = NULL;
+ struct net_device *odev = pkt_dev->odev;
+ int (*xmit)(struct sk_buff *, struct net_device *)
+ = odev->netdev_ops->ndo_start_xmit;
+ struct netdev_queue *txq;
__u64 idle_start = 0;
+ u16 queue_map;
int ret;
- odev = pkt_dev->odev;
-
if (pkt_dev->delay_us || pkt_dev->delay_ns) {
u64 now;
}
}
- if ((netif_queue_stopped(odev) ||
- (pkt_dev->skb &&
- netif_subqueue_stopped(odev, pkt_dev->skb))) ||
+ if (!pkt_dev->skb) {
+ set_cur_queue_map(pkt_dev);
+ queue_map = pkt_dev->cur_queue_map;
+ } else {
+ queue_map = skb_get_queue_mapping(pkt_dev->skb);
+ }
+
+ txq = netdev_get_tx_queue(odev, queue_map);
+ if (netif_tx_queue_stopped(txq) ||
+ netif_tx_queue_frozen(txq) ||
need_resched()) {
idle_start = getCurUs();
if (!netif_running(odev)) {
pktgen_stop_device(pkt_dev);
- if (pkt_dev->skb)
- kfree_skb(pkt_dev->skb);
+ kfree_skb(pkt_dev->skb);
pkt_dev->skb = NULL;
goto out;
}
pkt_dev->idle_acc += getCurUs() - idle_start;
- if (netif_queue_stopped(odev) ||
- netif_subqueue_stopped(odev, pkt_dev->skb)) {
+ if (netif_tx_queue_stopped(txq) ||
+ netif_tx_queue_frozen(txq)) {
pkt_dev->next_tx_us = getCurUs(); /* TODO */
pkt_dev->next_tx_ns = 0;
goto out; /* Try the next interface */
if ((++pkt_dev->clone_count >= pkt_dev->clone_skb)
|| (!pkt_dev->skb)) {
/* build a new pkt */
- if (pkt_dev->skb)
- kfree_skb(pkt_dev->skb);
+ kfree_skb(pkt_dev->skb);
pkt_dev->skb = fill_packet(odev, pkt_dev);
if (pkt_dev->skb == NULL) {
}
}
- netif_tx_lock_bh(odev);
- if (!netif_queue_stopped(odev) &&
- !netif_subqueue_stopped(odev, pkt_dev->skb)) {
+ /* fill_packet() might have changed the queue */
+ queue_map = skb_get_queue_mapping(pkt_dev->skb);
+ txq = netdev_get_tx_queue(odev, queue_map);
+
+ __netif_tx_lock_bh(txq);
+ if (!netif_tx_queue_stopped(txq) &&
+ !netif_tx_queue_frozen(txq)) {
atomic_inc(&(pkt_dev->skb->users));
retry_now:
- ret = odev->hard_start_xmit(pkt_dev->skb, odev);
+ ret = (*xmit)(pkt_dev->skb, odev);
if (likely(ret == NETDEV_TX_OK)) {
+ txq_trans_update(txq);
pkt_dev->last_ok = 1;
pkt_dev->sofar++;
pkt_dev->seq_num++;
pkt_dev->next_tx_ns = 0;
}
- netif_tx_unlock_bh(odev);
+ __netif_tx_unlock_bh(txq);
/* If pkt_dev->count is zero, then run forever */
if ((pkt_dev->count != 0) && (pkt_dev->sofar >= pkt_dev->count)) {
/* Done with this */
pktgen_stop_device(pkt_dev);
- if (pkt_dev->skb)
- kfree_skb(pkt_dev->skb);
+ kfree_skb(pkt_dev->skb);
pkt_dev->skb = NULL;
}
out:;
BUG_ON(smp_processor_id() != cpu);
init_waitqueue_head(&t->queue);
+ complete(&t->start_done);
pr_debug("pktgen: starting pktgen/%d: pid=%d\n", cpu, task_pid_nr(current));
if (err)
goto out1;
- pkt_dev->entry = proc_create(ifname, 0600,
- pg_proc_dir, &pktgen_if_fops);
+ pkt_dev->entry = proc_create_data(ifname, 0600, pg_proc_dir,
+ &pktgen_if_fops, pkt_dev);
if (!pkt_dev->entry) {
printk(KERN_ERR "pktgen: cannot create %s/%s procfs entry.\n",
PG_PROC_DIR, ifname);
err = -EINVAL;
goto out2;
}
- pkt_dev->entry->data = pkt_dev;
#ifdef CONFIG_XFRM
pkt_dev->ipsmode = XFRM_MODE_TRANSPORT;
pkt_dev->ipsproto = IPPROTO_ESP;
#ifdef CONFIG_XFRM
free_SAs(pkt_dev);
#endif
- if (pkt_dev->flows)
- vfree(pkt_dev->flows);
+ vfree(pkt_dev->flows);
kfree(pkt_dev);
return err;
}
INIT_LIST_HEAD(&t->if_list);
list_add_tail(&t->th_list, &pktgen_threads);
+ init_completion(&t->start_done);
p = kthread_create(pktgen_thread_worker, t, "kpktgend_%d", cpu);
if (IS_ERR(p)) {
kthread_bind(p, cpu);
t->tsk = p;
- pe = proc_create(t->tsk->comm, 0600, pg_proc_dir, &pktgen_thread_fops);
+ pe = proc_create_data(t->tsk->comm, 0600, pg_proc_dir,
+ &pktgen_thread_fops, t);
if (!pe) {
printk(KERN_ERR "pktgen: cannot create %s/%s procfs entry.\n",
PG_PROC_DIR, t->tsk->comm);
return -EINVAL;
}
- pe->data = t;
-
wake_up_process(p);
+ wait_for_completion(&t->start_done);
return 0;
}
#ifdef CONFIG_XFRM
free_SAs(pkt_dev);
#endif
- if (pkt_dev->flows)
- vfree(pkt_dev->flows);
+ vfree(pkt_dev->flows);
kfree(pkt_dev);
return 0;
}
pg_proc_dir = proc_mkdir(PG_PROC_DIR, init_net.proc_net);
if (!pg_proc_dir)
return -ENODEV;
- pg_proc_dir->owner = THIS_MODULE;
pe = proc_create(PGCTRL, 0600, pg_proc_dir, &pktgen_fops);
if (pe == NULL) {
return -EINVAL;
}
- pe->data = NULL;
-
/* Register us to receive netdevice events */
register_netdevice_notifier(&pktgen_notifier_block);