SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/rcutorture.c

   1 /*
   2  * Read-Copy Update module-based torture test facility
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License as published by
   6  * the Free Software Foundation; either version 2 of the License, or
   7  * (at your option) any later version.
   8  *
   9  * This program is distributed in the hope that it will be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write to the Free Software
  16  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  17  *
  18  * Copyright (C) IBM Corporation, 2005
  19  *
  20  * Authors: Paul E. McKenney <paulmck@us.ibm.com>
  21  *
  22  * See also:  Documentation/RCU/torture.txt
  23  */
  24 #include <linux/types.h>
  25 #include <linux/kernel.h>
  26 #include <linux/init.h>
  27 #include <linux/module.h>
  28 #include <linux/kthread.h>
  29 #include <linux/err.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/smp.h>
  32 #include <linux/rcupdate.h>
  33 #include <linux/interrupt.h>
  34 #include <linux/sched.h>
  35 #include <asm/atomic.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/completion.h>
  39 #include <linux/moduleparam.h>
  40 #include <linux/percpu.h>
  41 #include <linux/notifier.h>
  42 #include <linux/cpu.h>
  43 #include <linux/random.h>
  44 #include <linux/delay.h>
  45 #include <linux/byteorder/swabb.h>
  46 #include <linux/stat.h>
  47 #include <linux/srcu.h>
  48
  49 MODULE_LICENSE("GPL");
  50 MODULE_AUTHOR("Paul E. McKenney <paulmck@us.ibm.com>");
  51
  52 static int nreaders = -1;       /* # reader threads, defaults to 2*ncpus */
  53 static int stat_interval;       /* Interval between stats, in seconds. */
  54                                 /*  Defaults to "only at end of test". */
  55 static int verbose;             /* Print more debug info. */
  56 static int test_no_idle_hz;     /* Test RCU's support for tickless idle CPUs. */
  57 static int shuffle_interval = 5; /* Interval between shuffles (in sec)*/
  58 static char *torture_type = "rcu"; /* What to torture: rcu, rcu_bh, srcu. */
  59
  60 module_param(nreaders, int, 0);
  61 MODULE_PARM_DESC(nreaders, "Number of RCU reader threads");
  62 module_param(stat_interval, int, 0);
  63 MODULE_PARM_DESC(stat_interval, "Number of seconds between stats printk()s");
  64 module_param(verbose, bool, 0);
  65 MODULE_PARM_DESC(verbose, "Enable verbose debugging printk()s");
  66 module_param(test_no_idle_hz, bool, 0);
  67 MODULE_PARM_DESC(test_no_idle_hz, "Test support for tickless idle CPUs");
  68 module_param(shuffle_interval, int, 0);
  69 MODULE_PARM_DESC(shuffle_interval, "Number of seconds between shuffles");
  70 module_param(torture_type, charp, 0);
  71 MODULE_PARM_DESC(torture_type, "Type of RCU to torture (rcu, rcu_bh, srcu)");
  72
  73 #define TORTURE_FLAG "-torture:"
  74 #define PRINTK_STRING(s) \
  75         do { printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  76 #define VERBOSE_PRINTK_STRING(s) \
  77         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG s "\n", torture_type); } while (0)
  78 #define VERBOSE_PRINTK_ERRSTRING(s) \
  79         do { if (verbose) printk(KERN_ALERT "%s" TORTURE_FLAG "!!! " s "\n", torture_type); } while (0)
  80
  81 static char printk_buf[4096];
  82
  83 static int nrealreaders;
  84 static struct task_struct *writer_task;
  85 static struct task_struct **reader_tasks;
  86 static struct task_struct *stats_task;
  87 static struct task_struct *shuffler_task;
  88
  89 #define RCU_TORTURE_PIPE_LEN 10
  90
  91 struct rcu_torture {
  92         struct rcu_head rtort_rcu;
  93         int rtort_pipe_count;
  94         struct list_head rtort_free;
  95         int rtort_mbtest;
  96 };
  97
  98 static int fullstop = 0;        /* stop generating callbacks at test end. */
  99 static LIST_HEAD(rcu_torture_freelist);
 100 static struct rcu_torture *rcu_torture_current = NULL;
 101 static long rcu_torture_current_version = 0;
 102 static struct rcu_torture rcu_tortures[10 * RCU_TORTURE_PIPE_LEN];
 103 static DEFINE_SPINLOCK(rcu_torture_lock);
 104 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_count) =
 105         { 0 };
 106 static DEFINE_PER_CPU(long [RCU_TORTURE_PIPE_LEN + 1], rcu_torture_batch) =
 107         { 0 };
 108 static atomic_t rcu_torture_wcount[RCU_TORTURE_PIPE_LEN + 1];
 109 static atomic_t n_rcu_torture_alloc;
 110 static atomic_t n_rcu_torture_alloc_fail;
 111 static atomic_t n_rcu_torture_free;
 112 static atomic_t n_rcu_torture_mberror;
 113 static atomic_t n_rcu_torture_error;
 114
 115 /*
 116  * Allocate an element from the rcu_tortures pool.
 117  */
 118 static struct rcu_torture *
 119 rcu_torture_alloc(void)
 120 {
 121         struct list_head *p;
 122
 123         spin_lock_bh(&rcu_torture_lock);
 124         if (list_empty(&rcu_torture_freelist)) {
 125                 atomic_inc(&n_rcu_torture_alloc_fail);
 126                 spin_unlock_bh(&rcu_torture_lock);
 127                 return NULL;
 128         }
 129         atomic_inc(&n_rcu_torture_alloc);
 130         p = rcu_torture_freelist.next;
 131         list_del_init(p);
 132         spin_unlock_bh(&rcu_torture_lock);
 133         return container_of(p, struct rcu_torture, rtort_free);
 134 }
 135
 136 /*
 137  * Free an element to the rcu_tortures pool.
 138  */
 139 static void
 140 rcu_torture_free(struct rcu_torture *p)
 141 {
 142         atomic_inc(&n_rcu_torture_free);
 143         spin_lock_bh(&rcu_torture_lock);
 144         list_add_tail(&p->rtort_free, &rcu_torture_freelist);
 145         spin_unlock_bh(&rcu_torture_lock);
 146 }
 147
 148 struct rcu_random_state {
 149         unsigned long rrs_state;
 150         unsigned long rrs_count;
 151 };
 152
 153 #define RCU_RANDOM_MULT 39916801  /* prime */
 154 #define RCU_RANDOM_ADD  479001701 /* prime */
 155 #define RCU_RANDOM_REFRESH 10000
 156
 157 #define DEFINE_RCU_RANDOM(name) struct rcu_random_state name = { 0, 0 }
 158
 159 /*
 160  * Crude but fast random-number generator.  Uses a linear congruential
 161  * generator, with occasional help from get_random_bytes().
 162  */
 163 static long
 164 rcu_random(struct rcu_random_state *rrsp)
 165 {
 166         long refresh;
 167
 168         if (--rrsp->rrs_count < 0) {
 169                 get_random_bytes(&refresh, sizeof(refresh));
 170                 rrsp->rrs_state += refresh;
 171                 rrsp->rrs_count = RCU_RANDOM_REFRESH;
 172         }
 173         rrsp->rrs_state = rrsp->rrs_state * RCU_RANDOM_MULT + RCU_RANDOM_ADD;
 174         return swahw32(rrsp->rrs_state);
 175 }
 176
 177 /*
 178  * Operations vector for selecting different types of tests.
 179  */
 180
 181 struct rcu_torture_ops {
 182         void (*init)(void);
 183         void (*cleanup)(void);
 184         int (*readlock)(void);
 185         void (*readdelay)(struct rcu_random_state *rrsp);
 186         void (*readunlock)(int idx);
 187         int (*completed)(void);
 188         void (*deferredfree)(struct rcu_torture *p);
 189         int (*stats)(char *page);
 190         char *name;
 191 };
 192 static struct rcu_torture_ops *cur_ops = NULL;
 193
 194 /*
 195  * Definitions for rcu torture testing.
 196  */
 197
 198 static int rcu_torture_read_lock(void) __acquires(RCU)
 199 {
 200         rcu_read_lock();
 201         return 0;
 202 }
 203
 204 static void rcu_read_delay(struct rcu_random_state *rrsp)
 205 {
 206         long delay;
 207         const long longdelay = 200;
 208
 209         /* We want there to be long-running readers, but not all the time. */
 210
 211         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay);
 212         if (!delay)
 213                 udelay(longdelay);
 214 }
 215
 216 static void rcu_torture_read_unlock(int idx) __releases(RCU)
 217 {
 218         rcu_read_unlock();
 219 }
 220
 221 static int rcu_torture_completed(void)
 222 {
 223         return rcu_batches_completed();
 224 }
 225
 226 static void
 227 rcu_torture_cb(struct rcu_head *p)
 228 {
 229         int i;
 230         struct rcu_torture *rp = container_of(p, struct rcu_torture, rtort_rcu);
 231
 232         if (fullstop) {
 233                 /* Test is ending, just drop callbacks on the floor. */
 234                 /* The next initialization will pick up the pieces. */
 235                 return;
 236         }
 237         i = rp->rtort_pipe_count;
 238         if (i > RCU_TORTURE_PIPE_LEN)
 239                 i = RCU_TORTURE_PIPE_LEN;
 240         atomic_inc(&rcu_torture_wcount[i]);
 241         if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 242                 rp->rtort_mbtest = 0;
 243                 rcu_torture_free(rp);
 244         } else
 245                 cur_ops->deferredfree(rp);
 246 }
 247
 248 static void rcu_torture_deferred_free(struct rcu_torture *p)
 249 {
 250         call_rcu(&p->rtort_rcu, rcu_torture_cb);
 251 }
 252
 253 static struct rcu_torture_ops rcu_ops = {
 254         .init = NULL,
 255         .cleanup = NULL,
 256         .readlock = rcu_torture_read_lock,
 257         .readdelay = rcu_read_delay,
 258         .readunlock = rcu_torture_read_unlock,
 259         .completed = rcu_torture_completed,
 260         .deferredfree = rcu_torture_deferred_free,
 261         .stats = NULL,
 262         .name = "rcu"
 263 };
 264
 265 /*
 266  * Definitions for rcu_bh torture testing.
 267  */
 268
 269 static int rcu_bh_torture_read_lock(void) __acquires(RCU_BH)
 270 {
 271         rcu_read_lock_bh();
 272         return 0;
 273 }
 274
 275 static void rcu_bh_torture_read_unlock(int idx) __releases(RCU_BH)
 276 {
 277         rcu_read_unlock_bh();
 278 }
 279
 280 static int rcu_bh_torture_completed(void)
 281 {
 282         return rcu_batches_completed_bh();
 283 }
 284
 285 static void rcu_bh_torture_deferred_free(struct rcu_torture *p)
 286 {
 287         call_rcu_bh(&p->rtort_rcu, rcu_torture_cb);
 288 }
 289
 290 static struct rcu_torture_ops rcu_bh_ops = {
 291         .init = NULL,
 292         .cleanup = NULL,
 293         .readlock = rcu_bh_torture_read_lock,
 294         .readdelay = rcu_read_delay,  /* just reuse rcu's version. */
 295         .readunlock = rcu_bh_torture_read_unlock,
 296         .completed = rcu_bh_torture_completed,
 297         .deferredfree = rcu_bh_torture_deferred_free,
 298         .stats = NULL,
 299         .name = "rcu_bh"
 300 };
 301
 302 /*
 303  * Definitions for srcu torture testing.
 304  */
 305
 306 static struct srcu_struct srcu_ctl;
 307 static struct list_head srcu_removed;
 308
 309 static void srcu_torture_init(void)
 310 {
 311         init_srcu_struct(&srcu_ctl);
 312         INIT_LIST_HEAD(&srcu_removed);
 313 }
 314
 315 static void srcu_torture_cleanup(void)
 316 {
 317         synchronize_srcu(&srcu_ctl);
 318         cleanup_srcu_struct(&srcu_ctl);
 319 }
 320
 321 static int srcu_torture_read_lock(void)
 322 {
 323         return srcu_read_lock(&srcu_ctl);
 324 }
 325
 326 static void srcu_read_delay(struct rcu_random_state *rrsp)
 327 {
 328         long delay;
 329         const long uspertick = 1000000 / HZ;
 330         const long longdelay = 10;
 331
 332         /* We want there to be long-running readers, but not all the time. */
 333
 334         delay = rcu_random(rrsp) % (nrealreaders * 2 * longdelay * uspertick);
 335         if (!delay)
 336                 schedule_timeout_interruptible(longdelay);
 337 }
 338
 339 static void srcu_torture_read_unlock(int idx)
 340 {
 341         srcu_read_unlock(&srcu_ctl, idx);
 342 }
 343
 344 static int srcu_torture_completed(void)
 345 {
 346         return srcu_batches_completed(&srcu_ctl);
 347 }
 348
 349 static void srcu_torture_deferred_free(struct rcu_torture *p)
 350 {
 351         int i;
 352         struct rcu_torture *rp;
 353         struct rcu_torture *rp1;
 354
 355         synchronize_srcu(&srcu_ctl);
 356         list_add(&p->rtort_free, &srcu_removed);
 357         list_for_each_entry_safe(rp, rp1, &srcu_removed, rtort_free) {
 358                 i = rp->rtort_pipe_count;
 359                 if (i > RCU_TORTURE_PIPE_LEN)
 360                         i = RCU_TORTURE_PIPE_LEN;
 361                 atomic_inc(&rcu_torture_wcount[i]);
 362                 if (++rp->rtort_pipe_count >= RCU_TORTURE_PIPE_LEN) {
 363                         rp->rtort_mbtest = 0;
 364                         list_del(&rp->rtort_free);
 365                         rcu_torture_free(rp);
 366                 }
 367         }
 368 }
 369
 370 static int srcu_torture_stats(char *page)
 371 {
 372         int cnt = 0;
 373         int cpu;
 374         int idx = srcu_ctl.completed & 0x1;
 375
 376         cnt += sprintf(&page[cnt], "%s%s per-CPU(idx=%d):",
 377                        torture_type, TORTURE_FLAG, idx);
 378         for_each_possible_cpu(cpu) {
 379                 cnt += sprintf(&page[cnt], " %d(%d,%d)", cpu,
 380                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[!idx],
 381                                per_cpu_ptr(srcu_ctl.per_cpu_ref, cpu)->c[idx]);
 382         }
 383         cnt += sprintf(&page[cnt], "\n");
 384         return cnt;
 385 }
 386
 387 static struct rcu_torture_ops srcu_ops = {
 388         .init = srcu_torture_init,
 389         .cleanup = srcu_torture_cleanup,
 390         .readlock = srcu_torture_read_lock,
 391         .readdelay = srcu_read_delay,
 392         .readunlock = srcu_torture_read_unlock,
 393         .completed = srcu_torture_completed,
 394         .deferredfree = srcu_torture_deferred_free,
 395         .stats = srcu_torture_stats,
 396         .name = "srcu"
 397 };
 398
 399 static struct rcu_torture_ops *torture_ops[] =
 400         { &rcu_ops, &rcu_bh_ops, &srcu_ops, NULL };
 401
 402 /*
 403  * RCU torture writer kthread.  Repeatedly substitutes a new structure
 404  * for that pointed to by rcu_torture_current, freeing the old structure
 405  * after a series of grace periods (the "pipeline").
 406  */
 407 static int
 408 rcu_torture_writer(void *arg)
 409 {
 410         int i;
 411         long oldbatch = rcu_batches_completed();
 412         struct rcu_torture *rp;
 413         struct rcu_torture *old_rp;
 414         static DEFINE_RCU_RANDOM(rand);
 415
 416         VERBOSE_PRINTK_STRING("rcu_torture_writer task started");
 417         set_user_nice(current, 19);
 418
 419         do {
 420                 schedule_timeout_uninterruptible(1);
 421                 if ((rp = rcu_torture_alloc()) == NULL)
 422                         continue;
 423                 rp->rtort_pipe_count = 0;
 424                 udelay(rcu_random(&rand) & 0x3ff);
 425                 old_rp = rcu_torture_current;
 426                 rp->rtort_mbtest = 1;
 427                 rcu_assign_pointer(rcu_torture_current, rp);
 428                 smp_wmb();
 429                 if (old_rp != NULL) {
 430                         i = old_rp->rtort_pipe_count;
 431                         if (i > RCU_TORTURE_PIPE_LEN)
 432                                 i = RCU_TORTURE_PIPE_LEN;
 433                         atomic_inc(&rcu_torture_wcount[i]);
 434                         old_rp->rtort_pipe_count++;
 435                         cur_ops->deferredfree(old_rp);
 436                 }
 437                 rcu_torture_current_version++;
 438                 oldbatch = cur_ops->completed();
 439         } while (!kthread_should_stop() && !fullstop);
 440         VERBOSE_PRINTK_STRING("rcu_torture_writer task stopping");
 441         while (!kthread_should_stop())
 442                 schedule_timeout_uninterruptible(1);
 443         return 0;
 444 }
 445
 446 /*
 447  * RCU torture reader kthread.  Repeatedly dereferences rcu_torture_current,
 448  * incrementing the corresponding element of the pipeline array.  The
 449  * counter in the element should never be greater than 1, otherwise, the
 450  * RCU implementation is broken.
 451  */
 452 static int
 453 rcu_torture_reader(void *arg)
 454 {
 455         int completed;
 456         int idx;
 457         DEFINE_RCU_RANDOM(rand);
 458         struct rcu_torture *p;
 459         int pipe_count;
 460
 461         VERBOSE_PRINTK_STRING("rcu_torture_reader task started");
 462         set_user_nice(current, 19);
 463
 464         do {
 465                 idx = cur_ops->readlock();
 466                 completed = cur_ops->completed();
 467                 p = rcu_dereference(rcu_torture_current);
 468                 if (p == NULL) {
 469                         /* Wait for rcu_torture_writer to get underway */
 470                         cur_ops->readunlock(idx);
 471                         schedule_timeout_interruptible(HZ);
 472                         continue;
 473                 }
 474                 if (p->rtort_mbtest == 0)
 475                         atomic_inc(&n_rcu_torture_mberror);
 476                 cur_ops->readdelay(&rand);
 477                 preempt_disable();
 478                 pipe_count = p->rtort_pipe_count;
 479                 if (pipe_count > RCU_TORTURE_PIPE_LEN) {
 480                         /* Should not happen, but... */
 481                         pipe_count = RCU_TORTURE_PIPE_LEN;
 482                 }
 483                 ++__get_cpu_var(rcu_torture_count)[pipe_count];
 484                 completed = cur_ops->completed() - completed;
 485                 if (completed > RCU_TORTURE_PIPE_LEN) {
 486                         /* Should not happen, but... */
 487                         completed = RCU_TORTURE_PIPE_LEN;
 488                 }
 489                 ++__get_cpu_var(rcu_torture_batch)[completed];
 490                 preempt_enable();
 491                 cur_ops->readunlock(idx);
 492                 schedule();
 493         } while (!kthread_should_stop() && !fullstop);
 494         VERBOSE_PRINTK_STRING("rcu_torture_reader task stopping");
 495         while (!kthread_should_stop())
 496                 schedule_timeout_uninterruptible(1);
 497         return 0;
 498 }
 499
 500 /*
 501  * Create an RCU-torture statistics message in the specified buffer.
 502  */
 503 static int
 504 rcu_torture_printk(char *page)
 505 {
 506         int cnt = 0;
 507         int cpu;
 508         int i;
 509         long pipesummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 510         long batchsummary[RCU_TORTURE_PIPE_LEN + 1] = { 0 };
 511
 512         for_each_possible_cpu(cpu) {
 513                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 514                         pipesummary[i] += per_cpu(rcu_torture_count, cpu)[i];
 515                         batchsummary[i] += per_cpu(rcu_torture_batch, cpu)[i];
 516                 }
 517         }
 518         for (i = RCU_TORTURE_PIPE_LEN - 1; i >= 0; i--) {
 519                 if (pipesummary[i] != 0)
 520                         break;
 521         }
 522         cnt += sprintf(&page[cnt], "%s%s ", torture_type, TORTURE_FLAG);
 523         cnt += sprintf(&page[cnt],
 524                        "rtc: %p ver: %ld tfle: %d rta: %d rtaf: %d rtf: %d "
 525                        "rtmbe: %d",
 526                        rcu_torture_current,
 527                        rcu_torture_current_version,
 528                        list_empty(&rcu_torture_freelist),
 529                        atomic_read(&n_rcu_torture_alloc),
 530                        atomic_read(&n_rcu_torture_alloc_fail),
 531                        atomic_read(&n_rcu_torture_free),
 532                        atomic_read(&n_rcu_torture_mberror));
 533         if (atomic_read(&n_rcu_torture_mberror) != 0)
 534                 cnt += sprintf(&page[cnt], " !!!");
 535         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 536         if (i > 1) {
 537                 cnt += sprintf(&page[cnt], "!!! ");
 538                 atomic_inc(&n_rcu_torture_error);
 539         }
 540         cnt += sprintf(&page[cnt], "Reader Pipe: ");
 541         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 542                 cnt += sprintf(&page[cnt], " %ld", pipesummary[i]);
 543         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 544         cnt += sprintf(&page[cnt], "Reader Batch: ");
 545         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 546                 cnt += sprintf(&page[cnt], " %ld", batchsummary[i]);
 547         cnt += sprintf(&page[cnt], "\n%s%s ", torture_type, TORTURE_FLAG);
 548         cnt += sprintf(&page[cnt], "Free-Block Circulation: ");
 549         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 550                 cnt += sprintf(&page[cnt], " %d",
 551                                atomic_read(&rcu_torture_wcount[i]));
 552         }
 553         cnt += sprintf(&page[cnt], "\n");
 554         if (cur_ops->stats != NULL)
 555                 cnt += cur_ops->stats(&page[cnt]);
 556         return cnt;
 557 }
 558
 559 /*
 560  * Print torture statistics.  Caller must ensure that there is only
 561  * one call to this function at a given time!!!  This is normally
 562  * accomplished by relying on the module system to only have one copy
 563  * of the module loaded, and then by giving the rcu_torture_stats
 564  * kthread full control (or the init/cleanup functions when rcu_torture_stats
 565  * thread is not running).
 566  */
 567 static void
 568 rcu_torture_stats_print(void)
 569 {
 570         int cnt;
 571
 572         cnt = rcu_torture_printk(printk_buf);
 573         printk(KERN_ALERT "%s", printk_buf);
 574 }
 575
 576 /*
 577  * Periodically prints torture statistics, if periodic statistics printing
 578  * was specified via the stat_interval module parameter.
 579  *
 580  * No need to worry about fullstop here, since this one doesn't reference
 581  * volatile state or register callbacks.
 582  */
 583 static int
 584 rcu_torture_stats(void *arg)
 585 {
 586         VERBOSE_PRINTK_STRING("rcu_torture_stats task started");
 587         do {
 588                 schedule_timeout_interruptible(stat_interval * HZ);
 589                 rcu_torture_stats_print();
 590         } while (!kthread_should_stop());
 591         VERBOSE_PRINTK_STRING("rcu_torture_stats task stopping");
 592         return 0;
 593 }
 594
 595 static int rcu_idle_cpu;        /* Force all torture tasks off this CPU */
 596
 597 /* Shuffle tasks such that we allow @rcu_idle_cpu to become idle. A special case
 598  * is when @rcu_idle_cpu = -1, when we allow the tasks to run on all CPUs.
 599  */
 600 static void rcu_torture_shuffle_tasks(void)
 601 {
 602         cpumask_t tmp_mask = CPU_MASK_ALL;
 603         int i;
 604
 605         lock_cpu_hotplug();
 606
 607         /* No point in shuffling if there is only one online CPU (ex: UP) */
 608         if (num_online_cpus() == 1) {
 609                 unlock_cpu_hotplug();
 610                 return;
 611         }
 612
 613         if (rcu_idle_cpu != -1)
 614                 cpu_clear(rcu_idle_cpu, tmp_mask);
 615
 616         set_cpus_allowed(current, tmp_mask);
 617
 618         if (reader_tasks != NULL) {
 619                 for (i = 0; i < nrealreaders; i++)
 620                         if (reader_tasks[i])
 621                                 set_cpus_allowed(reader_tasks[i], tmp_mask);
 622         }
 623
 624         if (writer_task)
 625                 set_cpus_allowed(writer_task, tmp_mask);
 626
 627         if (stats_task)
 628                 set_cpus_allowed(stats_task, tmp_mask);
 629
 630         if (rcu_idle_cpu == -1)
 631                 rcu_idle_cpu = num_online_cpus() - 1;
 632         else
 633                 rcu_idle_cpu--;
 634
 635         unlock_cpu_hotplug();
 636 }
 637
 638 /* Shuffle tasks across CPUs, with the intent of allowing each CPU in the
 639  * system to become idle at a time and cut off its timer ticks. This is meant
 640  * to test the support for such tickless idle CPU in RCU.
 641  */
 642 static int
 643 rcu_torture_shuffle(void *arg)
 644 {
 645         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task started");
 646         do {
 647                 schedule_timeout_interruptible(shuffle_interval * HZ);
 648                 rcu_torture_shuffle_tasks();
 649         } while (!kthread_should_stop());
 650         VERBOSE_PRINTK_STRING("rcu_torture_shuffle task stopping");
 651         return 0;
 652 }
 653
 654 static inline void
 655 rcu_torture_print_module_parms(char *tag)
 656 {
 657         printk(KERN_ALERT "%s" TORTURE_FLAG "--- %s: nreaders=%d "
 658                 "stat_interval=%d verbose=%d test_no_idle_hz=%d "
 659                 "shuffle_interval = %d\n",
 660                 torture_type, tag, nrealreaders, stat_interval, verbose,
 661                 test_no_idle_hz, shuffle_interval);
 662 }
 663
 664 static void
 665 rcu_torture_cleanup(void)
 666 {
 667         int i;
 668
 669         fullstop = 1;
 670         if (shuffler_task != NULL) {
 671                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_shuffle task");
 672                 kthread_stop(shuffler_task);
 673         }
 674         shuffler_task = NULL;
 675
 676         if (writer_task != NULL) {
 677                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_writer task");
 678                 kthread_stop(writer_task);
 679         }
 680         writer_task = NULL;
 681
 682         if (reader_tasks != NULL) {
 683                 for (i = 0; i < nrealreaders; i++) {
 684                         if (reader_tasks[i] != NULL) {
 685                                 VERBOSE_PRINTK_STRING(
 686                                         "Stopping rcu_torture_reader task");
 687                                 kthread_stop(reader_tasks[i]);
 688                         }
 689                         reader_tasks[i] = NULL;
 690                 }
 691                 kfree(reader_tasks);
 692                 reader_tasks = NULL;
 693         }
 694         rcu_torture_current = NULL;
 695
 696         if (stats_task != NULL) {
 697                 VERBOSE_PRINTK_STRING("Stopping rcu_torture_stats task");
 698                 kthread_stop(stats_task);
 699         }
 700         stats_task = NULL;
 701
 702         /* Wait for all RCU callbacks to fire.  */
 703         rcu_barrier();
 704
 705         rcu_torture_stats_print();  /* -After- the stats thread is stopped! */
 706
 707         if (cur_ops->cleanup != NULL)
 708                 cur_ops->cleanup();
 709         if (atomic_read(&n_rcu_torture_error))
 710                 rcu_torture_print_module_parms("End of test: FAILURE");
 711         else
 712                 rcu_torture_print_module_parms("End of test: SUCCESS");
 713 }
 714
 715 static int
 716 rcu_torture_init(void)
 717 {
 718         int i;
 719         int cpu;
 720         int firsterr = 0;
 721
 722         /* Process args and tell the world that the torturer is on the job. */
 723
 724         for (i = 0; cur_ops = torture_ops[i], cur_ops != NULL; i++) {
 725                 cur_ops = torture_ops[i];
 726                 if (strcmp(torture_type, cur_ops->name) == 0) {
 727                         break;
 728                 }
 729         }
 730         if (cur_ops == NULL) {
 731                 printk(KERN_ALERT "rcutorture: invalid torture type: \"%s\"\n",
 732                        torture_type);
 733                 return (-EINVAL);
 734         }
 735         if (cur_ops->init != NULL)
 736                 cur_ops->init(); /* no "goto unwind" prior to this point!!! */
 737
 738         if (nreaders >= 0)
 739                 nrealreaders = nreaders;
 740         else
 741                 nrealreaders = 2 * num_online_cpus();
 742         rcu_torture_print_module_parms("Start of test");
 743         fullstop = 0;
 744
 745         /* Set up the freelist. */
 746
 747         INIT_LIST_HEAD(&rcu_torture_freelist);
 748         for (i = 0; i < sizeof(rcu_tortures) / sizeof(rcu_tortures[0]); i++) {
 749                 rcu_tortures[i].rtort_mbtest = 0;
 750                 list_add_tail(&rcu_tortures[i].rtort_free,
 751                               &rcu_torture_freelist);
 752         }
 753
 754         /* Initialize the statistics so that each run gets its own numbers. */
 755
 756         rcu_torture_current = NULL;
 757         rcu_torture_current_version = 0;
 758         atomic_set(&n_rcu_torture_alloc, 0);
 759         atomic_set(&n_rcu_torture_alloc_fail, 0);
 760         atomic_set(&n_rcu_torture_free, 0);
 761         atomic_set(&n_rcu_torture_mberror, 0);
 762         atomic_set(&n_rcu_torture_error, 0);
 763         for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++)
 764                 atomic_set(&rcu_torture_wcount[i], 0);
 765         for_each_possible_cpu(cpu) {
 766                 for (i = 0; i < RCU_TORTURE_PIPE_LEN + 1; i++) {
 767                         per_cpu(rcu_torture_count, cpu)[i] = 0;
 768                         per_cpu(rcu_torture_batch, cpu)[i] = 0;
 769                 }
 770         }
 771
 772         /* Start up the kthreads. */
 773
 774         VERBOSE_PRINTK_STRING("Creating rcu_torture_writer task");
 775         writer_task = kthread_run(rcu_torture_writer, NULL,
 776                                   "rcu_torture_writer");
 777         if (IS_ERR(writer_task)) {
 778                 firsterr = PTR_ERR(writer_task);
 779                 VERBOSE_PRINTK_ERRSTRING("Failed to create writer");
 780                 writer_task = NULL;
 781                 goto unwind;
 782         }
 783         reader_tasks = kmalloc(nrealreaders * sizeof(reader_tasks[0]),
 784                                GFP_KERNEL);
 785         if (reader_tasks == NULL) {
 786                 VERBOSE_PRINTK_ERRSTRING("out of memory");
 787                 firsterr = -ENOMEM;
 788                 goto unwind;
 789         }
 790         for (i = 0; i < nrealreaders; i++) {
 791                 VERBOSE_PRINTK_STRING("Creating rcu_torture_reader task");
 792                 reader_tasks[i] = kthread_run(rcu_torture_reader, NULL,
 793                                               "rcu_torture_reader");
 794                 if (IS_ERR(reader_tasks[i])) {
 795                         firsterr = PTR_ERR(reader_tasks[i]);
 796                         VERBOSE_PRINTK_ERRSTRING("Failed to create reader");
 797                         reader_tasks[i] = NULL;
 798                         goto unwind;
 799                 }
 800         }
 801         if (stat_interval > 0) {
 802                 VERBOSE_PRINTK_STRING("Creating rcu_torture_stats task");
 803                 stats_task = kthread_run(rcu_torture_stats, NULL,
 804                                         "rcu_torture_stats");
 805                 if (IS_ERR(stats_task)) {
 806                         firsterr = PTR_ERR(stats_task);
 807                         VERBOSE_PRINTK_ERRSTRING("Failed to create stats");
 808                         stats_task = NULL;
 809                         goto unwind;
 810                 }
 811         }
 812         if (test_no_idle_hz) {
 813                 rcu_idle_cpu = num_online_cpus() - 1;
 814                 /* Create the shuffler thread */
 815                 shuffler_task = kthread_run(rcu_torture_shuffle, NULL,
 816                                           "rcu_torture_shuffle");
 817                 if (IS_ERR(shuffler_task)) {
 818                         firsterr = PTR_ERR(shuffler_task);
 819                         VERBOSE_PRINTK_ERRSTRING("Failed to create shuffler");
 820                         shuffler_task = NULL;
 821                         goto unwind;
 822                 }
 823         }
 824         return 0;
 825
 826 unwind:
 827         rcu_torture_cleanup();
 828         return firsterr;
 829 }
 830
 831 module_init(rcu_torture_init);
 832 module_exit(rcu_torture_cleanup);