arch/alpha/kernel/smp.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *      linux/arch/alpha/kernel/smp.c
   4  *
   5  *      2001-07-09 Phil Ezolt ([email protected])
   6  *            Renamed modified smp_call_function to smp_call_function_on_cpu()
   7  *            Created an function that conforms to the old calling convention
   8  *            of smp_call_function().
   9  *
  10  *            This is helpful for DCPI.
  11  *
  12  */
  13
  14 #include <linux/errno.h>
  15 #include <linux/kernel.h>
  16 #include <linux/kernel_stat.h>
  17 #include <linux/module.h>
  18 #include <linux/sched/mm.h>
  19 #include <linux/mm.h>
  20 #include <linux/err.h>
  21 #include <linux/threads.h>
  22 #include <linux/smp.h>
  23 #include <linux/interrupt.h>
  24 #include <linux/init.h>
  25 #include <linux/delay.h>
  26 #include <linux/spinlock.h>
  27 #include <linux/irq.h>
  28 #include <linux/cache.h>
  29 #include <linux/profile.h>
  30 #include <linux/bitops.h>
  31 #include <linux/cpu.h>
  32
  33 #include <asm/hwrpb.h>
  34 #include <asm/ptrace.h>
  35 #include <linux/atomic.h>
  36
  37 #include <asm/io.h>
  38 #include <asm/irq.h>
  39 #include <asm/mmu_context.h>
  40 #include <asm/tlbflush.h>
  41
  42 #include "proto.h"
  43 #include "irq_impl.h"
  44
  45
  46 #define DEBUG_SMP 0
  47 #if DEBUG_SMP
  48 #define DBGS(args)      printk args
  49 #else
  50 #define DBGS(args)
  51 #endif
  52
  53 /* A collection of per-processor data.  */
  54 struct cpuinfo_alpha cpu_data[NR_CPUS];
  55 EXPORT_SYMBOL(cpu_data);
  56
  57 /* A collection of single bit ipi messages.  */
  58 static struct {
  59         unsigned long bits ____cacheline_aligned;
  60 } ipi_data[NR_CPUS] __cacheline_aligned;
  61
  62 enum ipi_message_type {
  63         IPI_RESCHEDULE,
  64         IPI_CALL_FUNC,
  65         IPI_CPU_STOP,
  66 };
  67
  68 /* Set to a secondary's cpuid when it comes online.  */
  69 static int smp_secondary_alive = 0;
  70
  71 int smp_num_probed;             /* Internal processor count */
  72 int smp_num_cpus = 1;           /* Number that came online.  */
  73 EXPORT_SYMBOL(smp_num_cpus);
  74
  75 /*
  76  * Called by both boot and secondaries to move global data into
  77  *  per-processor storage.
  78  */
  79 static inline void __init
  80 smp_store_cpu_info(int cpuid)
  81 {
  82         cpu_data[cpuid].loops_per_jiffy = loops_per_jiffy;
  83         cpu_data[cpuid].last_asn = ASN_FIRST_VERSION;
  84         cpu_data[cpuid].need_new_asn = 0;
  85         cpu_data[cpuid].asn_lock = 0;
  86 }
  87
  88 /*
  89  * Ideally sets up per-cpu profiling hooks.  Doesn't do much now...
  90  */
  91 static inline void __init
  92 smp_setup_percpu_timer(int cpuid)
  93 {
  94         cpu_data[cpuid].prof_counter = 1;
  95         cpu_data[cpuid].prof_multiplier = 1;
  96 }
  97
  98 static void __init
  99 wait_boot_cpu_to_stop(int cpuid)
 100 {
 101         unsigned long stop = jiffies + 10*HZ;
 102
 103         while (time_before(jiffies, stop)) {
 104                 if (!smp_secondary_alive)
 105                         return;
 106                 barrier();
 107         }
 108
 109         printk("wait_boot_cpu_to_stop: FAILED on CPU %d, hanging now\n", cpuid);
 110         for (;;)
 111                 barrier();
 112 }
 113
 114 /*
 115  * Where secondaries begin a life of C.
 116  */
 117 void __init
 118 smp_callin(void)
 119 {
 120         int cpuid = hard_smp_processor_id();
 121
 122         if (cpu_online(cpuid)) {
 123                 printk("??, cpu 0x%x already present??\n", cpuid);
 124                 BUG();
 125         }
 126         set_cpu_online(cpuid, true);
 127
 128         /* Turn on machine checks.  */
 129         wrmces(7);
 130
 131         /* Set trap vectors.  */
 132         trap_init();
 133
 134         /* Set interrupt vector.  */
 135         wrent(entInt, 0);
 136
 137         /* Get our local ticker going. */
 138         smp_setup_percpu_timer(cpuid);
 139         init_clockevent();
 140
 141         /* Call platform-specific callin, if specified */
 142         if (alpha_mv.smp_callin)
 143                 alpha_mv.smp_callin();
 144
 145         /* All kernel threads share the same mm context.  */
 146         mmgrab(&init_mm);
 147         current->active_mm = &init_mm;
 148
 149         /* inform the notifiers about the new cpu */
 150         notify_cpu_starting(cpuid);
 151
 152         /* Must have completely accurate bogos.  */
 153         local_irq_enable();
 154
 155         /* Wait boot CPU to stop with irq enabled before running
 156            calibrate_delay. */
 157         wait_boot_cpu_to_stop(cpuid);
 158         mb();
 159         calibrate_delay();
 160
 161         smp_store_cpu_info(cpuid);
 162         /* Allow master to continue only after we written loops_per_jiffy.  */
 163         wmb();
 164         smp_secondary_alive = 1;
 165
 166         DBGS(("smp_callin: commencing CPU %d current %p active_mm %p\n",
 167               cpuid, current, current->active_mm));
 168
 169         preempt_disable();
 170         cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
 171 }
 172
 173 /* Wait until hwrpb->txrdy is clear for cpu.  Return -1 on timeout.  */
 174 static int
 175 wait_for_txrdy (unsigned long cpumask)
 176 {
 177         unsigned long timeout;
 178
 179         if (!(hwrpb->txrdy & cpumask))
 180                 return 0;
 181
 182         timeout = jiffies + 10*HZ;
 183         while (time_before(jiffies, timeout)) {
 184                 if (!(hwrpb->txrdy & cpumask))
 185                         return 0;
 186                 udelay(10);
 187                 barrier();
 188         }
 189
 190         return -1;
 191 }
 192
 193 /*
 194  * Send a message to a secondary's console.  "START" is one such
 195  * interesting message.  ;-)
 196  */
 197 static void
 198 send_secondary_console_msg(char *str, int cpuid)
 199 {
 200         struct percpu_struct *cpu;
 201         register char *cp1, *cp2;
 202         unsigned long cpumask;
 203         size_t len;
 204
 205         cpu = (struct percpu_struct *)
 206                 ((char*)hwrpb
 207                  + hwrpb->processor_offset
 208                  + cpuid * hwrpb->processor_size);
 209
 210         cpumask = (1UL << cpuid);
 211         if (wait_for_txrdy(cpumask))
 212                 goto timeout;
 213
 214         cp2 = str;
 215         len = strlen(cp2);
 216         *(unsigned int *)&cpu->ipc_buffer[0] = len;
 217         cp1 = (char *) &cpu->ipc_buffer[1];
 218         memcpy(cp1, cp2, len);
 219
 220         /* atomic test and set */
 221         wmb();
 222         set_bit(cpuid, &hwrpb->rxrdy);
 223
 224         if (wait_for_txrdy(cpumask))
 225                 goto timeout;
 226         return;
 227
 228  timeout:
 229         printk("Processor %x not ready\n", cpuid);
 230 }
 231
 232 /*
 233  * A secondary console wants to send a message.  Receive it.
 234  */
 235 static void
 236 recv_secondary_console_msg(void)
 237 {
 238         int mycpu, i, cnt;
 239         unsigned long txrdy = hwrpb->txrdy;
 240         char *cp1, *cp2, buf[80];
 241         struct percpu_struct *cpu;
 242
 243         DBGS(("recv_secondary_console_msg: TXRDY 0x%lx.\n", txrdy));
 244
 245         mycpu = hard_smp_processor_id();
 246
 247         for (i = 0; i < NR_CPUS; i++) {
 248                 if (!(txrdy & (1UL << i)))
 249                         continue;
 250
 251                 DBGS(("recv_secondary_console_msg: "
 252                       "TXRDY contains CPU %d.\n", i));
 253
 254                 cpu = (struct percpu_struct *)
 255                   ((char*)hwrpb
 256                    + hwrpb->processor_offset
 257                    + i * hwrpb->processor_size);
 258
 259                 DBGS(("recv_secondary_console_msg: on %d from %d"
 260                       " HALT_REASON 0x%lx FLAGS 0x%lx\n",
 261                       mycpu, i, cpu->halt_reason, cpu->flags));
 262
 263                 cnt = cpu->ipc_buffer[0] >> 32;
 264                 if (cnt <= 0 || cnt >= 80)
 265                         strcpy(buf, "<<< BOGUS MSG >>>");
 266                 else {
 267                         cp1 = (char *) &cpu->ipc_buffer[1];
 268                         cp2 = buf;
 269                         memcpy(cp2, cp1, cnt);
 270                         cp2[cnt] = '\0';
 271
 272                         while ((cp2 = strchr(cp2, '\r')) != 0) {
 273                                 *cp2 = ' ';
 274                                 if (cp2[1] == '\n')
 275                                         cp2[1] = ' ';
 276                         }
 277                 }
 278
 279                 DBGS((KERN_INFO "recv_secondary_console_msg: on %d "
 280                       "message is '%s'\n", mycpu, buf));
 281         }
 282
 283         hwrpb->txrdy = 0;
 284 }
 285
 286 /*
 287  * Convince the console to have a secondary cpu begin execution.
 288  */
 289 static int
 290 secondary_cpu_start(int cpuid, struct task_struct *idle)
 291 {
 292         struct percpu_struct *cpu;
 293         struct pcb_struct *hwpcb, *ipcb;
 294         unsigned long timeout;
 295
 296         cpu = (struct percpu_struct *)
 297                 ((char*)hwrpb
 298                  + hwrpb->processor_offset
 299                  + cpuid * hwrpb->processor_size);
 300         hwpcb = (struct pcb_struct *) cpu->hwpcb;
 301         ipcb = &task_thread_info(idle)->pcb;
 302
 303         /* Initialize the CPU's HWPCB to something just good enough for
 304            us to get started.  Immediately after starting, we'll swpctx
 305            to the target idle task's pcb.  Reuse the stack in the mean
 306            time.  Precalculate the target PCBB.  */
 307         hwpcb->ksp = (unsigned long)ipcb + sizeof(union thread_union) - 16;
 308         hwpcb->usp = 0;
 309         hwpcb->ptbr = ipcb->ptbr;
 310         hwpcb->pcc = 0;
 311         hwpcb->asn = 0;
 312         hwpcb->unique = virt_to_phys(ipcb);
 313         hwpcb->flags = ipcb->flags;
 314         hwpcb->res1 = hwpcb->res2 = 0;
 315
 316 #if 0
 317         DBGS(("KSP 0x%lx PTBR 0x%lx VPTBR 0x%lx UNIQUE 0x%lx\n",
 318               hwpcb->ksp, hwpcb->ptbr, hwrpb->vptb, hwpcb->unique));
 319 #endif
 320         DBGS(("Starting secondary cpu %d: state 0x%lx pal_flags 0x%lx\n",
 321               cpuid, idle->state, ipcb->flags));
 322
 323         /* Setup HWRPB fields that SRM uses to activate secondary CPU */
 324         hwrpb->CPU_restart = __smp_callin;
 325         hwrpb->CPU_restart_data = (unsigned long) __smp_callin;
 326
 327         /* Recalculate and update the HWRPB checksum */
 328         hwrpb_update_checksum(hwrpb);
 329
 330         /*
 331          * Send a "start" command to the specified processor.
 332          */
 333
 334         /* SRM III 3.4.1.3 */
 335         cpu->flags |= 0x22;     /* turn on Context Valid and Restart Capable */
 336         cpu->flags &= ~1;       /* turn off Bootstrap In Progress */
 337         wmb();
 338
 339         send_secondary_console_msg("START\r\n", cpuid);
 340
 341         /* Wait 10 seconds for an ACK from the console.  */
 342         timeout = jiffies + 10*HZ;
 343         while (time_before(jiffies, timeout)) {
 344                 if (cpu->flags & 1)
 345                         goto started;
 346                 udelay(10);
 347                 barrier();
 348         }
 349         printk(KERN_ERR "SMP: Processor %d failed to start.\n", cpuid);
 350         return -1;
 351
 352  started:
 353         DBGS(("secondary_cpu_start: SUCCESS for CPU %d!!!\n", cpuid));
 354         return 0;
 355 }
 356
 357 /*
 358  * Bring one cpu online.
 359  */
 360 static int
 361 smp_boot_one_cpu(int cpuid, struct task_struct *idle)
 362 {
 363         unsigned long timeout;
 364
 365         /* Signal the secondary to wait a moment.  */
 366         smp_secondary_alive = -1;
 367
 368         /* Whirrr, whirrr, whirrrrrrrrr... */
 369         if (secondary_cpu_start(cpuid, idle))
 370                 return -1;
 371
 372         /* Notify the secondary CPU it can run calibrate_delay.  */
 373         mb();
 374         smp_secondary_alive = 0;
 375
 376         /* We've been acked by the console; wait one second for
 377            the task to start up for real.  */
 378         timeout = jiffies + 1*HZ;
 379         while (time_before(jiffies, timeout)) {
 380                 if (smp_secondary_alive == 1)
 381                         goto alive;
 382                 udelay(10);
 383                 barrier();
 384         }
 385
 386         /* We failed to boot the CPU.  */
 387
 388         printk(KERN_ERR "SMP: Processor %d is stuck.\n", cpuid);
 389         return -1;
 390
 391  alive:
 392         /* Another "Red Snapper". */
 393         return 0;
 394 }
 395
 396 /*
 397  * Called from setup_arch.  Detect an SMP system and which processors
 398  * are present.
 399  */
 400 void __init
 401 setup_smp(void)
 402 {
 403         struct percpu_struct *cpubase, *cpu;
 404         unsigned long i;
 405
 406         if (boot_cpuid != 0) {
 407                 printk(KERN_WARNING "SMP: Booting off cpu %d instead of 0?\n",
 408                        boot_cpuid);
 409         }
 410
 411         if (hwrpb->nr_processors > 1) {
 412                 int boot_cpu_palrev;
 413
 414                 DBGS(("setup_smp: nr_processors %ld\n",
 415                       hwrpb->nr_processors));
 416
 417                 cpubase = (struct percpu_struct *)
 418                         ((char*)hwrpb + hwrpb->processor_offset);
 419                 boot_cpu_palrev = cpubase->pal_revision;
 420
 421                 for (i = 0; i < hwrpb->nr_processors; i++) {
 422                         cpu = (struct percpu_struct *)
 423                                 ((char *)cpubase + i*hwrpb->processor_size);
 424                         if ((cpu->flags & 0x1cc) == 0x1cc) {
 425                                 smp_num_probed++;
 426                                 set_cpu_possible(i, true);
 427                                 set_cpu_present(i, true);
 428                                 cpu->pal_revision = boot_cpu_palrev;
 429                         }
 430
 431                         DBGS(("setup_smp: CPU %d: flags 0x%lx type 0x%lx\n",
 432                               i, cpu->flags, cpu->type));
 433                         DBGS(("setup_smp: CPU %d: PAL rev 0x%lx\n",
 434                               i, cpu->pal_revision));
 435                 }
 436         } else {
 437                 smp_num_probed = 1;
 438         }
 439
 440         printk(KERN_INFO "SMP: %d CPUs probed -- cpu_present_mask = %lx\n",
 441                smp_num_probed, cpumask_bits(cpu_present_mask)[0]);
 442 }
 443
 444 /*
 445  * Called by smp_init prepare the secondaries
 446  */
 447 void __init
 448 smp_prepare_cpus(unsigned int max_cpus)
 449 {
 450         /* Take care of some initial bookkeeping.  */
 451         memset(ipi_data, 0, sizeof(ipi_data));
 452
 453         current_thread_info()->cpu = boot_cpuid;
 454
 455         smp_store_cpu_info(boot_cpuid);
 456         smp_setup_percpu_timer(boot_cpuid);
 457
 458         /* Nothing to do on a UP box, or when told not to.  */
 459         if (smp_num_probed == 1 || max_cpus == 0) {
 460                 init_cpu_possible(cpumask_of(boot_cpuid));
 461                 init_cpu_present(cpumask_of(boot_cpuid));
 462                 printk(KERN_INFO "SMP mode deactivated.\n");
 463                 return;
 464         }
 465
 466         printk(KERN_INFO "SMP starting up secondaries.\n");
 467
 468         smp_num_cpus = smp_num_probed;
 469 }
 470
 471 void
 472 smp_prepare_boot_cpu(void)
 473 {
 474 }
 475
 476 int
 477 __cpu_up(unsigned int cpu, struct task_struct *tidle)
 478 {
 479         smp_boot_one_cpu(cpu, tidle);
 480
 481         return cpu_online(cpu) ? 0 : -ENOSYS;
 482 }
 483
 484 void __init
 485 smp_cpus_done(unsigned int max_cpus)
 486 {
 487         int cpu;
 488         unsigned long bogosum = 0;
 489
 490         for(cpu = 0; cpu < NR_CPUS; cpu++)
 491                 if (cpu_online(cpu))
 492                         bogosum += cpu_data[cpu].loops_per_jiffy;
 493
 494         printk(KERN_INFO "SMP: Total of %d processors activated "
 495                "(%lu.%02lu BogoMIPS).\n",
 496                num_online_cpus(),
 497                (bogosum + 2500) / (500000/HZ),
 498                ((bogosum + 2500) / (5000/HZ)) % 100);
 499 }
 500
 501 int
 502 setup_profiling_timer(unsigned int multiplier)
 503 {
 504         return -EINVAL;
 505 }
 506
 507 static void
 508 send_ipi_message(const struct cpumask *to_whom, enum ipi_message_type operation)
 509 {
 510         int i;
 511
 512         mb();
 513         for_each_cpu(i, to_whom)
 514                 set_bit(operation, &ipi_data[i].bits);
 515
 516         mb();
 517         for_each_cpu(i, to_whom)
 518                 wripir(i);
 519 }
 520
 521 void
 522 handle_ipi(struct pt_regs *regs)
 523 {
 524         int this_cpu = smp_processor_id();
 525         unsigned long *pending_ipis = &ipi_data[this_cpu].bits;
 526         unsigned long ops;
 527
 528 #if 0
 529         DBGS(("handle_ipi: on CPU %d ops 0x%lx PC 0x%lx\n",
 530               this_cpu, *pending_ipis, regs->pc));
 531 #endif
 532
 533         mb();   /* Order interrupt and bit testing. */
 534         while ((ops = xchg(pending_ipis, 0)) != 0) {
 535           mb(); /* Order bit clearing and data access. */
 536           do {
 537                 unsigned long which;
 538
 539                 which = ops & -ops;
 540                 ops &= ~which;
 541                 which = __ffs(which);
 542
 543                 switch (which) {
 544                 case IPI_RESCHEDULE:
 545                         scheduler_ipi();
 546                         break;
 547
 548                 case IPI_CALL_FUNC:
 549                         generic_smp_call_function_interrupt();
 550                         break;
 551
 552                 case IPI_CPU_STOP:
 553                         halt();
 554
 555                 default:
 556                         printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
 557                                this_cpu, which);
 558                         break;
 559                 }
 560           } while (ops);
 561
 562           mb(); /* Order data access and bit testing. */
 563         }
 564
 565         cpu_data[this_cpu].ipi_count++;
 566
 567         if (hwrpb->txrdy)
 568                 recv_secondary_console_msg();
 569 }
 570
 571 void
 572 smp_send_reschedule(int cpu)
 573 {
 574 #ifdef DEBUG_IPI_MSG
 575         if (cpu == hard_smp_processor_id())
 576                 printk(KERN_WARNING
 577                        "smp_send_reschedule: Sending IPI to self.\n");
 578 #endif
 579         send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
 580 }
 581
 582 void
 583 smp_send_stop(void)
 584 {
 585         cpumask_t to_whom;
 586         cpumask_copy(&to_whom, cpu_possible_mask);
 587         cpumask_clear_cpu(smp_processor_id(), &to_whom);
 588 #ifdef DEBUG_IPI_MSG
 589         if (hard_smp_processor_id() != boot_cpu_id)
 590                 printk(KERN_WARNING "smp_send_stop: Not on boot cpu.\n");
 591 #endif
 592         send_ipi_message(&to_whom, IPI_CPU_STOP);
 593 }
 594
 595 void arch_send_call_function_ipi_mask(const struct cpumask *mask)
 596 {
 597         send_ipi_message(mask, IPI_CALL_FUNC);
 598 }
 599
 600 void arch_send_call_function_single_ipi(int cpu)
 601 {
 602         send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
 603 }
 604
 605 static void
 606 ipi_imb(void *ignored)
 607 {
 608         imb();
 609 }
 610
 611 void
 612 smp_imb(void)
 613 {
 614         /* Must wait other processors to flush their icache before continue. */
 615         on_each_cpu(ipi_imb, NULL, 1);
 616 }
 617 EXPORT_SYMBOL(smp_imb);
 618
 619 static void
 620 ipi_flush_tlb_all(void *ignored)
 621 {
 622         tbia();
 623 }
 624
 625 void
 626 flush_tlb_all(void)
 627 {
 628         /* Although we don't have any data to pass, we do want to
 629            synchronize with the other processors.  */
 630         on_each_cpu(ipi_flush_tlb_all, NULL, 1);
 631 }
 632
 633 #define asn_locked() (cpu_data[smp_processor_id()].asn_lock)
 634
 635 static void
 636 ipi_flush_tlb_mm(void *x)
 637 {
 638         struct mm_struct *mm = (struct mm_struct *) x;
 639         if (mm == current->active_mm && !asn_locked())
 640                 flush_tlb_current(mm);
 641         else
 642                 flush_tlb_other(mm);
 643 }
 644
 645 void
 646 flush_tlb_mm(struct mm_struct *mm)
 647 {
 648         preempt_disable();
 649
 650         if (mm == current->active_mm) {
 651                 flush_tlb_current(mm);
 652                 if (atomic_read(&mm->mm_users) <= 1) {
 653                         int cpu, this_cpu = smp_processor_id();
 654                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 655                                 if (!cpu_online(cpu) || cpu == this_cpu)
 656                                         continue;
 657                                 if (mm->context[cpu])
 658                                         mm->context[cpu] = 0;
 659                         }
 660                         preempt_enable();
 661                         return;
 662                 }
 663         }
 664
 665         smp_call_function(ipi_flush_tlb_mm, mm, 1);
 666
 667         preempt_enable();
 668 }
 669 EXPORT_SYMBOL(flush_tlb_mm);
 670
 671 struct flush_tlb_page_struct {
 672         struct vm_area_struct *vma;
 673         struct mm_struct *mm;
 674         unsigned long addr;
 675 };
 676
 677 static void
 678 ipi_flush_tlb_page(void *x)
 679 {
 680         struct flush_tlb_page_struct *data = (struct flush_tlb_page_struct *)x;
 681         struct mm_struct * mm = data->mm;
 682
 683         if (mm == current->active_mm && !asn_locked())
 684                 flush_tlb_current_page(mm, data->vma, data->addr);
 685         else
 686                 flush_tlb_other(mm);
 687 }
 688
 689 void
 690 flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
 691 {
 692         struct flush_tlb_page_struct data;
 693         struct mm_struct *mm = vma->vm_mm;
 694
 695         preempt_disable();
 696
 697         if (mm == current->active_mm) {
 698                 flush_tlb_current_page(mm, vma, addr);
 699                 if (atomic_read(&mm->mm_users) <= 1) {
 700                         int cpu, this_cpu = smp_processor_id();
 701                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 702                                 if (!cpu_online(cpu) || cpu == this_cpu)
 703                                         continue;
 704                                 if (mm->context[cpu])
 705                                         mm->context[cpu] = 0;
 706                         }
 707                         preempt_enable();
 708                         return;
 709                 }
 710         }
 711
 712         data.vma = vma;
 713         data.mm = mm;
 714         data.addr = addr;
 715
 716         smp_call_function(ipi_flush_tlb_page, &data, 1);
 717
 718         preempt_enable();
 719 }
 720 EXPORT_SYMBOL(flush_tlb_page);
 721
 722 void
 723 flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)
 724 {
 725         /* On the Alpha we always flush the whole user tlb.  */
 726         flush_tlb_mm(vma->vm_mm);
 727 }
 728 EXPORT_SYMBOL(flush_tlb_range);
 729
 730 static void
 731 ipi_flush_icache_page(void *x)
 732 {
 733         struct mm_struct *mm = (struct mm_struct *) x;
 734         if (mm == current->active_mm && !asn_locked())
 735                 __load_new_mm_context(mm);
 736         else
 737                 flush_tlb_other(mm);
 738 }
 739
 740 void
 741 flush_icache_user_page(struct vm_area_struct *vma, struct page *page,
 742                         unsigned long addr, int len)
 743 {
 744         struct mm_struct *mm = vma->vm_mm;
 745
 746         if ((vma->vm_flags & VM_EXEC) == 0)
 747                 return;
 748
 749         preempt_disable();
 750
 751         if (mm == current->active_mm) {
 752                 __load_new_mm_context(mm);
 753                 if (atomic_read(&mm->mm_users) <= 1) {
 754                         int cpu, this_cpu = smp_processor_id();
 755                         for (cpu = 0; cpu < NR_CPUS; cpu++) {
 756                                 if (!cpu_online(cpu) || cpu == this_cpu)
 757                                         continue;
 758                                 if (mm->context[cpu])
 759                                         mm->context[cpu] = 0;
 760                         }
 761                         preempt_enable();
 762                         return;
 763                 }
 764         }
 765
 766         smp_call_function(ipi_flush_icache_page, mm, 1);
 767
 768         preempt_enable();
 769 }