projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Fix invalid #if in vnc.c when debugging is enabled (Alexander Graf)
[qemu.git] / exec.c
CommitLineData
54936004 1/*
fd6ce8f6 2 * virtual page mapping and translated block handling
5fafdf24 3 *
54936004
FB		 4 * Copyright (c) 2003 Fabrice Bellard
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2 of the License, or (at your option) any later version.
10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
fad6cb1a 18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301 USA
54936004 19 */
67b915a5 20#include "config.h"
d5a8f07c 21#ifdef _WIN32
4fddf62a 22#define WIN32_LEAN_AND_MEAN
d5a8f07c
FB		 23#include <windows.h>
24#else
a98d49b1 25#include <sys/types.h>
d5a8f07c
FB		 26#include <sys/mman.h>
27#endif
54936004
FB		 28#include <stdlib.h>
29#include <stdio.h>
30#include <stdarg.h>
31#include <string.h>
32#include <errno.h>
33#include <unistd.h>
34#include <inttypes.h>
35
6180a181
FB		 36#include "cpu.h"
37#include "exec-all.h"
ca10f867 38#include "qemu-common.h"
b67d9a52 39#include "tcg.h"
b3c7724c 40#include "hw/hw.h"
74576198 41#include "osdep.h"
7ba1e619 42#include "kvm.h"
53a5960a
PB		 43#if defined(CONFIG_USER_ONLY)
44#include <qemu.h>
45#endif
54936004 46
fd6ce8f6 47//#define DEBUG_TB_INVALIDATE
66e85a21 48//#define DEBUG_FLUSH
9fa3e853 49//#define DEBUG_TLB
67d3b957 50//#define DEBUG_UNASSIGNED
fd6ce8f6
FB		 51
52/* make various TB consistency checks */
5fafdf24
TS		 53//#define DEBUG_TB_CHECK
54//#define DEBUG_TLB_CHECK
fd6ce8f6 55
1196be37 56//#define DEBUG_IOPORT
db7b5426 57//#define DEBUG_SUBPAGE
1196be37 58
99773bd4
PB		 59#if !defined(CONFIG_USER_ONLY)
60/* TB consistency checks only implemented for usermode emulation. */
61#undef DEBUG_TB_CHECK
62#endif
63
9fa3e853
FB		 64#define SMC_BITMAP_USE_THRESHOLD 10
65
66#define MMAP_AREA_START 0x00000000
67#define MMAP_AREA_END 0xa8000000
fd6ce8f6 68
108c49b8
FB		 69#if defined(TARGET_SPARC64)
70#define TARGET_PHYS_ADDR_SPACE_BITS 41
5dcb6b91
BS		 71#elif defined(TARGET_SPARC)
72#define TARGET_PHYS_ADDR_SPACE_BITS 36
bedb69ea
JM		 73#elif defined(TARGET_ALPHA)
74#define TARGET_PHYS_ADDR_SPACE_BITS 42
75#define TARGET_VIRT_ADDR_SPACE_BITS 42
108c49b8
FB		 76#elif defined(TARGET_PPC64)
77#define TARGET_PHYS_ADDR_SPACE_BITS 42
00f82b8a
AJ		 78#elif defined(TARGET_X86_64) && !defined(USE_KQEMU)
79#define TARGET_PHYS_ADDR_SPACE_BITS 42
80#elif defined(TARGET_I386) && !defined(USE_KQEMU)
81#define TARGET_PHYS_ADDR_SPACE_BITS 36
108c49b8
FB		 82#else
83/* Note: for compatibility with kqemu, we use 32 bits for x86_64 */
84#define TARGET_PHYS_ADDR_SPACE_BITS 32
85#endif
86
bdaf78e0 87static TranslationBlock *tbs;
26a5f13b 88int code_gen_max_blocks;
9fa3e853 89TranslationBlock *tb_phys_hash[CODE_GEN_PHYS_HASH_SIZE];
bdaf78e0 90static int nb_tbs;
eb51d102
FB		 91/* any access to the tbs or the page table must use this lock */
92spinlock_t tb_lock = SPIN_LOCK_UNLOCKED;
fd6ce8f6 93
141ac468
BS		 94#if defined(__arm__) || defined(__sparc_v9__)
95/* The prologue must be reachable with a direct jump. ARM and Sparc64
96 have limited branch ranges (possibly also PPC) so place it in a
d03d860b
BS		 97 section close to code segment. */
98#define code_gen_section \
99 __attribute__((__section__(".gen_code"))) \
100 __attribute__((aligned (32)))
101#else
102#define code_gen_section \
103 __attribute__((aligned (32)))
104#endif
105
106uint8_t code_gen_prologue[1024] code_gen_section;
bdaf78e0
BS		 107static uint8_t *code_gen_buffer;
108static unsigned long code_gen_buffer_size;
26a5f13b 109/* threshold to flush the translated code buffer */
bdaf78e0 110static unsigned long code_gen_buffer_max_size;
fd6ce8f6
FB		 111uint8_t *code_gen_ptr;
112
e2eef170 113#if !defined(CONFIG_USER_ONLY)
00f82b8a 114ram_addr_t phys_ram_size;
9fa3e853
FB		 115int phys_ram_fd;
116uint8_t *phys_ram_base;
1ccde1cb 117uint8_t *phys_ram_dirty;
74576198 118static int in_migration;
e9a1ab19 119static ram_addr_t phys_ram_alloc_offset = 0;
e2eef170 120#endif
9fa3e853 121
6a00d601
FB		 122CPUState *first_cpu;
123/* current CPU in the current thread. It is only valid inside
124 cpu_exec() */
5fafdf24 125CPUState *cpu_single_env;
2e70f6ef 126/* 0 = Do not count executed instructions.
bf20dc07 127 1 = Precise instruction counting.
2e70f6ef
PB		 128 2 = Adaptive rate instruction counting. */
129int use_icount = 0;
130/* Current instruction counter. While executing translated code this may
131 include some instructions that have not yet been executed. */
132int64_t qemu_icount;
6a00d601 133
54936004 134typedef struct PageDesc {
92e873b9 135 /* list of TBs intersecting this ram page */
fd6ce8f6 136 TranslationBlock *first_tb;
9fa3e853
FB		 137 /* in order to optimize self modifying code, we count the number
138 of lookups we do to a given page to use a bitmap */
139 unsigned int code_write_count;
140 uint8_t *code_bitmap;
141#if defined(CONFIG_USER_ONLY)
142 unsigned long flags;
143#endif
54936004
FB		 144} PageDesc;
145
92e873b9 146typedef struct PhysPageDesc {
0f459d16 147 /* offset in host memory of the page + io_index in the low bits */
00f82b8a 148 ram_addr_t phys_offset;
8da3ff18 149 ram_addr_t region_offset;
92e873b9
FB		 150} PhysPageDesc;
151
54936004 152#define L2_BITS 10
bedb69ea
JM		 153#if defined(CONFIG_USER_ONLY) && defined(TARGET_VIRT_ADDR_SPACE_BITS)
154/* XXX: this is a temporary hack for alpha target.
155 * In the future, this is to be replaced by a multi-level table
156 * to actually be able to handle the complete 64 bits address space.
157 */
158#define L1_BITS (TARGET_VIRT_ADDR_SPACE_BITS - L2_BITS - TARGET_PAGE_BITS)
159#else
03875444 160#define L1_BITS (32 - L2_BITS - TARGET_PAGE_BITS)
bedb69ea 161#endif
54936004
FB		 162
163#define L1_SIZE (1 << L1_BITS)
164#define L2_SIZE (1 << L2_BITS)
165
83fb7adf
FB		 166unsigned long qemu_real_host_page_size;
167unsigned long qemu_host_page_bits;
168unsigned long qemu_host_page_size;
169unsigned long qemu_host_page_mask;
54936004 170
92e873b9 171/* XXX: for system emulation, it could just be an array */
54936004 172static PageDesc *l1_map[L1_SIZE];
bdaf78e0 173static PhysPageDesc **l1_phys_map;
54936004 174
e2eef170
PB		 175#if !defined(CONFIG_USER_ONLY)
176static void io_mem_init(void);
177
33417e70 178/* io memory support */
33417e70
FB		 179CPUWriteMemoryFunc *io_mem_write[IO_MEM_NB_ENTRIES][4];
180CPUReadMemoryFunc *io_mem_read[IO_MEM_NB_ENTRIES][4];
a4193c8a 181void *io_mem_opaque[IO_MEM_NB_ENTRIES];
33417e70 182static int io_mem_nb;
6658ffb8
PB		 183static int io_mem_watch;
184#endif
33417e70 185
34865134 186/* log support */
d9b630fd 187static const char *logfilename = "/tmp/qemu.log";
34865134
FB		 188FILE *logfile;
189int loglevel;
e735b91c 190static int log_append = 0;
34865134 191
e3db7226
FB		 192/* statistics */
193static int tlb_flush_count;
194static int tb_flush_count;
195static int tb_phys_invalidate_count;
196
db7b5426
BS		 197#define SUBPAGE_IDX(addr) ((addr) & ~TARGET_PAGE_MASK)
198typedef struct subpage_t {
199 target_phys_addr_t base;
3ee89922
BS		 200 CPUReadMemoryFunc **mem_read[TARGET_PAGE_SIZE][4];
201 CPUWriteMemoryFunc **mem_write[TARGET_PAGE_SIZE][4];
202 void *opaque[TARGET_PAGE_SIZE][2][4];
8da3ff18 203 ram_addr_t region_offset[TARGET_PAGE_SIZE][2][4];
db7b5426
BS		 204} subpage_t;
205
7cb69cae
FB		 206#ifdef _WIN32
207static void map_exec(void *addr, long size)
208{
209 DWORD old_protect;
210 VirtualProtect(addr, size,
211 PAGE_EXECUTE_READWRITE, &old_protect);
212
213}
214#else
215static void map_exec(void *addr, long size)
216{
4369415f 217 unsigned long start, end, page_size;
7cb69cae 218
4369415f 219 page_size = getpagesize();
7cb69cae 220 start = (unsigned long)addr;
4369415f 221 start &= ~(page_size - 1);
7cb69cae
FB		 222
223 end = (unsigned long)addr + size;
4369415f
FB		 224 end += page_size - 1;
225 end &= ~(page_size - 1);
7cb69cae
FB		 226
227 mprotect((void *)start, end - start,
228 PROT_READ | PROT_WRITE | PROT_EXEC);
229}
230#endif
231
b346ff46 232static void page_init(void)
54936004 233{
83fb7adf 234 /* NOTE: we can always suppose that qemu_host_page_size >=
54936004 235 TARGET_PAGE_SIZE */
c2b48b69
AL		 236#ifdef _WIN32
237 {
238 SYSTEM_INFO system_info;
239
240 GetSystemInfo(&system_info);
241 qemu_real_host_page_size = system_info.dwPageSize;
242 }
243#else
244 qemu_real_host_page_size = getpagesize();
245#endif
83fb7adf
FB		 246 if (qemu_host_page_size == 0)
247 qemu_host_page_size = qemu_real_host_page_size;
248 if (qemu_host_page_size < TARGET_PAGE_SIZE)
249 qemu_host_page_size = TARGET_PAGE_SIZE;
250 qemu_host_page_bits = 0;
251 while ((1 << qemu_host_page_bits) < qemu_host_page_size)
252 qemu_host_page_bits++;
253 qemu_host_page_mask = ~(qemu_host_page_size - 1);
108c49b8
FB		 254 l1_phys_map = qemu_vmalloc(L1_SIZE * sizeof(void *));
255 memset(l1_phys_map, 0, L1_SIZE * sizeof(void *));
50a9569b
AZ		 256
257#if !defined(_WIN32) && defined(CONFIG_USER_ONLY)
258 {
259 long long startaddr, endaddr;
260 FILE *f;
261 int n;
262
c8a706fe 263 mmap_lock();
0776590d 264 last_brk = (unsigned long)sbrk(0);
50a9569b
AZ		 265 f = fopen("/proc/self/maps", "r");
266 if (f) {
267 do {
268 n = fscanf (f, "%llx-%llx %*[^\n]\n", &startaddr, &endaddr);
269 if (n == 2) {
e0b8d65a
BS		 270 startaddr = MIN(startaddr,
271 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
272 endaddr = MIN(endaddr,
273 (1ULL << TARGET_PHYS_ADDR_SPACE_BITS) - 1);
b5fc909e 274 page_set_flags(startaddr & TARGET_PAGE_MASK,
50a9569b
AZ		 275 TARGET_PAGE_ALIGN(endaddr),
276 PAGE_RESERVED);
277 }
278 } while (!feof(f));
279 fclose(f);
280 }
c8a706fe 281 mmap_unlock();
50a9569b
AZ		 282 }
283#endif
54936004
FB		 284}
285
434929bf 286static inline PageDesc **page_l1_map(target_ulong index)
54936004 287{
17e2377a
PB		 288#if TARGET_LONG_BITS > 32
289 /* Host memory outside guest VM. For 32-bit targets we have already
290 excluded high addresses. */
d8173e0f 291 if (index > ((target_ulong)L2_SIZE * L1_SIZE))
17e2377a
PB		 292 return NULL;
293#endif
434929bf
AL		 294 return &l1_map[index >> L2_BITS];
295}
296
297static inline PageDesc *page_find_alloc(target_ulong index)
298{
299 PageDesc **lp, *p;
300 lp = page_l1_map(index);
301 if (!lp)
302 return NULL;
303
54936004
FB		 304 p = *lp;
305 if (!p) {
306 /* allocate if not found */
17e2377a 307#if defined(CONFIG_USER_ONLY)
17e2377a
PB		 308 size_t len = sizeof(PageDesc) * L2_SIZE;
309 /* Don't use qemu_malloc because it may recurse. */
310 p = mmap(0, len, PROT_READ | PROT_WRITE,
311 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
54936004 312 *lp = p;
fb1c2cd7
AJ		 313 if (h2g_valid(p)) {
314 unsigned long addr = h2g(p);
17e2377a
PB		 315 page_set_flags(addr & TARGET_PAGE_MASK,
316 TARGET_PAGE_ALIGN(addr + len),
317 PAGE_RESERVED);
318 }
319#else
320 p = qemu_mallocz(sizeof(PageDesc) * L2_SIZE);
321 *lp = p;
322#endif
54936004
FB		 323 }
324 return p + (index & (L2_SIZE - 1));
325}
326
00f82b8a 327static inline PageDesc *page_find(target_ulong index)
54936004 328{
434929bf
AL		 329 PageDesc **lp, *p;
330 lp = page_l1_map(index);
331 if (!lp)
332 return NULL;
54936004 333
434929bf 334 p = *lp;
54936004
FB		 335 if (!p)
336 return 0;
fd6ce8f6
FB		 337 return p + (index & (L2_SIZE - 1));
338}
339
108c49b8 340static PhysPageDesc *phys_page_find_alloc(target_phys_addr_t index, int alloc)
92e873b9 341{
108c49b8 342 void **lp, **p;
e3f4e2a4 343 PhysPageDesc *pd;
92e873b9 344
108c49b8
FB		 345 p = (void **)l1_phys_map;
346#if TARGET_PHYS_ADDR_SPACE_BITS > 32
347
348#if TARGET_PHYS_ADDR_SPACE_BITS > (32 + L1_BITS)
349#error unsupported TARGET_PHYS_ADDR_SPACE_BITS
350#endif
351 lp = p + ((index >> (L1_BITS + L2_BITS)) & (L1_SIZE - 1));
92e873b9
FB		 352 p = *lp;
353 if (!p) {
354 /* allocate if not found */
108c49b8
FB		 355 if (!alloc)
356 return NULL;
357 p = qemu_vmalloc(sizeof(void *) * L1_SIZE);
358 memset(p, 0, sizeof(void *) * L1_SIZE);
359 *lp = p;
360 }
361#endif
362 lp = p + ((index >> L2_BITS) & (L1_SIZE - 1));
e3f4e2a4
PB		 363 pd = *lp;
364 if (!pd) {
365 int i;
108c49b8
FB		 366 /* allocate if not found */
367 if (!alloc)
368 return NULL;
e3f4e2a4
PB		 369 pd = qemu_vmalloc(sizeof(PhysPageDesc) * L2_SIZE);
370 *lp = pd;
371 for (i = 0; i < L2_SIZE; i++)
372 pd[i].phys_offset = IO_MEM_UNASSIGNED;
92e873b9 373 }
e3f4e2a4 374 return ((PhysPageDesc *)pd) + (index & (L2_SIZE - 1));
92e873b9
FB		 375}
376
108c49b8 377static inline PhysPageDesc *phys_page_find(target_phys_addr_t index)
92e873b9 378{
108c49b8 379 return phys_page_find_alloc(index, 0);
92e873b9
FB		 380}
381
9fa3e853 382#if !defined(CONFIG_USER_ONLY)
6a00d601 383static void tlb_protect_code(ram_addr_t ram_addr);
5fafdf24 384static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
3a7d929e 385 target_ulong vaddr);
c8a706fe
PB		 386#define mmap_lock() do { } while(0)
387#define mmap_unlock() do { } while(0)
9fa3e853 388#endif
fd6ce8f6 389
4369415f
FB		 390#define DEFAULT_CODE_GEN_BUFFER_SIZE (32 * 1024 * 1024)
391
392#if defined(CONFIG_USER_ONLY)
393/* Currently it is not recommanded to allocate big chunks of data in
394 user mode. It will change when a dedicated libc will be used */
395#define USE_STATIC_CODE_GEN_BUFFER
396#endif
397
398#ifdef USE_STATIC_CODE_GEN_BUFFER
399static uint8_t static_code_gen_buffer[DEFAULT_CODE_GEN_BUFFER_SIZE];
400#endif
401
8fcd3692 402static void code_gen_alloc(unsigned long tb_size)
26a5f13b 403{
4369415f
FB		 404#ifdef USE_STATIC_CODE_GEN_BUFFER
405 code_gen_buffer = static_code_gen_buffer;
406 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
407 map_exec(code_gen_buffer, code_gen_buffer_size);
408#else
26a5f13b
FB		 409 code_gen_buffer_size = tb_size;
410 if (code_gen_buffer_size == 0) {
4369415f
FB		 411#if defined(CONFIG_USER_ONLY)
412 /* in user mode, phys_ram_size is not meaningful */
413 code_gen_buffer_size = DEFAULT_CODE_GEN_BUFFER_SIZE;
414#else
26a5f13b 415 /* XXX: needs ajustments */
174a9a1f 416 code_gen_buffer_size = (unsigned long)(phys_ram_size / 4);
4369415f 417#endif
26a5f13b
FB		 418 }
419 if (code_gen_buffer_size < MIN_CODE_GEN_BUFFER_SIZE)
420 code_gen_buffer_size = MIN_CODE_GEN_BUFFER_SIZE;
421 /* The code gen buffer location may have constraints depending on
422 the host cpu and OS */
423#if defined(__linux__)
424 {
425 int flags;
141ac468
BS		 426 void *start = NULL;
427
26a5f13b
FB		 428 flags = MAP_PRIVATE | MAP_ANONYMOUS;
429#if defined(__x86_64__)
430 flags |= MAP_32BIT;
431 /* Cannot map more than that */
432 if (code_gen_buffer_size > (800 * 1024 * 1024))
433 code_gen_buffer_size = (800 * 1024 * 1024);
141ac468
BS		 434#elif defined(__sparc_v9__)
435 // Map the buffer below 2G, so we can use direct calls and branches
436 flags |= MAP_FIXED;
437 start = (void *) 0x60000000UL;
438 if (code_gen_buffer_size > (512 * 1024 * 1024))
439 code_gen_buffer_size = (512 * 1024 * 1024);
1cb0661e 440#elif defined(__arm__)
63d41246 441 /* Map the buffer below 32M, so we can use direct calls and branches */
1cb0661e
AZ		 442 flags |= MAP_FIXED;
443 start = (void *) 0x01000000UL;
444 if (code_gen_buffer_size > 16 * 1024 * 1024)
445 code_gen_buffer_size = 16 * 1024 * 1024;
26a5f13b 446#endif
141ac468
BS		 447 code_gen_buffer = mmap(start, code_gen_buffer_size,
448 PROT_WRITE | PROT_READ | PROT_EXEC,
26a5f13b
FB		 449 flags, -1, 0);
450 if (code_gen_buffer == MAP_FAILED) {
451 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
452 exit(1);
453 }
454 }
06e67a82
AL		 455#elif defined(__FreeBSD__)
456 {
457 int flags;
458 void *addr = NULL;
459 flags = MAP_PRIVATE | MAP_ANONYMOUS;
460#if defined(__x86_64__)
461 /* FreeBSD doesn't have MAP_32BIT, use MAP_FIXED and assume
462 * 0x40000000 is free */
463 flags |= MAP_FIXED;
464 addr = (void *)0x40000000;
465 /* Cannot map more than that */
466 if (code_gen_buffer_size > (800 * 1024 * 1024))
467 code_gen_buffer_size = (800 * 1024 * 1024);
468#endif
469 code_gen_buffer = mmap(addr, code_gen_buffer_size,
470 PROT_WRITE | PROT_READ | PROT_EXEC,
471 flags, -1, 0);
472 if (code_gen_buffer == MAP_FAILED) {
473 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
474 exit(1);
475 }
476 }
26a5f13b
FB		 477#else
478 code_gen_buffer = qemu_malloc(code_gen_buffer_size);
479 if (!code_gen_buffer) {
480 fprintf(stderr, "Could not allocate dynamic translator buffer\n");
481 exit(1);
482 }
483 map_exec(code_gen_buffer, code_gen_buffer_size);
484#endif
4369415f 485#endif /* !USE_STATIC_CODE_GEN_BUFFER */
26a5f13b
FB		 486 map_exec(code_gen_prologue, sizeof(code_gen_prologue));
487 code_gen_buffer_max_size = code_gen_buffer_size -
488 code_gen_max_block_size();
489 code_gen_max_blocks = code_gen_buffer_size / CODE_GEN_AVG_BLOCK_SIZE;
490 tbs = qemu_malloc(code_gen_max_blocks * sizeof(TranslationBlock));
491}
492
493/* Must be called before using the QEMU cpus. 'tb_size' is the size
494 (in bytes) allocated to the translation buffer. Zero means default
495 size. */
496void cpu_exec_init_all(unsigned long tb_size)
497{
26a5f13b
FB		 498 cpu_gen_init();
499 code_gen_alloc(tb_size);
500 code_gen_ptr = code_gen_buffer;
4369415f 501 page_init();
e2eef170 502#if !defined(CONFIG_USER_ONLY)
26a5f13b 503 io_mem_init();
e2eef170 504#endif
26a5f13b
FB		 505}
506
9656f324
PB		 507#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
508
509#define CPU_COMMON_SAVE_VERSION 1
510
511static void cpu_common_save(QEMUFile *f, void *opaque)
512{
513 CPUState *env = opaque;
514
515 qemu_put_be32s(f, &env->halted);
516 qemu_put_be32s(f, &env->interrupt_request);
517}
518
519static int cpu_common_load(QEMUFile *f, void *opaque, int version_id)
520{
521 CPUState *env = opaque;
522
523 if (version_id != CPU_COMMON_SAVE_VERSION)
524 return -EINVAL;
525
526 qemu_get_be32s(f, &env->halted);
75f482ae 527 qemu_get_be32s(f, &env->interrupt_request);
9656f324
PB		 528 tlb_flush(env, 1);
529
530 return 0;
531}
532#endif
533
6a00d601 534void cpu_exec_init(CPUState *env)
fd6ce8f6 535{
6a00d601
FB		 536 CPUState **penv;
537 int cpu_index;
538
6a00d601
FB		 539 env->next_cpu = NULL;
540 penv = &first_cpu;
541 cpu_index = 0;
542 while (*penv != NULL) {
543 penv = (CPUState **)&(*penv)->next_cpu;
544 cpu_index++;
545 }
546 env->cpu_index = cpu_index;
c0ce998e
AL		 547 TAILQ_INIT(&env->breakpoints);
548 TAILQ_INIT(&env->watchpoints);
6a00d601 549 *penv = env;
b3c7724c 550#if defined(CPU_SAVE_VERSION) && !defined(CONFIG_USER_ONLY)
9656f324
PB		 551 register_savevm("cpu_common", cpu_index, CPU_COMMON_SAVE_VERSION,
552 cpu_common_save, cpu_common_load, env);
b3c7724c
PB		 553 register_savevm("cpu", cpu_index, CPU_SAVE_VERSION,
554 cpu_save, cpu_load, env);
555#endif
fd6ce8f6
FB		 556}
557
9fa3e853
FB		 558static inline void invalidate_page_bitmap(PageDesc *p)
559{
560 if (p->code_bitmap) {
59817ccb 561 qemu_free(p->code_bitmap);
9fa3e853
FB		 562 p->code_bitmap = NULL;
563 }
564 p->code_write_count = 0;
565}
566
fd6ce8f6
FB		 567/* set to NULL all the 'first_tb' fields in all PageDescs */
568static void page_flush_tb(void)
569{
570 int i, j;
571 PageDesc *p;
572
573 for(i = 0; i < L1_SIZE; i++) {
574 p = l1_map[i];
575 if (p) {
9fa3e853
FB		 576 for(j = 0; j < L2_SIZE; j++) {
577 p->first_tb = NULL;
578 invalidate_page_bitmap(p);
579 p++;
580 }
fd6ce8f6
FB		 581 }
582 }
583}
584
585/* flush all the translation blocks */
d4e8164f 586/* XXX: tb_flush is currently not thread safe */
6a00d601 587void tb_flush(CPUState *env1)
fd6ce8f6 588{
6a00d601 589 CPUState *env;
0124311e 590#if defined(DEBUG_FLUSH)
ab3d1727
BS		 591 printf("qemu: flush code_size=%ld nb_tbs=%d avg_tb_size=%ld\n",
592 (unsigned long)(code_gen_ptr - code_gen_buffer),
593 nb_tbs, nb_tbs > 0 ?
594 ((unsigned long)(code_gen_ptr - code_gen_buffer)) / nb_tbs : 0);
fd6ce8f6 595#endif
26a5f13b 596 if ((unsigned long)(code_gen_ptr - code_gen_buffer) > code_gen_buffer_size)
a208e54a
PB		 597 cpu_abort(env1, "Internal error: code buffer overflow\n");
598
fd6ce8f6 599 nb_tbs = 0;
3b46e624 600
6a00d601
FB		 601 for(env = first_cpu; env != NULL; env = env->next_cpu) {
602 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
603 }
9fa3e853 604
8a8a608f 605 memset (tb_phys_hash, 0, CODE_GEN_PHYS_HASH_SIZE * sizeof (void *));
fd6ce8f6 606 page_flush_tb();
9fa3e853 607
fd6ce8f6 608 code_gen_ptr = code_gen_buffer;
d4e8164f
FB		 609 /* XXX: flush processor icache at this point if cache flush is
610 expensive */
e3db7226 611 tb_flush_count++;
fd6ce8f6
FB		 612}
613
614#ifdef DEBUG_TB_CHECK
615
bc98a7ef 616static void tb_invalidate_check(target_ulong address)
fd6ce8f6
FB		 617{
618 TranslationBlock *tb;
619 int i;
620 address &= TARGET_PAGE_MASK;
99773bd4
PB		 621 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
622 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
fd6ce8f6
FB		 623 if (!(address + TARGET_PAGE_SIZE <= tb->pc ||
624 address >= tb->pc + tb->size)) {
625 printf("ERROR invalidate: address=%08lx PC=%08lx size=%04x\n",
99773bd4 626 address, (long)tb->pc, tb->size);
fd6ce8f6
FB		 627 }
628 }
629 }
630}
631
632/* verify that all the pages have correct rights for code */
633static void tb_page_check(void)
634{
635 TranslationBlock *tb;
636 int i, flags1, flags2;
3b46e624 637
99773bd4
PB		 638 for(i = 0;i < CODE_GEN_PHYS_HASH_SIZE; i++) {
639 for(tb = tb_phys_hash[i]; tb != NULL; tb = tb->phys_hash_next) {
fd6ce8f6
FB		 640 flags1 = page_get_flags(tb->pc);
641 flags2 = page_get_flags(tb->pc + tb->size - 1);
642 if ((flags1 & PAGE_WRITE) || (flags2 & PAGE_WRITE)) {
643 printf("ERROR page flags: PC=%08lx size=%04x f1=%x f2=%x\n",
99773bd4 644 (long)tb->pc, tb->size, flags1, flags2);
fd6ce8f6
FB		 645 }
646 }
647 }
648}
649
bdaf78e0 650static void tb_jmp_check(TranslationBlock *tb)
d4e8164f
FB		 651{
652 TranslationBlock *tb1;
653 unsigned int n1;
654
655 /* suppress any remaining jumps to this TB */
656 tb1 = tb->jmp_first;
657 for(;;) {
658 n1 = (long)tb1 & 3;
659 tb1 = (TranslationBlock *)((long)tb1 & ~3);
660 if (n1 == 2)
661 break;
662 tb1 = tb1->jmp_next[n1];
663 }
664 /* check end of list */
665 if (tb1 != tb) {
666 printf("ERROR: jmp_list from 0x%08lx\n", (long)tb);
667 }
668}
669
fd6ce8f6
FB		 670#endif
671
672/* invalidate one TB */
673static inline void tb_remove(TranslationBlock **ptb, TranslationBlock *tb,
674 int next_offset)
675{
676 TranslationBlock *tb1;
677 for(;;) {
678 tb1 = *ptb;
679 if (tb1 == tb) {
680 *ptb = *(TranslationBlock **)((char *)tb1 + next_offset);
681 break;
682 }
683 ptb = (TranslationBlock **)((char *)tb1 + next_offset);
684 }
685}
686
9fa3e853
FB		 687static inline void tb_page_remove(TranslationBlock **ptb, TranslationBlock *tb)
688{
689 TranslationBlock *tb1;
690 unsigned int n1;
691
692 for(;;) {
693 tb1 = *ptb;
694 n1 = (long)tb1 & 3;
695 tb1 = (TranslationBlock *)((long)tb1 & ~3);
696 if (tb1 == tb) {
697 *ptb = tb1->page_next[n1];
698 break;
699 }
700 ptb = &tb1->page_next[n1];
701 }
702}
703
d4e8164f
FB		 704static inline void tb_jmp_remove(TranslationBlock *tb, int n)
705{
706 TranslationBlock *tb1, **ptb;
707 unsigned int n1;
708
709 ptb = &tb->jmp_next[n];
710 tb1 = *ptb;
711 if (tb1) {
712 /* find tb(n) in circular list */
713 for(;;) {
714 tb1 = *ptb;
715 n1 = (long)tb1 & 3;
716 tb1 = (TranslationBlock *)((long)tb1 & ~3);
717 if (n1 == n && tb1 == tb)
718 break;
719 if (n1 == 2) {
720 ptb = &tb1->jmp_first;
721 } else {
722 ptb = &tb1->jmp_next[n1];
723 }
724 }
725 /* now we can suppress tb(n) from the list */
726 *ptb = tb->jmp_next[n];
727
728 tb->jmp_next[n] = NULL;
729 }
730}
731
732/* reset the jump entry 'n' of a TB so that it is not chained to
733 another TB */
734static inline void tb_reset_jump(TranslationBlock *tb, int n)
735{
736 tb_set_jmp_target(tb, n, (unsigned long)(tb->tc_ptr + tb->tb_next_offset[n]));
737}
738
2e70f6ef 739void tb_phys_invalidate(TranslationBlock *tb, target_ulong page_addr)
fd6ce8f6 740{
6a00d601 741 CPUState *env;
8a40a180 742 PageDesc *p;
d4e8164f 743 unsigned int h, n1;
00f82b8a 744 target_phys_addr_t phys_pc;
8a40a180 745 TranslationBlock *tb1, *tb2;
3b46e624 746
8a40a180
FB		 747 /* remove the TB from the hash list */
748 phys_pc = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
749 h = tb_phys_hash_func(phys_pc);
5fafdf24 750 tb_remove(&tb_phys_hash[h], tb,
8a40a180
FB		 751 offsetof(TranslationBlock, phys_hash_next));
752
753 /* remove the TB from the page list */
754 if (tb->page_addr[0] != page_addr) {
755 p = page_find(tb->page_addr[0] >> TARGET_PAGE_BITS);
756 tb_page_remove(&p->first_tb, tb);
757 invalidate_page_bitmap(p);
758 }
759 if (tb->page_addr[1] != -1 && tb->page_addr[1] != page_addr) {
760 p = page_find(tb->page_addr[1] >> TARGET_PAGE_BITS);
761 tb_page_remove(&p->first_tb, tb);
762 invalidate_page_bitmap(p);
763 }
764
36bdbe54 765 tb_invalidated_flag = 1;
59817ccb 766
fd6ce8f6 767 /* remove the TB from the hash list */
8a40a180 768 h = tb_jmp_cache_hash_func(tb->pc);
6a00d601
FB		 769 for(env = first_cpu; env != NULL; env = env->next_cpu) {
770 if (env->tb_jmp_cache[h] == tb)
771 env->tb_jmp_cache[h] = NULL;
772 }
d4e8164f
FB		 773
774 /* suppress this TB from the two jump lists */
775 tb_jmp_remove(tb, 0);
776 tb_jmp_remove(tb, 1);
777
778 /* suppress any remaining jumps to this TB */
779 tb1 = tb->jmp_first;
780 for(;;) {
781 n1 = (long)tb1 & 3;
782 if (n1 == 2)
783 break;
784 tb1 = (TranslationBlock *)((long)tb1 & ~3);
785 tb2 = tb1->jmp_next[n1];
786 tb_reset_jump(tb1, n1);
787 tb1->jmp_next[n1] = NULL;
788 tb1 = tb2;
789 }
790 tb->jmp_first = (TranslationBlock *)((long)tb | 2); /* fail safe */
9fa3e853 791
e3db7226 792 tb_phys_invalidate_count++;
9fa3e853
FB		 793}
794
795static inline void set_bits(uint8_t *tab, int start, int len)
796{
797 int end, mask, end1;
798
799 end = start + len;
800 tab += start >> 3;
801 mask = 0xff << (start & 7);
802 if ((start & ~7) == (end & ~7)) {
803 if (start < end) {
804 mask &= ~(0xff << (end & 7));
805 *tab |= mask;
806 }
807 } else {
808 *tab++ |= mask;
809 start = (start + 8) & ~7;
810 end1 = end & ~7;
811 while (start < end1) {
812 *tab++ = 0xff;
813 start += 8;
814 }
815 if (start < end) {
816 mask = ~(0xff << (end & 7));
817 *tab |= mask;
818 }
819 }
820}
821
822static void build_page_bitmap(PageDesc *p)
823{
824 int n, tb_start, tb_end;
825 TranslationBlock *tb;
3b46e624 826
b2a7081a 827 p->code_bitmap = qemu_mallocz(TARGET_PAGE_SIZE / 8);
9fa3e853
FB		 828 if (!p->code_bitmap)
829 return;
9fa3e853
FB		 830
831 tb = p->first_tb;
832 while (tb != NULL) {
833 n = (long)tb & 3;
834 tb = (TranslationBlock *)((long)tb & ~3);
835 /* NOTE: this is subtle as a TB may span two physical pages */
836 if (n == 0) {
837 /* NOTE: tb_end may be after the end of the page, but
838 it is not a problem */
839 tb_start = tb->pc & ~TARGET_PAGE_MASK;
840 tb_end = tb_start + tb->size;
841 if (tb_end > TARGET_PAGE_SIZE)
842 tb_end = TARGET_PAGE_SIZE;
843 } else {
844 tb_start = 0;
845 tb_end = ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
846 }
847 set_bits(p->code_bitmap, tb_start, tb_end - tb_start);
848 tb = tb->page_next[n];
849 }
850}
851
2e70f6ef
PB		 852TranslationBlock *tb_gen_code(CPUState *env,
853 target_ulong pc, target_ulong cs_base,
854 int flags, int cflags)
d720b93d
FB		 855{
856 TranslationBlock *tb;
857 uint8_t *tc_ptr;
858 target_ulong phys_pc, phys_page2, virt_page2;
859 int code_gen_size;
860
c27004ec
FB		 861 phys_pc = get_phys_addr_code(env, pc);
862 tb = tb_alloc(pc);
d720b93d
FB		 863 if (!tb) {
864 /* flush must be done */
865 tb_flush(env);
866 /* cannot fail at this point */
c27004ec 867 tb = tb_alloc(pc);
2e70f6ef
PB		 868 /* Don't forget to invalidate previous TB info. */
869 tb_invalidated_flag = 1;
d720b93d
FB		 870 }
871 tc_ptr = code_gen_ptr;
872 tb->tc_ptr = tc_ptr;
873 tb->cs_base = cs_base;
874 tb->flags = flags;
875 tb->cflags = cflags;
d07bde88 876 cpu_gen_code(env, tb, &code_gen_size);
d720b93d 877 code_gen_ptr = (void *)(((unsigned long)code_gen_ptr + code_gen_size + CODE_GEN_ALIGN - 1) & ~(CODE_GEN_ALIGN - 1));
3b46e624 878
d720b93d 879 /* check next page if needed */
c27004ec 880 virt_page2 = (pc + tb->size - 1) & TARGET_PAGE_MASK;
d720b93d 881 phys_page2 = -1;
c27004ec 882 if ((pc & TARGET_PAGE_MASK) != virt_page2) {
d720b93d
FB		 883 phys_page2 = get_phys_addr_code(env, virt_page2);
884 }
885 tb_link_phys(tb, phys_pc, phys_page2);
2e70f6ef 886 return tb;
d720b93d 887}
3b46e624 888
9fa3e853
FB		 889/* invalidate all TBs which intersect with the target physical page
890 starting in range [start;end[. NOTE: start and end must refer to
d720b93d
FB		 891 the same physical page. 'is_cpu_write_access' should be true if called
892 from a real cpu write access: the virtual CPU will exit the current
893 TB if code is modified inside this TB. */
00f82b8a 894void tb_invalidate_phys_page_range(target_phys_addr_t start, target_phys_addr_t end,
d720b93d
FB		 895 int is_cpu_write_access)
896{
6b917547 897 TranslationBlock *tb, *tb_next, *saved_tb;
d720b93d 898 CPUState *env = cpu_single_env;
9fa3e853 899 target_ulong tb_start, tb_end;
6b917547
AL		 900 PageDesc *p;
901 int n;
902#ifdef TARGET_HAS_PRECISE_SMC
903 int current_tb_not_found = is_cpu_write_access;
904 TranslationBlock *current_tb = NULL;
905 int current_tb_modified = 0;
906 target_ulong current_pc = 0;
907 target_ulong current_cs_base = 0;
908 int current_flags = 0;
909#endif /* TARGET_HAS_PRECISE_SMC */
9fa3e853
FB		 910
911 p = page_find(start >> TARGET_PAGE_BITS);
5fafdf24 912 if (!p)
9fa3e853 913 return;
5fafdf24 914 if (!p->code_bitmap &&
d720b93d
FB		 915 ++p->code_write_count >= SMC_BITMAP_USE_THRESHOLD &&
916 is_cpu_write_access) {
9fa3e853
FB		 917 /* build code bitmap */
918 build_page_bitmap(p);
919 }
920
921 /* we remove all the TBs in the range [start, end[ */
922 /* XXX: see if in some cases it could be faster to invalidate all the code */
923 tb = p->first_tb;
924 while (tb != NULL) {
925 n = (long)tb & 3;
926 tb = (TranslationBlock *)((long)tb & ~3);
927 tb_next = tb->page_next[n];
928 /* NOTE: this is subtle as a TB may span two physical pages */
929 if (n == 0) {
930 /* NOTE: tb_end may be after the end of the page, but
931 it is not a problem */
932 tb_start = tb->page_addr[0] + (tb->pc & ~TARGET_PAGE_MASK);
933 tb_end = tb_start + tb->size;
934 } else {
935 tb_start = tb->page_addr[1];
936 tb_end = tb_start + ((tb->pc + tb->size) & ~TARGET_PAGE_MASK);
937 }
938 if (!(tb_end <= start || tb_start >= end)) {
d720b93d
FB		 939#ifdef TARGET_HAS_PRECISE_SMC
940 if (current_tb_not_found) {
941 current_tb_not_found = 0;
942 current_tb = NULL;
2e70f6ef 943 if (env->mem_io_pc) {
d720b93d 944 /* now we have a real cpu fault */
2e70f6ef 945 current_tb = tb_find_pc(env->mem_io_pc);
d720b93d
FB		 946 }
947 }
948 if (current_tb == tb &&
2e70f6ef 949 (current_tb->cflags & CF_COUNT_MASK) != 1) {
d720b93d
FB		 950 /* If we are modifying the current TB, we must stop
951 its execution. We could be more precise by checking
952 that the modification is after the current PC, but it
953 would require a specialized function to partially
954 restore the CPU state */
3b46e624 955
d720b93d 956 current_tb_modified = 1;
5fafdf24 957 cpu_restore_state(current_tb, env,
2e70f6ef 958 env->mem_io_pc, NULL);
6b917547
AL		 959 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
960 &current_flags);
d720b93d
FB		 961 }
962#endif /* TARGET_HAS_PRECISE_SMC */
6f5a9f7e
FB		 963 /* we need to do that to handle the case where a signal
964 occurs while doing tb_phys_invalidate() */
965 saved_tb = NULL;
966 if (env) {
967 saved_tb = env->current_tb;
968 env->current_tb = NULL;
969 }
9fa3e853 970 tb_phys_invalidate(tb, -1);
6f5a9f7e
FB		 971 if (env) {
972 env->current_tb = saved_tb;
973 if (env->interrupt_request && env->current_tb)
974 cpu_interrupt(env, env->interrupt_request);
975 }
9fa3e853
FB		 976 }
977 tb = tb_next;
978 }
979#if !defined(CONFIG_USER_ONLY)
980 /* if no code remaining, no need to continue to use slow writes */
981 if (!p->first_tb) {
982 invalidate_page_bitmap(p);
d720b93d 983 if (is_cpu_write_access) {
2e70f6ef 984 tlb_unprotect_code_phys(env, start, env->mem_io_vaddr);
d720b93d
FB		 985 }
986 }
987#endif
988#ifdef TARGET_HAS_PRECISE_SMC
989 if (current_tb_modified) {
990 /* we generate a block containing just the instruction
991 modifying the memory. It will ensure that it cannot modify
992 itself */
ea1c1802 993 env->current_tb = NULL;
2e70f6ef 994 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
d720b93d 995 cpu_resume_from_signal(env, NULL);
9fa3e853 996 }
fd6ce8f6 997#endif
9fa3e853 998}
fd6ce8f6 999
9fa3e853 1000/* len must be <= 8 and start must be a multiple of len */
00f82b8a 1001static inline void tb_invalidate_phys_page_fast(target_phys_addr_t start, int len)
9fa3e853
FB		 1002{
1003 PageDesc *p;
1004 int offset, b;
59817ccb 1005#if 0
a4193c8a 1006 if (1) {
93fcfe39
AL		 1007 qemu_log("modifying code at 0x%x size=%d EIP=%x PC=%08x\n",
1008 cpu_single_env->mem_io_vaddr, len,
1009 cpu_single_env->eip,
1010 cpu_single_env->eip + (long)cpu_single_env->segs[R_CS].base);
59817ccb
FB		 1011 }
1012#endif
9fa3e853 1013 p = page_find(start >> TARGET_PAGE_BITS);
5fafdf24 1014 if (!p)
9fa3e853
FB		 1015 return;
1016 if (p->code_bitmap) {
1017 offset = start & ~TARGET_PAGE_MASK;
1018 b = p->code_bitmap[offset >> 3] >> (offset & 7);
1019 if (b & ((1 << len) - 1))
1020 goto do_invalidate;
1021 } else {
1022 do_invalidate:
d720b93d 1023 tb_invalidate_phys_page_range(start, start + len, 1);
9fa3e853
FB		 1024 }
1025}
1026
9fa3e853 1027#if !defined(CONFIG_SOFTMMU)
00f82b8a 1028static void tb_invalidate_phys_page(target_phys_addr_t addr,
d720b93d 1029 unsigned long pc, void *puc)
9fa3e853 1030{
6b917547 1031 TranslationBlock *tb;
9fa3e853 1032 PageDesc *p;
6b917547 1033 int n;
d720b93d 1034#ifdef TARGET_HAS_PRECISE_SMC
6b917547 1035 TranslationBlock *current_tb = NULL;
d720b93d 1036 CPUState *env = cpu_single_env;
6b917547
AL		 1037 int current_tb_modified = 0;
1038 target_ulong current_pc = 0;
1039 target_ulong current_cs_base = 0;
1040 int current_flags = 0;
d720b93d 1041#endif
9fa3e853
FB		 1042
1043 addr &= TARGET_PAGE_MASK;
1044 p = page_find(addr >> TARGET_PAGE_BITS);
5fafdf24 1045 if (!p)
9fa3e853
FB		 1046 return;
1047 tb = p->first_tb;
d720b93d
FB		 1048#ifdef TARGET_HAS_PRECISE_SMC
1049 if (tb && pc != 0) {
1050 current_tb = tb_find_pc(pc);
1051 }
1052#endif
9fa3e853
FB		 1053 while (tb != NULL) {
1054 n = (long)tb & 3;
1055 tb = (TranslationBlock *)((long)tb & ~3);
d720b93d
FB		 1056#ifdef TARGET_HAS_PRECISE_SMC
1057 if (current_tb == tb &&
2e70f6ef 1058 (current_tb->cflags & CF_COUNT_MASK) != 1) {
d720b93d
FB		 1059 /* If we are modifying the current TB, we must stop
1060 its execution. We could be more precise by checking
1061 that the modification is after the current PC, but it
1062 would require a specialized function to partially
1063 restore the CPU state */
3b46e624 1064
d720b93d
FB		 1065 current_tb_modified = 1;
1066 cpu_restore_state(current_tb, env, pc, puc);
6b917547
AL		 1067 cpu_get_tb_cpu_state(env, &current_pc, &current_cs_base,
1068 &current_flags);
d720b93d
FB		 1069 }
1070#endif /* TARGET_HAS_PRECISE_SMC */
9fa3e853
FB		 1071 tb_phys_invalidate(tb, addr);
1072 tb = tb->page_next[n];
1073 }
fd6ce8f6 1074 p->first_tb = NULL;
d720b93d
FB		 1075#ifdef TARGET_HAS_PRECISE_SMC
1076 if (current_tb_modified) {
1077 /* we generate a block containing just the instruction
1078 modifying the memory. It will ensure that it cannot modify
1079 itself */
ea1c1802 1080 env->current_tb = NULL;
2e70f6ef 1081 tb_gen_code(env, current_pc, current_cs_base, current_flags, 1);
d720b93d
FB		 1082 cpu_resume_from_signal(env, puc);
1083 }
1084#endif
fd6ce8f6 1085}
9fa3e853 1086#endif
fd6ce8f6
FB		 1087
1088/* add the tb in the target page and protect it if necessary */
5fafdf24 1089static inline void tb_alloc_page(TranslationBlock *tb,
53a5960a 1090 unsigned int n, target_ulong page_addr)
fd6ce8f6
FB		 1091{
1092 PageDesc *p;
9fa3e853
FB		 1093 TranslationBlock *last_first_tb;
1094
1095 tb->page_addr[n] = page_addr;
3a7d929e 1096 p = page_find_alloc(page_addr >> TARGET_PAGE_BITS);
9fa3e853
FB		 1097 tb->page_next[n] = p->first_tb;
1098 last_first_tb = p->first_tb;
1099 p->first_tb = (TranslationBlock *)((long)tb | n);
1100 invalidate_page_bitmap(p);
fd6ce8f6 1101
107db443 1102#if defined(TARGET_HAS_SMC) || 1
d720b93d 1103
9fa3e853 1104#if defined(CONFIG_USER_ONLY)
fd6ce8f6 1105 if (p->flags & PAGE_WRITE) {
53a5960a
PB		 1106 target_ulong addr;
1107 PageDesc *p2;
9fa3e853
FB		 1108 int prot;
1109
fd6ce8f6
FB		 1110 /* force the host page as non writable (writes will have a
1111 page fault + mprotect overhead) */
53a5960a 1112 page_addr &= qemu_host_page_mask;
fd6ce8f6 1113 prot = 0;
53a5960a
PB		 1114 for(addr = page_addr; addr < page_addr + qemu_host_page_size;
1115 addr += TARGET_PAGE_SIZE) {
1116
1117 p2 = page_find (addr >> TARGET_PAGE_BITS);
1118 if (!p2)
1119 continue;
1120 prot |= p2->flags;
1121 p2->flags &= ~PAGE_WRITE;
1122 page_get_flags(addr);
1123 }
5fafdf24 1124 mprotect(g2h(page_addr), qemu_host_page_size,
fd6ce8f6
FB		 1125 (prot & PAGE_BITS) & ~PAGE_WRITE);
1126#ifdef DEBUG_TB_INVALIDATE
ab3d1727 1127 printf("protecting code page: 0x" TARGET_FMT_lx "\n",
53a5960a 1128 page_addr);
fd6ce8f6 1129#endif
fd6ce8f6 1130 }
9fa3e853
FB		 1131#else
1132 /* if some code is already present, then the pages are already
1133 protected. So we handle the case where only the first TB is
1134 allocated in a physical page */
1135 if (!last_first_tb) {
6a00d601 1136 tlb_protect_code(page_addr);
9fa3e853
FB		 1137 }
1138#endif
d720b93d
FB		 1139
1140#endif /* TARGET_HAS_SMC */
fd6ce8f6
FB		 1141}
1142
1143/* Allocate a new translation block. Flush the translation buffer if
1144 too many translation blocks or too much generated code. */
c27004ec 1145TranslationBlock *tb_alloc(target_ulong pc)
fd6ce8f6
FB		 1146{
1147 TranslationBlock *tb;
fd6ce8f6 1148
26a5f13b
FB		 1149 if (nb_tbs >= code_gen_max_blocks ||
1150 (code_gen_ptr - code_gen_buffer) >= code_gen_buffer_max_size)
d4e8164f 1151 return NULL;
fd6ce8f6
FB		 1152 tb = &tbs[nb_tbs++];
1153 tb->pc = pc;
b448f2f3 1154 tb->cflags = 0;
d4e8164f
FB		 1155 return tb;
1156}
1157
2e70f6ef
PB		 1158void tb_free(TranslationBlock *tb)
1159{
bf20dc07 1160 /* In practice this is mostly used for single use temporary TB
2e70f6ef
PB		 1161 Ignore the hard cases and just back up if this TB happens to
1162 be the last one generated. */
1163 if (nb_tbs > 0 && tb == &tbs[nb_tbs - 1]) {
1164 code_gen_ptr = tb->tc_ptr;
1165 nb_tbs--;
1166 }
1167}
1168
9fa3e853
FB		 1169/* add a new TB and link it to the physical page tables. phys_page2 is
1170 (-1) to indicate that only one page contains the TB. */
5fafdf24 1171void tb_link_phys(TranslationBlock *tb,
9fa3e853 1172 target_ulong phys_pc, target_ulong phys_page2)
d4e8164f 1173{
9fa3e853
FB		 1174 unsigned int h;
1175 TranslationBlock **ptb;
1176
c8a706fe
PB		 1177 /* Grab the mmap lock to stop another thread invalidating this TB
1178 before we are done. */
1179 mmap_lock();
9fa3e853
FB		 1180 /* add in the physical hash table */
1181 h = tb_phys_hash_func(phys_pc);
1182 ptb = &tb_phys_hash[h];
1183 tb->phys_hash_next = *ptb;
1184 *ptb = tb;
fd6ce8f6
FB		 1185
1186 /* add in the page list */
9fa3e853
FB		 1187 tb_alloc_page(tb, 0, phys_pc & TARGET_PAGE_MASK);
1188 if (phys_page2 != -1)
1189 tb_alloc_page(tb, 1, phys_page2);
1190 else
1191 tb->page_addr[1] = -1;
9fa3e853 1192
d4e8164f
FB		 1193 tb->jmp_first = (TranslationBlock *)((long)tb | 2);
1194 tb->jmp_next[0] = NULL;
1195 tb->jmp_next[1] = NULL;
1196
1197 /* init original jump addresses */
1198 if (tb->tb_next_offset[0] != 0xffff)
1199 tb_reset_jump(tb, 0);
1200 if (tb->tb_next_offset[1] != 0xffff)
1201 tb_reset_jump(tb, 1);
8a40a180
FB		 1202
1203#ifdef DEBUG_TB_CHECK
1204 tb_page_check();
1205#endif
c8a706fe 1206 mmap_unlock();
fd6ce8f6
FB		 1207}
1208
9fa3e853
FB		 1209/* find the TB 'tb' such that tb[0].tc_ptr <= tc_ptr <
1210 tb[1].tc_ptr. Return NULL if not found */
1211TranslationBlock *tb_find_pc(unsigned long tc_ptr)
fd6ce8f6 1212{
9fa3e853
FB		 1213 int m_min, m_max, m;
1214 unsigned long v;
1215 TranslationBlock *tb;
a513fe19
FB		 1216
1217 if (nb_tbs <= 0)
1218 return NULL;
1219 if (tc_ptr < (unsigned long)code_gen_buffer ||
1220 tc_ptr >= (unsigned long)code_gen_ptr)
1221 return NULL;
1222 /* binary search (cf Knuth) */
1223 m_min = 0;
1224 m_max = nb_tbs - 1;
1225 while (m_min <= m_max) {
1226 m = (m_min + m_max) >> 1;
1227 tb = &tbs[m];
1228 v = (unsigned long)tb->tc_ptr;
1229 if (v == tc_ptr)
1230 return tb;
1231 else if (tc_ptr < v) {
1232 m_max = m - 1;
1233 } else {
1234 m_min = m + 1;
1235 }
5fafdf24 1236 }
a513fe19
FB		 1237 return &tbs[m_max];
1238}
7501267e 1239
ea041c0e
FB		 1240static void tb_reset_jump_recursive(TranslationBlock *tb);
1241
1242static inline void tb_reset_jump_recursive2(TranslationBlock *tb, int n)
1243{
1244 TranslationBlock *tb1, *tb_next, **ptb;
1245 unsigned int n1;
1246
1247 tb1 = tb->jmp_next[n];
1248 if (tb1 != NULL) {
1249 /* find head of list */
1250 for(;;) {
1251 n1 = (long)tb1 & 3;
1252 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1253 if (n1 == 2)
1254 break;
1255 tb1 = tb1->jmp_next[n1];
1256 }
1257 /* we are now sure now that tb jumps to tb1 */
1258 tb_next = tb1;
1259
1260 /* remove tb from the jmp_first list */
1261 ptb = &tb_next->jmp_first;
1262 for(;;) {
1263 tb1 = *ptb;
1264 n1 = (long)tb1 & 3;
1265 tb1 = (TranslationBlock *)((long)tb1 & ~3);
1266 if (n1 == n && tb1 == tb)
1267 break;
1268 ptb = &tb1->jmp_next[n1];
1269 }
1270 *ptb = tb->jmp_next[n];
1271 tb->jmp_next[n] = NULL;
3b46e624 1272
ea041c0e
FB		 1273 /* suppress the jump to next tb in generated code */
1274 tb_reset_jump(tb, n);
1275
0124311e 1276 /* suppress jumps in the tb on which we could have jumped */
ea041c0e
FB		 1277 tb_reset_jump_recursive(tb_next);
1278 }
1279}
1280
1281static void tb_reset_jump_recursive(TranslationBlock *tb)
1282{
1283 tb_reset_jump_recursive2(tb, 0);
1284 tb_reset_jump_recursive2(tb, 1);
1285}
1286
1fddef4b 1287#if defined(TARGET_HAS_ICE)
d720b93d
FB		 1288static void breakpoint_invalidate(CPUState *env, target_ulong pc)
1289{
9b3c35e0
JM		 1290 target_phys_addr_t addr;
1291 target_ulong pd;
c2f07f81
PB		 1292 ram_addr_t ram_addr;
1293 PhysPageDesc *p;
d720b93d 1294
c2f07f81
PB		 1295 addr = cpu_get_phys_page_debug(env, pc);
1296 p = phys_page_find(addr >> TARGET_PAGE_BITS);
1297 if (!p) {
1298 pd = IO_MEM_UNASSIGNED;
1299 } else {
1300 pd = p->phys_offset;
1301 }
1302 ram_addr = (pd & TARGET_PAGE_MASK) | (pc & ~TARGET_PAGE_MASK);
706cd4b5 1303 tb_invalidate_phys_page_range(ram_addr, ram_addr + 1, 0);
d720b93d 1304}
c27004ec 1305#endif
d720b93d 1306
6658ffb8 1307/* Add a watchpoint. */
a1d1bb31
AL		 1308int cpu_watchpoint_insert(CPUState *env, target_ulong addr, target_ulong len,
1309 int flags, CPUWatchpoint **watchpoint)
6658ffb8 1310{
b4051334 1311 target_ulong len_mask = ~(len - 1);
c0ce998e 1312 CPUWatchpoint *wp;
6658ffb8 1313
b4051334
AL		 1314 /* sanity checks: allow power-of-2 lengths, deny unaligned watchpoints */
1315 if ((len != 1 && len != 2 && len != 4 && len != 8) || (addr & ~len_mask)) {
1316 fprintf(stderr, "qemu: tried to set invalid watchpoint at "
1317 TARGET_FMT_lx ", len=" TARGET_FMT_lu "\n", addr, len);
1318 return -EINVAL;
1319 }
a1d1bb31
AL		 1320 wp = qemu_malloc(sizeof(*wp));
1321 if (!wp)
426cd5d6 1322 return -ENOMEM;
a1d1bb31
AL		 1323
1324 wp->vaddr = addr;
b4051334 1325 wp->len_mask = len_mask;
a1d1bb31
AL		 1326 wp->flags = flags;
1327
2dc9f411 1328 /* keep all GDB-injected watchpoints in front */
c0ce998e
AL		 1329 if (flags & BP_GDB)
1330 TAILQ_INSERT_HEAD(&env->watchpoints, wp, entry);
1331 else
1332 TAILQ_INSERT_TAIL(&env->watchpoints, wp, entry);
6658ffb8 1333
6658ffb8 1334 tlb_flush_page(env, addr);
a1d1bb31
AL		 1335
1336 if (watchpoint)
1337 *watchpoint = wp;
1338 return 0;
6658ffb8
PB		 1339}
1340
a1d1bb31
AL		 1341/* Remove a specific watchpoint. */
1342int cpu_watchpoint_remove(CPUState *env, target_ulong addr, target_ulong len,
1343 int flags)
6658ffb8 1344{
b4051334 1345 target_ulong len_mask = ~(len - 1);
a1d1bb31 1346 CPUWatchpoint *wp;
6658ffb8 1347
c0ce998e 1348 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
b4051334 1349 if (addr == wp->vaddr && len_mask == wp->len_mask
6e140f28 1350 && flags == (wp->flags & ~BP_WATCHPOINT_HIT)) {
a1d1bb31 1351 cpu_watchpoint_remove_by_ref(env, wp);
6658ffb8
PB		 1352 return 0;
1353 }
1354 }
a1d1bb31 1355 return -ENOENT;
6658ffb8
PB		 1356}
1357
a1d1bb31
AL		 1358/* Remove a specific watchpoint by reference. */
1359void cpu_watchpoint_remove_by_ref(CPUState *env, CPUWatchpoint *watchpoint)
1360{
c0ce998e 1361 TAILQ_REMOVE(&env->watchpoints, watchpoint, entry);
7d03f82f 1362
a1d1bb31
AL		 1363 tlb_flush_page(env, watchpoint->vaddr);
1364
1365 qemu_free(watchpoint);
1366}
1367
1368/* Remove all matching watchpoints. */
1369void cpu_watchpoint_remove_all(CPUState *env, int mask)
1370{
c0ce998e 1371 CPUWatchpoint *wp, *next;
a1d1bb31 1372
c0ce998e 1373 TAILQ_FOREACH_SAFE(wp, &env->watchpoints, entry, next) {
a1d1bb31
AL		 1374 if (wp->flags & mask)
1375 cpu_watchpoint_remove_by_ref(env, wp);
c0ce998e 1376 }
7d03f82f
EI		 1377}
1378
a1d1bb31
AL		 1379/* Add a breakpoint. */
1380int cpu_breakpoint_insert(CPUState *env, target_ulong pc, int flags,
1381 CPUBreakpoint **breakpoint)
4c3a88a2 1382{
1fddef4b 1383#if defined(TARGET_HAS_ICE)
c0ce998e 1384 CPUBreakpoint *bp;
3b46e624 1385
a1d1bb31
AL		 1386 bp = qemu_malloc(sizeof(*bp));
1387 if (!bp)
426cd5d6 1388 return -ENOMEM;
4c3a88a2 1389
a1d1bb31
AL		 1390 bp->pc = pc;
1391 bp->flags = flags;
1392
2dc9f411 1393 /* keep all GDB-injected breakpoints in front */
c0ce998e
AL		 1394 if (flags & BP_GDB)
1395 TAILQ_INSERT_HEAD(&env->breakpoints, bp, entry);
1396 else
1397 TAILQ_INSERT_TAIL(&env->breakpoints, bp, entry);
3b46e624 1398
d720b93d 1399 breakpoint_invalidate(env, pc);
a1d1bb31
AL		 1400
1401 if (breakpoint)
1402 *breakpoint = bp;
4c3a88a2
FB		 1403 return 0;
1404#else
a1d1bb31 1405 return -ENOSYS;
4c3a88a2
FB		 1406#endif
1407}
1408
a1d1bb31
AL		 1409/* Remove a specific breakpoint. */
1410int cpu_breakpoint_remove(CPUState *env, target_ulong pc, int flags)
1411{
7d03f82f 1412#if defined(TARGET_HAS_ICE)
a1d1bb31
AL		 1413 CPUBreakpoint *bp;
1414
c0ce998e 1415 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
a1d1bb31
AL		 1416 if (bp->pc == pc && bp->flags == flags) {
1417 cpu_breakpoint_remove_by_ref(env, bp);
1418 return 0;
1419 }
7d03f82f 1420 }
a1d1bb31
AL		 1421 return -ENOENT;
1422#else
1423 return -ENOSYS;
7d03f82f
EI		 1424#endif
1425}
1426
a1d1bb31
AL		 1427/* Remove a specific breakpoint by reference. */
1428void cpu_breakpoint_remove_by_ref(CPUState *env, CPUBreakpoint *breakpoint)
4c3a88a2 1429{
1fddef4b 1430#if defined(TARGET_HAS_ICE)
c0ce998e 1431 TAILQ_REMOVE(&env->breakpoints, breakpoint, entry);
d720b93d 1432
a1d1bb31
AL		 1433 breakpoint_invalidate(env, breakpoint->pc);
1434
1435 qemu_free(breakpoint);
1436#endif
1437}
1438
1439/* Remove all matching breakpoints. */
1440void cpu_breakpoint_remove_all(CPUState *env, int mask)
1441{
1442#if defined(TARGET_HAS_ICE)
c0ce998e 1443 CPUBreakpoint *bp, *next;
a1d1bb31 1444
c0ce998e 1445 TAILQ_FOREACH_SAFE(bp, &env->breakpoints, entry, next) {
a1d1bb31
AL		 1446 if (bp->flags & mask)
1447 cpu_breakpoint_remove_by_ref(env, bp);
c0ce998e 1448 }
4c3a88a2
FB		 1449#endif
1450}
1451
c33a346e
FB		 1452/* enable or disable single step mode. EXCP_DEBUG is returned by the
1453 CPU loop after each instruction */
1454void cpu_single_step(CPUState *env, int enabled)
1455{
1fddef4b 1456#if defined(TARGET_HAS_ICE)
c33a346e
FB		 1457 if (env->singlestep_enabled != enabled) {
1458 env->singlestep_enabled = enabled;
1459 /* must flush all the translated code to avoid inconsistancies */
9fa3e853 1460 /* XXX: only flush what is necessary */
0124311e 1461 tb_flush(env);
c33a346e
FB		 1462 }
1463#endif
1464}
1465
34865134
FB		 1466/* enable or disable low levels log */
1467void cpu_set_log(int log_flags)
1468{
1469 loglevel = log_flags;
1470 if (loglevel && !logfile) {
11fcfab4 1471 logfile = fopen(logfilename, log_append ? "a" : "w");
34865134
FB		 1472 if (!logfile) {
1473 perror(logfilename);
1474 _exit(1);
1475 }
9fa3e853
FB		 1476#if !defined(CONFIG_SOFTMMU)
1477 /* must avoid mmap() usage of glibc by setting a buffer "by hand" */
1478 {
b55266b5 1479 static char logfile_buf[4096];
9fa3e853
FB		 1480 setvbuf(logfile, logfile_buf, _IOLBF, sizeof(logfile_buf));
1481 }
1482#else
34865134 1483 setvbuf(logfile, NULL, _IOLBF, 0);
9fa3e853 1484#endif
e735b91c
PB		 1485 log_append = 1;
1486 }
1487 if (!loglevel && logfile) {
1488 fclose(logfile);
1489 logfile = NULL;
34865134
FB		 1490 }
1491}
1492
1493void cpu_set_log_filename(const char *filename)
1494{
1495 logfilename = strdup(filename);
e735b91c
PB		 1496 if (logfile) {
1497 fclose(logfile);
1498 logfile = NULL;
1499 }
1500 cpu_set_log(loglevel);
34865134 1501}
c33a346e 1502
0124311e 1503/* mask must never be zero, except for A20 change call */
68a79315 1504void cpu_interrupt(CPUState *env, int mask)
ea041c0e 1505{
d5975363 1506#if !defined(USE_NPTL)
ea041c0e 1507 TranslationBlock *tb;
15a51156 1508 static spinlock_t interrupt_lock = SPIN_LOCK_UNLOCKED;
d5975363 1509#endif
2e70f6ef 1510 int old_mask;
59817ccb 1511
2e70f6ef 1512 old_mask = env->interrupt_request;
d5975363 1513 /* FIXME: This is probably not threadsafe. A different thread could
bf20dc07 1514 be in the middle of a read-modify-write operation. */
68a79315 1515 env->interrupt_request |= mask;
d5975363
PB		 1516#if defined(USE_NPTL)
1517 /* FIXME: TB unchaining isn't SMP safe. For now just ignore the
1518 problem and hope the cpu will stop of its own accord. For userspace
1519 emulation this often isn't actually as bad as it sounds. Often
1520 signals are used primarily to interrupt blocking syscalls. */
1521#else
2e70f6ef 1522 if (use_icount) {
266910c4 1523 env->icount_decr.u16.high = 0xffff;
2e70f6ef
PB		 1524#ifndef CONFIG_USER_ONLY
1525 /* CPU_INTERRUPT_EXIT isn't a real interrupt. It just means
1526 an async event happened and we need to process it. */
1527 if (!can_do_io(env)
1528 && (mask & ~(old_mask | CPU_INTERRUPT_EXIT)) != 0) {
1529 cpu_abort(env, "Raised interrupt while not in I/O function");
1530 }
1531#endif
1532 } else {
1533 tb = env->current_tb;
1534 /* if the cpu is currently executing code, we must unlink it and
1535 all the potentially executing TB */
1536 if (tb && !testandset(&interrupt_lock)) {
1537 env->current_tb = NULL;
1538 tb_reset_jump_recursive(tb);
1539 resetlock(&interrupt_lock);
1540 }
ea041c0e 1541 }
d5975363 1542#endif
ea041c0e
FB		 1543}
1544
b54ad049
FB		 1545void cpu_reset_interrupt(CPUState *env, int mask)
1546{
1547 env->interrupt_request &= ~mask;
1548}
1549
c7cd6a37 1550const CPULogItem cpu_log_items[] = {
5fafdf24 1551 { CPU_LOG_TB_OUT_ASM, "out_asm",
f193c797
FB		 1552 "show generated host assembly code for each compiled TB" },
1553 { CPU_LOG_TB_IN_ASM, "in_asm",
1554 "show target assembly code for each compiled TB" },
5fafdf24 1555 { CPU_LOG_TB_OP, "op",
57fec1fe 1556 "show micro ops for each compiled TB" },
f193c797 1557 { CPU_LOG_TB_OP_OPT, "op_opt",
e01a1157
BS		 1558 "show micro ops "
1559#ifdef TARGET_I386
1560 "before eflags optimization and "
f193c797 1561#endif
e01a1157 1562 "after liveness analysis" },
f193c797
FB		 1563 { CPU_LOG_INT, "int",
1564 "show interrupts/exceptions in short format" },
1565 { CPU_LOG_EXEC, "exec",
1566 "show trace before each executed TB (lots of logs)" },
9fddaa0c 1567 { CPU_LOG_TB_CPU, "cpu",
e91c8a77 1568 "show CPU state before block translation" },
f193c797
FB		 1569#ifdef TARGET_I386
1570 { CPU_LOG_PCALL, "pcall",
1571 "show protected mode far calls/returns/exceptions" },
eca1bdf4
AL		 1572 { CPU_LOG_RESET, "cpu_reset",
1573 "show CPU state before CPU resets" },
f193c797 1574#endif
8e3a9fd2 1575#ifdef DEBUG_IOPORT
fd872598
FB		 1576 { CPU_LOG_IOPORT, "ioport",
1577 "show all i/o ports accesses" },
8e3a9fd2 1578#endif
f193c797
FB		 1579 { 0, NULL, NULL },
1580};
1581
1582static int cmp1(const char *s1, int n, const char *s2)
1583{
1584 if (strlen(s2) != n)
1585 return 0;
1586 return memcmp(s1, s2, n) == 0;
1587}
3b46e624 1588
f193c797
FB		 1589/* takes a comma separated list of log masks. Return 0 if error. */
1590int cpu_str_to_log_mask(const char *str)
1591{
c7cd6a37 1592 const CPULogItem *item;
f193c797
FB		 1593 int mask;
1594 const char *p, *p1;
1595
1596 p = str;
1597 mask = 0;
1598 for(;;) {
1599 p1 = strchr(p, ',');
1600 if (!p1)
1601 p1 = p + strlen(p);
8e3a9fd2
FB		 1602 if(cmp1(p,p1-p,"all")) {
1603 for(item = cpu_log_items; item->mask != 0; item++) {
1604 mask |= item->mask;
1605 }
1606 } else {
f193c797
FB		 1607 for(item = cpu_log_items; item->mask != 0; item++) {
1608 if (cmp1(p, p1 - p, item->name))
1609 goto found;
1610 }
1611 return 0;
8e3a9fd2 1612 }
f193c797
FB		 1613 found:
1614 mask |= item->mask;
1615 if (*p1 != ',')
1616 break;
1617 p = p1 + 1;
1618 }
1619 return mask;
1620}
ea041c0e 1621
7501267e
FB		 1622void cpu_abort(CPUState *env, const char *fmt, ...)
1623{
1624 va_list ap;
493ae1f0 1625 va_list ap2;
7501267e
FB		 1626
1627 va_start(ap, fmt);
493ae1f0 1628 va_copy(ap2, ap);
7501267e
FB		 1629 fprintf(stderr, "qemu: fatal: ");
1630 vfprintf(stderr, fmt, ap);
1631 fprintf(stderr, "\n");
1632#ifdef TARGET_I386
7fe48483
FB		 1633 cpu_dump_state(env, stderr, fprintf, X86_DUMP_FPU | X86_DUMP_CCOP);
1634#else
1635 cpu_dump_state(env, stderr, fprintf, 0);
7501267e 1636#endif
93fcfe39
AL		 1637 if (qemu_log_enabled()) {
1638 qemu_log("qemu: fatal: ");
1639 qemu_log_vprintf(fmt, ap2);
1640 qemu_log("\n");
f9373291 1641#ifdef TARGET_I386
93fcfe39 1642 log_cpu_state(env, X86_DUMP_FPU | X86_DUMP_CCOP);
f9373291 1643#else
93fcfe39 1644 log_cpu_state(env, 0);
f9373291 1645#endif
31b1a7b4 1646 qemu_log_flush();
93fcfe39 1647 qemu_log_close();
924edcae 1648 }
493ae1f0 1649 va_end(ap2);
f9373291 1650 va_end(ap);
7501267e
FB		 1651 abort();
1652}
1653
c5be9f08
TS		 1654CPUState *cpu_copy(CPUState *env)
1655{
01ba9816 1656 CPUState *new_env = cpu_init(env->cpu_model_str);
c5be9f08
TS		 1657 CPUState *next_cpu = new_env->next_cpu;
1658 int cpu_index = new_env->cpu_index;
5a38f081
AL		 1659#if defined(TARGET_HAS_ICE)
1660 CPUBreakpoint *bp;
1661 CPUWatchpoint *wp;
1662#endif
1663
c5be9f08 1664 memcpy(new_env, env, sizeof(CPUState));
5a38f081
AL		 1665
1666 /* Preserve chaining and index. */
c5be9f08
TS		 1667 new_env->next_cpu = next_cpu;
1668 new_env->cpu_index = cpu_index;
5a38f081
AL		 1669
1670 /* Clone all break/watchpoints.
1671 Note: Once we support ptrace with hw-debug register access, make sure
1672 BP_CPU break/watchpoints are handled correctly on clone. */
1673 TAILQ_INIT(&env->breakpoints);
1674 TAILQ_INIT(&env->watchpoints);
1675#if defined(TARGET_HAS_ICE)
1676 TAILQ_FOREACH(bp, &env->breakpoints, entry) {
1677 cpu_breakpoint_insert(new_env, bp->pc, bp->flags, NULL);
1678 }
1679 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
1680 cpu_watchpoint_insert(new_env, wp->vaddr, (~wp->len_mask) + 1,
1681 wp->flags, NULL);
1682 }
1683#endif
1684
c5be9f08
TS		 1685 return new_env;
1686}
1687
0124311e
FB		 1688#if !defined(CONFIG_USER_ONLY)
1689
5c751e99
EI		 1690static inline void tlb_flush_jmp_cache(CPUState *env, target_ulong addr)
1691{
1692 unsigned int i;
1693
1694 /* Discard jump cache entries for any tb which might potentially
1695 overlap the flushed page. */
1696 i = tb_jmp_cache_hash_page(addr - TARGET_PAGE_SIZE);
1697 memset (&env->tb_jmp_cache[i], 0,
1698 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1699
1700 i = tb_jmp_cache_hash_page(addr);
1701 memset (&env->tb_jmp_cache[i], 0,
1702 TB_JMP_PAGE_SIZE * sizeof(TranslationBlock *));
1703}
1704
ee8b7021
FB		 1705/* NOTE: if flush_global is true, also flush global entries (not
1706 implemented yet) */
1707void tlb_flush(CPUState *env, int flush_global)
33417e70 1708{
33417e70 1709 int i;
0124311e 1710
9fa3e853
FB		 1711#if defined(DEBUG_TLB)
1712 printf("tlb_flush:\n");
1713#endif
0124311e
FB		 1714 /* must reset current TB so that interrupts cannot modify the
1715 links while we are modifying them */
1716 env->current_tb = NULL;
1717
33417e70 1718 for(i = 0; i < CPU_TLB_SIZE; i++) {
84b7b8e7
FB		 1719 env->tlb_table[0][i].addr_read = -1;
1720 env->tlb_table[0][i].addr_write = -1;
1721 env->tlb_table[0][i].addr_code = -1;
1722 env->tlb_table[1][i].addr_read = -1;
1723 env->tlb_table[1][i].addr_write = -1;
1724 env->tlb_table[1][i].addr_code = -1;
6fa4cea9
JM		 1725#if (NB_MMU_MODES >= 3)
1726 env->tlb_table[2][i].addr_read = -1;
1727 env->tlb_table[2][i].addr_write = -1;
1728 env->tlb_table[2][i].addr_code = -1;
1729#if (NB_MMU_MODES == 4)
1730 env->tlb_table[3][i].addr_read = -1;
1731 env->tlb_table[3][i].addr_write = -1;
1732 env->tlb_table[3][i].addr_code = -1;
1733#endif
1734#endif
33417e70 1735 }
9fa3e853 1736
8a40a180 1737 memset (env->tb_jmp_cache, 0, TB_JMP_CACHE_SIZE * sizeof (void *));
9fa3e853 1738
0a962c02
FB		 1739#ifdef USE_KQEMU
1740 if (env->kqemu_enabled) {
1741 kqemu_flush(env, flush_global);
1742 }
9fa3e853 1743#endif
e3db7226 1744 tlb_flush_count++;
33417e70
FB		 1745}
1746
274da6b2 1747static inline void tlb_flush_entry(CPUTLBEntry *tlb_entry, target_ulong addr)
61382a50 1748{
5fafdf24 1749 if (addr == (tlb_entry->addr_read &
84b7b8e7 1750 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
5fafdf24 1751 addr == (tlb_entry->addr_write &
84b7b8e7 1752 (TARGET_PAGE_MASK | TLB_INVALID_MASK)) ||
5fafdf24 1753 addr == (tlb_entry->addr_code &
84b7b8e7
FB		 1754 (TARGET_PAGE_MASK | TLB_INVALID_MASK))) {
1755 tlb_entry->addr_read = -1;
1756 tlb_entry->addr_write = -1;
1757 tlb_entry->addr_code = -1;
1758 }
61382a50
FB		 1759}
1760
2e12669a 1761void tlb_flush_page(CPUState *env, target_ulong addr)
33417e70 1762{
8a40a180 1763 int i;
0124311e 1764
9fa3e853 1765#if defined(DEBUG_TLB)
108c49b8 1766 printf("tlb_flush_page: " TARGET_FMT_lx "\n", addr);
9fa3e853 1767#endif
0124311e
FB		 1768 /* must reset current TB so that interrupts cannot modify the
1769 links while we are modifying them */
1770 env->current_tb = NULL;
61382a50
FB		 1771
1772 addr &= TARGET_PAGE_MASK;
1773 i = (addr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
84b7b8e7
FB		 1774 tlb_flush_entry(&env->tlb_table[0][i], addr);
1775 tlb_flush_entry(&env->tlb_table[1][i], addr);
6fa4cea9
JM		 1776#if (NB_MMU_MODES >= 3)
1777 tlb_flush_entry(&env->tlb_table[2][i], addr);
1778#if (NB_MMU_MODES == 4)
1779 tlb_flush_entry(&env->tlb_table[3][i], addr);
1780#endif
1781#endif
0124311e 1782
5c751e99 1783 tlb_flush_jmp_cache(env, addr);
9fa3e853 1784
0a962c02
FB		 1785#ifdef USE_KQEMU
1786 if (env->kqemu_enabled) {
1787 kqemu_flush_page(env, addr);
1788 }
1789#endif
9fa3e853
FB		 1790}
1791
9fa3e853
FB		 1792/* update the TLBs so that writes to code in the virtual page 'addr'
1793 can be detected */
6a00d601 1794static void tlb_protect_code(ram_addr_t ram_addr)
9fa3e853 1795{
5fafdf24 1796 cpu_physical_memory_reset_dirty(ram_addr,
6a00d601
FB		 1797 ram_addr + TARGET_PAGE_SIZE,
1798 CODE_DIRTY_FLAG);
9fa3e853
FB		 1799}
1800
9fa3e853 1801/* update the TLB so that writes in physical page 'phys_addr' are no longer
3a7d929e 1802 tested for self modifying code */
5fafdf24 1803static void tlb_unprotect_code_phys(CPUState *env, ram_addr_t ram_addr,
3a7d929e 1804 target_ulong vaddr)
9fa3e853 1805{
3a7d929e 1806 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] |= CODE_DIRTY_FLAG;
1ccde1cb
FB		 1807}
1808
5fafdf24 1809static inline void tlb_reset_dirty_range(CPUTLBEntry *tlb_entry,
1ccde1cb
FB		 1810 unsigned long start, unsigned long length)
1811{
1812 unsigned long addr;
84b7b8e7
FB		 1813 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
1814 addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) + tlb_entry->addend;
1ccde1cb 1815 if ((addr - start) < length) {
0f459d16 1816 tlb_entry->addr_write = (tlb_entry->addr_write & TARGET_PAGE_MASK) | TLB_NOTDIRTY;
1ccde1cb
FB		 1817 }
1818 }
1819}
1820
3a7d929e 1821void cpu_physical_memory_reset_dirty(ram_addr_t start, ram_addr_t end,
0a962c02 1822 int dirty_flags)
1ccde1cb
FB		 1823{
1824 CPUState *env;
4f2ac237 1825 unsigned long length, start1;
0a962c02
FB		 1826 int i, mask, len;
1827 uint8_t *p;
1ccde1cb
FB		 1828
1829 start &= TARGET_PAGE_MASK;
1830 end = TARGET_PAGE_ALIGN(end);
1831
1832 length = end - start;
1833 if (length == 0)
1834 return;
0a962c02 1835 len = length >> TARGET_PAGE_BITS;
3a7d929e 1836#ifdef USE_KQEMU
6a00d601
FB		 1837 /* XXX: should not depend on cpu context */
1838 env = first_cpu;
3a7d929e 1839 if (env->kqemu_enabled) {
f23db169
FB		 1840 ram_addr_t addr;
1841 addr = start;
1842 for(i = 0; i < len; i++) {
1843 kqemu_set_notdirty(env, addr);
1844 addr += TARGET_PAGE_SIZE;
1845 }
3a7d929e
FB		 1846 }
1847#endif
f23db169
FB		 1848 mask = ~dirty_flags;
1849 p = phys_ram_dirty + (start >> TARGET_PAGE_BITS);
1850 for(i = 0; i < len; i++)
1851 p[i] &= mask;
1852
1ccde1cb
FB		 1853 /* we modify the TLB cache so that the dirty bit will be set again
1854 when accessing the range */
59817ccb 1855 start1 = start + (unsigned long)phys_ram_base;
6a00d601
FB		 1856 for(env = first_cpu; env != NULL; env = env->next_cpu) {
1857 for(i = 0; i < CPU_TLB_SIZE; i++)
84b7b8e7 1858 tlb_reset_dirty_range(&env->tlb_table[0][i], start1, length);
6a00d601 1859 for(i = 0; i < CPU_TLB_SIZE; i++)
84b7b8e7 1860 tlb_reset_dirty_range(&env->tlb_table[1][i], start1, length);
6fa4cea9
JM		 1861#if (NB_MMU_MODES >= 3)
1862 for(i = 0; i < CPU_TLB_SIZE; i++)
1863 tlb_reset_dirty_range(&env->tlb_table[2][i], start1, length);
1864#if (NB_MMU_MODES == 4)
1865 for(i = 0; i < CPU_TLB_SIZE; i++)
1866 tlb_reset_dirty_range(&env->tlb_table[3][i], start1, length);
1867#endif
1868#endif
6a00d601 1869 }
1ccde1cb
FB		 1870}
1871
74576198
AL		 1872int cpu_physical_memory_set_dirty_tracking(int enable)
1873{
1874 in_migration = enable;
1875 return 0;
1876}
1877
1878int cpu_physical_memory_get_dirty_tracking(void)
1879{
1880 return in_migration;
1881}
1882
2bec46dc
AL		 1883void cpu_physical_sync_dirty_bitmap(target_phys_addr_t start_addr, target_phys_addr_t end_addr)
1884{
1885 if (kvm_enabled())
1886 kvm_physical_sync_dirty_bitmap(start_addr, end_addr);
1887}
1888
3a7d929e
FB		 1889static inline void tlb_update_dirty(CPUTLBEntry *tlb_entry)
1890{
1891 ram_addr_t ram_addr;
1892
84b7b8e7 1893 if ((tlb_entry->addr_write & ~TARGET_PAGE_MASK) == IO_MEM_RAM) {
5fafdf24 1894 ram_addr = (tlb_entry->addr_write & TARGET_PAGE_MASK) +
3a7d929e
FB		 1895 tlb_entry->addend - (unsigned long)phys_ram_base;
1896 if (!cpu_physical_memory_is_dirty(ram_addr)) {
0f459d16 1897 tlb_entry->addr_write |= TLB_NOTDIRTY;
3a7d929e
FB		 1898 }
1899 }
1900}
1901
1902/* update the TLB according to the current state of the dirty bits */
1903void cpu_tlb_update_dirty(CPUState *env)
1904{
1905 int i;
1906 for(i = 0; i < CPU_TLB_SIZE; i++)
84b7b8e7 1907 tlb_update_dirty(&env->tlb_table[0][i]);
3a7d929e 1908 for(i = 0; i < CPU_TLB_SIZE; i++)
84b7b8e7 1909 tlb_update_dirty(&env->tlb_table[1][i]);
6fa4cea9
JM		 1910#if (NB_MMU_MODES >= 3)
1911 for(i = 0; i < CPU_TLB_SIZE; i++)
1912 tlb_update_dirty(&env->tlb_table[2][i]);
1913#if (NB_MMU_MODES == 4)
1914 for(i = 0; i < CPU_TLB_SIZE; i++)
1915 tlb_update_dirty(&env->tlb_table[3][i]);
1916#endif
1917#endif
3a7d929e
FB		 1918}
1919
0f459d16 1920static inline void tlb_set_dirty1(CPUTLBEntry *tlb_entry, target_ulong vaddr)
1ccde1cb 1921{
0f459d16
PB		 1922 if (tlb_entry->addr_write == (vaddr | TLB_NOTDIRTY))
1923 tlb_entry->addr_write = vaddr;
1ccde1cb
FB		 1924}
1925
0f459d16
PB		 1926/* update the TLB corresponding to virtual page vaddr
1927 so that it is no longer dirty */
1928static inline void tlb_set_dirty(CPUState *env, target_ulong vaddr)
1ccde1cb 1929{
1ccde1cb
FB		 1930 int i;
1931
0f459d16 1932 vaddr &= TARGET_PAGE_MASK;
1ccde1cb 1933 i = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
0f459d16
PB		 1934 tlb_set_dirty1(&env->tlb_table[0][i], vaddr);
1935 tlb_set_dirty1(&env->tlb_table[1][i], vaddr);
6fa4cea9 1936#if (NB_MMU_MODES >= 3)
0f459d16 1937 tlb_set_dirty1(&env->tlb_table[2][i], vaddr);
6fa4cea9 1938#if (NB_MMU_MODES == 4)
0f459d16 1939 tlb_set_dirty1(&env->tlb_table[3][i], vaddr);
6fa4cea9
JM		 1940#endif
1941#endif
9fa3e853
FB		 1942}
1943
59817ccb
FB		 1944/* add a new TLB entry. At most one entry for a given virtual address
1945 is permitted. Return 0 if OK or 2 if the page could not be mapped
1946 (can only happen in non SOFTMMU mode for I/O pages or pages
1947 conflicting with the host address space). */
5fafdf24
TS		 1948int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
1949 target_phys_addr_t paddr, int prot,
6ebbf390 1950 int mmu_idx, int is_softmmu)
9fa3e853 1951{
92e873b9 1952 PhysPageDesc *p;
4f2ac237 1953 unsigned long pd;
9fa3e853 1954 unsigned int index;
4f2ac237 1955 target_ulong address;
0f459d16 1956 target_ulong code_address;
108c49b8 1957 target_phys_addr_t addend;
9fa3e853 1958 int ret;
84b7b8e7 1959 CPUTLBEntry *te;
a1d1bb31 1960 CPUWatchpoint *wp;
0f459d16 1961 target_phys_addr_t iotlb;
9fa3e853 1962
92e873b9 1963 p = phys_page_find(paddr >> TARGET_PAGE_BITS);
9fa3e853
FB		 1964 if (!p) {
1965 pd = IO_MEM_UNASSIGNED;
9fa3e853
FB		 1966 } else {
1967 pd = p->phys_offset;
9fa3e853
FB		 1968 }
1969#if defined(DEBUG_TLB)
6ebbf390
JM		 1970 printf("tlb_set_page: vaddr=" TARGET_FMT_lx " paddr=0x%08x prot=%x idx=%d smmu=%d pd=0x%08lx\n",
1971 vaddr, (int)paddr, prot, mmu_idx, is_softmmu, pd);
9fa3e853
FB		 1972#endif
1973
1974 ret = 0;
0f459d16
PB		 1975 address = vaddr;
1976 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM && !(pd & IO_MEM_ROMD)) {
1977 /* IO memory case (romd handled later) */
1978 address |= TLB_MMIO;
1979 }
1980 addend = (unsigned long)phys_ram_base + (pd & TARGET_PAGE_MASK);
1981 if ((pd & ~TARGET_PAGE_MASK) <= IO_MEM_ROM) {
1982 /* Normal RAM. */
1983 iotlb = pd & TARGET_PAGE_MASK;
1984 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM)
1985 iotlb |= IO_MEM_NOTDIRTY;
1986 else
1987 iotlb |= IO_MEM_ROM;
1988 } else {
1989 /* IO handlers are currently passed a phsical address.
1990 It would be nice to pass an offset from the base address
1991 of that region. This would avoid having to special case RAM,
1992 and avoid full address decoding in every device.
1993 We can't use the high bits of pd for this because
1994 IO_MEM_ROMD uses these as a ram address. */
8da3ff18
PB		 1995 iotlb = (pd & ~TARGET_PAGE_MASK);
1996 if (p) {
8da3ff18
PB		 1997 iotlb += p->region_offset;
1998 } else {
1999 iotlb += paddr;
2000 }
0f459d16
PB		 2001 }
2002
2003 code_address = address;
2004 /* Make accesses to pages with watchpoints go via the
2005 watchpoint trap routines. */
c0ce998e 2006 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
a1d1bb31 2007 if (vaddr == (wp->vaddr & TARGET_PAGE_MASK)) {
0f459d16
PB		 2008 iotlb = io_mem_watch + paddr;
2009 /* TODO: The memory case can be optimized by not trapping
2010 reads of pages with a write breakpoint. */
2011 address |= TLB_MMIO;
6658ffb8 2012 }
0f459d16 2013 }
d79acba4 2014
0f459d16
PB		 2015 index = (vaddr >> TARGET_PAGE_BITS) & (CPU_TLB_SIZE - 1);
2016 env->iotlb[mmu_idx][index] = iotlb - vaddr;
2017 te = &env->tlb_table[mmu_idx][index];
2018 te->addend = addend - vaddr;
2019 if (prot & PAGE_READ) {
2020 te->addr_read = address;
2021 } else {
2022 te->addr_read = -1;
2023 }
5c751e99 2024
0f459d16
PB		 2025 if (prot & PAGE_EXEC) {
2026 te->addr_code = code_address;
2027 } else {
2028 te->addr_code = -1;
2029 }
2030 if (prot & PAGE_WRITE) {
2031 if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_ROM ||
2032 (pd & IO_MEM_ROMD)) {
2033 /* Write access calls the I/O callback. */
2034 te->addr_write = address | TLB_MMIO;
2035 } else if ((pd & ~TARGET_PAGE_MASK) == IO_MEM_RAM &&
2036 !cpu_physical_memory_is_dirty(pd)) {
2037 te->addr_write = address | TLB_NOTDIRTY;
9fa3e853 2038 } else {
0f459d16 2039 te->addr_write = address;
9fa3e853 2040 }
0f459d16
PB		 2041 } else {
2042 te->addr_write = -1;
9fa3e853 2043 }
9fa3e853
FB		 2044 return ret;
2045}
2046
0124311e
FB		 2047#else
2048
ee8b7021 2049void tlb_flush(CPUState *env, int flush_global)
0124311e
FB		 2050{
2051}
2052
2e12669a 2053void tlb_flush_page(CPUState *env, target_ulong addr)
0124311e
FB		 2054{
2055}
2056
5fafdf24
TS		 2057int tlb_set_page_exec(CPUState *env, target_ulong vaddr,
2058 target_phys_addr_t paddr, int prot,
6ebbf390 2059 int mmu_idx, int is_softmmu)
9fa3e853
FB		 2060{
2061 return 0;
2062}
0124311e 2063
9fa3e853
FB		 2064/* dump memory mappings */
2065void page_dump(FILE *f)
33417e70 2066{
9fa3e853
FB		 2067 unsigned long start, end;
2068 int i, j, prot, prot1;
2069 PageDesc *p;
33417e70 2070
9fa3e853
FB		 2071 fprintf(f, "%-8s %-8s %-8s %s\n",
2072 "start", "end", "size", "prot");
2073 start = -1;
2074 end = -1;
2075 prot = 0;
2076 for(i = 0; i <= L1_SIZE; i++) {
2077 if (i < L1_SIZE)
2078 p = l1_map[i];
2079 else
2080 p = NULL;
2081 for(j = 0;j < L2_SIZE; j++) {
2082 if (!p)
2083 prot1 = 0;
2084 else
2085 prot1 = p[j].flags;
2086 if (prot1 != prot) {
2087 end = (i << (32 - L1_BITS)) | (j << TARGET_PAGE_BITS);
2088 if (start != -1) {
2089 fprintf(f, "%08lx-%08lx %08lx %c%c%c\n",
5fafdf24 2090 start, end, end - start,
9fa3e853
FB		 2091 prot & PAGE_READ ? 'r' : '-',
2092 prot & PAGE_WRITE ? 'w' : '-',
2093 prot & PAGE_EXEC ? 'x' : '-');
2094 }
2095 if (prot1 != 0)
2096 start = end;
2097 else
2098 start = -1;
2099 prot = prot1;
2100 }
2101 if (!p)
2102 break;
2103 }
33417e70 2104 }
33417e70
FB		 2105}
2106
53a5960a 2107int page_get_flags(target_ulong address)
33417e70 2108{
9fa3e853
FB		 2109 PageDesc *p;
2110
2111 p = page_find(address >> TARGET_PAGE_BITS);
33417e70 2112 if (!p)
9fa3e853
FB		 2113 return 0;
2114 return p->flags;
2115}
2116
2117/* modify the flags of a page and invalidate the code if
2118 necessary. The flag PAGE_WRITE_ORG is positionned automatically
2119 depending on PAGE_WRITE */
53a5960a 2120void page_set_flags(target_ulong start, target_ulong end, int flags)
9fa3e853
FB		 2121{
2122 PageDesc *p;
53a5960a 2123 target_ulong addr;
9fa3e853 2124
c8a706fe 2125 /* mmap_lock should already be held. */
9fa3e853
FB		 2126 start = start & TARGET_PAGE_MASK;
2127 end = TARGET_PAGE_ALIGN(end);
2128 if (flags & PAGE_WRITE)
2129 flags |= PAGE_WRITE_ORG;
9fa3e853
FB		 2130 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2131 p = page_find_alloc(addr >> TARGET_PAGE_BITS);
17e2377a
PB		 2132 /* We may be called for host regions that are outside guest
2133 address space. */
2134 if (!p)
2135 return;
9fa3e853
FB		 2136 /* if the write protection is set, then we invalidate the code
2137 inside */
5fafdf24 2138 if (!(p->flags & PAGE_WRITE) &&
9fa3e853
FB		 2139 (flags & PAGE_WRITE) &&
2140 p->first_tb) {
d720b93d 2141 tb_invalidate_phys_page(addr, 0, NULL);
9fa3e853
FB		 2142 }
2143 p->flags = flags;
2144 }
33417e70
FB		 2145}
2146
3d97b40b
TS		 2147int page_check_range(target_ulong start, target_ulong len, int flags)
2148{
2149 PageDesc *p;
2150 target_ulong end;
2151 target_ulong addr;
2152
55f280c9
AZ		 2153 if (start + len < start)
2154 /* we've wrapped around */
2155 return -1;
2156
3d97b40b
TS		 2157 end = TARGET_PAGE_ALIGN(start+len); /* must do before we loose bits in the next step */
2158 start = start & TARGET_PAGE_MASK;
2159
3d97b40b
TS		 2160 for(addr = start; addr < end; addr += TARGET_PAGE_SIZE) {
2161 p = page_find(addr >> TARGET_PAGE_BITS);
2162 if( !p )
2163 return -1;
2164 if( !(p->flags & PAGE_VALID) )
2165 return -1;
2166
dae3270c 2167 if ((flags & PAGE_READ) && !(p->flags & PAGE_READ))
3d97b40b 2168 return -1;
dae3270c
FB		 2169 if (flags & PAGE_WRITE) {
2170 if (!(p->flags & PAGE_WRITE_ORG))
2171 return -1;
2172 /* unprotect the page if it was put read-only because it
2173 contains translated code */
2174 if (!(p->flags & PAGE_WRITE)) {
2175 if (!page_unprotect(addr, 0, NULL))
2176 return -1;
2177 }
2178 return 0;
2179 }
3d97b40b
TS		 2180 }
2181 return 0;
2182}
2183
9fa3e853
FB		 2184/* called from signal handler: invalidate the code and unprotect the
2185 page. Return TRUE if the fault was succesfully handled. */
53a5960a 2186int page_unprotect(target_ulong address, unsigned long pc, void *puc)
9fa3e853
FB		 2187{
2188 unsigned int page_index, prot, pindex;
2189 PageDesc *p, *p1;
53a5960a 2190 target_ulong host_start, host_end, addr;
9fa3e853 2191
c8a706fe
PB		 2192 /* Technically this isn't safe inside a signal handler. However we
2193 know this only ever happens in a synchronous SEGV handler, so in
2194 practice it seems to be ok. */
2195 mmap_lock();
2196
83fb7adf 2197 host_start = address & qemu_host_page_mask;
9fa3e853
FB		 2198 page_index = host_start >> TARGET_PAGE_BITS;
2199 p1 = page_find(page_index);
c8a706fe
PB		 2200 if (!p1) {
2201 mmap_unlock();
9fa3e853 2202 return 0;
c8a706fe 2203 }
83fb7adf 2204 host_end = host_start + qemu_host_page_size;
9fa3e853
FB		 2205 p = p1;
2206 prot = 0;
2207 for(addr = host_start;addr < host_end; addr += TARGET_PAGE_SIZE) {
2208 prot |= p->flags;
2209 p++;
2210 }
2211 /* if the page was really writable, then we change its
2212 protection back to writable */
2213 if (prot & PAGE_WRITE_ORG) {
2214 pindex = (address - host_start) >> TARGET_PAGE_BITS;
2215 if (!(p1[pindex].flags & PAGE_WRITE)) {
5fafdf24 2216 mprotect((void *)g2h(host_start), qemu_host_page_size,
9fa3e853
FB		 2217 (prot & PAGE_BITS) | PAGE_WRITE);
2218 p1[pindex].flags |= PAGE_WRITE;
2219 /* and since the content will be modified, we must invalidate
2220 the corresponding translated code. */
d720b93d 2221 tb_invalidate_phys_page(address, pc, puc);
9fa3e853
FB		 2222#ifdef DEBUG_TB_CHECK
2223 tb_invalidate_check(address);
2224#endif
c8a706fe 2225 mmap_unlock();
9fa3e853
FB		 2226 return 1;
2227 }
2228 }
c8a706fe 2229 mmap_unlock();
9fa3e853
FB		 2230 return 0;
2231}
2232
6a00d601
FB		 2233static inline void tlb_set_dirty(CPUState *env,
2234 unsigned long addr, target_ulong vaddr)
1ccde1cb
FB		 2235{
2236}
9fa3e853
FB		 2237#endif /* defined(CONFIG_USER_ONLY) */
2238
e2eef170 2239#if !defined(CONFIG_USER_ONLY)
8da3ff18 2240
db7b5426 2241static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
8da3ff18 2242 ram_addr_t memory, ram_addr_t region_offset);
00f82b8a 2243static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
8da3ff18 2244 ram_addr_t orig_memory, ram_addr_t region_offset);
db7b5426
BS		 2245#define CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2, \
2246 need_subpage) \
2247 do { \
2248 if (addr > start_addr) \
2249 start_addr2 = 0; \
2250 else { \
2251 start_addr2 = start_addr & ~TARGET_PAGE_MASK; \
2252 if (start_addr2 > 0) \
2253 need_subpage = 1; \
2254 } \
2255 \
49e9fba2 2256 if ((start_addr + orig_size) - addr >= TARGET_PAGE_SIZE) \
db7b5426
BS		 2257 end_addr2 = TARGET_PAGE_SIZE - 1; \
2258 else { \
2259 end_addr2 = (start_addr + orig_size - 1) & ~TARGET_PAGE_MASK; \
2260 if (end_addr2 < TARGET_PAGE_SIZE - 1) \
2261 need_subpage = 1; \
2262 } \
2263 } while (0)
2264
33417e70
FB		 2265/* register physical memory. 'size' must be a multiple of the target
2266 page size. If (phys_offset & ~TARGET_PAGE_MASK) != 0, then it is an
8da3ff18
PB		 2267 io memory page. The address used when calling the IO function is
2268 the offset from the start of the region, plus region_offset. Both
2269 start_region and regon_offset are rounded down to a page boundary
2270 before calculating this offset. This should not be a problem unless
2271 the low bits of start_addr and region_offset differ. */
2272void cpu_register_physical_memory_offset(target_phys_addr_t start_addr,
2273 ram_addr_t size,
2274 ram_addr_t phys_offset,
2275 ram_addr_t region_offset)
33417e70 2276{
108c49b8 2277 target_phys_addr_t addr, end_addr;
92e873b9 2278 PhysPageDesc *p;
9d42037b 2279 CPUState *env;
00f82b8a 2280 ram_addr_t orig_size = size;
db7b5426 2281 void *subpage;
33417e70 2282
da260249
FB		 2283#ifdef USE_KQEMU
2284 /* XXX: should not depend on cpu context */
2285 env = first_cpu;
2286 if (env->kqemu_enabled) {
2287 kqemu_set_phys_mem(start_addr, size, phys_offset);
2288 }
2289#endif
7ba1e619
AL		 2290 if (kvm_enabled())
2291 kvm_set_phys_mem(start_addr, size, phys_offset);
2292
8da3ff18 2293 region_offset &= TARGET_PAGE_MASK;
5fd386f6 2294 size = (size + TARGET_PAGE_SIZE - 1) & TARGET_PAGE_MASK;
49e9fba2
BS		 2295 end_addr = start_addr + (target_phys_addr_t)size;
2296 for(addr = start_addr; addr != end_addr; addr += TARGET_PAGE_SIZE) {
db7b5426
BS		 2297 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2298 if (p && p->phys_offset != IO_MEM_UNASSIGNED) {
00f82b8a 2299 ram_addr_t orig_memory = p->phys_offset;
db7b5426
BS		 2300 target_phys_addr_t start_addr2, end_addr2;
2301 int need_subpage = 0;
2302
2303 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr, end_addr2,
2304 need_subpage);
4254fab8 2305 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
db7b5426
BS		 2306 if (!(orig_memory & IO_MEM_SUBPAGE)) {
2307 subpage = subpage_init((addr & TARGET_PAGE_MASK),
8da3ff18
PB		 2308 &p->phys_offset, orig_memory,
2309 p->region_offset);
db7b5426
BS		 2310 } else {
2311 subpage = io_mem_opaque[(orig_memory & ~TARGET_PAGE_MASK)
2312 >> IO_MEM_SHIFT];
2313 }
8da3ff18
PB		 2314 subpage_register(subpage, start_addr2, end_addr2, phys_offset,
2315 region_offset);
2316 p->region_offset = 0;
db7b5426
BS		 2317 } else {
2318 p->phys_offset = phys_offset;
2319 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
2320 (phys_offset & IO_MEM_ROMD))
2321 phys_offset += TARGET_PAGE_SIZE;
2322 }
2323 } else {
2324 p = phys_page_find_alloc(addr >> TARGET_PAGE_BITS, 1);
2325 p->phys_offset = phys_offset;
8da3ff18 2326 p->region_offset = region_offset;
db7b5426 2327 if ((phys_offset & ~TARGET_PAGE_MASK) <= IO_MEM_ROM ||
8da3ff18 2328 (phys_offset & IO_MEM_ROMD)) {
db7b5426 2329 phys_offset += TARGET_PAGE_SIZE;
0e8f0967 2330 } else {
db7b5426
BS		 2331 target_phys_addr_t start_addr2, end_addr2;
2332 int need_subpage = 0;
2333
2334 CHECK_SUBPAGE(addr, start_addr, start_addr2, end_addr,
2335 end_addr2, need_subpage);
2336
4254fab8 2337 if (need_subpage || phys_offset & IO_MEM_SUBWIDTH) {
db7b5426 2338 subpage = subpage_init((addr & TARGET_PAGE_MASK),
8da3ff18
PB		 2339 &p->phys_offset, IO_MEM_UNASSIGNED,
2340 0);
db7b5426 2341 subpage_register(subpage, start_addr2, end_addr2,
8da3ff18
PB		 2342 phys_offset, region_offset);
2343 p->region_offset = 0;
db7b5426
BS		 2344 }
2345 }
2346 }
8da3ff18 2347 region_offset += TARGET_PAGE_SIZE;
33417e70 2348 }
3b46e624 2349
9d42037b
FB		 2350 /* since each CPU stores ram addresses in its TLB cache, we must
2351 reset the modified entries */
2352 /* XXX: slow ! */
2353 for(env = first_cpu; env != NULL; env = env->next_cpu) {
2354 tlb_flush(env, 1);
2355 }
33417e70
FB		 2356}
2357
ba863458 2358/* XXX: temporary until new memory mapping API */
00f82b8a 2359ram_addr_t cpu_get_physical_page_desc(target_phys_addr_t addr)
ba863458
FB		 2360{
2361 PhysPageDesc *p;
2362
2363 p = phys_page_find(addr >> TARGET_PAGE_BITS);
2364 if (!p)
2365 return IO_MEM_UNASSIGNED;
2366 return p->phys_offset;
2367}
2368
f65ed4c1
AL		 2369void qemu_register_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2370{
2371 if (kvm_enabled())
2372 kvm_coalesce_mmio_region(addr, size);
2373}
2374
2375void qemu_unregister_coalesced_mmio(target_phys_addr_t addr, ram_addr_t size)
2376{
2377 if (kvm_enabled())
2378 kvm_uncoalesce_mmio_region(addr, size);
2379}
2380
e9a1ab19 2381/* XXX: better than nothing */
00f82b8a 2382ram_addr_t qemu_ram_alloc(ram_addr_t size)
e9a1ab19
FB		 2383{
2384 ram_addr_t addr;
7fb4fdcf 2385 if ((phys_ram_alloc_offset + size) > phys_ram_size) {
012a7045 2386 fprintf(stderr, "Not enough memory (requested_size = %" PRIu64 ", max memory = %" PRIu64 ")\n",
ed441467 2387 (uint64_t)size, (uint64_t)phys_ram_size);
e9a1ab19
FB		 2388 abort();
2389 }
2390 addr = phys_ram_alloc_offset;
2391 phys_ram_alloc_offset = TARGET_PAGE_ALIGN(phys_ram_alloc_offset + size);
2392 return addr;
2393}
2394
2395void qemu_ram_free(ram_addr_t addr)
2396{
2397}
2398
a4193c8a 2399static uint32_t unassigned_mem_readb(void *opaque, target_phys_addr_t addr)
33417e70 2400{
67d3b957 2401#ifdef DEBUG_UNASSIGNED
ab3d1727 2402 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
b4f0a316 2403#endif
0a6f8a6d 2404#if defined(TARGET_SPARC)
e18231a3
BS		 2405 do_unassigned_access(addr, 0, 0, 0, 1);
2406#endif
2407 return 0;
2408}
2409
2410static uint32_t unassigned_mem_readw(void *opaque, target_phys_addr_t addr)
2411{
2412#ifdef DEBUG_UNASSIGNED
2413 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2414#endif
0a6f8a6d 2415#if defined(TARGET_SPARC)
e18231a3
BS		 2416 do_unassigned_access(addr, 0, 0, 0, 2);
2417#endif
2418 return 0;
2419}
2420
2421static uint32_t unassigned_mem_readl(void *opaque, target_phys_addr_t addr)
2422{
2423#ifdef DEBUG_UNASSIGNED
2424 printf("Unassigned mem read " TARGET_FMT_plx "\n", addr);
2425#endif
0a6f8a6d 2426#if defined(TARGET_SPARC)
e18231a3 2427 do_unassigned_access(addr, 0, 0, 0, 4);
67d3b957 2428#endif
33417e70
FB		 2429 return 0;
2430}
2431
a4193c8a 2432static void unassigned_mem_writeb(void *opaque, target_phys_addr_t addr, uint32_t val)
33417e70 2433{
67d3b957 2434#ifdef DEBUG_UNASSIGNED
ab3d1727 2435 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
67d3b957 2436#endif
0a6f8a6d 2437#if defined(TARGET_SPARC)
e18231a3
BS		 2438 do_unassigned_access(addr, 1, 0, 0, 1);
2439#endif
2440}
2441
2442static void unassigned_mem_writew(void *opaque, target_phys_addr_t addr, uint32_t val)
2443{
2444#ifdef DEBUG_UNASSIGNED
2445 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2446#endif
0a6f8a6d 2447#if defined(TARGET_SPARC)
e18231a3
BS		 2448 do_unassigned_access(addr, 1, 0, 0, 2);
2449#endif
2450}
2451
2452static void unassigned_mem_writel(void *opaque, target_phys_addr_t addr, uint32_t val)
2453{
2454#ifdef DEBUG_UNASSIGNED
2455 printf("Unassigned mem write " TARGET_FMT_plx " = 0x%x\n", addr, val);
2456#endif
0a6f8a6d 2457#if defined(TARGET_SPARC)
e18231a3 2458 do_unassigned_access(addr, 1, 0, 0, 4);
b4f0a316 2459#endif
33417e70
FB		 2460}
2461
2462static CPUReadMemoryFunc *unassigned_mem_read[3] = {
2463 unassigned_mem_readb,
e18231a3
BS		 2464 unassigned_mem_readw,
2465 unassigned_mem_readl,
33417e70
FB		 2466};
2467
2468static CPUWriteMemoryFunc *unassigned_mem_write[3] = {
2469 unassigned_mem_writeb,
e18231a3
BS		 2470 unassigned_mem_writew,
2471 unassigned_mem_writel,
33417e70
FB		 2472};
2473
0f459d16
PB		 2474static void notdirty_mem_writeb(void *opaque, target_phys_addr_t ram_addr,
2475 uint32_t val)
9fa3e853 2476{
3a7d929e 2477 int dirty_flags;
3a7d929e
FB		 2478 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2479 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
9fa3e853 2480#if !defined(CONFIG_USER_ONLY)
3a7d929e
FB		 2481 tb_invalidate_phys_page_fast(ram_addr, 1);
2482 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
9fa3e853 2483#endif
3a7d929e 2484 }
0f459d16 2485 stb_p(phys_ram_base + ram_addr, val);
f32fc648
FB		 2486#ifdef USE_KQEMU
2487 if (cpu_single_env->kqemu_enabled &&
2488 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2489 kqemu_modify_page(cpu_single_env, ram_addr);
2490#endif
f23db169
FB		 2491 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2492 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2493 /* we remove the notdirty callback only if the code has been
2494 flushed */
2495 if (dirty_flags == 0xff)
2e70f6ef 2496 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
9fa3e853
FB		 2497}
2498
0f459d16
PB		 2499static void notdirty_mem_writew(void *opaque, target_phys_addr_t ram_addr,
2500 uint32_t val)
9fa3e853 2501{
3a7d929e 2502 int dirty_flags;
3a7d929e
FB		 2503 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2504 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
9fa3e853 2505#if !defined(CONFIG_USER_ONLY)
3a7d929e
FB		 2506 tb_invalidate_phys_page_fast(ram_addr, 2);
2507 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
9fa3e853 2508#endif
3a7d929e 2509 }
0f459d16 2510 stw_p(phys_ram_base + ram_addr, val);
f32fc648
FB		 2511#ifdef USE_KQEMU
2512 if (cpu_single_env->kqemu_enabled &&
2513 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2514 kqemu_modify_page(cpu_single_env, ram_addr);
2515#endif
f23db169
FB		 2516 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2517 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2518 /* we remove the notdirty callback only if the code has been
2519 flushed */
2520 if (dirty_flags == 0xff)
2e70f6ef 2521 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
9fa3e853
FB		 2522}
2523
0f459d16
PB		 2524static void notdirty_mem_writel(void *opaque, target_phys_addr_t ram_addr,
2525 uint32_t val)
9fa3e853 2526{
3a7d929e 2527 int dirty_flags;
3a7d929e
FB		 2528 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
2529 if (!(dirty_flags & CODE_DIRTY_FLAG)) {
9fa3e853 2530#if !defined(CONFIG_USER_ONLY)
3a7d929e
FB		 2531 tb_invalidate_phys_page_fast(ram_addr, 4);
2532 dirty_flags = phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS];
9fa3e853 2533#endif
3a7d929e 2534 }
0f459d16 2535 stl_p(phys_ram_base + ram_addr, val);
f32fc648
FB		 2536#ifdef USE_KQEMU
2537 if (cpu_single_env->kqemu_enabled &&
2538 (dirty_flags & KQEMU_MODIFY_PAGE_MASK) != KQEMU_MODIFY_PAGE_MASK)
2539 kqemu_modify_page(cpu_single_env, ram_addr);
2540#endif
f23db169
FB		 2541 dirty_flags |= (0xff & ~CODE_DIRTY_FLAG);
2542 phys_ram_dirty[ram_addr >> TARGET_PAGE_BITS] = dirty_flags;
2543 /* we remove the notdirty callback only if the code has been
2544 flushed */
2545 if (dirty_flags == 0xff)
2e70f6ef 2546 tlb_set_dirty(cpu_single_env, cpu_single_env->mem_io_vaddr);
9fa3e853
FB		 2547}
2548
3a7d929e 2549static CPUReadMemoryFunc *error_mem_read[3] = {
9fa3e853
FB		 2550 NULL, /* never used */
2551 NULL, /* never used */
2552 NULL, /* never used */
2553};
2554
1ccde1cb
FB		 2555static CPUWriteMemoryFunc *notdirty_mem_write[3] = {
2556 notdirty_mem_writeb,
2557 notdirty_mem_writew,
2558 notdirty_mem_writel,
2559};
2560
0f459d16 2561/* Generate a debug exception if a watchpoint has been hit. */
b4051334 2562static void check_watchpoint(int offset, int len_mask, int flags)
0f459d16
PB		 2563{
2564 CPUState *env = cpu_single_env;
06d55cc1
AL		 2565 target_ulong pc, cs_base;
2566 TranslationBlock *tb;
0f459d16 2567 target_ulong vaddr;
a1d1bb31 2568 CPUWatchpoint *wp;
06d55cc1 2569 int cpu_flags;
0f459d16 2570
06d55cc1
AL		 2571 if (env->watchpoint_hit) {
2572 /* We re-entered the check after replacing the TB. Now raise
2573 * the debug interrupt so that is will trigger after the
2574 * current instruction. */
2575 cpu_interrupt(env, CPU_INTERRUPT_DEBUG);
2576 return;
2577 }
2e70f6ef 2578 vaddr = (env->mem_io_vaddr & TARGET_PAGE_MASK) + offset;
c0ce998e 2579 TAILQ_FOREACH(wp, &env->watchpoints, entry) {
b4051334
AL		 2580 if ((vaddr == (wp->vaddr & len_mask) ||
2581 (vaddr & wp->len_mask) == wp->vaddr) && (wp->flags & flags)) {
6e140f28
AL		 2582 wp->flags |= BP_WATCHPOINT_HIT;
2583 if (!env->watchpoint_hit) {
2584 env->watchpoint_hit = wp;
2585 tb = tb_find_pc(env->mem_io_pc);
2586 if (!tb) {
2587 cpu_abort(env, "check_watchpoint: could not find TB for "
2588 "pc=%p", (void *)env->mem_io_pc);
2589 }
2590 cpu_restore_state(tb, env, env->mem_io_pc, NULL);
2591 tb_phys_invalidate(tb, -1);
2592 if (wp->flags & BP_STOP_BEFORE_ACCESS) {
2593 env->exception_index = EXCP_DEBUG;
2594 } else {
2595 cpu_get_tb_cpu_state(env, &pc, &cs_base, &cpu_flags);
2596 tb_gen_code(env, pc, cs_base, cpu_flags, 1);
2597 }
2598 cpu_resume_from_signal(env, NULL);
06d55cc1 2599 }
6e140f28
AL		 2600 } else {
2601 wp->flags &= ~BP_WATCHPOINT_HIT;
0f459d16
PB		 2602 }
2603 }
2604}
2605
6658ffb8
PB		 2606/* Watchpoint access routines. Watchpoints are inserted using TLB tricks,
2607 so these check for a hit then pass through to the normal out-of-line
2608 phys routines. */
2609static uint32_t watch_mem_readb(void *opaque, target_phys_addr_t addr)
2610{
b4051334 2611 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_READ);
6658ffb8
PB		 2612 return ldub_phys(addr);
2613}
2614
2615static uint32_t watch_mem_readw(void *opaque, target_phys_addr_t addr)
2616{
b4051334 2617 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_READ);
6658ffb8
PB		 2618 return lduw_phys(addr);
2619}
2620
2621static uint32_t watch_mem_readl(void *opaque, target_phys_addr_t addr)
2622{
b4051334 2623 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_READ);
6658ffb8
PB		 2624 return ldl_phys(addr);
2625}
2626
6658ffb8
PB		 2627static void watch_mem_writeb(void *opaque, target_phys_addr_t addr,
2628 uint32_t val)
2629{
b4051334 2630 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x0, BP_MEM_WRITE);
6658ffb8
PB		 2631 stb_phys(addr, val);
2632}
2633
2634static void watch_mem_writew(void *opaque, target_phys_addr_t addr,
2635 uint32_t val)
2636{
b4051334 2637 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x1, BP_MEM_WRITE);
6658ffb8
PB		 2638 stw_phys(addr, val);
2639}
2640
2641static void watch_mem_writel(void *opaque, target_phys_addr_t addr,
2642 uint32_t val)
2643{
b4051334 2644 check_watchpoint(addr & ~TARGET_PAGE_MASK, ~0x3, BP_MEM_WRITE);
6658ffb8
PB		 2645 stl_phys(addr, val);
2646}
2647
2648static CPUReadMemoryFunc *watch_mem_read[3] = {
2649 watch_mem_readb,
2650 watch_mem_readw,
2651 watch_mem_readl,
2652};
2653
2654static CPUWriteMemoryFunc *watch_mem_write[3] = {
2655 watch_mem_writeb,
2656 watch_mem_writew,
2657 watch_mem_writel,
2658};
6658ffb8 2659
db7b5426
BS		 2660static inline uint32_t subpage_readlen (subpage_t *mmio, target_phys_addr_t addr,
2661 unsigned int len)
2662{
db7b5426
BS		 2663 uint32_t ret;
2664 unsigned int idx;
2665
8da3ff18 2666 idx = SUBPAGE_IDX(addr);
db7b5426
BS		 2667#if defined(DEBUG_SUBPAGE)
2668 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d\n", __func__,
2669 mmio, len, addr, idx);
2670#endif
8da3ff18
PB		 2671 ret = (**mmio->mem_read[idx][len])(mmio->opaque[idx][0][len],
2672 addr + mmio->region_offset[idx][0][len]);
db7b5426
BS		 2673
2674 return ret;
2675}
2676
2677static inline void subpage_writelen (subpage_t *mmio, target_phys_addr_t addr,
2678 uint32_t value, unsigned int len)
2679{
db7b5426
BS		 2680 unsigned int idx;
2681
8da3ff18 2682 idx = SUBPAGE_IDX(addr);
db7b5426
BS		 2683#if defined(DEBUG_SUBPAGE)
2684 printf("%s: subpage %p len %d addr " TARGET_FMT_plx " idx %d value %08x\n", __func__,
2685 mmio, len, addr, idx, value);
2686#endif
8da3ff18
PB		 2687 (**mmio->mem_write[idx][len])(mmio->opaque[idx][1][len],
2688 addr + mmio->region_offset[idx][1][len],
2689 value);
db7b5426
BS		 2690}
2691
2692static uint32_t subpage_readb (void *opaque, target_phys_addr_t addr)
2693{
2694#if defined(DEBUG_SUBPAGE)
2695 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2696#endif
2697
2698 return subpage_readlen(opaque, addr, 0);
2699}
2700
2701static void subpage_writeb (void *opaque, target_phys_addr_t addr,
2702 uint32_t value)
2703{
2704#if defined(DEBUG_SUBPAGE)
2705 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2706#endif
2707 subpage_writelen(opaque, addr, value, 0);
2708}
2709
2710static uint32_t subpage_readw (void *opaque, target_phys_addr_t addr)
2711{
2712#if defined(DEBUG_SUBPAGE)
2713 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2714#endif
2715
2716 return subpage_readlen(opaque, addr, 1);
2717}
2718
2719static void subpage_writew (void *opaque, target_phys_addr_t addr,
2720 uint32_t value)
2721{
2722#if defined(DEBUG_SUBPAGE)
2723 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2724#endif
2725 subpage_writelen(opaque, addr, value, 1);
2726}
2727
2728static uint32_t subpage_readl (void *opaque, target_phys_addr_t addr)
2729{
2730#if defined(DEBUG_SUBPAGE)
2731 printf("%s: addr " TARGET_FMT_plx "\n", __func__, addr);
2732#endif
2733
2734 return subpage_readlen(opaque, addr, 2);
2735}
2736
2737static void subpage_writel (void *opaque,
2738 target_phys_addr_t addr, uint32_t value)
2739{
2740#if defined(DEBUG_SUBPAGE)
2741 printf("%s: addr " TARGET_FMT_plx " val %08x\n", __func__, addr, value);
2742#endif
2743 subpage_writelen(opaque, addr, value, 2);
2744}
2745
2746static CPUReadMemoryFunc *subpage_read[] = {
2747 &subpage_readb,
2748 &subpage_readw,
2749 &subpage_readl,
2750};
2751
2752static CPUWriteMemoryFunc *subpage_write[] = {
2753 &subpage_writeb,
2754 &subpage_writew,
2755 &subpage_writel,
2756};
2757
2758static int subpage_register (subpage_t *mmio, uint32_t start, uint32_t end,
8da3ff18 2759 ram_addr_t memory, ram_addr_t region_offset)
db7b5426
BS		 2760{
2761 int idx, eidx;
4254fab8 2762 unsigned int i;
db7b5426
BS		 2763
2764 if (start >= TARGET_PAGE_SIZE || end >= TARGET_PAGE_SIZE)
2765 return -1;
2766 idx = SUBPAGE_IDX(start);
2767 eidx = SUBPAGE_IDX(end);
2768#if defined(DEBUG_SUBPAGE)
2769 printf("%s: %p start %08x end %08x idx %08x eidx %08x mem %d\n", __func__,
2770 mmio, start, end, idx, eidx, memory);
2771#endif
2772 memory >>= IO_MEM_SHIFT;
2773 for (; idx <= eidx; idx++) {
4254fab8 2774 for (i = 0; i < 4; i++) {
3ee89922
BS		 2775 if (io_mem_read[memory][i]) {
2776 mmio->mem_read[idx][i] = &io_mem_read[memory][i];
2777 mmio->opaque[idx][0][i] = io_mem_opaque[memory];
8da3ff18 2778 mmio->region_offset[idx][0][i] = region_offset;
3ee89922
BS		 2779 }
2780 if (io_mem_write[memory][i]) {
2781 mmio->mem_write[idx][i] = &io_mem_write[memory][i];
2782 mmio->opaque[idx][1][i] = io_mem_opaque[memory];
8da3ff18 2783 mmio->region_offset[idx][1][i] = region_offset;
3ee89922 2784 }
4254fab8 2785 }
db7b5426
BS		 2786 }
2787
2788 return 0;
2789}
2790
00f82b8a 2791static void *subpage_init (target_phys_addr_t base, ram_addr_t *phys,
8da3ff18 2792 ram_addr_t orig_memory, ram_addr_t region_offset)
db7b5426
BS		 2793{
2794 subpage_t *mmio;
2795 int subpage_memory;
2796
2797 mmio = qemu_mallocz(sizeof(subpage_t));
2798 if (mmio != NULL) {
2799 mmio->base = base;
2800 subpage_memory = cpu_register_io_memory(0, subpage_read, subpage_write, mmio);
2801#if defined(DEBUG_SUBPAGE)
2802 printf("%s: %p base " TARGET_FMT_plx " len %08x %d\n", __func__,
2803 mmio, base, TARGET_PAGE_SIZE, subpage_memory);
2804#endif
2805 *phys = subpage_memory | IO_MEM_SUBPAGE;
8da3ff18
PB		 2806 subpage_register(mmio, 0, TARGET_PAGE_SIZE - 1, orig_memory,
2807 region_offset);
db7b5426
BS		 2808 }
2809
2810 return mmio;
2811}
2812
33417e70
FB		 2813static void io_mem_init(void)
2814{
3a7d929e 2815 cpu_register_io_memory(IO_MEM_ROM >> IO_MEM_SHIFT, error_mem_read, unassigned_mem_write, NULL);
a4193c8a 2816 cpu_register_io_memory(IO_MEM_UNASSIGNED >> IO_MEM_SHIFT, unassigned_mem_read, unassigned_mem_write, NULL);
3a7d929e 2817 cpu_register_io_memory(IO_MEM_NOTDIRTY >> IO_MEM_SHIFT, error_mem_read, notdirty_mem_write, NULL);
1ccde1cb
FB		 2818 io_mem_nb = 5;
2819
0f459d16 2820 io_mem_watch = cpu_register_io_memory(0, watch_mem_read,
6658ffb8 2821 watch_mem_write, NULL);
1ccde1cb 2822 /* alloc dirty bits array */
0a962c02 2823 phys_ram_dirty = qemu_vmalloc(phys_ram_size >> TARGET_PAGE_BITS);
3a7d929e 2824 memset(phys_ram_dirty, 0xff, phys_ram_size >> TARGET_PAGE_BITS);
33417e70
FB		 2825}
2826
2827/* mem_read and mem_write are arrays of functions containing the
2828 function to access byte (index 0), word (index 1) and dword (index
3ee89922
BS		 2829 2). Functions can be omitted with a NULL function pointer. The
2830 registered functions may be modified dynamically later.
2831 If io_index is non zero, the corresponding io zone is
4254fab8
BS		 2832 modified. If it is zero, a new io zone is allocated. The return
2833 value can be used with cpu_register_physical_memory(). (-1) is
2834 returned if error. */
33417e70
FB		 2835int cpu_register_io_memory(int io_index,
2836 CPUReadMemoryFunc **mem_read,
a4193c8a
FB		 2837 CPUWriteMemoryFunc **mem_write,
2838 void *opaque)
33417e70 2839{
4254fab8 2840 int i, subwidth = 0;
33417e70
FB		 2841
2842 if (io_index <= 0) {
b5ff1b31 2843 if (io_mem_nb >= IO_MEM_NB_ENTRIES)
33417e70
FB		 2844 return -1;
2845 io_index = io_mem_nb++;
2846 } else {
2847 if (io_index >= IO_MEM_NB_ENTRIES)
2848 return -1;
2849 }
b5ff1b31 2850
33417e70 2851 for(i = 0;i < 3; i++) {
4254fab8
BS		 2852 if (!mem_read[i] || !mem_write[i])
2853 subwidth = IO_MEM_SUBWIDTH;
33417e70
FB		 2854 io_mem_read[io_index][i] = mem_read[i];
2855 io_mem_write[io_index][i] = mem_write[i];
2856 }
a4193c8a 2857 io_mem_opaque[io_index] = opaque;
4254fab8 2858 return (io_index << IO_MEM_SHIFT) | subwidth;
33417e70 2859}
61382a50 2860
8926b517
FB		 2861CPUWriteMemoryFunc **cpu_get_io_memory_write(int io_index)
2862{
2863 return io_mem_write[io_index >> IO_MEM_SHIFT];
2864}
2865
2866CPUReadMemoryFunc **cpu_get_io_memory_read(int io_index)
2867{
2868 return io_mem_read[io_index >> IO_MEM_SHIFT];
2869}
2870
e2eef170
PB		 2871#endif /* !defined(CONFIG_USER_ONLY) */
2872
13eb76e0
FB		 2873/* physical memory access (slow version, mainly for debug) */
2874#if defined(CONFIG_USER_ONLY)
5fafdf24 2875void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
13eb76e0
FB		 2876 int len, int is_write)
2877{
2878 int l, flags;
2879 target_ulong page;
53a5960a 2880 void * p;
13eb76e0
FB		 2881
2882 while (len > 0) {
2883 page = addr & TARGET_PAGE_MASK;
2884 l = (page + TARGET_PAGE_SIZE) - addr;
2885 if (l > len)
2886 l = len;
2887 flags = page_get_flags(page);
2888 if (!(flags & PAGE_VALID))
2889 return;
2890 if (is_write) {
2891 if (!(flags & PAGE_WRITE))
2892 return;
579a97f7 2893 /* XXX: this code should not depend on lock_user */
72fb7daa 2894 if (!(p = lock_user(VERIFY_WRITE, addr, l, 0)))
579a97f7
FB		 2895 /* FIXME - should this return an error rather than just fail? */
2896 return;
72fb7daa
AJ		 2897 memcpy(p, buf, l);
2898 unlock_user(p, addr, l);
13eb76e0
FB		 2899 } else {
2900 if (!(flags & PAGE_READ))
2901 return;
579a97f7 2902 /* XXX: this code should not depend on lock_user */
72fb7daa 2903 if (!(p = lock_user(VERIFY_READ, addr, l, 1)))
579a97f7
FB		 2904 /* FIXME - should this return an error rather than just fail? */
2905 return;
72fb7daa 2906 memcpy(buf, p, l);
5b257578 2907 unlock_user(p, addr, 0);
13eb76e0
FB		 2908 }
2909 len -= l;
2910 buf += l;
2911 addr += l;
2912 }
2913}
8df1cd07 2914
13eb76e0 2915#else
5fafdf24 2916void cpu_physical_memory_rw(target_phys_addr_t addr, uint8_t *buf,
13eb76e0
FB		 2917 int len, int is_write)
2918{
2919 int l, io_index;
2920 uint8_t *ptr;
2921 uint32_t val;
2e12669a
FB		 2922 target_phys_addr_t page;
2923 unsigned long pd;
92e873b9 2924 PhysPageDesc *p;
3b46e624 2925
13eb76e0
FB		 2926 while (len > 0) {
2927 page = addr & TARGET_PAGE_MASK;
2928 l = (page + TARGET_PAGE_SIZE) - addr;
2929 if (l > len)
2930 l = len;
92e873b9 2931 p = phys_page_find(page >> TARGET_PAGE_BITS);
13eb76e0
FB		 2932 if (!p) {
2933 pd = IO_MEM_UNASSIGNED;
2934 } else {
2935 pd = p->phys_offset;
2936 }
3b46e624 2937
13eb76e0 2938 if (is_write) {
3a7d929e 2939 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
13eb76e0 2940 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 2941 if (p)
2942 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
6a00d601
FB		 2943 /* XXX: could force cpu_single_env to NULL to avoid
2944 potential bugs */
13eb76e0 2945 if (l >= 4 && ((addr & 3) == 0)) {
1c213d19 2946 /* 32 bit write access */
c27004ec 2947 val = ldl_p(buf);
a4193c8a 2948 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
13eb76e0
FB		 2949 l = 4;
2950 } else if (l >= 2 && ((addr & 1) == 0)) {
1c213d19 2951 /* 16 bit write access */
c27004ec 2952 val = lduw_p(buf);
a4193c8a 2953 io_mem_write[io_index][1](io_mem_opaque[io_index], addr, val);
13eb76e0
FB		 2954 l = 2;
2955 } else {
1c213d19 2956 /* 8 bit write access */
c27004ec 2957 val = ldub_p(buf);
a4193c8a 2958 io_mem_write[io_index][0](io_mem_opaque[io_index], addr, val);
13eb76e0
FB		 2959 l = 1;
2960 }
2961 } else {
b448f2f3
FB		 2962 unsigned long addr1;
2963 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
13eb76e0 2964 /* RAM case */
b448f2f3 2965 ptr = phys_ram_base + addr1;
13eb76e0 2966 memcpy(ptr, buf, l);
3a7d929e
FB		 2967 if (!cpu_physical_memory_is_dirty(addr1)) {
2968 /* invalidate code */
2969 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
2970 /* set dirty bit */
5fafdf24 2971 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
f23db169 2972 (0xff & ~CODE_DIRTY_FLAG);
3a7d929e 2973 }
13eb76e0
FB		 2974 }
2975 } else {
5fafdf24 2976 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2a4188a3 2977 !(pd & IO_MEM_ROMD)) {
13eb76e0
FB		 2978 /* I/O case */
2979 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 2980 if (p)
2981 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
13eb76e0
FB		 2982 if (l >= 4 && ((addr & 3) == 0)) {
2983 /* 32 bit read access */
a4193c8a 2984 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
c27004ec 2985 stl_p(buf, val);
13eb76e0
FB		 2986 l = 4;
2987 } else if (l >= 2 && ((addr & 1) == 0)) {
2988 /* 16 bit read access */
a4193c8a 2989 val = io_mem_read[io_index][1](io_mem_opaque[io_index], addr);
c27004ec 2990 stw_p(buf, val);
13eb76e0
FB		 2991 l = 2;
2992 } else {
1c213d19 2993 /* 8 bit read access */
a4193c8a 2994 val = io_mem_read[io_index][0](io_mem_opaque[io_index], addr);
c27004ec 2995 stb_p(buf, val);
13eb76e0
FB		 2996 l = 1;
2997 }
2998 } else {
2999 /* RAM case */
5fafdf24 3000 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
13eb76e0
FB		 3001 (addr & ~TARGET_PAGE_MASK);
3002 memcpy(buf, ptr, l);
3003 }
3004 }
3005 len -= l;
3006 buf += l;
3007 addr += l;
3008 }
3009}
8df1cd07 3010
d0ecd2aa 3011/* used for ROM loading : can write in RAM and ROM */
5fafdf24 3012void cpu_physical_memory_write_rom(target_phys_addr_t addr,
d0ecd2aa
FB		 3013 const uint8_t *buf, int len)
3014{
3015 int l;
3016 uint8_t *ptr;
3017 target_phys_addr_t page;
3018 unsigned long pd;
3019 PhysPageDesc *p;
3b46e624 3020
d0ecd2aa
FB		 3021 while (len > 0) {
3022 page = addr & TARGET_PAGE_MASK;
3023 l = (page + TARGET_PAGE_SIZE) - addr;
3024 if (l > len)
3025 l = len;
3026 p = phys_page_find(page >> TARGET_PAGE_BITS);
3027 if (!p) {
3028 pd = IO_MEM_UNASSIGNED;
3029 } else {
3030 pd = p->phys_offset;
3031 }
3b46e624 3032
d0ecd2aa 3033 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM &&
2a4188a3
FB		 3034 (pd & ~TARGET_PAGE_MASK) != IO_MEM_ROM &&
3035 !(pd & IO_MEM_ROMD)) {
d0ecd2aa
FB		 3036 /* do nothing */
3037 } else {
3038 unsigned long addr1;
3039 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3040 /* ROM/RAM case */
3041 ptr = phys_ram_base + addr1;
3042 memcpy(ptr, buf, l);
3043 }
3044 len -= l;
3045 buf += l;
3046 addr += l;
3047 }
3048}
3049
6d16c2f8
AL		 3050typedef struct {
3051 void *buffer;
3052 target_phys_addr_t addr;
3053 target_phys_addr_t len;
3054} BounceBuffer;
3055
3056static BounceBuffer bounce;
3057
ba223c29
AL		 3058typedef struct MapClient {
3059 void *opaque;
3060 void (*callback)(void *opaque);
3061 LIST_ENTRY(MapClient) link;
3062} MapClient;
3063
3064static LIST_HEAD(map_client_list, MapClient) map_client_list
3065 = LIST_HEAD_INITIALIZER(map_client_list);
3066
3067void *cpu_register_map_client(void *opaque, void (*callback)(void *opaque))
3068{
3069 MapClient *client = qemu_malloc(sizeof(*client));
3070
3071 client->opaque = opaque;
3072 client->callback = callback;
3073 LIST_INSERT_HEAD(&map_client_list, client, link);
3074 return client;
3075}
3076
3077void cpu_unregister_map_client(void *_client)
3078{
3079 MapClient *client = (MapClient *)_client;
3080
3081 LIST_REMOVE(client, link);
3082}
3083
3084static void cpu_notify_map_clients(void)
3085{
3086 MapClient *client;
3087
3088 while (!LIST_EMPTY(&map_client_list)) {
3089 client = LIST_FIRST(&map_client_list);
3090 client->callback(client->opaque);
3091 LIST_REMOVE(client, link);
3092 }
3093}
3094
6d16c2f8
AL		 3095/* Map a physical memory region into a host virtual address.
3096 * May map a subset of the requested range, given by and returned in *plen.
3097 * May return NULL if resources needed to perform the mapping are exhausted.
3098 * Use only for reads OR writes - not for read-modify-write operations.
ba223c29
AL		 3099 * Use cpu_register_map_client() to know when retrying the map operation is
3100 * likely to succeed.
6d16c2f8
AL		 3101 */
3102void *cpu_physical_memory_map(target_phys_addr_t addr,
3103 target_phys_addr_t *plen,
3104 int is_write)
3105{
3106 target_phys_addr_t len = *plen;
3107 target_phys_addr_t done = 0;
3108 int l;
3109 uint8_t *ret = NULL;
3110 uint8_t *ptr;
3111 target_phys_addr_t page;
3112 unsigned long pd;
3113 PhysPageDesc *p;
3114 unsigned long addr1;
3115
3116 while (len > 0) {
3117 page = addr & TARGET_PAGE_MASK;
3118 l = (page + TARGET_PAGE_SIZE) - addr;
3119 if (l > len)
3120 l = len;
3121 p = phys_page_find(page >> TARGET_PAGE_BITS);
3122 if (!p) {
3123 pd = IO_MEM_UNASSIGNED;
3124 } else {
3125 pd = p->phys_offset;
3126 }
3127
3128 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3129 if (done || bounce.buffer) {
3130 break;
3131 }
3132 bounce.buffer = qemu_memalign(TARGET_PAGE_SIZE, TARGET_PAGE_SIZE);
3133 bounce.addr = addr;
3134 bounce.len = l;
3135 if (!is_write) {
3136 cpu_physical_memory_rw(addr, bounce.buffer, l, 0);
3137 }
3138 ptr = bounce.buffer;
3139 } else {
3140 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3141 ptr = phys_ram_base + addr1;
3142 }
3143 if (!done) {
3144 ret = ptr;
3145 } else if (ret + done != ptr) {
3146 break;
3147 }
3148
3149 len -= l;
3150 addr += l;
3151 done += l;
3152 }
3153 *plen = done;
3154 return ret;
3155}
3156
3157/* Unmaps a memory region previously mapped by cpu_physical_memory_map().
3158 * Will also mark the memory as dirty if is_write == 1. access_len gives
3159 * the amount of memory that was actually read or written by the caller.
3160 */
3161void cpu_physical_memory_unmap(void *buffer, target_phys_addr_t len,
3162 int is_write, target_phys_addr_t access_len)
3163{
3164 if (buffer != bounce.buffer) {
3165 if (is_write) {
3166 unsigned long addr1 = (uint8_t *)buffer - phys_ram_base;
3167 while (access_len) {
3168 unsigned l;
3169 l = TARGET_PAGE_SIZE;
3170 if (l > access_len)
3171 l = access_len;
3172 if (!cpu_physical_memory_is_dirty(addr1)) {
3173 /* invalidate code */
3174 tb_invalidate_phys_page_range(addr1, addr1 + l, 0);
3175 /* set dirty bit */
3176 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3177 (0xff & ~CODE_DIRTY_FLAG);
3178 }
3179 addr1 += l;
3180 access_len -= l;
3181 }
3182 }
3183 return;
3184 }
3185 if (is_write) {
3186 cpu_physical_memory_write(bounce.addr, bounce.buffer, access_len);
3187 }
3188 qemu_free(bounce.buffer);
3189 bounce.buffer = NULL;
ba223c29 3190 cpu_notify_map_clients();
6d16c2f8 3191}
d0ecd2aa 3192
8df1cd07
FB		 3193/* warning: addr must be aligned */
3194uint32_t ldl_phys(target_phys_addr_t addr)
3195{
3196 int io_index;
3197 uint8_t *ptr;
3198 uint32_t val;
3199 unsigned long pd;
3200 PhysPageDesc *p;
3201
3202 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3203 if (!p) {
3204 pd = IO_MEM_UNASSIGNED;
3205 } else {
3206 pd = p->phys_offset;
3207 }
3b46e624 3208
5fafdf24 3209 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
2a4188a3 3210 !(pd & IO_MEM_ROMD)) {
8df1cd07
FB		 3211 /* I/O case */
3212 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 3213 if (p)
3214 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
8df1cd07
FB		 3215 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3216 } else {
3217 /* RAM case */
5fafdf24 3218 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
8df1cd07
FB		 3219 (addr & ~TARGET_PAGE_MASK);
3220 val = ldl_p(ptr);
3221 }
3222 return val;
3223}
3224
84b7b8e7
FB		 3225/* warning: addr must be aligned */
3226uint64_t ldq_phys(target_phys_addr_t addr)
3227{
3228 int io_index;
3229 uint8_t *ptr;
3230 uint64_t val;
3231 unsigned long pd;
3232 PhysPageDesc *p;
3233
3234 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3235 if (!p) {
3236 pd = IO_MEM_UNASSIGNED;
3237 } else {
3238 pd = p->phys_offset;
3239 }
3b46e624 3240
2a4188a3
FB		 3241 if ((pd & ~TARGET_PAGE_MASK) > IO_MEM_ROM &&
3242 !(pd & IO_MEM_ROMD)) {
84b7b8e7
FB		 3243 /* I/O case */
3244 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 3245 if (p)
3246 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
84b7b8e7
FB		 3247#ifdef TARGET_WORDS_BIGENDIAN
3248 val = (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr) << 32;
3249 val |= io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4);
3250#else
3251 val = io_mem_read[io_index][2](io_mem_opaque[io_index], addr);
3252 val |= (uint64_t)io_mem_read[io_index][2](io_mem_opaque[io_index], addr + 4) << 32;
3253#endif
3254 } else {
3255 /* RAM case */
5fafdf24 3256 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
84b7b8e7
FB		 3257 (addr & ~TARGET_PAGE_MASK);
3258 val = ldq_p(ptr);
3259 }
3260 return val;
3261}
3262
aab33094
FB		 3263/* XXX: optimize */
3264uint32_t ldub_phys(target_phys_addr_t addr)
3265{
3266 uint8_t val;
3267 cpu_physical_memory_read(addr, &val, 1);
3268 return val;
3269}
3270
3271/* XXX: optimize */
3272uint32_t lduw_phys(target_phys_addr_t addr)
3273{
3274 uint16_t val;
3275 cpu_physical_memory_read(addr, (uint8_t *)&val, 2);
3276 return tswap16(val);
3277}
3278
8df1cd07
FB		 3279/* warning: addr must be aligned. The ram page is not masked as dirty
3280 and the code inside is not invalidated. It is useful if the dirty
3281 bits are used to track modified PTEs */
3282void stl_phys_notdirty(target_phys_addr_t addr, uint32_t val)
3283{
3284 int io_index;
3285 uint8_t *ptr;
3286 unsigned long pd;
3287 PhysPageDesc *p;
3288
3289 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3290 if (!p) {
3291 pd = IO_MEM_UNASSIGNED;
3292 } else {
3293 pd = p->phys_offset;
3294 }
3b46e624 3295
3a7d929e 3296 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
8df1cd07 3297 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 3298 if (p)
3299 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
8df1cd07
FB		 3300 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3301 } else {
74576198
AL		 3302 unsigned long addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3303 ptr = phys_ram_base + addr1;
8df1cd07 3304 stl_p(ptr, val);
74576198
AL		 3305
3306 if (unlikely(in_migration)) {
3307 if (!cpu_physical_memory_is_dirty(addr1)) {
3308 /* invalidate code */
3309 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3310 /* set dirty bit */
3311 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3312 (0xff & ~CODE_DIRTY_FLAG);
3313 }
3314 }
8df1cd07
FB		 3315 }
3316}
3317
bc98a7ef
JM		 3318void stq_phys_notdirty(target_phys_addr_t addr, uint64_t val)
3319{
3320 int io_index;
3321 uint8_t *ptr;
3322 unsigned long pd;
3323 PhysPageDesc *p;
3324
3325 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3326 if (!p) {
3327 pd = IO_MEM_UNASSIGNED;
3328 } else {
3329 pd = p->phys_offset;
3330 }
3b46e624 3331
bc98a7ef
JM		 3332 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
3333 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 3334 if (p)
3335 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
bc98a7ef
JM		 3336#ifdef TARGET_WORDS_BIGENDIAN
3337 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val >> 32);
3338 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val);
3339#else
3340 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3341 io_mem_write[io_index][2](io_mem_opaque[io_index], addr + 4, val >> 32);
3342#endif
3343 } else {
5fafdf24 3344 ptr = phys_ram_base + (pd & TARGET_PAGE_MASK) +
bc98a7ef
JM		 3345 (addr & ~TARGET_PAGE_MASK);
3346 stq_p(ptr, val);
3347 }
3348}
3349
8df1cd07 3350/* warning: addr must be aligned */
8df1cd07
FB		 3351void stl_phys(target_phys_addr_t addr, uint32_t val)
3352{
3353 int io_index;
3354 uint8_t *ptr;
3355 unsigned long pd;
3356 PhysPageDesc *p;
3357
3358 p = phys_page_find(addr >> TARGET_PAGE_BITS);
3359 if (!p) {
3360 pd = IO_MEM_UNASSIGNED;
3361 } else {
3362 pd = p->phys_offset;
3363 }
3b46e624 3364
3a7d929e 3365 if ((pd & ~TARGET_PAGE_MASK) != IO_MEM_RAM) {
8df1cd07 3366 io_index = (pd >> IO_MEM_SHIFT) & (IO_MEM_NB_ENTRIES - 1);
8da3ff18
PB		 3367 if (p)
3368 addr = (addr & ~TARGET_PAGE_MASK) + p->region_offset;
8df1cd07
FB		 3369 io_mem_write[io_index][2](io_mem_opaque[io_index], addr, val);
3370 } else {
3371 unsigned long addr1;
3372 addr1 = (pd & TARGET_PAGE_MASK) + (addr & ~TARGET_PAGE_MASK);
3373 /* RAM case */
3374 ptr = phys_ram_base + addr1;
3375 stl_p(ptr, val);
3a7d929e
FB		 3376 if (!cpu_physical_memory_is_dirty(addr1)) {
3377 /* invalidate code */
3378 tb_invalidate_phys_page_range(addr1, addr1 + 4, 0);
3379 /* set dirty bit */
f23db169
FB		 3380 phys_ram_dirty[addr1 >> TARGET_PAGE_BITS] |=
3381 (0xff & ~CODE_DIRTY_FLAG);
3a7d929e 3382 }
8df1cd07
FB		 3383 }
3384}
3385
aab33094
FB		 3386/* XXX: optimize */
3387void stb_phys(target_phys_addr_t addr, uint32_t val)
3388{
3389 uint8_t v = val;
3390 cpu_physical_memory_write(addr, &v, 1);
3391}
3392
3393/* XXX: optimize */
3394void stw_phys(target_phys_addr_t addr, uint32_t val)
3395{
3396 uint16_t v = tswap16(val);
3397 cpu_physical_memory_write(addr, (const uint8_t *)&v, 2);
3398}
3399
3400/* XXX: optimize */
3401void stq_phys(target_phys_addr_t addr, uint64_t val)
3402{
3403 val = tswap64(val);
3404 cpu_physical_memory_write(addr, (const uint8_t *)&val, 8);
3405}
3406
13eb76e0
FB		 3407#endif
3408
3409/* virtual memory access for debug */
5fafdf24 3410int cpu_memory_rw_debug(CPUState *env, target_ulong addr,
b448f2f3 3411 uint8_t *buf, int len, int is_write)
13eb76e0
FB		 3412{
3413 int l;
9b3c35e0
JM		 3414 target_phys_addr_t phys_addr;
3415 target_ulong page;
13eb76e0
FB		 3416
3417 while (len > 0) {
3418 page = addr & TARGET_PAGE_MASK;
3419 phys_addr = cpu_get_phys_page_debug(env, page);
3420 /* if no physical page mapped, return an error */
3421 if (phys_addr == -1)
3422 return -1;
3423 l = (page + TARGET_PAGE_SIZE) - addr;
3424 if (l > len)
3425 l = len;
5fafdf24 3426 cpu_physical_memory_rw(phys_addr + (addr & ~TARGET_PAGE_MASK),
b448f2f3 3427 buf, l, is_write);
13eb76e0
FB		 3428 len -= l;
3429 buf += l;
3430 addr += l;
3431 }
3432 return 0;
3433}
3434
2e70f6ef
PB		 3435/* in deterministic execution mode, instructions doing device I/Os
3436 must be at the end of the TB */
3437void cpu_io_recompile(CPUState *env, void *retaddr)
3438{
3439 TranslationBlock *tb;
3440 uint32_t n, cflags;
3441 target_ulong pc, cs_base;
3442 uint64_t flags;
3443
3444 tb = tb_find_pc((unsigned long)retaddr);
3445 if (!tb) {
3446 cpu_abort(env, "cpu_io_recompile: could not find TB for pc=%p",
3447 retaddr);
3448 }
3449 n = env->icount_decr.u16.low + tb->icount;
3450 cpu_restore_state(tb, env, (unsigned long)retaddr, NULL);
3451 /* Calculate how many instructions had been executed before the fault
bf20dc07 3452 occurred. */
2e70f6ef
PB		 3453 n = n - env->icount_decr.u16.low;
3454 /* Generate a new TB ending on the I/O insn. */
3455 n++;
3456 /* On MIPS and SH, delay slot instructions can only be restarted if
3457 they were already the first instruction in the TB. If this is not
bf20dc07 3458 the first instruction in a TB then re-execute the preceding
2e70f6ef
PB		 3459 branch. */
3460#if defined(TARGET_MIPS)
3461 if ((env->hflags & MIPS_HFLAG_BMASK) != 0 && n > 1) {
3462 env->active_tc.PC -= 4;
3463 env->icount_decr.u16.low++;
3464 env->hflags &= ~MIPS_HFLAG_BMASK;
3465 }
3466#elif defined(TARGET_SH4)
3467 if ((env->flags & ((DELAY_SLOT | DELAY_SLOT_CONDITIONAL))) != 0
3468 && n > 1) {
3469 env->pc -= 2;
3470 env->icount_decr.u16.low++;
3471 env->flags &= ~(DELAY_SLOT | DELAY_SLOT_CONDITIONAL);
3472 }
3473#endif
3474 /* This should never happen. */
3475 if (n > CF_COUNT_MASK)
3476 cpu_abort(env, "TB too big during recompile");
3477
3478 cflags = n | CF_LAST_IO;
3479 pc = tb->pc;
3480 cs_base = tb->cs_base;
3481 flags = tb->flags;
3482 tb_phys_invalidate(tb, -1);
3483 /* FIXME: In theory this could raise an exception. In practice
3484 we have already translated the block once so it's probably ok. */
3485 tb_gen_code(env, pc, cs_base, flags, cflags);
bf20dc07 3486 /* TODO: If env->pc != tb->pc (i.e. the faulting instruction was not
2e70f6ef
PB		 3487 the first in the TB) then we end up generating a whole new TB and
3488 repeating the fault, which is horribly inefficient.
3489 Better would be to execute just this insn uncached, or generate a
3490 second new TB. */
3491 cpu_resume_from_signal(env, NULL);
3492}
3493
e3db7226
FB		 3494void dump_exec_info(FILE *f,
3495 int (*cpu_fprintf)(FILE *f, const char *fmt, ...))
3496{
3497 int i, target_code_size, max_target_code_size;
3498 int direct_jmp_count, direct_jmp2_count, cross_page;
3499 TranslationBlock *tb;
3b46e624 3500
e3db7226
FB		 3501 target_code_size = 0;
3502 max_target_code_size = 0;
3503 cross_page = 0;
3504 direct_jmp_count = 0;
3505 direct_jmp2_count = 0;
3506 for(i = 0; i < nb_tbs; i++) {
3507 tb = &tbs[i];
3508 target_code_size += tb->size;
3509 if (tb->size > max_target_code_size)
3510 max_target_code_size = tb->size;
3511 if (tb->page_addr[1] != -1)
3512 cross_page++;
3513 if (tb->tb_next_offset[0] != 0xffff) {
3514 direct_jmp_count++;
3515 if (tb->tb_next_offset[1] != 0xffff) {
3516 direct_jmp2_count++;
3517 }
3518 }
3519 }
3520 /* XXX: avoid using doubles ? */
57fec1fe 3521 cpu_fprintf(f, "Translation buffer state:\n");
26a5f13b
FB		 3522 cpu_fprintf(f, "gen code size %ld/%ld\n",
3523 code_gen_ptr - code_gen_buffer, code_gen_buffer_max_size);
3524 cpu_fprintf(f, "TB count %d/%d\n",
3525 nb_tbs, code_gen_max_blocks);
5fafdf24 3526 cpu_fprintf(f, "TB avg target size %d max=%d bytes\n",
e3db7226
FB		 3527 nb_tbs ? target_code_size / nb_tbs : 0,
3528 max_target_code_size);
5fafdf24 3529 cpu_fprintf(f, "TB avg host size %d bytes (expansion ratio: %0.1f)\n",
e3db7226
FB		 3530 nb_tbs ? (code_gen_ptr - code_gen_buffer) / nb_tbs : 0,
3531 target_code_size ? (double) (code_gen_ptr - code_gen_buffer) / target_code_size : 0);
5fafdf24
TS		 3532 cpu_fprintf(f, "cross page TB count %d (%d%%)\n",
3533 cross_page,
e3db7226
FB		 3534 nb_tbs ? (cross_page * 100) / nb_tbs : 0);
3535 cpu_fprintf(f, "direct jump count %d (%d%%) (2 jumps=%d %d%%)\n",
5fafdf24 3536 direct_jmp_count,
e3db7226
FB		 3537 nb_tbs ? (direct_jmp_count * 100) / nb_tbs : 0,
3538 direct_jmp2_count,
3539 nb_tbs ? (direct_jmp2_count * 100) / nb_tbs : 0);
57fec1fe 3540 cpu_fprintf(f, "\nStatistics:\n");
e3db7226
FB		 3541 cpu_fprintf(f, "TB flush count %d\n", tb_flush_count);
3542 cpu_fprintf(f, "TB invalidate count %d\n", tb_phys_invalidate_count);
3543 cpu_fprintf(f, "TLB flush count %d\n", tlb_flush_count);
b67d9a52 3544 tcg_dump_info(f, cpu_fprintf);
e3db7226
FB		 3545}
3546
5fafdf24 3547#if !defined(CONFIG_USER_ONLY)
61382a50
FB		 3548
3549#define MMUSUFFIX _cmmu
3550#define GETPC() NULL
3551#define env cpu_single_env
b769d8fe 3552#define SOFTMMU_CODE_ACCESS
61382a50
FB		 3553
3554#define SHIFT 0
3555#include "softmmu_template.h"
3556
3557#define SHIFT 1
3558#include "softmmu_template.h"
3559
3560#define SHIFT 2
3561#include "softmmu_template.h"
3562
3563#define SHIFT 3
3564#include "softmmu_template.h"
3565
3566#undef env
3567
3568#endif
Empty description
This page took 0.782215 seconds and 4 git commands to generate.