X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/15417787212f321efa0592d64b89d6b189c58b41..c61177881cbda50704207dd9fb4811659bbf913e:/target/s390x/mem_helper.c diff --git a/target/s390x/mem_helper.c b/target/s390x/mem_helper.c index 402147e708..e21a47fb4d 100644 --- a/target/s390x/mem_helper.c +++ b/target/s390x/mem_helper.c @@ -20,7 +20,7 @@ #include "qemu/osdep.h" #include "cpu.h" -#include "exec/address-spaces.h" +#include "internal.h" #include "exec/helper-proto.h" #include "exec/exec-all.h" #include "exec/cpu_ldst.h" @@ -38,10 +38,10 @@ NULL, it means that the function was called in C code (i.e. not from generated code or from helper.c) */ /* XXX: fix it to restore all registers */ -void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type, - int mmu_idx, uintptr_t retaddr) +void tlb_fill(CPUState *cs, target_ulong addr, int size, + MMUAccessType access_type, int mmu_idx, uintptr_t retaddr) { - int ret = s390_cpu_handle_mmu_fault(cs, addr, access_type, mmu_idx); + int ret = s390_cpu_handle_mmu_fault(cs, addr, size, access_type, mmu_idx); if (unlikely(ret != 0)) { cpu_loop_exit_restore(cs, retaddr); } @@ -56,6 +56,17 @@ void tlb_fill(CPUState *cs, target_ulong addr, MMUAccessType access_type, #define HELPER_LOG(x...) #endif +static inline bool psw_key_valid(CPUS390XState *env, uint8_t psw_key) +{ + uint16_t pkm = env->cregs[3] >> 16; + + if (env->psw.mask & PSW_MASK_PSTATE) { + /* PSW key has range 0..15, it is valid if the bit is 1 in the PKM */ + return pkm & (0x80 >> psw_key); + } + return true; +} + /* Reduce the length so that addr + len doesn't cross a page boundary. */ static inline uint32_t adj_len_to_page(uint32_t len, uint64_t addr) { @@ -73,9 +84,7 @@ static inline void check_alignment(CPUS390XState *env, uint64_t v, int wordsize, uintptr_t ra) { if (v % wordsize) { - CPUState *cs = CPU(s390_env_get_cpu(env)); - cpu_restore_state(cs, ra); - program_interrupt(env, PGM_SPECIFICATION, 6); + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); } } @@ -133,6 +142,68 @@ static void fast_memset(CPUS390XState *env, uint64_t dest, uint8_t byte, } } +#ifndef CONFIG_USER_ONLY +static void fast_memmove_idx(CPUS390XState *env, uint64_t dest, uint64_t src, + uint32_t len, int dest_idx, int src_idx, + uintptr_t ra) +{ + TCGMemOpIdx oi_dest = make_memop_idx(MO_UB, dest_idx); + TCGMemOpIdx oi_src = make_memop_idx(MO_UB, src_idx); + uint32_t len_adj; + void *src_p; + void *dest_p; + uint8_t x; + + while (len > 0) { + src = wrap_address(env, src); + dest = wrap_address(env, dest); + src_p = tlb_vaddr_to_host(env, src, MMU_DATA_LOAD, src_idx); + dest_p = tlb_vaddr_to_host(env, dest, MMU_DATA_STORE, dest_idx); + + if (src_p && dest_p) { + /* Access to both whole pages granted. */ + len_adj = adj_len_to_page(adj_len_to_page(len, src), dest); + memmove(dest_p, src_p, len_adj); + } else { + /* We failed to get access to one or both whole pages. The next + read or write access will likely fill the QEMU TLB for the + next iteration. */ + len_adj = 1; + x = helper_ret_ldub_mmu(env, src, oi_src, ra); + helper_ret_stb_mmu(env, dest, x, oi_dest, ra); + } + src += len_adj; + dest += len_adj; + len -= len_adj; + } +} + +static int mmu_idx_from_as(uint8_t as) +{ + switch (as) { + case AS_PRIMARY: + return MMU_PRIMARY_IDX; + case AS_SECONDARY: + return MMU_SECONDARY_IDX; + case AS_HOME: + return MMU_HOME_IDX; + default: + /* FIXME AS_ACCREG */ + g_assert_not_reached(); + } +} + +static void fast_memmove_as(CPUS390XState *env, uint64_t dest, uint64_t src, + uint32_t len, uint8_t dest_as, uint8_t src_as, + uintptr_t ra) +{ + int src_idx = mmu_idx_from_as(src_as); + int dest_idx = mmu_idx_from_as(dest_as); + + fast_memmove_idx(env, dest, src, len, dest_idx, src_idx, ra); +} +#endif + static void fast_memmove(CPUS390XState *env, uint64_t dest, uint64_t src, uint32_t l, uintptr_t ra) { @@ -408,20 +479,6 @@ uint32_t HELPER(clm)(CPUS390XState *env, uint32_t r1, uint32_t mask, return cc; } -static inline uint64_t wrap_address(CPUS390XState *env, uint64_t a) -{ - if (!(env->psw.mask & PSW_MASK_64)) { - if (!(env->psw.mask & PSW_MASK_32)) { - /* 24-Bit mode */ - a &= 0x00ffffff; - } else { - /* 31-Bit mode */ - a &= 0x7fffffff; - } - } - return a; -} - static inline uint64_t get_address(CPUS390XState *env, int reg) { return wrap_address(env, env->regs[reg]); @@ -476,18 +533,20 @@ static inline void set_length(CPUS390XState *env, int reg, uint64_t length) } /* search string (c is byte to search, r2 is string, r1 end of string) */ -uint64_t HELPER(srst)(CPUS390XState *env, uint64_t r0, uint64_t end, - uint64_t str) +void HELPER(srst)(CPUS390XState *env, uint32_t r1, uint32_t r2) { uintptr_t ra = GETPC(); + uint64_t end, str; uint32_t len; - uint8_t v, c = r0; + uint8_t v, c = env->regs[0]; - str = wrap_address(env, str); - end = wrap_address(env, end); + /* Bits 32-55 must contain all 0. */ + if (env->regs[0] & 0xffffff00u) { + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); + } - /* Assume for now that R2 is unmodified. */ - env->retxl = str; + str = get_address(env, r2); + end = get_address(env, r1); /* Lest we fail to service interrupts in a timely manner, limit the amount of work we're willing to do. For now, let's cap at 8k. */ @@ -495,20 +554,60 @@ uint64_t HELPER(srst)(CPUS390XState *env, uint64_t r0, uint64_t end, if (str + len == end) { /* Character not found. R1 & R2 are unmodified. */ env->cc_op = 2; - return end; + return; } v = cpu_ldub_data_ra(env, str + len, ra); if (v == c) { /* Character found. Set R1 to the location; R2 is unmodified. */ env->cc_op = 1; - return str + len; + set_address(env, r1, str + len); + return; + } + } + + /* CPU-determined bytes processed. Advance R2 to next byte to process. */ + env->cc_op = 3; + set_address(env, r2, str + len); +} + +void HELPER(srstu)(CPUS390XState *env, uint32_t r1, uint32_t r2) +{ + uintptr_t ra = GETPC(); + uint32_t len; + uint16_t v, c = env->regs[0]; + uint64_t end, str, adj_end; + + /* Bits 32-47 of R0 must be zero. */ + if (env->regs[0] & 0xffff0000u) { + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); + } + + str = get_address(env, r2); + end = get_address(env, r1); + + /* If the LSB of the two addresses differ, use one extra byte. */ + adj_end = end + ((str ^ end) & 1); + + /* Lest we fail to service interrupts in a timely manner, limit the + amount of work we're willing to do. For now, let's cap at 8k. */ + for (len = 0; len < 0x2000; len += 2) { + if (str + len == adj_end) { + /* End of input found. */ + env->cc_op = 2; + return; + } + v = cpu_lduw_data_ra(env, str + len, ra); + if (v == c) { + /* Character found. Set R1 to the location; R2 is unmodified. */ + env->cc_op = 1; + set_address(env, r1, str + len); + return; } } /* CPU-determined bytes processed. Advance R2 to next byte to process. */ - env->retxl = str + len; env->cc_op = 3; - return end; + set_address(env, r2, str + len); } /* unsigned string compare (c is string terminator) */ @@ -593,6 +692,11 @@ void HELPER(lam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uintptr_t ra = GETPC(); int i; + if (a2 & 0x3) { + /* we either came here by lam or lamy, which have different lengths */ + s390_program_interrupt(env, PGM_SPECIFICATION, ILEN_AUTO, ra); + } + for (i = r1;; i = (i + 1) % 16) { env->aregs[i] = cpu_ldl_data_ra(env, a2, ra); a2 += 4; @@ -609,6 +713,10 @@ void HELPER(stam)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uintptr_t ra = GETPC(); int i; + if (a2 & 0x3) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } + for (i = r1;; i = (i + 1) % 16) { cpu_stl_data_ra(env, a2, env->aregs[i], ra); a2 += 4; @@ -1088,6 +1196,29 @@ uint32_t HELPER(unpku)(CPUS390XState *env, uint64_t dest, uint32_t destlen, return do_unpkau(env, dest, destlen, 2, src, GETPC()); } +uint32_t HELPER(tp)(CPUS390XState *env, uint64_t dest, uint32_t destlen) +{ + uintptr_t ra = GETPC(); + uint32_t cc = 0; + int i; + + for (i = 0; i < destlen; i++) { + uint8_t b = cpu_ldub_data_ra(env, dest + i, ra); + /* digit */ + cc |= (b & 0xf0) > 0x90 ? 2 : 0; + + if (i == (destlen - 1)) { + /* sign */ + cc |= (b & 0xf) < 0xa ? 1 : 0; + } else { + /* digit */ + cc |= (b & 0xf) > 0x9 ? 2 : 0; + } + } + + return cc; +} + static uint32_t do_helper_tr(CPUS390XState *env, uint32_t len, uint64_t array, uint64_t trans, uintptr_t ra) { @@ -1148,17 +1279,18 @@ uint64_t HELPER(tre)(CPUS390XState *env, uint64_t array, return array + i; } -static uint32_t do_helper_trt(CPUS390XState *env, uint32_t len, uint64_t array, - uint64_t trans, uintptr_t ra) +static inline uint32_t do_helper_trt(CPUS390XState *env, int len, + uint64_t array, uint64_t trans, + int inc, uintptr_t ra) { - uint32_t i; + int i; for (i = 0; i <= len; i++) { - uint8_t byte = cpu_ldub_data_ra(env, array + i, ra); + uint8_t byte = cpu_ldub_data_ra(env, array + i * inc, ra); uint8_t sbyte = cpu_ldub_data_ra(env, trans + byte, ra); if (sbyte != 0) { - set_address(env, 1, array + i); + set_address(env, 1, array + i * inc); env->regs[2] = deposit64(env->regs[2], 0, 8, sbyte); return (i == len) ? 2 : 1; } @@ -1170,11 +1302,71 @@ static uint32_t do_helper_trt(CPUS390XState *env, uint32_t len, uint64_t array, uint32_t HELPER(trt)(CPUS390XState *env, uint32_t len, uint64_t array, uint64_t trans) { - return do_helper_trt(env, len, array, trans, GETPC()); + return do_helper_trt(env, len, array, trans, 1, GETPC()); } -void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, - uint32_t r1, uint32_t r3) +uint32_t HELPER(trtr)(CPUS390XState *env, uint32_t len, uint64_t array, + uint64_t trans) +{ + return do_helper_trt(env, len, array, trans, -1, GETPC()); +} + +/* Translate one/two to one/two */ +uint32_t HELPER(trXX)(CPUS390XState *env, uint32_t r1, uint32_t r2, + uint32_t tst, uint32_t sizes) +{ + uintptr_t ra = GETPC(); + int dsize = (sizes & 1) ? 1 : 2; + int ssize = (sizes & 2) ? 1 : 2; + uint64_t tbl = get_address(env, 1); + uint64_t dst = get_address(env, r1); + uint64_t len = get_length(env, r1 + 1); + uint64_t src = get_address(env, r2); + uint32_t cc = 3; + int i; + + /* The lower address bits of TBL are ignored. For TROO, TROT, it's + the low 3 bits (double-word aligned). For TRTO, TRTT, it's either + the low 12 bits (4K, without ETF2-ENH) or 3 bits (with ETF2-ENH). */ + if (ssize == 2 && !s390_has_feat(S390_FEAT_ETF2_ENH)) { + tbl &= -4096; + } else { + tbl &= -8; + } + + check_alignment(env, len, ssize, ra); + + /* Lest we fail to service interrupts in a timely manner, */ + /* limit the amount of work we're willing to do. */ + for (i = 0; i < 0x2000; i++) { + uint16_t sval = cpu_ldusize_data_ra(env, src, ssize, ra); + uint64_t tble = tbl + (sval * dsize); + uint16_t dval = cpu_ldusize_data_ra(env, tble, dsize, ra); + if (dval == tst) { + cc = 1; + break; + } + cpu_stsize_data_ra(env, dst, dval, dsize, ra); + + len -= ssize; + src += ssize; + dst += dsize; + + if (len == 0) { + cc = 0; + break; + } + } + + set_address(env, r1, dst); + set_length(env, r1 + 1, len); + set_address(env, r2, src); + + return cc; +} + +static void do_cdsg(CPUS390XState *env, uint64_t addr, + uint32_t r1, uint32_t r3, bool parallel) { uintptr_t ra = GETPC(); Int128 cmpv = int128_make128(env->regs[r1 + 1], env->regs[r1]); @@ -1182,7 +1374,7 @@ void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, Int128 oldv; bool fail; - if (parallel_cpus) { + if (parallel) { #ifndef CONFIG_ATOMIC128 cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); #else @@ -1194,6 +1386,8 @@ void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, } else { uint64_t oldh, oldl; + check_alignment(env, addr, 16, ra); + oldh = cpu_ldq_data_ra(env, addr + 0, ra); oldl = cpu_ldq_data_ra(env, addr + 8, ra); @@ -1212,6 +1406,219 @@ void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, env->regs[r1 + 1] = int128_getlo(oldv); } +void HELPER(cdsg)(CPUS390XState *env, uint64_t addr, + uint32_t r1, uint32_t r3) +{ + do_cdsg(env, addr, r1, r3, false); +} + +void HELPER(cdsg_parallel)(CPUS390XState *env, uint64_t addr, + uint32_t r1, uint32_t r3) +{ + do_cdsg(env, addr, r1, r3, true); +} + +static uint32_t do_csst(CPUS390XState *env, uint32_t r3, uint64_t a1, + uint64_t a2, bool parallel) +{ +#if !defined(CONFIG_USER_ONLY) || defined(CONFIG_ATOMIC128) + uint32_t mem_idx = cpu_mmu_index(env, false); +#endif + uintptr_t ra = GETPC(); + uint32_t fc = extract32(env->regs[0], 0, 8); + uint32_t sc = extract32(env->regs[0], 8, 8); + uint64_t pl = get_address(env, 1) & -16; + uint64_t svh, svl; + uint32_t cc; + + /* Sanity check the function code and storage characteristic. */ + if (fc > 1 || sc > 3) { + if (!s390_has_feat(S390_FEAT_COMPARE_AND_SWAP_AND_STORE_2)) { + goto spec_exception; + } + if (fc > 2 || sc > 4 || (fc == 2 && (r3 & 1))) { + goto spec_exception; + } + } + + /* Sanity check the alignments. */ + if (extract32(a1, 0, 4 << fc) || extract32(a2, 0, 1 << sc)) { + goto spec_exception; + } + + /* Sanity check writability of the store address. */ +#ifndef CONFIG_USER_ONLY + probe_write(env, a2, 0, mem_idx, ra); +#endif + + /* Note that the compare-and-swap is atomic, and the store is atomic, but + the complete operation is not. Therefore we do not need to assert serial + context in order to implement this. That said, restart early if we can't + support either operation that is supposed to be atomic. */ + if (parallel) { + int mask = 0; +#if !defined(CONFIG_ATOMIC64) + mask = -8; +#elif !defined(CONFIG_ATOMIC128) + mask = -16; +#endif + if (((4 << fc) | (1 << sc)) & mask) { + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); + } + } + + /* All loads happen before all stores. For simplicity, load the entire + store value area from the parameter list. */ + svh = cpu_ldq_data_ra(env, pl + 16, ra); + svl = cpu_ldq_data_ra(env, pl + 24, ra); + + switch (fc) { + case 0: + { + uint32_t nv = cpu_ldl_data_ra(env, pl, ra); + uint32_t cv = env->regs[r3]; + uint32_t ov; + + if (parallel) { +#ifdef CONFIG_USER_ONLY + uint32_t *haddr = g2h(a1); + ov = atomic_cmpxchg__nocheck(haddr, cv, nv); +#else + TCGMemOpIdx oi = make_memop_idx(MO_TEUL | MO_ALIGN, mem_idx); + ov = helper_atomic_cmpxchgl_be_mmu(env, a1, cv, nv, oi, ra); +#endif + } else { + ov = cpu_ldl_data_ra(env, a1, ra); + cpu_stl_data_ra(env, a1, (ov == cv ? nv : ov), ra); + } + cc = (ov != cv); + env->regs[r3] = deposit64(env->regs[r3], 32, 32, ov); + } + break; + + case 1: + { + uint64_t nv = cpu_ldq_data_ra(env, pl, ra); + uint64_t cv = env->regs[r3]; + uint64_t ov; + + if (parallel) { +#ifdef CONFIG_ATOMIC64 +# ifdef CONFIG_USER_ONLY + uint64_t *haddr = g2h(a1); + ov = atomic_cmpxchg__nocheck(haddr, cv, nv); +# else + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN, mem_idx); + ov = helper_atomic_cmpxchgq_be_mmu(env, a1, cv, nv, oi, ra); +# endif +#else + /* Note that we asserted !parallel above. */ + g_assert_not_reached(); +#endif + } else { + ov = cpu_ldq_data_ra(env, a1, ra); + cpu_stq_data_ra(env, a1, (ov == cv ? nv : ov), ra); + } + cc = (ov != cv); + env->regs[r3] = ov; + } + break; + + case 2: + { + uint64_t nvh = cpu_ldq_data_ra(env, pl, ra); + uint64_t nvl = cpu_ldq_data_ra(env, pl + 8, ra); + Int128 nv = int128_make128(nvl, nvh); + Int128 cv = int128_make128(env->regs[r3 + 1], env->regs[r3]); + Int128 ov; + + if (parallel) { +#ifdef CONFIG_ATOMIC128 + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + ov = helper_atomic_cmpxchgo_be_mmu(env, a1, cv, nv, oi, ra); + cc = !int128_eq(ov, cv); +#else + /* Note that we asserted !parallel above. */ + g_assert_not_reached(); +#endif + } else { + uint64_t oh = cpu_ldq_data_ra(env, a1 + 0, ra); + uint64_t ol = cpu_ldq_data_ra(env, a1 + 8, ra); + + ov = int128_make128(ol, oh); + cc = !int128_eq(ov, cv); + if (cc) { + nv = ov; + } + + cpu_stq_data_ra(env, a1 + 0, int128_gethi(nv), ra); + cpu_stq_data_ra(env, a1 + 8, int128_getlo(nv), ra); + } + + env->regs[r3 + 0] = int128_gethi(ov); + env->regs[r3 + 1] = int128_getlo(ov); + } + break; + + default: + g_assert_not_reached(); + } + + /* Store only if the comparison succeeded. Note that above we use a pair + of 64-bit big-endian loads, so for sc < 3 we must extract the value + from the most-significant bits of svh. */ + if (cc == 0) { + switch (sc) { + case 0: + cpu_stb_data_ra(env, a2, svh >> 56, ra); + break; + case 1: + cpu_stw_data_ra(env, a2, svh >> 48, ra); + break; + case 2: + cpu_stl_data_ra(env, a2, svh >> 32, ra); + break; + case 3: + cpu_stq_data_ra(env, a2, svh, ra); + break; + case 4: + if (parallel) { +#ifdef CONFIG_ATOMIC128 + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + Int128 sv = int128_make128(svl, svh); + helper_atomic_sto_be_mmu(env, a2, sv, oi, ra); +#else + /* Note that we asserted !parallel above. */ + g_assert_not_reached(); +#endif + } else { + cpu_stq_data_ra(env, a2 + 0, svh, ra); + cpu_stq_data_ra(env, a2 + 8, svl, ra); + } + break; + default: + g_assert_not_reached(); + } + } + + return cc; + + spec_exception: + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); + g_assert_not_reached(); +} + +uint32_t HELPER(csst)(CPUS390XState *env, uint32_t r3, uint64_t a1, uint64_t a2) +{ + return do_csst(env, r3, a1, a2, false); +} + +uint32_t HELPER(csst_parallel)(CPUS390XState *env, uint32_t r3, uint64_t a1, + uint64_t a2) +{ + return do_csst(env, r3, a1, a2, true); +} + #if !defined(CONFIG_USER_ONLY) void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) { @@ -1221,6 +1628,10 @@ void HELPER(lctlg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uint64_t src = a2; uint32_t i; + if (src & 0x7) { + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); + } + for (i = r1;; i = (i + 1) % 16) { uint64_t val = cpu_ldq_data_ra(env, src, ra); if (env->cregs[i] != val && i >= 9 && i <= 11) { @@ -1251,6 +1662,10 @@ void HELPER(lctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uint64_t src = a2; uint32_t i; + if (src & 0x3) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } + for (i = r1;; i = (i + 1) % 16) { uint32_t val = cpu_ldl_data_ra(env, src, ra); if ((uint32_t)env->cregs[i] != val && i >= 9 && i <= 11) { @@ -1278,6 +1693,10 @@ void HELPER(stctg)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uint64_t dest = a2; uint32_t i; + if (dest & 0x7) { + s390_program_interrupt(env, PGM_SPECIFICATION, 6, ra); + } + for (i = r1;; i = (i + 1) % 16) { cpu_stq_data_ra(env, dest, env->cregs[i], ra); dest += sizeof(uint64_t); @@ -1294,6 +1713,10 @@ void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uint64_t dest = a2; uint32_t i; + if (dest & 0x3) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } + for (i = r1;; i = (i + 1) % 16) { cpu_stl_data_ra(env, dest, env->cregs[i], ra); dest += sizeof(uint32_t); @@ -1307,37 +1730,55 @@ void HELPER(stctl)(CPUS390XState *env, uint32_t r1, uint64_t a2, uint32_t r3) uint32_t HELPER(testblock)(CPUS390XState *env, uint64_t real_addr) { uintptr_t ra = GETPC(); - CPUState *cs = CPU(s390_env_get_cpu(env)); - uint64_t abs_addr; int i; - real_addr = wrap_address(env, real_addr); - abs_addr = mmu_real2abs(env, real_addr) & TARGET_PAGE_MASK; - if (!address_space_access_valid(&address_space_memory, abs_addr, - TARGET_PAGE_SIZE, true)) { - cpu_restore_state(cs, ra); - program_interrupt(env, PGM_ADDRESSING, 4); - return 1; - } - - /* Check low-address protection */ - if ((env->cregs[0] & CR0_LOWPROT) && real_addr < 0x2000) { - cpu_restore_state(cs, ra); - program_interrupt(env, PGM_PROTECTION, 4); - return 1; - } + real_addr = wrap_address(env, real_addr) & TARGET_PAGE_MASK; for (i = 0; i < TARGET_PAGE_SIZE; i += 8) { - stq_phys(cs->as, abs_addr + i, 0); + cpu_stq_real_ra(env, real_addr + i, 0, ra); } return 0; } -uint32_t HELPER(tprot)(uint64_t a1, uint64_t a2) +uint32_t HELPER(tprot)(CPUS390XState *env, uint64_t a1, uint64_t a2) { - /* XXX implement */ - return 0; + S390CPU *cpu = s390_env_get_cpu(env); + CPUState *cs = CPU(cpu); + + /* + * TODO: we currently don't handle all access protection types + * (including access-list and key-controlled) as well as AR mode. + */ + if (!s390_cpu_virt_mem_check_write(cpu, a1, 0, 1)) { + /* Fetching permitted; storing permitted */ + return 0; + } + + if (env->int_pgm_code == PGM_PROTECTION) { + /* retry if reading is possible */ + cs->exception_index = 0; + if (!s390_cpu_virt_mem_check_read(cpu, a1, 0, 1)) { + /* Fetching permitted; storing not permitted */ + return 1; + } + } + + switch (env->int_pgm_code) { + case PGM_PROTECTION: + /* Fetching not permitted; storing not permitted */ + cs->exception_index = 0; + return 2; + case PGM_ADDRESSING: + case PGM_TRANS_SPEC: + /* exceptions forwarded to the guest */ + s390_cpu_virt_mem_handle_exc(cpu, GETPC()); + return 0; + } + + /* Translation not available */ + cs->exception_index = 0; + return 3; } /* insert storage key extended */ @@ -1469,38 +1910,94 @@ uint32_t HELPER(mvcp)(CPUS390XState *env, uint64_t l, uint64_t a1, uint64_t a2) return cc; } +void HELPER(idte)(CPUS390XState *env, uint64_t r1, uint64_t r2, uint32_t m4) +{ + CPUState *cs = CPU(s390_env_get_cpu(env)); + const uintptr_t ra = GETPC(); + uint64_t table, entry, raddr; + uint16_t entries, i, index = 0; + + if (r2 & 0xff000) { + s390_program_interrupt(env, PGM_SPECIFICATION, 4, ra); + } + + if (!(r2 & 0x800)) { + /* invalidation-and-clearing operation */ + table = r1 & ASCE_ORIGIN; + entries = (r2 & 0x7ff) + 1; + + switch (r1 & ASCE_TYPE_MASK) { + case ASCE_TYPE_REGION1: + index = (r2 >> 53) & 0x7ff; + break; + case ASCE_TYPE_REGION2: + index = (r2 >> 42) & 0x7ff; + break; + case ASCE_TYPE_REGION3: + index = (r2 >> 31) & 0x7ff; + break; + case ASCE_TYPE_SEGMENT: + index = (r2 >> 20) & 0x7ff; + break; + } + for (i = 0; i < entries; i++) { + /* addresses are not wrapped in 24/31bit mode but table index is */ + raddr = table + ((index + i) & 0x7ff) * sizeof(entry); + entry = cpu_ldq_real_ra(env, raddr, ra); + if (!(entry & REGION_ENTRY_INV)) { + /* we are allowed to not store if already invalid */ + entry |= REGION_ENTRY_INV; + cpu_stq_real_ra(env, raddr, entry, ra); + } + } + } + + /* We simply flush the complete tlb, therefore we can ignore r3. */ + if (m4 & 1) { + tlb_flush(cs); + } else { + tlb_flush_all_cpus_synced(cs); + } +} + /* invalidate pte */ void HELPER(ipte)(CPUS390XState *env, uint64_t pto, uint64_t vaddr, uint32_t m4) { CPUState *cs = CPU(s390_env_get_cpu(env)); + const uintptr_t ra = GETPC(); uint64_t page = vaddr & TARGET_PAGE_MASK; uint64_t pte_addr, pte; /* Compute the page table entry address */ - pte_addr = (pto & _SEGMENT_ENTRY_ORIGIN); + pte_addr = (pto & SEGMENT_ENTRY_ORIGIN); pte_addr += (vaddr & VADDR_PX) >> 9; /* Mark the page table entry as invalid */ - pte = ldq_phys(cs->as, pte_addr); - pte |= _PAGE_INVALID; - stq_phys(cs->as, pte_addr, pte); + pte = cpu_ldq_real_ra(env, pte_addr, ra); + pte |= PAGE_INVALID; + cpu_stq_real_ra(env, pte_addr, pte, ra); /* XXX we exploit the fact that Linux passes the exact virtual address here - it's not obliged to! */ - /* XXX: the LC bit should be considered as 0 if the local-TLB-clearing - facility is not installed. */ - if (m4 & 1) { - tlb_flush_page(cs, page); - } else { - tlb_flush_page_all_cpus_synced(cs, page); - } - - /* XXX 31-bit hack */ if (m4 & 1) { - tlb_flush_page(cs, page ^ 0x80000000); + if (vaddr & ~VADDR_PX) { + tlb_flush_page(cs, page); + /* XXX 31-bit hack */ + tlb_flush_page(cs, page ^ 0x80000000); + } else { + /* looks like we don't have a valid virtual address */ + tlb_flush(cs); + } } else { - tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000); + if (vaddr & ~VADDR_PX) { + tlb_flush_page_all_cpus_synced(cs, page); + /* XXX 31-bit hack */ + tlb_flush_page_all_cpus_synced(cs, page ^ 0x80000000); + } else { + /* looks like we don't have a valid virtual address */ + tlb_flush_all_cpus_synced(cs); + } } } @@ -1523,24 +2020,18 @@ void HELPER(purge)(CPUS390XState *env) /* load using real address */ uint64_t HELPER(lura)(CPUS390XState *env, uint64_t addr) { - CPUState *cs = CPU(s390_env_get_cpu(env)); - - return (uint32_t)ldl_phys(cs->as, wrap_address(env, addr)); + return cpu_ldl_real_ra(env, wrap_address(env, addr), GETPC()); } uint64_t HELPER(lurag)(CPUS390XState *env, uint64_t addr) { - CPUState *cs = CPU(s390_env_get_cpu(env)); - - return ldq_phys(cs->as, wrap_address(env, addr)); + return cpu_ldq_real_ra(env, wrap_address(env, addr), GETPC()); } /* store using real address */ void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1) { - CPUState *cs = CPU(s390_env_get_cpu(env)); - - stl_phys(cs->as, wrap_address(env, addr), (uint32_t)v1); + cpu_stl_real_ra(env, wrap_address(env, addr), (uint32_t)v1, GETPC()); if ((env->psw.mask & PSW_MASK_PER) && (env->cregs[9] & PER_CR9_EVENT_STORE) && @@ -1553,9 +2044,7 @@ void HELPER(stura)(CPUS390XState *env, uint64_t addr, uint64_t v1) void HELPER(sturg)(CPUS390XState *env, uint64_t addr, uint64_t v1) { - CPUState *cs = CPU(s390_env_get_cpu(env)); - - stq_phys(cs->as, wrap_address(env, addr), v1); + cpu_stq_real_ra(env, wrap_address(env, addr), v1, GETPC()); if ((env->psw.mask & PSW_MASK_PER) && (env->cregs[9] & PER_CR9_EVENT_STORE) && @@ -1577,8 +2066,7 @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) /* XXX incomplete - has more corner cases */ if (!(env->psw.mask & PSW_MASK_64) && (addr >> 32)) { - cpu_restore_state(cs, GETPC()); - program_interrupt(env, PGM_SPECIAL_OP, 2); + s390_program_interrupt(env, PGM_SPECIAL_OP, 2, GETPC()); } old_exc = cs->exception_index; @@ -1597,6 +2085,79 @@ uint64_t HELPER(lra)(CPUS390XState *env, uint64_t addr) } #endif +/* load pair from quadword */ +static uint64_t do_lpq(CPUS390XState *env, uint64_t addr, bool parallel) +{ + uintptr_t ra = GETPC(); + uint64_t hi, lo; + + if (parallel) { +#ifndef CONFIG_ATOMIC128 + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + Int128 v = helper_atomic_ldo_be_mmu(env, addr, oi, ra); + hi = int128_gethi(v); + lo = int128_getlo(v); +#endif + } else { + check_alignment(env, addr, 16, ra); + + hi = cpu_ldq_data_ra(env, addr + 0, ra); + lo = cpu_ldq_data_ra(env, addr + 8, ra); + } + + env->retxl = lo; + return hi; +} + +uint64_t HELPER(lpq)(CPUS390XState *env, uint64_t addr) +{ + return do_lpq(env, addr, false); +} + +uint64_t HELPER(lpq_parallel)(CPUS390XState *env, uint64_t addr) +{ + return do_lpq(env, addr, true); +} + +/* store pair to quadword */ +static void do_stpq(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high, bool parallel) +{ + uintptr_t ra = GETPC(); + + if (parallel) { +#ifndef CONFIG_ATOMIC128 + cpu_loop_exit_atomic(ENV_GET_CPU(env), ra); +#else + int mem_idx = cpu_mmu_index(env, false); + TCGMemOpIdx oi = make_memop_idx(MO_TEQ | MO_ALIGN_16, mem_idx); + + Int128 v = int128_make128(low, high); + helper_atomic_sto_be_mmu(env, addr, v, oi, ra); +#endif + } else { + check_alignment(env, addr, 16, ra); + + cpu_stq_data_ra(env, addr + 0, high, ra); + cpu_stq_data_ra(env, addr + 8, low, ra); + } +} + +void HELPER(stpq)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) +{ + do_stpq(env, addr, low, high, false); +} + +void HELPER(stpq_parallel)(CPUS390XState *env, uint64_t addr, + uint64_t low, uint64_t high) +{ + do_stpq(env, addr, low, high, true); +} + /* Execute instruction. This instruction executes an insn modified with the contents of r1. It does not change the executed instruction in memory; it does not change the program counter. @@ -1638,7 +2199,6 @@ void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr) [0x6] = do_helper_oc, [0x7] = do_helper_xc, [0xc] = do_helper_tr, - [0xd] = do_helper_trt, }; dx_helper helper = dx[opc & 0xf]; @@ -1668,3 +2228,398 @@ void HELPER(ex)(CPUS390XState *env, uint32_t ilen, uint64_t r1, uint64_t addr) that requires such execution. */ env->ex_value = insn | ilen; } + +uint32_t HELPER(mvcos)(CPUS390XState *env, uint64_t dest, uint64_t src, + uint64_t len) +{ + const uint8_t psw_key = (env->psw.mask & PSW_MASK_KEY) >> PSW_SHIFT_KEY; + const uint8_t psw_as = (env->psw.mask & PSW_MASK_ASC) >> PSW_SHIFT_ASC; + const uint64_t r0 = env->regs[0]; + const uintptr_t ra = GETPC(); + uint8_t dest_key, dest_as, dest_k, dest_a; + uint8_t src_key, src_as, src_k, src_a; + uint64_t val; + int cc = 0; + + HELPER_LOG("%s dest %" PRIx64 ", src %" PRIx64 ", len %" PRIx64 "\n", + __func__, dest, src, len); + + if (!(env->psw.mask & PSW_MASK_DAT)) { + s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra); + } + + /* OAC (operand access control) for the first operand -> dest */ + val = (r0 & 0xffff0000ULL) >> 16; + dest_key = (val >> 12) & 0xf; + dest_as = (val >> 6) & 0x3; + dest_k = (val >> 1) & 0x1; + dest_a = val & 0x1; + + /* OAC (operand access control) for the second operand -> src */ + val = (r0 & 0x0000ffffULL); + src_key = (val >> 12) & 0xf; + src_as = (val >> 6) & 0x3; + src_k = (val >> 1) & 0x1; + src_a = val & 0x1; + + if (!dest_k) { + dest_key = psw_key; + } + if (!src_k) { + src_key = psw_key; + } + if (!dest_a) { + dest_as = psw_as; + } + if (!src_a) { + src_as = psw_as; + } + + if (dest_a && dest_as == AS_HOME && (env->psw.mask & PSW_MASK_PSTATE)) { + s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra); + } + if (!(env->cregs[0] & CR0_SECONDARY) && + (dest_as == AS_SECONDARY || src_as == AS_SECONDARY)) { + s390_program_interrupt(env, PGM_SPECIAL_OP, 6, ra); + } + if (!psw_key_valid(env, dest_key) || !psw_key_valid(env, src_key)) { + s390_program_interrupt(env, PGM_PRIVILEGED, 6, ra); + } + + len = wrap_length(env, len); + if (len > 4096) { + cc = 3; + len = 4096; + } + + /* FIXME: AR-mode and proper problem state mode (using PSW keys) missing */ + if (src_as == AS_ACCREG || dest_as == AS_ACCREG || + (env->psw.mask & PSW_MASK_PSTATE)) { + qemu_log_mask(LOG_UNIMP, "%s: AR-mode and PSTATE support missing\n", + __func__); + s390_program_interrupt(env, PGM_ADDRESSING, 6, ra); + } + + /* FIXME: a) LAP + * b) Access using correct keys + * c) AR-mode + */ +#ifdef CONFIG_USER_ONLY + /* psw keys are never valid in user mode, we will never reach this */ + g_assert_not_reached(); +#else + fast_memmove_as(env, dest, src, len, dest_as, src_as, ra); +#endif + + return cc; +} + +/* Decode a Unicode character. A return value < 0 indicates success, storing + the UTF-32 result into OCHAR and the input length into OLEN. A return + value >= 0 indicates failure, and the CC value to be returned. */ +typedef int (*decode_unicode_fn)(CPUS390XState *env, uint64_t addr, + uint64_t ilen, bool enh_check, uintptr_t ra, + uint32_t *ochar, uint32_t *olen); + +/* Encode a Unicode character. A return value < 0 indicates success, storing + the bytes into ADDR and the output length into OLEN. A return value >= 0 + indicates failure, and the CC value to be returned. */ +typedef int (*encode_unicode_fn)(CPUS390XState *env, uint64_t addr, + uint64_t ilen, uintptr_t ra, uint32_t c, + uint32_t *olen); + +static int decode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, + bool enh_check, uintptr_t ra, + uint32_t *ochar, uint32_t *olen) +{ + uint8_t s0, s1, s2, s3; + uint32_t c, l; + + if (ilen < 1) { + return 0; + } + s0 = cpu_ldub_data_ra(env, addr, ra); + if (s0 <= 0x7f) { + /* one byte character */ + l = 1; + c = s0; + } else if (s0 <= (enh_check ? 0xc1 : 0xbf)) { + /* invalid character */ + return 2; + } else if (s0 <= 0xdf) { + /* two byte character */ + l = 2; + if (ilen < 2) { + return 0; + } + s1 = cpu_ldub_data_ra(env, addr + 1, ra); + c = s0 & 0x1f; + c = (c << 6) | (s1 & 0x3f); + if (enh_check && (s1 & 0xc0) != 0x80) { + return 2; + } + } else if (s0 <= 0xef) { + /* three byte character */ + l = 3; + if (ilen < 3) { + return 0; + } + s1 = cpu_ldub_data_ra(env, addr + 1, ra); + s2 = cpu_ldub_data_ra(env, addr + 2, ra); + c = s0 & 0x0f; + c = (c << 6) | (s1 & 0x3f); + c = (c << 6) | (s2 & 0x3f); + /* Fold the byte-by-byte range descriptions in the PoO into + tests against the complete value. It disallows encodings + that could be smaller, and the UTF-16 surrogates. */ + if (enh_check + && ((s1 & 0xc0) != 0x80 + || (s2 & 0xc0) != 0x80 + || c < 0x1000 + || (c >= 0xd800 && c <= 0xdfff))) { + return 2; + } + } else if (s0 <= (enh_check ? 0xf4 : 0xf7)) { + /* four byte character */ + l = 4; + if (ilen < 4) { + return 0; + } + s1 = cpu_ldub_data_ra(env, addr + 1, ra); + s2 = cpu_ldub_data_ra(env, addr + 2, ra); + s3 = cpu_ldub_data_ra(env, addr + 3, ra); + c = s0 & 0x07; + c = (c << 6) | (s1 & 0x3f); + c = (c << 6) | (s2 & 0x3f); + c = (c << 6) | (s3 & 0x3f); + /* See above. */ + if (enh_check + && ((s1 & 0xc0) != 0x80 + || (s2 & 0xc0) != 0x80 + || (s3 & 0xc0) != 0x80 + || c < 0x010000 + || c > 0x10ffff)) { + return 2; + } + } else { + /* invalid character */ + return 2; + } + + *ochar = c; + *olen = l; + return -1; +} + +static int decode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, + bool enh_check, uintptr_t ra, + uint32_t *ochar, uint32_t *olen) +{ + uint16_t s0, s1; + uint32_t c, l; + + if (ilen < 2) { + return 0; + } + s0 = cpu_lduw_data_ra(env, addr, ra); + if ((s0 & 0xfc00) != 0xd800) { + /* one word character */ + l = 2; + c = s0; + } else { + /* two word character */ + l = 4; + if (ilen < 4) { + return 0; + } + s1 = cpu_lduw_data_ra(env, addr + 2, ra); + c = extract32(s0, 6, 4) + 1; + c = (c << 6) | (s0 & 0x3f); + c = (c << 10) | (s1 & 0x3ff); + if (enh_check && (s1 & 0xfc00) != 0xdc00) { + /* invalid surrogate character */ + return 2; + } + } + + *ochar = c; + *olen = l; + return -1; +} + +static int decode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, + bool enh_check, uintptr_t ra, + uint32_t *ochar, uint32_t *olen) +{ + uint32_t c; + + if (ilen < 4) { + return 0; + } + c = cpu_ldl_data_ra(env, addr, ra); + if ((c >= 0xd800 && c <= 0xdbff) || c > 0x10ffff) { + /* invalid unicode character */ + return 2; + } + + *ochar = c; + *olen = 4; + return -1; +} + +static int encode_utf8(CPUS390XState *env, uint64_t addr, uint64_t ilen, + uintptr_t ra, uint32_t c, uint32_t *olen) +{ + uint8_t d[4]; + uint32_t l, i; + + if (c <= 0x7f) { + /* one byte character */ + l = 1; + d[0] = c; + } else if (c <= 0x7ff) { + /* two byte character */ + l = 2; + d[1] = 0x80 | extract32(c, 0, 6); + d[0] = 0xc0 | extract32(c, 6, 5); + } else if (c <= 0xffff) { + /* three byte character */ + l = 3; + d[2] = 0x80 | extract32(c, 0, 6); + d[1] = 0x80 | extract32(c, 6, 6); + d[0] = 0xe0 | extract32(c, 12, 4); + } else { + /* four byte character */ + l = 4; + d[3] = 0x80 | extract32(c, 0, 6); + d[2] = 0x80 | extract32(c, 6, 6); + d[1] = 0x80 | extract32(c, 12, 6); + d[0] = 0xf0 | extract32(c, 18, 3); + } + + if (ilen < l) { + return 1; + } + for (i = 0; i < l; ++i) { + cpu_stb_data_ra(env, addr + i, d[i], ra); + } + + *olen = l; + return -1; +} + +static int encode_utf16(CPUS390XState *env, uint64_t addr, uint64_t ilen, + uintptr_t ra, uint32_t c, uint32_t *olen) +{ + uint16_t d0, d1; + + if (c <= 0xffff) { + /* one word character */ + if (ilen < 2) { + return 1; + } + cpu_stw_data_ra(env, addr, c, ra); + *olen = 2; + } else { + /* two word character */ + if (ilen < 4) { + return 1; + } + d1 = 0xdc00 | extract32(c, 0, 10); + d0 = 0xd800 | extract32(c, 10, 6); + d0 = deposit32(d0, 6, 4, extract32(c, 16, 5) - 1); + cpu_stw_data_ra(env, addr + 0, d0, ra); + cpu_stw_data_ra(env, addr + 2, d1, ra); + *olen = 4; + } + + return -1; +} + +static int encode_utf32(CPUS390XState *env, uint64_t addr, uint64_t ilen, + uintptr_t ra, uint32_t c, uint32_t *olen) +{ + if (ilen < 4) { + return 1; + } + cpu_stl_data_ra(env, addr, c, ra); + *olen = 4; + return -1; +} + +static inline uint32_t convert_unicode(CPUS390XState *env, uint32_t r1, + uint32_t r2, uint32_t m3, uintptr_t ra, + decode_unicode_fn decode, + encode_unicode_fn encode) +{ + uint64_t dst = get_address(env, r1); + uint64_t dlen = get_length(env, r1 + 1); + uint64_t src = get_address(env, r2); + uint64_t slen = get_length(env, r2 + 1); + bool enh_check = m3 & 1; + int cc, i; + + /* Lest we fail to service interrupts in a timely manner, limit the + amount of work we're willing to do. For now, let's cap at 256. */ + for (i = 0; i < 256; ++i) { + uint32_t c, ilen, olen; + + cc = decode(env, src, slen, enh_check, ra, &c, &ilen); + if (unlikely(cc >= 0)) { + break; + } + cc = encode(env, dst, dlen, ra, c, &olen); + if (unlikely(cc >= 0)) { + break; + } + + src += ilen; + slen -= ilen; + dst += olen; + dlen -= olen; + cc = 3; + } + + set_address(env, r1, dst); + set_length(env, r1 + 1, dlen); + set_address(env, r2, src); + set_length(env, r2 + 1, slen); + + return cc; +} + +uint32_t HELPER(cu12)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf8, encode_utf16); +} + +uint32_t HELPER(cu14)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf8, encode_utf32); +} + +uint32_t HELPER(cu21)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf16, encode_utf8); +} + +uint32_t HELPER(cu24)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf16, encode_utf32); +} + +uint32_t HELPER(cu41)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf32, encode_utf8); +} + +uint32_t HELPER(cu42)(CPUS390XState *env, uint32_t r1, uint32_t r2, uint32_t m3) +{ + return convert_unicode(env, r1, r2, m3, GETPC(), + decode_utf32, encode_utf16); +}