2 * Lockstep Execution Plugin
4 * Allows you to execute two QEMU instances in lockstep and report
5 * when their execution diverges. This is mainly useful for developers
6 * who want to see where a change to TCG code generation has
7 * introduced a subtle and hard to find bug.
10 * - single-threaded linux-user apps only with non-deterministic syscalls
11 * - no MTTCG enabled system emulation (icount may help)
13 * While icount makes things more deterministic it doesn't mean a
14 * particular run may execute the exact same sequence of blocks. An
15 * asynchronous event (for example X11 graphics update) may cause a
16 * block to end early and a new partial block to start. This means
17 * serial only test cases are a better bet. -d nochain may also help.
19 * This code is not thread safe!
21 * Copyright (c) 2020 Linaro Ltd
23 * SPDX-License-Identifier: GPL-2.0-or-later
29 #include <sys/socket.h>
34 #include <qemu-plugin.h>
36 QEMU_PLUGIN_EXPORT int qemu_plugin_version = QEMU_PLUGIN_VERSION;
38 /* saved so we can uninstall later */
39 static qemu_plugin_id_t our_id;
41 static unsigned long bb_count;
42 static unsigned long insn_count;
44 /* Information about a translated block */
50 /* Information about an execution state in the log */
53 unsigned long insn_count;
54 unsigned long block_count;
57 /* The execution state we compare */
60 unsigned long insn_count;
68 /* list of translated block info */
69 static GSList *blocks;
71 /* execution log and points of divergence */
72 static GSList *log, *divergence_log;
75 static char *path_to_unlink;
79 static void plugin_cleanup(qemu_plugin_id_t id)
81 /* Free our block data */
82 g_slist_free_full(blocks, &g_free);
83 g_slist_free_full(log, &g_free);
84 g_slist_free(divergence_log);
88 unlink(path_to_unlink);
92 static void plugin_exit(qemu_plugin_id_t id, void *p)
94 g_autoptr(GString) out = g_string_new("No divergence :-)\n");
95 g_string_append_printf(out, "Executed %ld/%d blocks\n",
96 bb_count, g_slist_length(log));
97 g_string_append_printf(out, "Executed ~%ld instructions\n", insn_count);
98 qemu_plugin_outs(out->str);
103 static void report_divergance(ExecState *us, ExecState *them)
105 DivergeState divrec = { log, 0 };
106 g_autoptr(GString) out = g_string_new("");
107 bool diverged = false;
110 * If we have diverged before did we get back on track or are we
111 * totally loosing it?
113 if (divergence_log) {
114 DivergeState *last = (DivergeState *) divergence_log->data;
117 for (entry = log; g_slist_next(entry); entry = g_slist_next(entry)) {
118 if (entry == last->log_pos) {
125 * If the last two records are so close it is likely we will
126 * not recover synchronisation with the other end.
128 if (divrec.distance == 1 && last->distance == 1) {
132 divergence_log = g_slist_prepend(divergence_log,
133 g_memdup(&divrec, sizeof(divrec)));
135 /* Output short log entry of going out of sync... */
136 if (verbose || divrec.distance == 1 || diverged) {
137 g_string_printf(out, "@ 0x%016lx vs 0x%016lx (%d/%d since last)\n",
138 us->pc, them->pc, g_slist_length(divergence_log),
140 qemu_plugin_outs(out->str);
147 g_string_printf(out, "Δ insn_count @ 0x%016lx (%ld) vs 0x%016lx (%ld)\n",
148 us->pc, us->insn_count, them->pc, them->insn_count);
150 for (entry = log, i = 0;
151 g_slist_next(entry) && i < 5;
152 entry = g_slist_next(entry), i++) {
153 ExecInfo *prev = (ExecInfo *) entry->data;
154 g_string_append_printf(out,
155 " previously @ 0x%016lx/%ld (%ld insns)\n",
156 prev->block->pc, prev->block->insns,
159 qemu_plugin_outs(out->str);
160 qemu_plugin_outs("too much divergence... giving up.");
161 qemu_plugin_uninstall(our_id, plugin_cleanup);
165 static void vcpu_tb_exec(unsigned int cpu_index, void *udata)
167 BlockInfo *bi = (BlockInfo *) udata;
173 us.insn_count = insn_count;
176 * Write our current position to the other end. If we fail the
177 * other end has probably died and we should shut down gracefully.
179 bytes = write(socket_fd, &us, sizeof(ExecState));
180 if (bytes < sizeof(ExecState)) {
181 qemu_plugin_outs(bytes < 0 ?
182 "problem writing to socket" :
183 "wrote less than expected to socket");
184 qemu_plugin_uninstall(our_id, plugin_cleanup);
189 * Now read where our peer has reached. Again a failure probably
190 * indicates the other end died and we should close down cleanly.
192 bytes = read(socket_fd, &them, sizeof(ExecState));
193 if (bytes < sizeof(ExecState)) {
194 qemu_plugin_outs(bytes < 0 ?
195 "problem reading from socket" :
196 "read less than expected");
197 qemu_plugin_uninstall(our_id, plugin_cleanup);
202 * Compare and report if we have diverged.
204 if (us.pc != them.pc) {
205 report_divergance(&us, &them);
209 * Assume this block will execute fully and record it
210 * in the execution log.
212 insn_count += bi->insns;
214 exec = g_new0(ExecInfo, 1);
216 exec->insn_count = insn_count;
217 exec->block_count = bb_count;
218 log = g_slist_prepend(log, exec);
221 static void vcpu_tb_trans(qemu_plugin_id_t id, struct qemu_plugin_tb *tb)
223 BlockInfo *bi = g_new0(BlockInfo, 1);
224 bi->pc = qemu_plugin_tb_vaddr(tb);
225 bi->insns = qemu_plugin_tb_n_insns(tb);
227 /* save a reference so we can free later */
228 blocks = g_slist_prepend(blocks, bi);
229 qemu_plugin_register_vcpu_tb_exec_cb(tb, vcpu_tb_exec,
230 QEMU_PLUGIN_CB_NO_REGS, (void *)bi);
235 * Instead of encoding master/slave status into what is essentially
236 * two peers we shall just take the simple approach of checking for
237 * the existence of the pipe and assuming if it's not there we are the
240 static bool setup_socket(const char *path)
242 struct sockaddr_un sockaddr;
245 fd = socket(AF_UNIX, SOCK_STREAM, 0);
247 perror("create socket");
251 sockaddr.sun_family = AF_UNIX;
252 g_strlcpy(sockaddr.sun_path, path, sizeof(sockaddr.sun_path) - 1);
253 if (bind(fd, (struct sockaddr *)&sockaddr, sizeof(sockaddr)) < 0) {
254 perror("bind socket");
259 /* remember to clean-up */
260 path_to_unlink = g_strdup(path);
262 if (listen(fd, 1) < 0) {
263 perror("listen socket");
268 socket_fd = accept(fd, NULL, NULL);
269 if (socket_fd < 0 && errno != EINTR) {
270 perror("accept socket");
275 qemu_plugin_outs("setup_socket::ready\n");
281 static bool connect_socket(const char *path)
284 struct sockaddr_un sockaddr;
286 fd = socket(AF_UNIX, SOCK_STREAM, 0);
288 perror("create socket");
292 sockaddr.sun_family = AF_UNIX;
293 g_strlcpy(sockaddr.sun_path, path, sizeof(sockaddr.sun_path) - 1);
295 if (connect(fd, (struct sockaddr *)&sockaddr, sizeof(sockaddr)) < 0) {
296 perror("failed to connect");
301 qemu_plugin_outs("connect_socket::ready\n");
307 static bool setup_unix_socket(const char *path)
309 if (g_file_test(path, G_FILE_TEST_EXISTS)) {
310 return connect_socket(path);
312 return setup_socket(path);
317 QEMU_PLUGIN_EXPORT int qemu_plugin_install(qemu_plugin_id_t id,
318 const qemu_info_t *info,
319 int argc, char **argv)
322 g_autofree char *sock_path = NULL;
324 for (i = 0; i < argc; i++) {
326 g_autofree char **tokens = g_strsplit(p, "=", 2);
328 if (g_strcmp0(tokens[0], "verbose") == 0) {
329 if (!qemu_plugin_bool_parse(tokens[0], tokens[1], &verbose)) {
330 fprintf(stderr, "boolean argument parsing failed: %s\n", p);
333 } else if (g_strcmp0(tokens[0], "sockpath") == 0) {
334 sock_path = tokens[1];
336 fprintf(stderr, "option parsing failed: %s\n", p);
341 if (sock_path == NULL) {
342 fprintf(stderr, "Need a socket path to talk to other instance.\n");
346 if (!setup_unix_socket(sock_path)) {
347 fprintf(stderr, "Failed to setup socket for communications.\n");
353 qemu_plugin_register_vcpu_tb_trans_cb(id, vcpu_tb_trans);
354 qemu_plugin_register_atexit_cb(id, plugin_exit, NULL);