arch/um/drivers/virtio_uml.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Virtio vhost-user driver
   4  *
   5  * Copyright(c) 2019 Intel Corporation
   6  *
   7  * This driver allows virtio devices to be used over a vhost-user socket.
   8  *
   9  * Guest devices can be instantiated by kernel module or command line
  10  * parameters. One device will be created for each parameter. Syntax:
  11  *
  12  *              virtio_uml.device=<socket>:<virtio_id>[:<platform_id>]
  13  * where:
  14  *              <socket>        := vhost-user socket path to connect
  15  *              <virtio_id>     := virtio device id (as in virtio_ids.h)
  16  *              <platform_id>   := (optional) platform device id
  17  *
  18  * example:
  19  *              virtio_uml.device=/var/uml.socket:1
  20  *
  21  * Based on Virtio MMIO driver by Pawel Moll, copyright 2011-2014, ARM Ltd.
  22  */
  23 #include <linux/module.h>
  24 #include <linux/of.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <linux/virtio.h>
  28 #include <linux/virtio_config.h>
  29 #include <linux/virtio_ring.h>
  30 #include <linux/time-internal.h>
  31 #include <linux/virtio-uml.h>
  32 #include <shared/as-layout.h>
  33 #include <irq_kern.h>
  34 #include <init.h>
  35 #include <os.h>
  36 #include "vhost_user.h"
  37
  38 #define MAX_SUPPORTED_QUEUE_SIZE        256
  39
  40 #define to_virtio_uml_device(_vdev) \
  41         container_of(_vdev, struct virtio_uml_device, vdev)
  42
  43 struct virtio_uml_platform_data {
  44         u32 virtio_device_id;
  45         const char *socket_path;
  46         struct work_struct conn_broken_wk;
  47         struct platform_device *pdev;
  48 };
  49
  50 struct virtio_uml_device {
  51         struct virtio_device vdev;
  52         struct platform_device *pdev;
  53         struct virtio_uml_platform_data *pdata;
  54
  55         raw_spinlock_t sock_lock;
  56         int sock, req_fd, irq;
  57         u64 features;
  58         u64 protocol_features;
  59         u64 max_vqs;
  60         u8 status;
  61         u8 registered:1;
  62         u8 suspended:1;
  63         u8 no_vq_suspend:1;
  64
  65         u8 config_changed_irq:1;
  66         uint64_t vq_irq_vq_map;
  67         int recv_rc;
  68 };
  69
  70 struct virtio_uml_vq_info {
  71         int kick_fd, call_fd;
  72         char name[32];
  73         bool suspended;
  74 };
  75
  76 #define vu_err(vu_dev, ...)     dev_err(&(vu_dev)->pdev->dev, ##__VA_ARGS__)
  77
  78 /* Vhost-user protocol */
  79
  80 static int full_sendmsg_fds(int fd, const void *buf, unsigned int len,
  81                             const int *fds, unsigned int fds_num)
  82 {
  83         int rc;
  84
  85         do {
  86                 rc = os_sendmsg_fds(fd, buf, len, fds, fds_num);
  87                 if (rc > 0) {
  88                         buf += rc;
  89                         len -= rc;
  90                         fds = NULL;
  91                         fds_num = 0;
  92                 }
  93         } while (len && (rc >= 0 || rc == -EINTR));
  94
  95         if (rc < 0)
  96                 return rc;
  97         return 0;
  98 }
  99
 100 static int full_read(int fd, void *buf, int len, bool abortable)
 101 {
 102         int rc;
 103
 104         if (!len)
 105                 return 0;
 106
 107         do {
 108                 rc = os_read_file(fd, buf, len);
 109                 if (rc > 0) {
 110                         buf += rc;
 111                         len -= rc;
 112                 }
 113         } while (len && (rc > 0 || rc == -EINTR || (!abortable && rc == -EAGAIN)));
 114
 115         if (rc < 0)
 116                 return rc;
 117         if (rc == 0)
 118                 return -ECONNRESET;
 119         return 0;
 120 }
 121
 122 static int vhost_user_recv_header(int fd, struct vhost_user_msg *msg)
 123 {
 124         return full_read(fd, msg, sizeof(msg->header), true);
 125 }
 126
 127 static int vhost_user_recv(struct virtio_uml_device *vu_dev,
 128                            int fd, struct vhost_user_msg *msg,
 129                            size_t max_payload_size, bool wait)
 130 {
 131         size_t size;
 132         int rc;
 133
 134         /*
 135          * In virtio time-travel mode, we're handling all the vhost-user
 136          * FDs by polling them whenever appropriate. However, we may get
 137          * into a situation where we're sending out an interrupt message
 138          * to a device (e.g. a net device) and need to handle a simulation
 139          * time message while doing so, e.g. one that tells us to update
 140          * our idea of how long we can run without scheduling.
 141          *
 142          * Thus, we need to not just read() from the given fd, but need
 143          * to also handle messages for the simulation time - this function
 144          * does that for us while waiting for the given fd to be readable.
 145          */
 146         if (wait)
 147                 time_travel_wait_readable(fd);
 148
 149         rc = vhost_user_recv_header(fd, msg);
 150
 151         if (rc)
 152                 return rc;
 153         size = msg->header.size;
 154         if (size > max_payload_size)
 155                 return -EPROTO;
 156         return full_read(fd, &msg->payload, size, false);
 157 }
 158
 159 static void vhost_user_check_reset(struct virtio_uml_device *vu_dev,
 160                                    int rc)
 161 {
 162         struct virtio_uml_platform_data *pdata = vu_dev->pdata;
 163
 164         if (rc != -ECONNRESET)
 165                 return;
 166
 167         if (!vu_dev->registered)
 168                 return;
 169
 170         vu_dev->registered = 0;
 171
 172         schedule_work(&pdata->conn_broken_wk);
 173 }
 174
 175 static int vhost_user_recv_resp(struct virtio_uml_device *vu_dev,
 176                                 struct vhost_user_msg *msg,
 177                                 size_t max_payload_size)
 178 {
 179         int rc = vhost_user_recv(vu_dev, vu_dev->sock, msg,
 180                                  max_payload_size, true);
 181
 182         if (rc) {
 183                 vhost_user_check_reset(vu_dev, rc);
 184                 return rc;
 185         }
 186
 187         if (msg->header.flags != (VHOST_USER_FLAG_REPLY | VHOST_USER_VERSION))
 188                 return -EPROTO;
 189
 190         return 0;
 191 }
 192
 193 static int vhost_user_recv_u64(struct virtio_uml_device *vu_dev,
 194                                u64 *value)
 195 {
 196         struct vhost_user_msg msg;
 197         int rc = vhost_user_recv_resp(vu_dev, &msg,
 198                                       sizeof(msg.payload.integer));
 199
 200         if (rc)
 201                 return rc;
 202         if (msg.header.size != sizeof(msg.payload.integer))
 203                 return -EPROTO;
 204         *value = msg.payload.integer;
 205         return 0;
 206 }
 207
 208 static int vhost_user_recv_req(struct virtio_uml_device *vu_dev,
 209                                struct vhost_user_msg *msg,
 210                                size_t max_payload_size)
 211 {
 212         int rc = vhost_user_recv(vu_dev, vu_dev->req_fd, msg,
 213                                  max_payload_size, false);
 214
 215         if (rc)
 216                 return rc;
 217
 218         if ((msg->header.flags & ~VHOST_USER_FLAG_NEED_REPLY) !=
 219                         VHOST_USER_VERSION)
 220                 return -EPROTO;
 221
 222         return 0;
 223 }
 224
 225 static int vhost_user_send(struct virtio_uml_device *vu_dev,
 226                            bool need_response, struct vhost_user_msg *msg,
 227                            int *fds, size_t num_fds)
 228 {
 229         size_t size = sizeof(msg->header) + msg->header.size;
 230         unsigned long flags;
 231         bool request_ack;
 232         int rc;
 233
 234         msg->header.flags |= VHOST_USER_VERSION;
 235
 236         /*
 237          * The need_response flag indicates that we already need a response,
 238          * e.g. to read the features. In these cases, don't request an ACK as
 239          * it is meaningless. Also request an ACK only if supported.
 240          */
 241         request_ack = !need_response;
 242         if (!(vu_dev->protocol_features &
 243                         BIT_ULL(VHOST_USER_PROTOCOL_F_REPLY_ACK)))
 244                 request_ack = false;
 245
 246         if (request_ack)
 247                 msg->header.flags |= VHOST_USER_FLAG_NEED_REPLY;
 248
 249         raw_spin_lock_irqsave(&vu_dev->sock_lock, flags);
 250         rc = full_sendmsg_fds(vu_dev->sock, msg, size, fds, num_fds);
 251         if (rc < 0)
 252                 goto out;
 253
 254         if (request_ack) {
 255                 uint64_t status;
 256
 257                 rc = vhost_user_recv_u64(vu_dev, &status);
 258                 if (rc)
 259                         goto out;
 260
 261                 if (status) {
 262                         vu_err(vu_dev, "slave reports error: %llu\n", status);
 263                         rc = -EIO;
 264                         goto out;
 265                 }
 266         }
 267
 268 out:
 269         raw_spin_unlock_irqrestore(&vu_dev->sock_lock, flags);
 270         return rc;
 271 }
 272
 273 static int vhost_user_send_no_payload(struct virtio_uml_device *vu_dev,
 274                                       bool need_response, u32 request)
 275 {
 276         struct vhost_user_msg msg = {
 277                 .header.request = request,
 278         };
 279
 280         return vhost_user_send(vu_dev, need_response, &msg, NULL, 0);
 281 }
 282
 283 static int vhost_user_send_no_payload_fd(struct virtio_uml_device *vu_dev,
 284                                          u32 request, int fd)
 285 {
 286         struct vhost_user_msg msg = {
 287                 .header.request = request,
 288         };
 289
 290         return vhost_user_send(vu_dev, false, &msg, &fd, 1);
 291 }
 292
 293 static int vhost_user_send_u64(struct virtio_uml_device *vu_dev,
 294                                u32 request, u64 value)
 295 {
 296         struct vhost_user_msg msg = {
 297                 .header.request = request,
 298                 .header.size = sizeof(msg.payload.integer),
 299                 .payload.integer = value,
 300         };
 301
 302         return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 303 }
 304
 305 static int vhost_user_set_owner(struct virtio_uml_device *vu_dev)
 306 {
 307         return vhost_user_send_no_payload(vu_dev, false, VHOST_USER_SET_OWNER);
 308 }
 309
 310 static int vhost_user_get_features(struct virtio_uml_device *vu_dev,
 311                                    u64 *features)
 312 {
 313         int rc = vhost_user_send_no_payload(vu_dev, true,
 314                                             VHOST_USER_GET_FEATURES);
 315
 316         if (rc)
 317                 return rc;
 318         return vhost_user_recv_u64(vu_dev, features);
 319 }
 320
 321 static int vhost_user_set_features(struct virtio_uml_device *vu_dev,
 322                                    u64 features)
 323 {
 324         return vhost_user_send_u64(vu_dev, VHOST_USER_SET_FEATURES, features);
 325 }
 326
 327 static int vhost_user_get_protocol_features(struct virtio_uml_device *vu_dev,
 328                                             u64 *protocol_features)
 329 {
 330         int rc = vhost_user_send_no_payload(vu_dev, true,
 331                         VHOST_USER_GET_PROTOCOL_FEATURES);
 332
 333         if (rc)
 334                 return rc;
 335         return vhost_user_recv_u64(vu_dev, protocol_features);
 336 }
 337
 338 static int vhost_user_set_protocol_features(struct virtio_uml_device *vu_dev,
 339                                             u64 protocol_features)
 340 {
 341         return vhost_user_send_u64(vu_dev, VHOST_USER_SET_PROTOCOL_FEATURES,
 342                                    protocol_features);
 343 }
 344
 345 static int vhost_user_get_queue_num(struct virtio_uml_device *vu_dev,
 346                                     u64 *queue_num)
 347 {
 348         int rc = vhost_user_send_no_payload(vu_dev, true,
 349                         VHOST_USER_GET_QUEUE_NUM);
 350
 351         if (rc)
 352                 return rc;
 353         return vhost_user_recv_u64(vu_dev, queue_num);
 354 }
 355
 356 static void vhost_user_reply(struct virtio_uml_device *vu_dev,
 357                              struct vhost_user_msg *msg, int response)
 358 {
 359         struct vhost_user_msg reply = {
 360                 .payload.integer = response,
 361         };
 362         size_t size = sizeof(reply.header) + sizeof(reply.payload.integer);
 363         int rc;
 364
 365         reply.header = msg->header;
 366         reply.header.flags &= ~VHOST_USER_FLAG_NEED_REPLY;
 367         reply.header.flags |= VHOST_USER_FLAG_REPLY;
 368         reply.header.size = sizeof(reply.payload.integer);
 369
 370         rc = full_sendmsg_fds(vu_dev->req_fd, &reply, size, NULL, 0);
 371
 372         if (rc)
 373                 vu_err(vu_dev,
 374                        "sending reply to slave request failed: %d (size %zu)\n",
 375                        rc, size);
 376 }
 377
 378 static irqreturn_t vu_req_read_message(struct virtio_uml_device *vu_dev,
 379                                        struct time_travel_event *ev)
 380 {
 381         struct virtqueue *vq;
 382         int response = 1;
 383         struct {
 384                 struct vhost_user_msg msg;
 385                 u8 extra_payload[512];
 386         } msg;
 387         int rc;
 388         irqreturn_t irq_rc = IRQ_NONE;
 389
 390         while (1) {
 391                 rc = vhost_user_recv_req(vu_dev, &msg.msg,
 392                                          sizeof(msg.msg.payload) +
 393                                          sizeof(msg.extra_payload));
 394                 if (rc)
 395                         break;
 396
 397                 switch (msg.msg.header.request) {
 398                 case VHOST_USER_SLAVE_CONFIG_CHANGE_MSG:
 399                         vu_dev->config_changed_irq = true;
 400                         response = 0;
 401                         break;
 402                 case VHOST_USER_SLAVE_VRING_CALL:
 403                         virtio_device_for_each_vq((&vu_dev->vdev), vq) {
 404                                 if (vq->index == msg.msg.payload.vring_state.index) {
 405                                         response = 0;
 406                                         vu_dev->vq_irq_vq_map |= BIT_ULL(vq->index);
 407                                         break;
 408                                 }
 409                         }
 410                         break;
 411                 case VHOST_USER_SLAVE_IOTLB_MSG:
 412                         /* not supported - VIRTIO_F_ACCESS_PLATFORM */
 413                 case VHOST_USER_SLAVE_VRING_HOST_NOTIFIER_MSG:
 414                         /* not supported - VHOST_USER_PROTOCOL_F_HOST_NOTIFIER */
 415                 default:
 416                         vu_err(vu_dev, "unexpected slave request %d\n",
 417                                msg.msg.header.request);
 418                 }
 419
 420                 if (ev && !vu_dev->suspended)
 421                         time_travel_add_irq_event(ev);
 422
 423                 if (msg.msg.header.flags & VHOST_USER_FLAG_NEED_REPLY)
 424                         vhost_user_reply(vu_dev, &msg.msg, response);
 425                 irq_rc = IRQ_HANDLED;
 426         }
 427         /* mask EAGAIN as we try non-blocking read until socket is empty */
 428         vu_dev->recv_rc = (rc == -EAGAIN) ? 0 : rc;
 429         return irq_rc;
 430 }
 431
 432 static irqreturn_t vu_req_interrupt(int irq, void *data)
 433 {
 434         struct virtio_uml_device *vu_dev = data;
 435         irqreturn_t ret = IRQ_HANDLED;
 436
 437         if (!um_irq_timetravel_handler_used())
 438                 ret = vu_req_read_message(vu_dev, NULL);
 439
 440         if (vu_dev->recv_rc) {
 441                 vhost_user_check_reset(vu_dev, vu_dev->recv_rc);
 442         } else if (vu_dev->vq_irq_vq_map) {
 443                 struct virtqueue *vq;
 444
 445                 virtio_device_for_each_vq((&vu_dev->vdev), vq) {
 446                         if (vu_dev->vq_irq_vq_map & BIT_ULL(vq->index))
 447                                 vring_interrupt(0 /* ignored */, vq);
 448                 }
 449                 vu_dev->vq_irq_vq_map = 0;
 450         } else if (vu_dev->config_changed_irq) {
 451                 virtio_config_changed(&vu_dev->vdev);
 452                 vu_dev->config_changed_irq = false;
 453         }
 454
 455         return ret;
 456 }
 457
 458 static void vu_req_interrupt_comm_handler(int irq, int fd, void *data,
 459                                           struct time_travel_event *ev)
 460 {
 461         vu_req_read_message(data, ev);
 462 }
 463
 464 static int vhost_user_init_slave_req(struct virtio_uml_device *vu_dev)
 465 {
 466         int rc, req_fds[2];
 467
 468         /* Use a pipe for slave req fd, SIGIO is not supported for eventfd */
 469         rc = os_pipe(req_fds, true, true);
 470         if (rc < 0)
 471                 return rc;
 472         vu_dev->req_fd = req_fds[0];
 473
 474         rc = um_request_irq_tt(UM_IRQ_ALLOC, vu_dev->req_fd, IRQ_READ,
 475                                vu_req_interrupt, IRQF_SHARED,
 476                                vu_dev->pdev->name, vu_dev,
 477                                vu_req_interrupt_comm_handler);
 478         if (rc < 0)
 479                 goto err_close;
 480
 481         vu_dev->irq = rc;
 482
 483         rc = vhost_user_send_no_payload_fd(vu_dev, VHOST_USER_SET_SLAVE_REQ_FD,
 484                                            req_fds[1]);
 485         if (rc)
 486                 goto err_free_irq;
 487
 488         goto out;
 489
 490 err_free_irq:
 491         um_free_irq(vu_dev->irq, vu_dev);
 492 err_close:
 493         os_close_file(req_fds[0]);
 494 out:
 495         /* Close unused write end of request fds */
 496         os_close_file(req_fds[1]);
 497         return rc;
 498 }
 499
 500 static int vhost_user_init(struct virtio_uml_device *vu_dev)
 501 {
 502         int rc = vhost_user_set_owner(vu_dev);
 503
 504         if (rc)
 505                 return rc;
 506         rc = vhost_user_get_features(vu_dev, &vu_dev->features);
 507         if (rc)
 508                 return rc;
 509
 510         if (vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)) {
 511                 rc = vhost_user_get_protocol_features(vu_dev,
 512                                 &vu_dev->protocol_features);
 513                 if (rc)
 514                         return rc;
 515                 vu_dev->protocol_features &= VHOST_USER_SUPPORTED_PROTOCOL_F;
 516                 rc = vhost_user_set_protocol_features(vu_dev,
 517                                 vu_dev->protocol_features);
 518                 if (rc)
 519                         return rc;
 520         }
 521
 522         if (vu_dev->protocol_features &
 523                         BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 524                 rc = vhost_user_init_slave_req(vu_dev);
 525                 if (rc)
 526                         return rc;
 527         }
 528
 529         if (vu_dev->protocol_features &
 530                         BIT_ULL(VHOST_USER_PROTOCOL_F_MQ)) {
 531                 rc = vhost_user_get_queue_num(vu_dev, &vu_dev->max_vqs);
 532                 if (rc)
 533                         return rc;
 534         } else {
 535                 vu_dev->max_vqs = U64_MAX;
 536         }
 537
 538         return 0;
 539 }
 540
 541 static void vhost_user_get_config(struct virtio_uml_device *vu_dev,
 542                                   u32 offset, void *buf, u32 len)
 543 {
 544         u32 cfg_size = offset + len;
 545         struct vhost_user_msg *msg;
 546         size_t payload_size = sizeof(msg->payload.config) + cfg_size;
 547         size_t msg_size = sizeof(msg->header) + payload_size;
 548         int rc;
 549
 550         if (!(vu_dev->protocol_features &
 551               BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG)))
 552                 return;
 553
 554         msg = kzalloc(msg_size, GFP_KERNEL);
 555         if (!msg)
 556                 return;
 557         msg->header.request = VHOST_USER_GET_CONFIG;
 558         msg->header.size = payload_size;
 559         msg->payload.config.offset = 0;
 560         msg->payload.config.size = cfg_size;
 561
 562         rc = vhost_user_send(vu_dev, true, msg, NULL, 0);
 563         if (rc) {
 564                 vu_err(vu_dev, "sending VHOST_USER_GET_CONFIG failed: %d\n",
 565                        rc);
 566                 goto free;
 567         }
 568
 569         rc = vhost_user_recv_resp(vu_dev, msg, msg_size);
 570         if (rc) {
 571                 vu_err(vu_dev,
 572                        "receiving VHOST_USER_GET_CONFIG response failed: %d\n",
 573                        rc);
 574                 goto free;
 575         }
 576
 577         if (msg->header.size != payload_size ||
 578             msg->payload.config.size != cfg_size) {
 579                 rc = -EPROTO;
 580                 vu_err(vu_dev,
 581                        "Invalid VHOST_USER_GET_CONFIG sizes (payload %d expected %zu, config %u expected %u)\n",
 582                        msg->header.size, payload_size,
 583                        msg->payload.config.size, cfg_size);
 584                 goto free;
 585         }
 586         memcpy(buf, msg->payload.config.payload + offset, len);
 587
 588 free:
 589         kfree(msg);
 590 }
 591
 592 static void vhost_user_set_config(struct virtio_uml_device *vu_dev,
 593                                   u32 offset, const void *buf, u32 len)
 594 {
 595         struct vhost_user_msg *msg;
 596         size_t payload_size = sizeof(msg->payload.config) + len;
 597         size_t msg_size = sizeof(msg->header) + payload_size;
 598         int rc;
 599
 600         if (!(vu_dev->protocol_features &
 601               BIT_ULL(VHOST_USER_PROTOCOL_F_CONFIG)))
 602                 return;
 603
 604         msg = kzalloc(msg_size, GFP_KERNEL);
 605         if (!msg)
 606                 return;
 607         msg->header.request = VHOST_USER_SET_CONFIG;
 608         msg->header.size = payload_size;
 609         msg->payload.config.offset = offset;
 610         msg->payload.config.size = len;
 611         memcpy(msg->payload.config.payload, buf, len);
 612
 613         rc = vhost_user_send(vu_dev, false, msg, NULL, 0);
 614         if (rc)
 615                 vu_err(vu_dev, "sending VHOST_USER_SET_CONFIG failed: %d\n",
 616                        rc);
 617
 618         kfree(msg);
 619 }
 620
 621 static int vhost_user_init_mem_region(u64 addr, u64 size, int *fd_out,
 622                                       struct vhost_user_mem_region *region_out)
 623 {
 624         unsigned long long mem_offset;
 625         int rc = phys_mapping(addr, &mem_offset);
 626
 627         if (WARN(rc < 0, "phys_mapping of 0x%llx returned %d\n", addr, rc))
 628                 return -EFAULT;
 629         *fd_out = rc;
 630         region_out->guest_addr = addr;
 631         region_out->user_addr = addr;
 632         region_out->size = size;
 633         region_out->mmap_offset = mem_offset;
 634
 635         /* Ensure mapping is valid for the entire region */
 636         rc = phys_mapping(addr + size - 1, &mem_offset);
 637         if (WARN(rc != *fd_out, "phys_mapping of 0x%llx failed: %d != %d\n",
 638                  addr + size - 1, rc, *fd_out))
 639                 return -EFAULT;
 640         return 0;
 641 }
 642
 643 static int vhost_user_set_mem_table(struct virtio_uml_device *vu_dev)
 644 {
 645         struct vhost_user_msg msg = {
 646                 .header.request = VHOST_USER_SET_MEM_TABLE,
 647                 .header.size = offsetof(typeof(msg.payload.mem_regions), regions[1]),
 648                 .payload.mem_regions.num = 1,
 649         };
 650         unsigned long reserved = uml_reserved - uml_physmem;
 651         int fds[2];
 652         int rc;
 653
 654         /*
 655          * This is a bit tricky, see also the comment with setup_physmem().
 656          *
 657          * Essentially, setup_physmem() uses a file to mmap() our physmem,
 658          * but the code and data we *already* have is omitted. To us, this
 659          * is no difference, since they both become part of our address
 660          * space and memory consumption. To somebody looking in from the
 661          * outside, however, it is different because the part of our memory
 662          * consumption that's already part of the binary (code/data) is not
 663          * mapped from the file, so it's not visible to another mmap from
 664          * the file descriptor.
 665          *
 666          * Thus, don't advertise this space to the vhost-user slave. This
 667          * means that the slave will likely abort or similar when we give
 668          * it an address from the hidden range, since it's not marked as
 669          * a valid address, but at least that way we detect the issue and
 670          * don't just have the slave read an all-zeroes buffer from the
 671          * shared memory file, or write something there that we can never
 672          * see (depending on the direction of the virtqueue traffic.)
 673          *
 674          * Since we usually don't want to use .text for virtio buffers,
 675          * this effectively means that you cannot use
 676          *  1) global variables, which are in the .bss and not in the shm
 677          *     file-backed memory
 678          *  2) the stack in some processes, depending on where they have
 679          *     their stack (or maybe only no interrupt stack?)
 680          *
 681          * The stack is already not typically valid for DMA, so this isn't
 682          * much of a restriction, but global variables might be encountered.
 683          *
 684          * It might be possible to fix it by copying around the data that's
 685          * between bss_start and where we map the file now, but it's not
 686          * something that you typically encounter with virtio drivers, so
 687          * it didn't seem worthwhile.
 688          */
 689         rc = vhost_user_init_mem_region(reserved, physmem_size - reserved,
 690                                         &fds[0],
 691                                         &msg.payload.mem_regions.regions[0]);
 692
 693         if (rc < 0)
 694                 return rc;
 695
 696         return vhost_user_send(vu_dev, false, &msg, fds,
 697                                msg.payload.mem_regions.num);
 698 }
 699
 700 static int vhost_user_set_vring_state(struct virtio_uml_device *vu_dev,
 701                                       u32 request, u32 index, u32 num)
 702 {
 703         struct vhost_user_msg msg = {
 704                 .header.request = request,
 705                 .header.size = sizeof(msg.payload.vring_state),
 706                 .payload.vring_state.index = index,
 707                 .payload.vring_state.num = num,
 708         };
 709
 710         return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 711 }
 712
 713 static int vhost_user_set_vring_num(struct virtio_uml_device *vu_dev,
 714                                     u32 index, u32 num)
 715 {
 716         return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_NUM,
 717                                           index, num);
 718 }
 719
 720 static int vhost_user_set_vring_base(struct virtio_uml_device *vu_dev,
 721                                      u32 index, u32 offset)
 722 {
 723         return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_BASE,
 724                                           index, offset);
 725 }
 726
 727 static int vhost_user_set_vring_addr(struct virtio_uml_device *vu_dev,
 728                                      u32 index, u64 desc, u64 used, u64 avail,
 729                                      u64 log)
 730 {
 731         struct vhost_user_msg msg = {
 732                 .header.request = VHOST_USER_SET_VRING_ADDR,
 733                 .header.size = sizeof(msg.payload.vring_addr),
 734                 .payload.vring_addr.index = index,
 735                 .payload.vring_addr.desc = desc,
 736                 .payload.vring_addr.used = used,
 737                 .payload.vring_addr.avail = avail,
 738                 .payload.vring_addr.log = log,
 739         };
 740
 741         return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 742 }
 743
 744 static int vhost_user_set_vring_fd(struct virtio_uml_device *vu_dev,
 745                                    u32 request, int index, int fd)
 746 {
 747         struct vhost_user_msg msg = {
 748                 .header.request = request,
 749                 .header.size = sizeof(msg.payload.integer),
 750                 .payload.integer = index,
 751         };
 752
 753         if (index & ~VHOST_USER_VRING_INDEX_MASK)
 754                 return -EINVAL;
 755         if (fd < 0) {
 756                 msg.payload.integer |= VHOST_USER_VRING_POLL_MASK;
 757                 return vhost_user_send(vu_dev, false, &msg, NULL, 0);
 758         }
 759         return vhost_user_send(vu_dev, false, &msg, &fd, 1);
 760 }
 761
 762 static int vhost_user_set_vring_call(struct virtio_uml_device *vu_dev,
 763                                      int index, int fd)
 764 {
 765         return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_CALL,
 766                                        index, fd);
 767 }
 768
 769 static int vhost_user_set_vring_kick(struct virtio_uml_device *vu_dev,
 770                                      int index, int fd)
 771 {
 772         return vhost_user_set_vring_fd(vu_dev, VHOST_USER_SET_VRING_KICK,
 773                                        index, fd);
 774 }
 775
 776 static int vhost_user_set_vring_enable(struct virtio_uml_device *vu_dev,
 777                                        u32 index, bool enable)
 778 {
 779         if (!(vu_dev->features & BIT_ULL(VHOST_USER_F_PROTOCOL_FEATURES)))
 780                 return 0;
 781
 782         return vhost_user_set_vring_state(vu_dev, VHOST_USER_SET_VRING_ENABLE,
 783                                           index, enable);
 784 }
 785
 786
 787 /* Virtio interface */
 788
 789 static bool vu_notify(struct virtqueue *vq)
 790 {
 791         struct virtio_uml_vq_info *info = vq->priv;
 792         const uint64_t n = 1;
 793         int rc;
 794
 795         if (info->suspended)
 796                 return true;
 797
 798         time_travel_propagate_time();
 799
 800         if (info->kick_fd < 0) {
 801                 struct virtio_uml_device *vu_dev;
 802
 803                 vu_dev = to_virtio_uml_device(vq->vdev);
 804
 805                 return vhost_user_set_vring_state(vu_dev, VHOST_USER_VRING_KICK,
 806                                                   vq->index, 0) == 0;
 807         }
 808
 809         do {
 810                 rc = os_write_file(info->kick_fd, &n, sizeof(n));
 811         } while (rc == -EINTR);
 812         return !WARN(rc != sizeof(n), "write returned %d\n", rc);
 813 }
 814
 815 static irqreturn_t vu_interrupt(int irq, void *opaque)
 816 {
 817         struct virtqueue *vq = opaque;
 818         struct virtio_uml_vq_info *info = vq->priv;
 819         uint64_t n;
 820         int rc;
 821         irqreturn_t ret = IRQ_NONE;
 822
 823         do {
 824                 rc = os_read_file(info->call_fd, &n, sizeof(n));
 825                 if (rc == sizeof(n))
 826                         ret |= vring_interrupt(irq, vq);
 827         } while (rc == sizeof(n) || rc == -EINTR);
 828         WARN(rc != -EAGAIN, "read returned %d\n", rc);
 829         return ret;
 830 }
 831
 832
 833 static void vu_get(struct virtio_device *vdev, unsigned offset,
 834                    void *buf, unsigned len)
 835 {
 836         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 837
 838         vhost_user_get_config(vu_dev, offset, buf, len);
 839 }
 840
 841 static void vu_set(struct virtio_device *vdev, unsigned offset,
 842                    const void *buf, unsigned len)
 843 {
 844         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 845
 846         vhost_user_set_config(vu_dev, offset, buf, len);
 847 }
 848
 849 static u8 vu_get_status(struct virtio_device *vdev)
 850 {
 851         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 852
 853         return vu_dev->status;
 854 }
 855
 856 static void vu_set_status(struct virtio_device *vdev, u8 status)
 857 {
 858         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 859
 860         vu_dev->status = status;
 861 }
 862
 863 static void vu_reset(struct virtio_device *vdev)
 864 {
 865         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 866
 867         vu_dev->status = 0;
 868 }
 869
 870 static void vu_del_vq(struct virtqueue *vq)
 871 {
 872         struct virtio_uml_vq_info *info = vq->priv;
 873
 874         if (info->call_fd >= 0) {
 875                 struct virtio_uml_device *vu_dev;
 876
 877                 vu_dev = to_virtio_uml_device(vq->vdev);
 878
 879                 um_free_irq(vu_dev->irq, vq);
 880                 os_close_file(info->call_fd);
 881         }
 882
 883         if (info->kick_fd >= 0)
 884                 os_close_file(info->kick_fd);
 885
 886         vring_del_virtqueue(vq);
 887         kfree(info);
 888 }
 889
 890 static void vu_del_vqs(struct virtio_device *vdev)
 891 {
 892         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 893         struct virtqueue *vq, *n;
 894         u64 features;
 895
 896         /* Note: reverse order as a workaround to a decoding bug in snabb */
 897         list_for_each_entry_reverse(vq, &vdev->vqs, list)
 898                 WARN_ON(vhost_user_set_vring_enable(vu_dev, vq->index, false));
 899
 900         /* Ensure previous messages have been processed */
 901         WARN_ON(vhost_user_get_features(vu_dev, &features));
 902
 903         list_for_each_entry_safe(vq, n, &vdev->vqs, list)
 904                 vu_del_vq(vq);
 905 }
 906
 907 static int vu_setup_vq_call_fd(struct virtio_uml_device *vu_dev,
 908                                struct virtqueue *vq)
 909 {
 910         struct virtio_uml_vq_info *info = vq->priv;
 911         int call_fds[2];
 912         int rc, irq;
 913
 914         /* no call FD needed/desired in this case */
 915         if (vu_dev->protocol_features &
 916                         BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS) &&
 917             vu_dev->protocol_features &
 918                         BIT_ULL(VHOST_USER_PROTOCOL_F_SLAVE_REQ)) {
 919                 info->call_fd = -1;
 920                 return 0;
 921         }
 922
 923         /* Use a pipe for call fd, since SIGIO is not supported for eventfd */
 924         rc = os_pipe(call_fds, true, true);
 925         if (rc < 0)
 926                 return rc;
 927
 928         info->call_fd = call_fds[0];
 929         irq = um_request_irq(vu_dev->irq, info->call_fd, IRQ_READ,
 930                              vu_interrupt, IRQF_SHARED, info->name, vq);
 931         if (irq < 0) {
 932                 rc = irq;
 933                 goto close_both;
 934         }
 935
 936         rc = vhost_user_set_vring_call(vu_dev, vq->index, call_fds[1]);
 937         if (rc)
 938                 goto release_irq;
 939
 940         vu_dev->irq = irq;
 941
 942         goto out;
 943
 944 release_irq:
 945         um_free_irq(irq, vq);
 946 close_both:
 947         os_close_file(call_fds[0]);
 948 out:
 949         /* Close (unused) write end of call fds */
 950         os_close_file(call_fds[1]);
 951
 952         return rc;
 953 }
 954
 955 static struct virtqueue *vu_setup_vq(struct virtio_device *vdev,
 956                                      unsigned index, vq_callback_t *callback,
 957                                      const char *name, bool ctx)
 958 {
 959         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
 960         struct platform_device *pdev = vu_dev->pdev;
 961         struct virtio_uml_vq_info *info;
 962         struct virtqueue *vq;
 963         int num = MAX_SUPPORTED_QUEUE_SIZE;
 964         int rc;
 965
 966         info = kzalloc(sizeof(*info), GFP_KERNEL);
 967         if (!info) {
 968                 rc = -ENOMEM;
 969                 goto error_kzalloc;
 970         }
 971         snprintf(info->name, sizeof(info->name), "%s.%d-%s", pdev->name,
 972                  pdev->id, name);
 973
 974         vq = vring_create_virtqueue(index, num, PAGE_SIZE, vdev, true, true,
 975                                     ctx, vu_notify, callback, info->name);
 976         if (!vq) {
 977                 rc = -ENOMEM;
 978                 goto error_create;
 979         }
 980         vq->priv = info;
 981         vq->num_max = num;
 982         num = virtqueue_get_vring_size(vq);
 983
 984         if (vu_dev->protocol_features &
 985                         BIT_ULL(VHOST_USER_PROTOCOL_F_INBAND_NOTIFICATIONS)) {
 986                 info->kick_fd = -1;
 987         } else {
 988                 rc = os_eventfd(0, 0);
 989                 if (rc < 0)
 990                         goto error_kick;
 991                 info->kick_fd = rc;
 992         }
 993
 994         rc = vu_setup_vq_call_fd(vu_dev, vq);
 995         if (rc)
 996                 goto error_call;
 997
 998         rc = vhost_user_set_vring_num(vu_dev, index, num);
 999         if (rc)
1000                 goto error_setup;
1001
1002         rc = vhost_user_set_vring_base(vu_dev, index, 0);
1003         if (rc)
1004                 goto error_setup;
1005
1006         rc = vhost_user_set_vring_addr(vu_dev, index,
1007                                        virtqueue_get_desc_addr(vq),
1008                                        virtqueue_get_used_addr(vq),
1009                                        virtqueue_get_avail_addr(vq),
1010                                        (u64) -1);
1011         if (rc)
1012                 goto error_setup;
1013
1014         return vq;
1015
1016 error_setup:
1017         if (info->call_fd >= 0) {
1018                 um_free_irq(vu_dev->irq, vq);
1019                 os_close_file(info->call_fd);
1020         }
1021 error_call:
1022         if (info->kick_fd >= 0)
1023                 os_close_file(info->kick_fd);
1024 error_kick:
1025         vring_del_virtqueue(vq);
1026 error_create:
1027         kfree(info);
1028 error_kzalloc:
1029         return ERR_PTR(rc);
1030 }
1031
1032 static int vu_find_vqs(struct virtio_device *vdev, unsigned nvqs,
1033                        struct virtqueue *vqs[],
1034                        struct virtqueue_info vqs_info[],
1035                        struct irq_affinity *desc)
1036 {
1037         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1038         int i, queue_idx = 0, rc;
1039         struct virtqueue *vq;
1040
1041         /* not supported for now */
1042         if (WARN(nvqs > 64 || nvqs > vu_dev->max_vqs,
1043                  "%d VQs requested, only up to 64 or %lld supported\n",
1044                  nvqs, vu_dev->max_vqs))
1045                 return -EINVAL;
1046
1047         rc = vhost_user_set_mem_table(vu_dev);
1048         if (rc)
1049                 return rc;
1050
1051         for (i = 0; i < nvqs; ++i) {
1052                 struct virtqueue_info *vqi = &vqs_info[i];
1053
1054                 if (!vqi->name) {
1055                         vqs[i] = NULL;
1056                         continue;
1057                 }
1058
1059                 vqs[i] = vu_setup_vq(vdev, queue_idx++, vqi->callback,
1060                                      vqi->name, vqi->ctx);
1061                 if (IS_ERR(vqs[i])) {
1062                         rc = PTR_ERR(vqs[i]);
1063                         goto error_setup;
1064                 }
1065         }
1066
1067         list_for_each_entry(vq, &vdev->vqs, list) {
1068                 struct virtio_uml_vq_info *info = vq->priv;
1069
1070                 if (info->kick_fd >= 0) {
1071                         rc = vhost_user_set_vring_kick(vu_dev, vq->index,
1072                                                        info->kick_fd);
1073                         if (rc)
1074                                 goto error_setup;
1075                 }
1076
1077                 rc = vhost_user_set_vring_enable(vu_dev, vq->index, true);
1078                 if (rc)
1079                         goto error_setup;
1080         }
1081
1082         return 0;
1083
1084 error_setup:
1085         vu_del_vqs(vdev);
1086         return rc;
1087 }
1088
1089 static u64 vu_get_features(struct virtio_device *vdev)
1090 {
1091         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1092
1093         return vu_dev->features;
1094 }
1095
1096 static int vu_finalize_features(struct virtio_device *vdev)
1097 {
1098         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1099         u64 supported = vdev->features & VHOST_USER_SUPPORTED_F;
1100
1101         vring_transport_features(vdev);
1102         vu_dev->features = vdev->features | supported;
1103
1104         return vhost_user_set_features(vu_dev, vu_dev->features);
1105 }
1106
1107 static const char *vu_bus_name(struct virtio_device *vdev)
1108 {
1109         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1110
1111         return vu_dev->pdev->name;
1112 }
1113
1114 static const struct virtio_config_ops virtio_uml_config_ops = {
1115         .get = vu_get,
1116         .set = vu_set,
1117         .get_status = vu_get_status,
1118         .set_status = vu_set_status,
1119         .reset = vu_reset,
1120         .find_vqs = vu_find_vqs,
1121         .del_vqs = vu_del_vqs,
1122         .get_features = vu_get_features,
1123         .finalize_features = vu_finalize_features,
1124         .bus_name = vu_bus_name,
1125 };
1126
1127 static void virtio_uml_release_dev(struct device *d)
1128 {
1129         struct virtio_device *vdev =
1130                         container_of(d, struct virtio_device, dev);
1131         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1132
1133         time_travel_propagate_time();
1134
1135         /* might not have been opened due to not negotiating the feature */
1136         if (vu_dev->req_fd >= 0) {
1137                 um_free_irq(vu_dev->irq, vu_dev);
1138                 os_close_file(vu_dev->req_fd);
1139         }
1140
1141         os_close_file(vu_dev->sock);
1142         kfree(vu_dev);
1143 }
1144
1145 void virtio_uml_set_no_vq_suspend(struct virtio_device *vdev,
1146                                   bool no_vq_suspend)
1147 {
1148         struct virtio_uml_device *vu_dev = to_virtio_uml_device(vdev);
1149
1150         if (WARN_ON(vdev->config != &virtio_uml_config_ops))
1151                 return;
1152
1153         vu_dev->no_vq_suspend = no_vq_suspend;
1154         dev_info(&vdev->dev, "%sabled VQ suspend\n",
1155                  no_vq_suspend ? "dis" : "en");
1156 }
1157
1158 static void vu_of_conn_broken(struct work_struct *wk)
1159 {
1160         struct virtio_uml_platform_data *pdata;
1161         struct virtio_uml_device *vu_dev;
1162
1163         pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk);
1164
1165         vu_dev = platform_get_drvdata(pdata->pdev);
1166
1167         virtio_break_device(&vu_dev->vdev);
1168
1169         /*
1170          * We can't remove the device from the devicetree so the only thing we
1171          * can do is warn.
1172          */
1173         WARN_ON(1);
1174 }
1175
1176 /* Platform device */
1177
1178 static struct virtio_uml_platform_data *
1179 virtio_uml_create_pdata(struct platform_device *pdev)
1180 {
1181         struct device_node *np = pdev->dev.of_node;
1182         struct virtio_uml_platform_data *pdata;
1183         int ret;
1184
1185         if (!np)
1186                 return ERR_PTR(-EINVAL);
1187
1188         pdata = devm_kzalloc(&pdev->dev, sizeof(*pdata), GFP_KERNEL);
1189         if (!pdata)
1190                 return ERR_PTR(-ENOMEM);
1191
1192         INIT_WORK(&pdata->conn_broken_wk, vu_of_conn_broken);
1193         pdata->pdev = pdev;
1194
1195         ret = of_property_read_string(np, "socket-path", &pdata->socket_path);
1196         if (ret)
1197                 return ERR_PTR(ret);
1198
1199         ret = of_property_read_u32(np, "virtio-device-id",
1200                                    &pdata->virtio_device_id);
1201         if (ret)
1202                 return ERR_PTR(ret);
1203
1204         return pdata;
1205 }
1206
1207 static int virtio_uml_probe(struct platform_device *pdev)
1208 {
1209         struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1210         struct virtio_uml_device *vu_dev;
1211         int rc;
1212
1213         if (!pdata) {
1214                 pdata = virtio_uml_create_pdata(pdev);
1215                 if (IS_ERR(pdata))
1216                         return PTR_ERR(pdata);
1217         }
1218
1219         vu_dev = kzalloc(sizeof(*vu_dev), GFP_KERNEL);
1220         if (!vu_dev)
1221                 return -ENOMEM;
1222
1223         vu_dev->pdata = pdata;
1224         vu_dev->vdev.dev.parent = &pdev->dev;
1225         vu_dev->vdev.dev.release = virtio_uml_release_dev;
1226         vu_dev->vdev.config = &virtio_uml_config_ops;
1227         vu_dev->vdev.id.device = pdata->virtio_device_id;
1228         vu_dev->vdev.id.vendor = VIRTIO_DEV_ANY_ID;
1229         vu_dev->pdev = pdev;
1230         vu_dev->req_fd = -1;
1231         vu_dev->irq = UM_IRQ_ALLOC;
1232
1233         time_travel_propagate_time();
1234
1235         do {
1236                 rc = os_connect_socket(pdata->socket_path);
1237         } while (rc == -EINTR);
1238         if (rc < 0)
1239                 goto error_free;
1240         vu_dev->sock = rc;
1241
1242         raw_spin_lock_init(&vu_dev->sock_lock);
1243
1244         rc = vhost_user_init(vu_dev);
1245         if (rc)
1246                 goto error_init;
1247
1248         platform_set_drvdata(pdev, vu_dev);
1249
1250         device_set_wakeup_capable(&vu_dev->vdev.dev, true);
1251
1252         rc = register_virtio_device(&vu_dev->vdev);
1253         if (rc)
1254                 put_device(&vu_dev->vdev.dev);
1255         vu_dev->registered = 1;
1256         return rc;
1257
1258 error_init:
1259         os_close_file(vu_dev->sock);
1260 error_free:
1261         kfree(vu_dev);
1262         return rc;
1263 }
1264
1265 static void virtio_uml_remove(struct platform_device *pdev)
1266 {
1267         struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1268
1269         unregister_virtio_device(&vu_dev->vdev);
1270 }
1271
1272 /* Command line device list */
1273
1274 static void vu_cmdline_release_dev(struct device *d)
1275 {
1276 }
1277
1278 static struct device vu_cmdline_parent = {
1279         .init_name = "virtio-uml-cmdline",
1280         .release = vu_cmdline_release_dev,
1281 };
1282
1283 static bool vu_cmdline_parent_registered;
1284 static int vu_cmdline_id;
1285
1286 static int vu_unregister_cmdline_device(struct device *dev, void *data)
1287 {
1288         struct platform_device *pdev = to_platform_device(dev);
1289         struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1290
1291         kfree(pdata->socket_path);
1292         platform_device_unregister(pdev);
1293         return 0;
1294 }
1295
1296 static void vu_conn_broken(struct work_struct *wk)
1297 {
1298         struct virtio_uml_platform_data *pdata;
1299         struct virtio_uml_device *vu_dev;
1300
1301         pdata = container_of(wk, struct virtio_uml_platform_data, conn_broken_wk);
1302
1303         vu_dev = platform_get_drvdata(pdata->pdev);
1304
1305         virtio_break_device(&vu_dev->vdev);
1306
1307         vu_unregister_cmdline_device(&pdata->pdev->dev, NULL);
1308 }
1309
1310 static int vu_cmdline_set(const char *device, const struct kernel_param *kp)
1311 {
1312         const char *ids = strchr(device, ':');
1313         unsigned int virtio_device_id;
1314         int processed, consumed, err;
1315         char *socket_path;
1316         struct virtio_uml_platform_data pdata, *ppdata;
1317         struct platform_device *pdev;
1318
1319         if (!ids || ids == device)
1320                 return -EINVAL;
1321
1322         processed = sscanf(ids, ":%u%n:%d%n",
1323                            &virtio_device_id, &consumed,
1324                            &vu_cmdline_id, &consumed);
1325
1326         if (processed < 1 || ids[consumed])
1327                 return -EINVAL;
1328
1329         if (!vu_cmdline_parent_registered) {
1330                 err = device_register(&vu_cmdline_parent);
1331                 if (err) {
1332                         pr_err("Failed to register parent device!\n");
1333                         put_device(&vu_cmdline_parent);
1334                         return err;
1335                 }
1336                 vu_cmdline_parent_registered = true;
1337         }
1338
1339         socket_path = kmemdup_nul(device, ids - device, GFP_KERNEL);
1340         if (!socket_path)
1341                 return -ENOMEM;
1342
1343         pdata.virtio_device_id = (u32) virtio_device_id;
1344         pdata.socket_path = socket_path;
1345
1346         pr_info("Registering device virtio-uml.%d id=%d at %s\n",
1347                 vu_cmdline_id, virtio_device_id, socket_path);
1348
1349         pdev = platform_device_register_data(&vu_cmdline_parent, "virtio-uml",
1350                                              vu_cmdline_id++, &pdata,
1351                                              sizeof(pdata));
1352         err = PTR_ERR_OR_ZERO(pdev);
1353         if (err)
1354                 goto free;
1355
1356         ppdata = pdev->dev.platform_data;
1357         ppdata->pdev = pdev;
1358         INIT_WORK(&ppdata->conn_broken_wk, vu_conn_broken);
1359
1360         return 0;
1361
1362 free:
1363         kfree(socket_path);
1364         return err;
1365 }
1366
1367 static int vu_cmdline_get_device(struct device *dev, void *data)
1368 {
1369         struct platform_device *pdev = to_platform_device(dev);
1370         struct virtio_uml_platform_data *pdata = pdev->dev.platform_data;
1371         char *buffer = data;
1372         unsigned int len = strlen(buffer);
1373
1374         snprintf(buffer + len, PAGE_SIZE - len, "%s:%d:%d\n",
1375                  pdata->socket_path, pdata->virtio_device_id, pdev->id);
1376         return 0;
1377 }
1378
1379 static int vu_cmdline_get(char *buffer, const struct kernel_param *kp)
1380 {
1381         buffer[0] = '\0';
1382         if (vu_cmdline_parent_registered)
1383                 device_for_each_child(&vu_cmdline_parent, buffer,
1384                                       vu_cmdline_get_device);
1385         return strlen(buffer) + 1;
1386 }
1387
1388 static const struct kernel_param_ops vu_cmdline_param_ops = {
1389         .set = vu_cmdline_set,
1390         .get = vu_cmdline_get,
1391 };
1392
1393 device_param_cb(device, &vu_cmdline_param_ops, NULL, S_IRUSR);
1394 __uml_help(vu_cmdline_param_ops,
1395 "virtio_uml.device=<socket>:<virtio_id>[:<platform_id>]\n"
1396 "    Configure a virtio device over a vhost-user socket.\n"
1397 "    See virtio_ids.h for a list of possible virtio device id values.\n"
1398 "    Optionally use a specific platform_device id.\n\n"
1399 );
1400
1401
1402 static void vu_unregister_cmdline_devices(void)
1403 {
1404         if (vu_cmdline_parent_registered) {
1405                 device_for_each_child(&vu_cmdline_parent, NULL,
1406                                       vu_unregister_cmdline_device);
1407                 device_unregister(&vu_cmdline_parent);
1408                 vu_cmdline_parent_registered = false;
1409         }
1410 }
1411
1412 /* Platform driver */
1413
1414 static const struct of_device_id virtio_uml_match[] = {
1415         { .compatible = "virtio,uml", },
1416         { }
1417 };
1418 MODULE_DEVICE_TABLE(of, virtio_uml_match);
1419
1420 static int virtio_uml_suspend(struct platform_device *pdev, pm_message_t state)
1421 {
1422         struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1423
1424         if (!vu_dev->no_vq_suspend) {
1425                 struct virtqueue *vq;
1426
1427                 virtio_device_for_each_vq((&vu_dev->vdev), vq) {
1428                         struct virtio_uml_vq_info *info = vq->priv;
1429
1430                         info->suspended = true;
1431                         vhost_user_set_vring_enable(vu_dev, vq->index, false);
1432                 }
1433         }
1434
1435         if (!device_may_wakeup(&vu_dev->vdev.dev)) {
1436                 vu_dev->suspended = true;
1437                 return 0;
1438         }
1439
1440         return irq_set_irq_wake(vu_dev->irq, 1);
1441 }
1442
1443 static int virtio_uml_resume(struct platform_device *pdev)
1444 {
1445         struct virtio_uml_device *vu_dev = platform_get_drvdata(pdev);
1446
1447         if (!vu_dev->no_vq_suspend) {
1448                 struct virtqueue *vq;
1449
1450                 virtio_device_for_each_vq((&vu_dev->vdev), vq) {
1451                         struct virtio_uml_vq_info *info = vq->priv;
1452
1453                         info->suspended = false;
1454                         vhost_user_set_vring_enable(vu_dev, vq->index, true);
1455                 }
1456         }
1457
1458         vu_dev->suspended = false;
1459
1460         if (!device_may_wakeup(&vu_dev->vdev.dev))
1461                 return 0;
1462
1463         return irq_set_irq_wake(vu_dev->irq, 0);
1464 }
1465
1466 static struct platform_driver virtio_uml_driver = {
1467         .probe = virtio_uml_probe,
1468         .remove = virtio_uml_remove,
1469         .driver = {
1470                 .name = "virtio-uml",
1471                 .of_match_table = virtio_uml_match,
1472         },
1473         .suspend = virtio_uml_suspend,
1474         .resume = virtio_uml_resume,
1475 };
1476
1477 static int __init virtio_uml_init(void)
1478 {
1479         return platform_driver_register(&virtio_uml_driver);
1480 }
1481
1482 static void __exit virtio_uml_exit(void)
1483 {
1484         platform_driver_unregister(&virtio_uml_driver);
1485         vu_unregister_cmdline_devices();
1486 }
1487
1488 module_init(virtio_uml_init);
1489 module_exit(virtio_uml_exit);
1490 __uml_exitcall(virtio_uml_exit);
1491
1492 MODULE_DESCRIPTION("UML driver for vhost-user virtio devices");
1493 MODULE_LICENSE("GPL");