fbmem: don't allow too huge resolutions

syzbot is reporting page fault at vga16fb_fillrect() [1], for
vga16fb_check_var() is failing to detect multiplication overflow.

  if (vxres * vyres > maxmem) {
    vyres = maxmem / vxres;
    if (vyres < yres)
      return -ENOMEM;
  }

Since no module would accept too huge resolutions where multiplication
overflow happens, let's reject in the common path.

Link: https://syzkaller.appspot.com/bug?extid=04168c8063cfdde1db5e
Reported-by: syzbot <[email protected]>
Debugged-by: Randy Dunlap <[email protected]>
Signed-off-by: Tetsuo Handa <[email protected]>
Reviewed-by: Geert Uytterhoeven <[email protected]>
Cc: [email protected]
Signed-off-by: Daniel Vetter <[email protected]>
Link: https://patchwork.freedesktop.org/patch/msgid/[email protected]

diff --git a/drivers/video/fbdev/core/fbmem.c b/drivers/video/fbdev/core/fbmem.c
index 71fb710f1ce3a2c09959342601aa4fc463296bba..7420d2c16e47e34f8acf94fd70f7417133e62773 100644 (file)
--- a/drivers/video/fbdev/core/fbmem.c
+++ b/drivers/video/fbdev/core/fbmem.c
@@ -962,6 +962,7 @@ fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var)
        struct fb_var_screeninfo old_var;
        struct fb_videomode mode;
        struct fb_event event;
+       u32 unused;
 
        if (var->activate & FB_ACTIVATE_INV_MODE) {
                struct fb_videomode mode1, mode2;
@@ -1008,6 +1009,11 @@ fb_set_var(struct fb_info *info, struct fb_var_screeninfo *var)
        if (var->xres < 8 || var->yres < 8)
                return -EINVAL;
 
+       /* Too huge resolution causes multiplication overflow. */
+       if (check_mul_overflow(var->xres, var->yres, &unused) ||
+           check_mul_overflow(var->xres_virtual, var->yres_virtual, &unused))
+               return -EINVAL;
+
        ret = info->fbops->fb_check_var(var, info);
 
        if (ret)