From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Frediano Ziglio <fziglio@redhat.com>
Date: Mon, 29 Feb 2016 14:24:03 +0000
Subject: [PATCH] create a function to validate surface parameters

Make possible to reuse it outside red-parse-qxl.c

Signed-off-by: Frediano Ziglio <fziglio@redhat.com>
---
 server/red_parse_qxl.c | 50 ++++++++++++++++++++++++++++++++------------------
 server/red_parse_qxl.h |  5 +++++
 2 files changed, 37 insertions(+), 18 deletions(-)

diff --git a/server/red_parse_qxl.c b/server/red_parse_qxl.c
index bd0c408..7dc6a4d 100644
--- a/server/red_parse_qxl.c
+++ b/server/red_parse_qxl.c
@@ -19,7 +19,6 @@
 #include <config.h>
 #endif
 
-#include <stdbool.h>
 #include <inttypes.h>
 #include <glib.h>
 #include "common/lz_common.h"
@@ -1306,13 +1305,41 @@ static unsigned int surface_format_to_bpp(uint32_t format)
     return 0;
 }
 
+bool red_validate_surface(uint32_t width, uint32_t height,
+                          int32_t stride, uint32_t format)
+{
+    unsigned int bpp;
+    uint64_t size;
+
+    bpp = surface_format_to_bpp(format);
+
+    /* check if format is valid */
+    if (!bpp) {
+        return false;
+    }
+
+    /* check stride is larger than required bytes */
+    size = ((uint64_t) width * bpp + 7u) / 8u;
+    /* the uint32_t conversion is here to avoid problems with -2^31 value */
+    if (stride == G_MININT32 || size > (uint32_t) abs(stride)) {
+        return false;
+    }
+
+    /* the multiplication can overflow, also abs(-2^31) may return a negative value */
+    size = (uint64_t) height * abs(stride);
+    if (size > MAX_DATA_CHUNK) {
+        return false;
+    }
+
+    return true;
+}
+
 int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,
                         RedSurfaceCmd *red, QXLPHYSICAL addr)
 {
     QXLSurfaceCmd *qxl;
     uint64_t size;
     int error;
-    unsigned int bpp;
 
     qxl = (QXLSurfaceCmd *)get_virt(slots, addr, sizeof(*qxl), group_id,
                                     &error);
@@ -1331,26 +1358,13 @@ int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,
         red->u.surface_create.width  = qxl->u.surface_create.width;
         red->u.surface_create.height = qxl->u.surface_create.height;
         red->u.surface_create.stride = qxl->u.surface_create.stride;
-        bpp = surface_format_to_bpp(red->u.surface_create.format);
 
-        /* check if format is valid */
-        if (!bpp) {
+        if (!red_validate_surface(red->u.surface_create.width, red->u.surface_create.height,
+                                  red->u.surface_create.stride, red->u.surface_create.format)) {
             return 1;
         }
 
-        /* check stride is larger than required bytes */
-        size = ((uint64_t) red->u.surface_create.width * bpp + 7u) / 8u;
-        /* the uint32_t conversion is here to avoid problems with -2^31 value */
-        if (red->u.surface_create.stride == G_MININT32
-            || size > (uint32_t) abs(red->u.surface_create.stride)) {
-            return 1;
-        }
-
-        /* the multiplication can overflow, also abs(-2^31) may return a negative value */
-        size = (uint64_t) red->u.surface_create.height * abs(red->u.surface_create.stride);
-        if (size > MAX_DATA_CHUNK) {
-            return 1;
-        }
+        size = red->u.surface_create.height * abs(red->u.surface_create.stride);
         red->u.surface_create.data =
             (uint8_t*)get_virt(slots, qxl->u.surface_create.data, size, group_id, &error);
         if (error) {
diff --git a/server/red_parse_qxl.h b/server/red_parse_qxl.h
index 3adc9fa..e18d8d0 100644
--- a/server/red_parse_qxl.h
+++ b/server/red_parse_qxl.h
@@ -19,6 +19,8 @@
 #ifndef RED_ABI_TRANSLATE_H
 #define RED_ABI_TRANSLATE_H
 
+#include <stdbool.h>
+
 #include <spice/qxl_dev.h>
 #include "red_common.h"
 #include "red_memslots.h"
@@ -128,6 +130,9 @@ int red_get_message(RedMemSlotInfo *slots, int group_id,
                     RedMessage *red, QXLPHYSICAL addr);
 void red_put_message(RedMessage *red);
 
+bool red_validate_surface(uint32_t width, uint32_t height,
+                          int32_t stride, uint32_t format);
+
 int red_get_surface_cmd(RedMemSlotInfo *slots, int group_id,
                         RedSurfaceCmd *red, QXLPHYSICAL addr);
 void red_put_surface_cmd(RedSurfaceCmd *red);