6963023: ZDI-CAN-809: Sun JRE JPEGImageWriter.writeImage Remote Code Execution Vulnerability
authorbae
Wed, 30 Jun 2010 11:32:38 +0400
changeset 6865 e7987a9205f4
parent 6864 192093b11c67
child 6867 3e770ac705b6
6963023: ZDI-CAN-809: Sun JRE JPEGImageWriter.writeImage Remote Code Execution Vulnerability Reviewed-by: prr
jdk/src/share/native/sun/awt/image/jpeg/imageioJPEG.c
--- a/jdk/src/share/native/sun/awt/image/jpeg/imageioJPEG.c	Mon Jun 28 13:07:23 2010 -0700
+++ b/jdk/src/share/native/sun/awt/image/jpeg/imageioJPEG.c	Wed Jun 30 11:32:38 2010 +0400
@@ -2614,7 +2614,8 @@
     JSAMPROW scanLinePtr;
     int i, j;
     int pixelStride;
-    unsigned char *in, *out, *pixelLimit;
+    unsigned char *in, *out, *pixelLimit, *scanLineLimit;
+    unsigned int scanLineSize, pixelBufferSize;
     int targetLine;
     pixelBufferPtr pb;
     sun_jpeg_error_ptr jerr;
@@ -2650,19 +2651,25 @@
 
     }
 
+    scanLineSize = destWidth * numBands;
     if ((inCs < 0) || (inCs > JCS_YCCK) ||
         (outCs < 0) || (outCs > JCS_YCCK) ||
         (numBands < 1) || (numBands > MAX_BANDS) ||
         (srcWidth < 0) ||
         (destWidth < 0) || (destWidth > srcWidth) ||
         (destHeight < 0) ||
-        (stepX < 0) || (stepY < 0))
+        (stepX < 0) || (stepY < 0) ||
+        ((scanLineSize / numBands) < destWidth))  /* destWidth causes an integer overflow */
     {
         JNU_ThrowByName(env, "javax/imageio/IIOException",
                         "Invalid argument to native writeImage");
         return JNI_FALSE;
     }
 
+    if (stepX > srcWidth) {
+        stepX = srcWidth;
+    }
+
     bandSize = (*env)->GetIntArrayElements(env, bandSizes, NULL);
 
     for (i = 0; i < numBands; i++) {
@@ -2710,7 +2717,7 @@
     }
 
     // Allocate a 1-scanline buffer
-    scanLinePtr = (JSAMPROW)malloc(destWidth*numBands);
+    scanLinePtr = (JSAMPROW)malloc(scanLineSize);
     if (scanLinePtr == NULL) {
         RELEASE_ARRAYS(env, data, (const JOCTET *)(dest->next_output_byte));
         JNU_ThrowByName( env,
@@ -2718,6 +2725,7 @@
                          "Writing JPEG Stream");
         return data->abortFlag;
     }
+    scanLineLimit = scanLinePtr + scanLineSize;
 
     /* Establish the setjmp return context for sun_jpeg_error_exit to use. */
     jerr = (sun_jpeg_error_ptr) cinfo->err;
@@ -2866,6 +2874,8 @@
     }
 
     targetLine = 0;
+    pixelBufferSize = srcWidth * numBands;
+    pixelStride = numBands * stepX;
 
     // for each line in destHeight
     while ((data->abortFlag == JNI_FALSE)
@@ -2886,9 +2896,9 @@
 
         in = data->pixelBuf.buf.bp;
         out = scanLinePtr;
-        pixelLimit = in + srcWidth*numBands;
-        pixelStride = numBands*stepX;
-        for (; in < pixelLimit; in += pixelStride) {
+        pixelLimit = in + ((pixelBufferSize > data->pixelBuf.byteBufferLength) ?
+                           data->pixelBuf.byteBufferLength : pixelBufferSize);
+        for (; (in < pixelLimit) && (out < scanLineLimit); in += pixelStride) {
             for (i = 0; i < numBands; i++) {
                 if (scale !=NULL && scale[i] != NULL) {
                     *out++ = scale[i][*(in+i)];