jdk-sandbox: comparison jdk/src/solaris/native/sun/java2d/x11/XRBackendNative.c

equal deleted inserted replaced

-:f013dabb0bf8
+:1f1914231324
 #define REQUIRED_XRENDER_VER3 3
 #define PKGINFO_LINE_LEN_MAX 256
 #define PKGINFO_LINE_CNT_MAX 50
+/*
+* X protocol uses (u_int16)length to specify the length in 4 bytes quantities
+* of the whole request.  Both XRenderFillRectangles() and XFillRectangles()
+* have provisions to fragment into several requests if the number of rectangles
+* plus the current x request does not fit into 65535*4 bytes.  While
+* XRenderCreateLinearGradient() and XRenderCreateRadialGradient() have
+* provisions to gracefully degrade if the resulting request would exceed
+* 65535*4 bytes.
+*
+* Below, we define a cap of 65535*4 bytes for the maximum X request payload
+* allowed for Non-(XRenderFillRectangles() or XFillRectangles()) API calls,
+* just to be conservative.  This is offset by the size of our maximum x*Req
+* type in this compilation unit, which is xRenderCreateRadiaGradientReq.
+*
+* Note that sizeof(xRenderCreateRadiaGradientReq) = 36
+*/
+#define MAX_PAYLOAD (262140u - 36u)
+#define MAXUINT (0xffffffffu)
 static jboolean IsXRenderAvailable(jboolean verbose) {
 void *xrenderlib;
 int major_opcode, first_event, first_error;
 color.blue = blue;
 if (rectCnt <= 256) {
 xRects = &sRects[0];
 } else {
+if (MAXUINT / sizeof(XRectangle) < (unsigned)rectCnt) {
+/* rectCnt too big, integer overflow */
+return;
+}
 xRects = (XRectangle *) malloc(sizeof(XRectangle) * rectCnt);
 if (xRects == NULL) {
 return;
 }
 }
 Picture gradient = 0;
 XRenderColor *colors;
 XFixed *stops;
 XLinearGradient grad;
+if (MAX_PAYLOAD / (sizeof(XRenderColor) + sizeof(XFixed))
+< (unsigned)numStops) {
+/* numStops too big, payload overflow */
+return -1;
+}
 if ((pixels = (jshort *)
 (*env)->GetPrimitiveArrayCritical(env, pixelsArray, NULL)) == NULL) {
 return -1;
 }
 if ((fractions = (jfloat *)
 grad.p2.y = y2;
 /*TODO optimized & malloc check*/
 colors = (XRenderColor *) malloc(numStops * sizeof(XRenderColor));
 stops =  (XFixed *) malloc(numStops * sizeof(XFixed));
+if (colors == NULL || stops == NULL) {
+if (colors != NULL) {
+free(colors);
+}
+if (stops != NULL) {
+free(stops);
+}
+(*env)->ReleasePrimitiveArrayCritical(env, pixelsArray, pixels, JNI_ABORT);
+(*env)->ReleasePrimitiveArrayCritical(env, fractionsArray, fractions, JNI_ABORT);
+return -1;
+}
 for (i=0; i < numStops; i++) {
 stops[i] = XDoubleToFixed(fractions[i]);
 colors[i].alpha = pixels[i*4 + 0];
 colors[i].red = pixels[i*4 + 1];
 Picture gradient = 0;
 XRenderColor *colors;
 XFixed *stops;
 XRadialGradient grad;
+if (MAX_PAYLOAD / (sizeof(XRenderColor) + sizeof(XFixed))
+< (unsigned)numStops) {
+/* numStops too big, payload overflow */
+return -1;
+}
 if ((pixels =
 (jshort *)(*env)->GetPrimitiveArrayCritical(env, pixelsArray, NULL)) == NULL) {
 return -1;
 }
 grad.outer.radius = outerRadius;
 /*TODO optimized & malloc check*/
 colors = (XRenderColor *) malloc(numStops * sizeof(XRenderColor));
 stops =  (XFixed *) malloc(numStops * sizeof(XFixed));
+if (colors == NULL || stops == NULL) {
+if (colors != NULL) {
+free(colors);
+}
+if (stops != NULL) {
+free(stops);
+}
+(*env)->ReleasePrimitiveArrayCritical(env, pixelsArray, pixels, JNI_ABORT);
+(*env)->ReleasePrimitiveArrayCritical(env, fractionsArray, fractions, JNI_ABORT);
+return -1;
+}
 for (i=0; i < numStops; i++) {
 stops[i] = XDoubleToFixed(fractions[i]);
 colors[i].alpha = pixels[i*4 + 0];
 colors[i].red = pixels[i*4 + 1];
 jbyteArray pixelDataArray, int pixelDataLength) {
 jlong *glyphInfoPtrs;
 unsigned char *pixelData;
 int i;
+if (MAX_PAYLOAD / (sizeof(XGlyphInfo) + sizeof(Glyph))
+< (unsigned)glyphCnt) {
+/* glyphCnt too big, payload overflow */
+return;
+}
 XGlyphInfo *xginfo = (XGlyphInfo *) malloc(sizeof(XGlyphInfo) * glyphCnt);
 Glyph *gid = (Glyph *) malloc(sizeof(Glyph) * glyphCnt);
 if (xginfo == NULL || gid == NULL) {
 if (xginfo != NULL) {
 }
 JNIEXPORT void JNICALL
 Java_sun_java2d_xr_XRBackendNative_XRFreeGlyphsNative
 (JNIEnv *env, jclass cls, jint glyphSet, jintArray gidArray, jint glyphCnt) {
+if (MAX_PAYLOAD / sizeof(Glyph) < (unsigned)glyphCnt) {
+/* glyphCnt too big, payload overflow */
+return;
+}
 /* The glyph ids are 32 bit but may be stored in a 64 bit long on
 * a 64 bit architecture. So optimise the 32 bit case to avoid
 * extra stack or heap allocations by directly referencing the
 * underlying Java array and only allocate on 64 bit.
 unsigned int *xids;
 XGlyphElt32 selts[24];
 unsigned int sids[256];
 int charCnt = 0;
+if ((MAX_PAYLOAD / sizeof(XGlyphElt32) < (unsigned)eltCnt)
+|| (MAX_PAYLOAD / sizeof(unsigned int) < (unsigned)glyphCnt)
+|| ((MAX_PAYLOAD - sizeof(XGlyphElt32)*(unsigned)eltCnt) /
+sizeof(unsigned int) < (unsigned)glyphCnt))
+{
+/* (eltCnt, glyphCnt) too big, payload overflow */
+return;
+}
 if (eltCnt <= 24) {
 xelts = &selts[0];
 }else {
 xelts = (XGlyphElt32 *) malloc(sizeof(XGlyphElt32) * eltCnt);
 if (xelts == NULL) {
 XRectangle sRects[256];
 if (rectCnt <= 256) {
 xRects = &sRects[0];
 } else {
+if (MAXUINT / sizeof(XRectangle) < (unsigned)rectCnt) {
+/* rectCnt too big, integer overflow */
+return;
+}
 xRects = (XRectangle *) malloc(sizeof(XRectangle) * rectCnt);
 if (xRects == NULL) {
 return;
 }
 }

changeset 19009	1f1914231324
parent 19006	fcda25164f7d
child 19171	14d41f09941b