# HG changeset patch
# User mbaesken
# Date 1567670383 -7200
# Node ID 9726449d264488689b54fb15dff7db8fb7dbda3f
# Parent  c8bc506106e3786fae799fe1825ab96d297b7975
8230466: check malloc/calloc results in jdk.hotspot.agent
Reviewed-by: cjplummer, ysuenaga, sspitsyn

diff -r c8bc506106e3 -r 9726449d2644 src/jdk.hotspot.agent/linux/native/libsaproc/symtab.c
--- a/src/jdk.hotspot.agent/linux/native/libsaproc/symtab.c	Thu Sep 05 11:09:12 2019 +0200
+++ b/src/jdk.hotspot.agent/linux/native/libsaproc/symtab.c	Thu Sep 05 09:59:43 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2014, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -210,6 +210,9 @@
                                 + strlen(".debug/")
                                 + strlen(debug_file_directory)
                                 + 2);
+  if (debug_pathname == NULL) {
+    return -1;
+  }
   strcpy(debug_pathname, name);
   char *last_slash = strrchr(debug_pathname, '/');
   if (last_slash == NULL) {
@@ -279,6 +282,9 @@
 
   filename = malloc(strlen (debug_file_directory) + (sizeof "/.build-id/" - 1) + 1
                     + 2 * size + (sizeof ".debug" - 1) + 1);
+  if (filename == NULL) {
+    return NULL;
+  }
   s = filename + sprintf (filename, "%s/.build-id/", debug_file_directory);
   if (size > 0)
     {
@@ -305,7 +311,9 @@
     = (unsigned char*)(note+1) + note->n_namesz;
   char *filename
     = (build_id_to_debug_filename (note->n_descsz, bytes));
-
+  if (filename == NULL) {
+    return NULL;
+  }
   fd = pathmap_open(filename);
   if (fd >= 0) {
     symtab = build_symtab_internal(fd, NULL, /* try_debuginfo */ false);
@@ -417,6 +425,10 @@
       htab_sz = n*1.25;
 
       symtab->hash_table = (struct hsearch_data*) calloc(1, sizeof(struct hsearch_data));
+      if (symtab->hash_table == NULL) {
+        goto bad;
+      }
+
       rslt = hcreate_r(n, symtab->hash_table);
       // guarantee(rslt, "unexpected failure: hcreate_r");
 
@@ -426,11 +438,17 @@
       // strings will not be destroyed by elf_end.
       size = scn_cache[shdr->sh_link].c_shdr->sh_size;
       symtab->strs = (char *)malloc(size);
+      if (symtab->strs == NULL) {
+        goto bad;
+      }
       memcpy(symtab->strs, scn_cache[shdr->sh_link].c_data, size);
 
       // allocate memory for storing symbol offset and size;
       symtab->num_symbols = n;
       symtab->symbols = (struct elf_symbol *)calloc(n , sizeof(struct elf_symbol));
+      if (symtab->symbols == NULL) {
+        goto bad;
+      }
 
       // copy symbols info our symtab and enter them info the hash table
       for (j = 0; j < n; j++, syms++) {
@@ -512,6 +530,11 @@
       symtab = prev_symtab;
     }
   }
+  goto quit;
+
+bad:
+  destroy_symtab(symtab);
+  symtab = NULL;
 
 quit:
   if (shbuf) free(shbuf);
diff -r c8bc506106e3 -r 9726449d2644 src/jdk.hotspot.agent/macosx/native/libsaproc/MacosxDebuggerLocal.m
--- a/src/jdk.hotspot.agent/macosx/native/libsaproc/MacosxDebuggerLocal.m	Thu Sep 05 11:09:12 2019 +0200
+++ b/src/jdk.hotspot.agent/macosx/native/libsaproc/MacosxDebuggerLocal.m	Thu Sep 05 09:59:43 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2002, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2002, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -373,7 +373,16 @@
 
   // Allocate storage for pages and flags.
   pages = malloc(pageCount * sizeof(vm_offset_t));
+  if (pages == NULL) {
+    (*env)->DeleteLocalRef(env, array);
+    return NULL;
+  }
   mapped = calloc(pageCount, sizeof(int));
+  if (mapped == NULL) {
+    (*env)->DeleteLocalRef(env, array);
+    free(pages);
+    return NULL;
+  }
 
   task_t gTask = getTask(env, this_obj);
   // Try to read each of the pages.
diff -r c8bc506106e3 -r 9726449d2644 src/jdk.hotspot.agent/macosx/native/libsaproc/symtab.c
--- a/src/jdk.hotspot.agent/macosx/native/libsaproc/symtab.c	Thu Sep 05 11:09:12 2019 +0200
+++ b/src/jdk.hotspot.agent/macosx/native/libsaproc/symtab.c	Thu Sep 05 09:59:43 2019 +0200
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2003, 2019, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -69,18 +69,22 @@
     if (is_debug()) {
       DBT rkey, rvalue;
       char* tmp = (char *)malloc(strlen(symtab->symbols[i].name) + 1);
-      strcpy(tmp, symtab->symbols[i].name);
-      rkey.data = tmp;
-      rkey.size = strlen(tmp) + 1;
-      (*symtab->hash_table->get)(symtab->hash_table, &rkey, &rvalue, 0);
-      // we may get a copy back so compare contents
-      symtab_symbol *res = (symtab_symbol *)rvalue.data;
-      if (strcmp(res->name, symtab->symbols[i].name)  ||
+      if (tmp == NULL) {
+        print_debug("error allocating array in build_search_table\n");
+      } else {
+        strcpy(tmp, symtab->symbols[i].name);
+        rkey.data = tmp;
+        rkey.size = strlen(tmp) + 1;
+        (*symtab->hash_table->get)(symtab->hash_table, &rkey, &rvalue, 0);
+        // we may get a copy back so compare contents
+        symtab_symbol *res = (symtab_symbol *)rvalue.data;
+        if (strcmp(res->name, symtab->symbols[i].name)  ||
           res->offset != symtab->symbols[i].offset    ||
           res->size != symtab->symbols[i].size) {
-        print_debug("error to get hash_table value!\n");
+            print_debug("error to get hash_table value!\n");
+        }
+        free(tmp);
       }
-      free(tmp);
     }
   }
 }