--- a/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/Config.java	Fri Jun 08 11:08:38 2018 +0200
+++ b/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/Config.java	Fri Jun 08 11:11:06 2018 +0200
@@ -45,6 +45,7 @@
 
     final int NREGION                   = 10;
     final int MAX_BACKREF_NUM           = 1000;
+    final int MAX_CAPTURE_GROUP_NUM     = 0x8000;
     final int MAX_REPEAT_NUM            = 100000;
     final int MAX_MULTI_BYTE_RANGES_NUM = 10000;
 

--- a/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/ScanEnvironment.java	Fri Jun 08 11:08:38 2018 +0200
+++ b/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/ScanEnvironment.java	Fri Jun 08 11:11:06 2018 +0200
@@ -62,6 +62,9 @@
     }
 
     public int addMemEntry() {
+        if (numMem >= Config.MAX_CAPTURE_GROUP_NUM) {
+            throw new InternalException(ErrorMessages.ERR_TOO_MANY_CAPTURE_GROUPS);
+        }
         if (numMem++ == 0) {
             memNodes = new Node[SCANENV_MEMNODES_SIZE];
         } else if (numMem >= memNodes.length) {

--- a/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/exception/ErrorMessages.java	Fri Jun 08 11:08:38 2018 +0200
+++ b/src/jdk.scripting.nashorn/share/classes/jdk/nashorn/internal/runtime/regexp/joni/exception/ErrorMessages.java	Fri Jun 08 11:11:06 2018 +0200
@@ -31,6 +31,7 @@
     final String ERR_PARSER_BUG = "internal parser error (bug)";
     final String ERR_UNDEFINED_BYTECODE = "undefined bytecode (bug)";
     final String ERR_UNEXPECTED_BYTECODE = "unexpected bytecode (bug)";
+    final String ERR_TOO_MANY_CAPTURE_GROUPS = "too many capture groups";
 
     /* syntax error */
     final String ERR_END_PATTERN_AT_LEFT_BRACE = "end pattern at left brace";

--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/nashorn/script/basic/JDK-8204290.js	Fri Jun 08 11:11:06 2018 +0200
@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * 
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ * 
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * JDK-8204290: Add check to limit number of capture groups
+ *
+ * @test
+ * @run
+ */
+
+try {
+    new RegExp("()".repeat(0x8001));
+    fail("Expected exception");
+} catch (e) {
+    Assert.assertTrue(e instanceof SyntaxError);
+    Assert.assertEquals(e.message, "too many capture groups");
+}
+