diff -r e7fdc9d9c376 -r 64413aaea8ed test/jdk/jdk/jfr/event/metadata/TestLookForUntestedEvents.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/jdk/jdk/jfr/event/metadata/TestLookForUntestedEvents.java Tue Nov 27 15:52:34 2018 -0800 @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2018, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package jdk.jfr.event.metadata; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Field; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import jdk.jfr.EventType; +import jdk.jfr.Experimental; +import jdk.jfr.FlightRecorder; +import jdk.test.lib.jfr.EventNames; +import jdk.test.lib.Utils; + +/** + * @test Check for JFR events not covered by tests + * @key jfr + * @requires vm.hasJFR + * @library /test/lib /test/jdk + * @run main jdk.jfr.event.metadata.TestLookForUntestedEvents + */ +public class TestLookForUntestedEvents { + private static final Path jfrTestRoot = Paths.get(Utils.TEST_SRC).getParent().getParent(); + private static final String MSG_SEPARATOR = "=========================="; + private static Set jfrEventTypes = new HashSet<>(); + + private static final Set knownEventsMissingFromEventNames = new HashSet<>( + Arrays.asList( + // The Z* events below should be marked as experimental; see: JDK-8213966 + "ZStatisticsSampler", "ZStatisticsCounter", + "ZPageAllocation", "ZThreadPhase" + ) + ); + + private static final Set hardToTestEvents = new HashSet<>( + Arrays.asList( + "DataLoss", "IntFlag", "ReservedStackActivation", + "DoubleFlag", "UnsignedLongFlagChanged", "IntFlagChanged", + "UnsignedIntFlag", "UnsignedIntFlagChanged", "DoubleFlagChanged") + ); + + // GC uses specific framework to test the events, instead of using event names literally. + // GC tests were inspected, as well as runtime output of GC tests. + // The following events below are know to be covered based on that inspection. + private static final Set coveredGcEvents = new HashSet<>( + Arrays.asList( + "MetaspaceGCThreshold", "MetaspaceAllocationFailure", "MetaspaceOOM", + "MetaspaceChunkFreeListSummary", "G1HeapSummary", "ParallelOldGarbageCollection", + "OldGarbageCollection", "G1GarbageCollection", "GCPhasePause", + "GCPhasePauseLevel1", "GCPhasePauseLevel2", "GCPhasePauseLevel3", + "GCPhasePauseLevel4", "GCPhaseConcurrent") + ); + + // This is a "known failure list" for this test. + // NOTE: if the event is not covered, a bug should be open, and bug number + // noted in the comments for this set. + private static final Set knownNotCoveredEvents = new HashSet<>( + // DumpReason: JDK-8213918, Shutdown: JDK-8213917 + Arrays.asList("DumpReason", "Shutdown") + ); + + + public static void main(String[] args) throws Exception { + for (EventType type : FlightRecorder.getFlightRecorder().getEventTypes()) { + if (type.getAnnotation(Experimental.class) == null) { + jfrEventTypes.add(type.getName().replace("jdk.", "")); + } + } + + checkEventNamesClass(); + lookForEventsNotCoveredByTests(); + } + + // Look thru JFR tests to make sure JFR events are referenced in the tests + private static void lookForEventsNotCoveredByTests() throws Exception { + List paths = Files.walk(jfrTestRoot) + .filter(Files::isRegularFile) + .filter(path -> isJavaFile(path)) + .collect(Collectors.toList()); + + Set eventsNotCoveredByTest = new HashSet<>(jfrEventTypes); + for (String event : jfrEventTypes) { + for (Path p : paths) { + if (findStringInFile(p, event)) { + eventsNotCoveredByTest.remove(event); + break; + } + } + } + + // Account for hard-to-test, experimental and GC tested events + eventsNotCoveredByTest.removeAll(hardToTestEvents); + eventsNotCoveredByTest.removeAll(coveredGcEvents); + eventsNotCoveredByTest.removeAll(knownNotCoveredEvents); + + if (!eventsNotCoveredByTest.isEmpty()) { + print(MSG_SEPARATOR + " Events not covered by test"); + for (String event: eventsNotCoveredByTest) { + print(event); + } + print(MSG_SEPARATOR); + throw new RuntimeException("Found JFR events not covered by tests"); + } + } + + // Make sure all the JFR events are accounted for in jdk.test.lib.jfr.EventNames + private static void checkEventNamesClass() throws Exception { + // jdk.test.lib.jfr.EventNames + Set eventsFromEventNamesClass = new HashSet<>(); + for (Field f : EventNames.class.getFields()) { + String name = f.getName(); + if (!name.equals("PREFIX")) { + String eventName = (String) f.get(null); + eventName = eventName.replace(EventNames.PREFIX, ""); + eventsFromEventNamesClass.add(eventName); + } + } + + // Account for the events that are known to be missing from the EventNames.java + eventsFromEventNamesClass.addAll(knownEventsMissingFromEventNames); + + if (!jfrEventTypes.equals(eventsFromEventNamesClass)) { + String exceptionMsg = "Events declared in jdk.test.lib.jfr.EventNames differ " + + "from events returned by FlightRecorder.getEventTypes()"; + print(MSG_SEPARATOR); + print(exceptionMsg); + print(""); + printSetDiff(jfrEventTypes, eventsFromEventNamesClass, + "jfrEventTypes", "eventsFromEventNamesClass"); + print(""); + + print("This could be because:"); + print("1) You forgot to write a unit test. Please do so in test/jdk/jdk/jfr/event/"); + print("2) You wrote a unit test, but you didn't reference the event in"); + print(" test/lib/jdk/test/lib/jfr/EventNames.java. "); + print("3) It is not feasible to test the event, not even a sanity test. "); + print(" Add the event name to test/lib/jdk/test/lib/jfr/EventNames.java "); + print(" and a short comment why it can't be tested"); + print("4) The event is experimental. Please add 'experimental=\"true\"' to "); + print(" element in metadata.xml if it is a native event, or @Experimental if it is a "); + print(" Java event. The event will now not show up in JMC"); + System.out.println(MSG_SEPARATOR); + throw new RuntimeException(exceptionMsg); + } + } + + // ================ Helper methods + private static boolean isJavaFile(Path p) { + String fileName = p.getFileName().toString(); + int i = fileName.lastIndexOf('.'); + if ( (i < 0) || (i > fileName.length()) ) { + return false; + } + return "java".equals(fileName.substring(i+1)); + } + + private static boolean findStringInFile(Path p, String searchTerm) throws IOException { + long c = 0; + try (Stream stream = Files.lines(p)) { + c = stream + .filter(line -> line.contains(searchTerm)) + .count(); + } + return (c != 0); + } + + private static void printSetDiff(Set a, Set b, + String setAName, String setBName) { + if (a.size() > b.size()) { + a.removeAll(b); + System.out.printf("Set %s has more elements than set %s:", setAName, setBName); + System.out.println(); + printSet(a); + } else { + b.removeAll(a); + System.out.printf("Set %s has more elements than set %s:", setBName, setAName); + System.out.println(); + printSet(b); + } + } + + private static void printSet(Set set) { + for (String e : set) { + System.out.println(e); + } + } + + private static void print(String s) { + System.out.println(s); + } +}