OPTIMUS-36426 export changes from 489132a74cc30c74a7bafec0e01e84fc510…

…f0890

.gitattributes

@@ -1,24 +1,21 @@
 # Declare files that will always have LF line endings on checkout
-*.sh text eol=lf
-*.md text eol=lf
-*.java text eol=lf
-*.scala text eol=lf
-*.groovy text eol=lf
 *.conf text eol=lf
-*.txt text eol=lf
-*.proto text eol=lf
-*.fbs text eol=lf
-*.sql text eol=lf
 *.csv text eol=lf
-*.xml text eol=lf
+*.fbs text eol=lf
+*.groovy text eol=lf
+*.java text eol=lf
 *.ksh text eol=lf
+*.md text eol=lf
+*.pl text eol=lf
+*.pm text eol=lf
+*.proto text eol=lf
 *.py text eol=lf
+*.scala text eol=lf
+*.sh text eol=lf
+*.sql text eol=lf
+*.txt text eol=lf
+*.xml text eol=lf
 
 # Similarly for CRLF
 *.bat text eol=crlf
 *.cmd text eol=crlf
-# Perl scripts should always have LF (Unix default) line ending
-*.pl text eol=lf
-*.pm text eol=lf
-
-

NOTICE

@@ -2,7 +2,7 @@ Optimus Cirrus
 
 Copyright 2021 Morgan Stanley.
 
-This product includes software developed at Morgan Stanley.
+This product includes software developed at Morgan Stanley. Please refer to the LICENSE file for the details and the bundled licenses.
 
 This product includes software developed at the Apache Software Foundation (http://www.apache.org/), including:
   - Apache ZooKeeper (https://zookeeper.apache.org/)
@@ -11,6 +11,9 @@ This product bundles software from http://github.com/scalamacros/macrology201, w
 For details, see optimus/platform/projects/core_macro/src/main/scala/optimus/utils/internal/Splicer.scala
 
 This product includes software developed by other groups and added in accordance with their Apache 2.0 licenses. Projects include:
-  - https://github.com/spray/spray-json (in spray_json module; refer to https://github.com/spray/spray-json/pull/222/files)
+  - https://github.com/spray/spray-json (in optimus.platform.spray_json module; refer to https://github.com/spray/spray-json/pull/222/files)
   - https://github.com/corretto/hotpatch-for-apache-log4j2 (in optimus/platform/projects/entityagent/src/main/java/optimus/JndiDefanger.java)
   - https://github.com/thombergs/diffparser (in git-utils/src/main/scala/optimus/git/diffparser/DiffParser.java)
+  - https://github.com/gradle/gradle (in optimus.buildtool.htmlreport), specifically from the 'core' and 'testing-base' subprojects
+  - https://github.com/scala/scala (in optimus/buildtool/projects/core/src/main/scala/optimus/buildtool/utils/parser/Parser.scala)
+  - https://github.com/pelamfi/pelam-scala-incubator (in optimus/buildtool/projects/runconf/src/main/scala/optimus/buildtool/runconf/compile/TopologicalSort.scala)

optimus/platform/projects/alarms/src/main/scala/optimus/exceptions/RTListStatic.scala

@@ -40,6 +40,7 @@ object RTListStatic {
     "org.apache.commons.math3.exception.NumberIsTooLargeException" ::
     "org.apache.commons.lang.NotImplementedException" ::
     "org.apache.commons.lang3.NotImplementedException" ::
+    "org.xml.sax.SAXParseException" ::
     Nil).toSet
 
   // Not so sure about these. Please add comments when augmenting.

optimus/platform/projects/breadcrumbs/src/main/scala/optimus/breadcrumbs/crumbs/Properties.scala

@@ -753,6 +753,7 @@ object Properties extends KnownProperties {
 
   val oldCacheSize = propI
   val newCacheSize = propI
+
 }
 
 final case class RequestsStallInfo(pluginType: StallPlugin.Value, reqCount: Int, req: Seq[String]) {

optimus/platform/projects/entityagent/src/main/java/optimus/EntityAgent.java

@@ -31,7 +31,6 @@
 import optimus.graph.chaos.ChaosMonkeyInjector;
 import optimus.junit.CachingJunitRunnerInjector;
 import optimus.systemexit.ExitInterceptProp;
-import optimus.systemexit.SystemExitTransformer;
 import optimus.testidle.TestIdle;
 import optimus.testidle.TestIdleTransformer;
 
@@ -171,7 +170,7 @@ public static void premain(String agentArgs, Instrumentation instrumentation) th
     InstrumentationConfig.init();
 
     // To patch java core classes to call our methods, we need to explicitly put the jar containing those methods on the
-    // botstrap class loader path.  To find that jar, we let our current classloader search for a class we added just to be
+    // bootstrap class loader path.  To find that jar, we let our current classloader search for a class we added just to be
     // searched for.
 
     var allowCorePatches = false;
@@ -232,10 +231,6 @@ public byte[] transform(ClassLoader loader,
       customTransformers.put("optimus/core/TPDMask", new TPDMaskTransfomer());
     }
 
-    if (System.getProperty(ExitInterceptProp.name) != null) {
-      instrumentation.addTransformer(new SystemExitTransformer(), instrumentation.isRetransformClassesSupported());
-      instrumentation.retransformClasses(java.lang.System.class);
-    }
     if (System.getProperty(TestIdle.monitorIdlePropertyName) != null) {
       instrumentation.addTransformer(new TestIdleTransformer(), true);
     }
@@ -248,6 +243,12 @@ public byte[] transform(ClassLoader loader,
 
     EntityAgentTransformer transformer = new EntityAgentTransformer(allowCorePatches);
     instrumentation.addTransformer(transformer, true);
+
+    if (System.getProperty(ExitInterceptProp.name) != null) {
+      InstrumentationConfig.addSystemExitPrefix();
+      instrumentation.retransformClasses(java.lang.System.class);
+    }
+
     if (instrumentation.isNativeMethodPrefixSupported()) instrumentation.setNativeMethodPrefix(transformer, nativePrefix);
     else logErrMsg("Native Method Prefix not supported. Skipping...");
   }

optimus/platform/projects/entityagent/src/main/java/optimus/debug/InstrumentationCmds.java

@@ -254,7 +254,9 @@ public static void prefixCallWithDumpOnTransitivelyCached(String methodToPatch)
   public static void prefixECCurrentWithTriggerIfInModuleCtor() {
     var moduleCtorTrigger = "optimus.debug.InstrumentedModuleCtor.trigger";
     prefixCall("optimus.graph.OGSchedulerContext.current", moduleCtorTrigger);
-    /* eventually we will detect the methods with an annotation:
+    /* eventually we will detect the methods with an annotation. Until then:
+      @see optimus.graph.OGSchedulerContext#_TRACESUPPORT_unsafe_current()
+
     prefixCall("optimus.platform.ScenarioStack.getNode", moduleCtorTrigger);
     prefixCall("optimus.platform.ScenarioStack.env", moduleCtorTrigger);
     prefixCall("optimus.platform.ScenarioStack.getTrackingNodeID", moduleCtorTrigger);
@@ -390,6 +392,6 @@ public static void verifyEntityDeepEqualityDuringCaching() {
     suffix.suffixWithThis = true;
     suffix.suffixWithReturnValue = true;
     suffix.suffixWithArgs = true;
-    suffix.suffixNoArgumentBoxing = true;
+    suffix.noArgumentBoxing = true;
   }
 }

optimus/platform/projects/entityagent/src/main/java/optimus/debug/InstrumentationConfig.java

@@ -20,6 +20,7 @@
 import java.util.function.Predicate;
 
 import optimus.EntityAgent;
+import optimus.systemexit.SystemExitReplacement;
 import org.objectweb.asm.Type;
 
 enum EntityInstrumentationType {
@@ -100,7 +101,8 @@ public class InstrumentationConfig {
   private final static String getNodeDesc = "(Loptimus/graph/PropertyNode;Loptimus/graph/OGSchedulerContext;)Loptimus/graph/Node;";
   private final static String verifySSGetNodeDesc = "(Loptimus/graph/Node;Loptimus/platform/ScenarioStack;Loptimus/graph/PropertyNode;Loptimus/graph/OGSchedulerContext;)V";
   private static final InstrumentationConfig.MethodRef verifyScenarioStackGetNode = new InstrumentationConfig.MethodRef(IS, "verifyScenarioStackGetNode", verifySSGetNodeDesc);
-  private static final InstrumentationConfig.MethodRef verifyRunAndWaitEntry = new InstrumentationConfig.MethodRef(IS, "runAndWaitEntry", "(Loptimus/graph/OGSchedulerContext;)V");
+  private static final InstrumentationConfig.MethodRef verifyRunAndWaitEntry =
+      new InstrumentationConfig.MethodRef(IS, "runAndWaitEntry", "(Loptimus/graph/OGSchedulerContext;Loptimus/graph/NodeTask;)V");
   private static final InstrumentationConfig.MethodRef verifyRunAndWaitExit = new InstrumentationConfig.MethodRef(IS, "runAndWaitExit", "(Loptimus/graph/OGSchedulerContext;)V");
   private static final InstrumentationConfig.MethodRef scenarioStackGetNode = new InstrumentationConfig.MethodRef(SS_TYPE, "getNode", getNodeDesc);
   private static final InstrumentationConfig.MethodRef runAndWait = new InstrumentationConfig.MethodRef(OGSC_TYPE, "runAndWait");
@@ -148,23 +150,31 @@ static void addModuleOrEntityExclusion(String className) {
 
   static void addVerifyScenarioStackCalls() {
     var ssHook = addSuffixCall(scenarioStackGetNode, verifyScenarioStackGetNode);
+    ssHook.noArgumentBoxing = true;
     ssHook.suffixWithThis = true;
     ssHook.suffixWithArgs = true;
-    ssHook.suffixNoArgumentBoxing = true;
     ssHook.suffixWithReturnValue = true;
 
-    var runAndWaitEntry = addPrefixCall(runAndWait, verifyRunAndWaitEntry, false, false);
+    var runAndWaitEntry = addPrefixCall(runAndWait, verifyRunAndWaitEntry, false, true);
+    runAndWaitEntry.noArgumentBoxing = true;
     runAndWaitEntry.prefixWithThis = true;
 
     var runAndWaitExit = addSuffixCall(runAndWait, verifyRunAndWaitExit);
-    runAndWaitExit.suffixNoArgumentBoxing = true;
     runAndWaitExit.suffixWithThis = true;
   }
 
   public static void recordConstructorInvocationSite(String className) {
     addRecordPrefixCallIntoMemberWithCallSite(__constructedAt, asMethodRef(className + ".<init>"));
   }
 
+  public static void addSystemExitPrefix() {
+    var systemExit = new MethodRef("java/lang/System", "exit");
+    var systemExitPrefix = new MethodRef("optimus/debug/InstrumentedSystem", "exitPrefix", "(I)V");
+    var prefix = addPrefixCall(systemExit, systemExitPrefix, false, true);
+    InstrumentedSystem.callback = SystemExitReplacement::exitImpl;
+    prefix.noArgumentBoxing = true;
+  }
+
   public static class MethodDesc {
     final public MethodRef ref;
     public volatile Object tag;   // Used to stash NodeName for JVMNodeTask [SEE_TRACE_AS_NODE]
@@ -212,12 +222,16 @@ static class MethodPatch {
     boolean suffixWithThis;
     boolean suffixWithReturnValue; // calls suffix, then it returns the original value
     boolean suffixWithArgs;
-    boolean suffixNoArgumentBoxing;
+    boolean noArgumentBoxing;
     boolean wrapWithTryCatch; // adds a try catch and invokes suffixOnException in case of exception thrown
     FieldRef storeToField;
     ClassPatch classPatch;
     BiPredicate<String, String> predicate;
 
+    boolean shouldInject(String name, String descriptor) {
+      return predicate == null || predicate.test(name, descriptor) || (classPatch.replaceObjectAsBase != null && name.equals("<init>"));
+    }
+
     MethodPatch(MethodRef from) { this.from = from; }
   }
 

optimus/platform/projects/entityagent/src/main/java/optimus/debug/InstrumentationInjector.java

@@ -263,7 +263,7 @@ public MethodVisitor visitMethod(int access, String name, String desc, String si
     if (classPatch.allMethodsPatch != null)
       return new InstrumentationInjectorMethodVisitor(classPatch.allMethodsPatch, mv, access, name, desc);
     MethodPatch methodPatch = classPatch.forMethod(name, desc);
-    if (methodPatch != null && (methodPatch.predicate == null || methodPatch.predicate.test(name, desc)))
+    if (methodPatch != null)
       return new InstrumentationInjectorMethodVisitor(methodPatch, mv, access, name, desc);
     else if (classPatch.cacheAllApplies && name.equals("apply") && isCreateEntityMethod(desc))
       return new InstrumentationInjectorMethodVisitor(patchForCachingMethod(className, name), mv, access, name, desc);
@@ -367,10 +367,13 @@ class InstrumentationInjectorMethodVisitor extends AdviceAdapter implements Opco
   private final Label tryBlockEnd = new Label();
   private final Label catchBlockStart = new Label();
   private final Label catchBlockEnd = new Label();
+  private final boolean doInject;
+
   InstrumentationInjectorMethodVisitor(MethodPatch patch, MethodVisitor mv, int access,
                                        String name, String descriptor) {
     super(ASM9, mv, access, name, descriptor);
     this.patch = patch;
+    this.doInject = patch.predicate == null || patch.predicate.test(name, methodDesc);
 
     if (patch.passLocalValue && !patch.localValueIsCallWithArgs) {
       localValueType = patch.prefix.descriptor != null
@@ -397,10 +400,11 @@ private void dupReturnValueOrNullForVoid(int opcode, boolean boxValueTypes) {
       visitInsn(ACONST_NULL);
     else if (opcode == ARETURN || opcode == ATHROW)
       dup();
-    else if (opcode == LRETURN || opcode == DRETURN)
-      dup2(); // double/long take two slots
     else {
-      dup();
+      if (opcode == LRETURN || opcode == DRETURN)
+        dup2(); // double/long take two slots
+      else
+        dup();
       if (boxValueTypes)
         box(Type.getReturnType(this.methodDesc));
     }
@@ -421,6 +425,16 @@ private String loadThisOrNull() {
     return OBJECT_DESC;
   }
 
+  private String loadArgsInlineOrAsArray() {
+    if (patch.noArgumentBoxing) {
+      loadArgs();
+      return "";  // Rely on descriptor explicitly supplied
+    } else {
+      loadArgArray();
+      return OBJECT_ARR_DESC;
+    }
+  }
+
   private String loadLocalValueIfRequested() {
     if (patch.passLocalValue) {
       mv.visitVarInsn(localValueType.getOpcode(ILOAD), __localValue);
@@ -461,10 +475,8 @@ private void injectMethodPrefix() {
     if(patch.prefixWithThis)
       descriptor += loadThisOrNull();
 
-    if (patch.prefixWithArgs) {
-      loadArgArray();
-      descriptor += OBJECT_ARR_DESC;
-    }
+    if (patch.prefixWithArgs)
+      descriptor += loadArgsInlineOrAsArray();
 
     if (patch.passLocalValue || patch.storeToField != null)
       descriptor += ")" + OBJECT_DESC;
@@ -515,7 +527,8 @@ private void injectMethodPrefix() {
   @Override
   public void visitCode() {
     super.visitCode();
-    injectMethodPrefix();
+    if (doInject)
+      injectMethodPrefix();
   }
 
   @Override
@@ -526,7 +539,7 @@ protected void onMethodEnter() {
 
   @Override
   protected void onMethodExit(int opcode) {
-    if (opcode == ATHROW && (patch.suffixNoArgumentBoxing || patch.wrapWithTryCatch))
+    if (opcode == ATHROW && (patch.noArgumentBoxing || patch.wrapWithTryCatch))
       return; // do not generate exit call at the point of exception throw
 
     if (patch.cacheInField != null) {
@@ -536,13 +549,13 @@ protected void onMethodExit(int opcode) {
       mv.visitFieldInsn(PUTFIELD, patch.from.cls, patch.cacheInField.name, patch.cacheInField.type);
     }
 
-    if (patch.suffix == null)
+    if (patch.suffix == null || !doInject)
       return;
 
     var descriptor = "(";
 
     if (patch.suffixWithReturnValue) {
-      dupReturnValueOrNullForVoid(opcode, !patch.suffixNoArgumentBoxing);
+      dupReturnValueOrNullForVoid(opcode, !patch.noArgumentBoxing);
       descriptor += OBJECT_DESC;
     }
 
@@ -555,7 +568,7 @@ protected void onMethodExit(int opcode) {
       descriptor += loadThisOrNull();
 
     if (patch.suffixWithArgs)
-      loadArgs();
+      descriptor += loadArgsInlineOrAsArray();
 
     descriptor += ")V";
 
@@ -576,6 +589,10 @@ public void visitMethodInsn(int opcodeAndSource, String owner, String name, Stri
 
   @Override
   public void visitMaxs(int maxStack, int maxLocals) {
+    if (!doInject) {
+      super.visitMaxs(maxStack, maxLocals);
+      return;
+    }
     if (patch.wrapWithTryCatch) {
       visitLabel(tryBlockEnd);
       mv.visitInsn(NOP);

optimus/platform/projects/entityagent/src/main/java/optimus/graph/DiagnosticSettings.java

@@ -68,6 +68,7 @@ public class DiagnosticSettings {
   final public static boolean isClassMonitorEnabled;
   final public static int showThisNumberOfTopUsedClasses;
   final public static boolean keepFullTraceFile;
+  final public static String traceToOverride;
   final public static String fullTraceDir;
   final public static boolean profileShowThreadSummary; // light profile shows per-thread breakdown at shutdown
   final public static boolean profilerDisableHotspotsCSV;
@@ -83,10 +84,20 @@ public class DiagnosticSettings {
    * __profileId, getProfileId, cctor are injected in CompletableNode derived classes to better attribute profile data
    */
   final public static boolean injectNodeMethods;  // Defaults to traceAvailable, can't be turned off by a "nmi_off" flag
+
   /**
    * Detect and report any non-RT behaviour
    */
   final public static boolean enableRTVerifier = getBoolProperty("optimus.rt.verifier", false);
+  /**
+   * Publish RT violations as crumbs in splunk
+   */
+  final public static boolean publishRTVerifierCrumbs = getBoolProperty("optimus.rt.verifier.crumbs", true);
+  /**
+   * Report RT violations to file
+   */
+  final public static boolean writeRTVerifierReport = getBoolProperty("optimus.rt.verifier.report", false);
+
   /**
    * Report exactly why a cross-scenario lookup failed (for nodes with favorReuse = true)
    */
@@ -148,6 +159,7 @@ public class DiagnosticSettings {
 
   public static double infoDumpPeriodicityHours = getDoubleProperty("optimus.diagnostic.dump.period.hours", 0.0);
   public static boolean fullHeapDumpOnKill = getBoolProperty("optimus.diagnostic.dump.heap", false);
+  public static boolean fakeOutOfMemoryErrorOnKill = getBoolProperty("optimus.diagnostic.dump.fake.oom", false);
   public static String infoDumpDir;
   public static boolean infoDumpOnShutdown = getBoolProperty("optimus.diagnostic.dump.shutdown", false);
 
@@ -460,6 +472,8 @@ private static boolean existsInCmdLine(String key) {
     classBiopsyClasses = getStringPropertyAsSet(AGENT_BIOPSY_CLASSES);
     classDumpEnabled = !classDumpClasses.isEmpty() || !classBiopsyClasses.isEmpty();
 
+    traceToOverride = getStringProperty("optimus.traceTo", null);
+
     // set this to true to avoid deleting the backing storage for the full trace: this is intended to be useful
     // when debugging the full trace profiler (the user-serviceable full trace is produced by --profile-graph
     // fulltrace))