Debug info V2 support (#160)

* renames (not compiling) * WIP * update GetStackFrames logic * WIP (breakpoint manager busted) * cleanup disassembly manager * WIP * update Source object logic in GetStackFrames * minor cleanup * WIP * array and map containers * WIP * Ignore return types for now * Updated how script identification works * cleanup * update lib-bctk * move changelog to repo root * Update docs * update changelog * update doc * ngd ent nuget link Co-authored-by: Harry <[email protected]>
neo-project · Mar 14, 2022 · be9a588 · be9a588
1 parent 50de777
commit be9a588
Show file tree

Hide file tree

Showing 23 changed files with 713 additions and 796 deletions.
diff --git a/.github/workflows/build-vscode.yml b/.github/workflows/build-vscode.yml
@@ -11,7 +11,7 @@ on:
       - 'release/**'
     paths-ignore:
       - 'readme.md'
-      - 'src/extension/CHANGELOG.md'
+      - 'CHANGELOG.md'
       - 'docs/**'
   workflow_call:
     outputs:
@@ -81,8 +81,10 @@ jobs:
 
     - name: copy debug adapter packages into extension folder
       run: cp adapters/*.nupkg src/extension/
-    - name: copy repo root readme into extension folder
-      run: cp readme.md src/extension/
+    - name: copy repo root readme + changelog into extension folder
+      run: |
+        cp readme.md src/extension/
+        cp CHANGELOG.md src/extension/
 
     - name: Install debug extension dependencies
       run: npm ci

diff --git a/src/extension/CHANGELOG.md → CHANGELOG.md b/src/extension/CHANGELOG.md → CHANGELOG.md
@@ -18,6 +18,18 @@ may not exactly match a publicly released version.
   marketplace, but will not have the prerelease string in the version number. For more
   details, please see [Versioning Strategy](https://github.com/neo-project/neo-debugger#versioning-strategy)
 
+## [Unreleased]
+
+### Changed
+
+* Neo Debug Info version 2 support (current v2 proposal: https://github.com/devhawk/proposals/blob/devhawk/cd2l/nep-19.md)
+* Associate all known debug infos with deployed contract hash on debugger startup (debug engine only)
+* Load all scripts into Disassembly manager on startup (debug engine only)
+
+### Removed
+
+* Hand-authored Storage Schema support has been replaces with Neo Debug Info v2 support.
+
 ## [3.2.34] - 2022-03-11
 
 ### Fixed

diff --git a/docs/storage-schema-overview.md b/docs/storage-schema-overview.md
diff --git a/src/adapter3/BreakpointManager.cs b/src/adapter3/BreakpointManager.cs
@@ -10,114 +10,136 @@ namespace NeoDebug.Neo3
 {
     class BreakpointManager
     {
-        private readonly DisassemblyManager disassemblyManager;
-        private readonly IReadOnlyList<DebugInfo> debugInfoList;
-        private readonly Dictionary<UInt160, ImmutableHashSet<int>> breakpointCache = new Dictionary<UInt160, ImmutableHashSet<int>>();
-        private readonly Dictionary<string, IReadOnlyList<SourceBreakpoint>> sourceBreakpointMap = new Dictionary<string, IReadOnlyList<SourceBreakpoint>>();
+        readonly DisassemblyManager disassemblyManager;
+        readonly Func<IEnumerable<KeyValuePair<UInt160, DebugInfo>>> getDebugInfos;
+        readonly Dictionary<string, IReadOnlySet<(UInt160 hash, int position)>> sourceBreakpoints = new();
+        readonly Dictionary<UInt160, IReadOnlySet<int>> disassemblyBreakpoints = new(); 
+        readonly Dictionary<UInt160, IReadOnlySet<int>> breakpointCache = new();
 
-        public BreakpointManager(DisassemblyManager disassemblyManager, IReadOnlyList<DebugInfo> debugInfoList)
+        public BreakpointManager(DisassemblyManager disassemblyManager, Func<IEnumerable<KeyValuePair<UInt160, DebugInfo>>> getDebugInfos)
         {
             this.disassemblyManager = disassemblyManager;
-            this.debugInfoList = debugInfoList;
+            this.getDebugInfos = getDebugInfos;
         }
 
         public IEnumerable<Breakpoint> SetBreakpoints(Source source, IReadOnlyList<SourceBreakpoint> sourceBreakpoints)
         {
-            breakpointCache.Clear();
-            this.sourceBreakpointMap[source.Path] = sourceBreakpoints;
-
-            if (UInt160.TryParse(source.Name, out var scriptHash))
+            if (source.SourceReference.HasValue)
             {
-                var lineMap = disassemblyManager.TryGetDisassembly(scriptHash, out var disassembly)
-                    ? disassembly.LineMap : ImmutableDictionary<int, int>.Empty;
+                if (disassemblyManager.TryGet(source.SourceReference.Value, out var disassembly))
+                {
+                    HashSet<int> breakpoints = new();
+                    foreach (var sbp in sourceBreakpoints)
+                    {
+                        var validated = disassembly.LineMap.TryGetValue(sbp.Line, out var address);
 
-                foreach (var sbp in sourceBreakpoints)
+                        breakpoints.Add(address);
+
+                        yield return new Breakpoint(validated)
+                        {
+                            Column = sbp.Column,
+                            Line = sbp.Line,
+                            Source = source
+                        };
+                    }
+                    disassemblyBreakpoints[disassembly.ScriptHash] = breakpoints;
+                }
+                else
                 {
-                    yield return new Breakpoint()
+                    foreach (var sbp in sourceBreakpoints)
                     {
-                        Verified = lineMap.TryGetValue(sbp.Line, out var _) ? true : false,
-                        Column = sbp.Column,
-                        Line = sbp.Line,
-                        Source = source
-                    };
+                        yield return new Breakpoint(false)
+                        {
+                            Column = sbp.Column,
+                            Line = sbp.Line,
+                            Source = source
+                        };
+                    }
                 }
             }
             else
             {
-                var sequencePoints = debugInfoList
-                    .SelectMany(d => d.Methods.SelectMany(m => m.SequencePoints).Select(sp => (d, sp)))
-                    .Where(t => t.sp.PathEquals(t.d, source.Path))
-                    .Select(t => t.sp)
-                    .ToImmutableList();
+                var sbpValidated = new bool[sourceBreakpoints.Count];
+                HashSet<(UInt160, int)> breakpoints = new();
+
+                foreach (var (scriptHash, debugInfo) in getDebugInfos())
+                {
+
+                    if (!TryFindDocumentIndex(debugInfo.Documents, source.Path, out var index)) continue;
+
+                    // TODO: Cache this?
+                    var pointLookup = debugInfo.Methods
+                        .SelectMany(m => m.SequencePoints)
+                        .Where(sp => sp.Document == index)
+                        .ToLookup(sp => sp.Start.line);
+
+                    for (int j = 0; j < sourceBreakpoints.Count; j++)
+                    {
+                        SourceBreakpoint? sbp = sourceBreakpoints[j];
+                        var validated = pointLookup.TryLookup(sbp.Line, out var points);
+
+                        if (validated)
+                        {
+                            sbpValidated[j] = true;
+                            breakpoints.Add((scriptHash, points.First().Address));
+                        }
+                    }
+                }
+
+                this.sourceBreakpoints[source.Path] = breakpoints;
 
-                foreach (var sbp in sourceBreakpoints)
+                for (int i = 0; i < sourceBreakpoints.Count; i++)
                 {
-                    yield return new Breakpoint()
+                    var sbp = sourceBreakpoints[i];
+                    yield return new Breakpoint(sbpValidated[i])
                     {
-                        Verified = sequencePoints.Any(sp => sp.Start.line == sbp.Line),
                         Column = sbp.Column,
                         Line = sbp.Line,
                         Source = source
                     };
                 }
             }
-        }
 
-        private ImmutableHashSet<int> GetBreakpoints(UInt160 scriptHash)
-        {
-            if (!breakpointCache.TryGetValue(scriptHash, out var breakpoints))
+            this.breakpointCache.Clear();
+
+            var srcBPs = this.sourceBreakpoints.SelectMany(kvp => kvp.Value);
+            var dsmBPs = this.disassemblyBreakpoints.SelectMany(bp => bp.Value.Select(p => (hash: bp.Key, position: p)));
+
+            var groupedBPs = srcBPs.Concat(dsmBPs).GroupBy(bp => bp.hash);
+            foreach (var contractBPs in groupedBPs)
             {
-                var builder = ImmutableHashSet.CreateBuilder<int>();
+                this.breakpointCache[contractBPs.Key] = contractBPs.Select(t => t.position)
+                    .Distinct()
+                    .ToHashSet();
+            }
 
-                foreach (var kvp in sourceBreakpointMap)
+            static bool TryFindDocumentIndex(IReadOnlyList<string> documents, string path, out int index)
+            {
+                for (int i = 0; i < documents.Count; i++)
                 {
-                    if (UInt160.TryParse(kvp.Key, out var sourceScriptHash))
-                    {
-                        if (sourceScriptHash == scriptHash)
-                        {
-                            var lineMap = disassemblyManager.TryGetDisassembly(scriptHash, out var disassembly)
-                                ? disassembly.LineMap : ImmutableDictionary<int, int>.Empty;
-
-                            foreach (var sbp in kvp.Value)
-                            {
-                                if (lineMap.TryGetValue(sbp.Line, out var address))
-                                {
-                                    builder.Add(address);
-                                }
-                            }
-                        }
-                    }
-                    else
+                    if (documents[i].Equals(path, StringComparison.OrdinalIgnoreCase))
                     {
-                        foreach (var debugInfo in debugInfoList)
-                        {
-                            var sequencePoints = debugInfo.Methods
-                                .SelectMany(m => m.SequencePoints)
-                                .Where(sp => sp.PathEquals(debugInfo, kvp.Key))
-                                .ToImmutableList();
-
-                            foreach (var sbp in kvp.Value)
-                            {
-                                var foundSP = sequencePoints.Find(sp => sp.Start.line == sbp.Line);
-
-                                if (foundSP != null)
-                                {
-                                    builder.Add(foundSP.Address);
-                                }
-                            }
-                        }
+                        index = i;
+                        return true;
                     }
                 }
 
-                breakpoints = builder.ToImmutable();
-                breakpointCache[scriptHash] = breakpoints;
+                index = 0;
+                return false;
             }
-
-            return breakpoints;
         }
 
-        public bool CheckBreakpoint(UInt160 scriptHash, int? instructionPointer)
-            => instructionPointer.HasValue
-                && GetBreakpoints(scriptHash).Contains(instructionPointer.Value);
+        public bool CheckBreakpoint(IExecutionContext? context)
+        {
+            if (context is null) return false;
+
+            if (breakpointCache.TryGetValue(context.ScriptHash, out var set)
+                && set.Contains(context.InstructionPointer))
+            {
+                return true;
+            }
+
+            return false;
+        }
     }
 }
diff --git a/src/adapter3/DebugApplicationEngine.ExecutionContextAdapter.cs b/src/adapter3/DebugApplicationEngine.ExecutionContextAdapter.cs
@@ -16,17 +16,17 @@ private class ExecutionContextAdapter : IExecutionContext
             public ExecutionContextAdapter(ExecutionContext context, IDictionary<UInt160, UInt160> scriptIdMap)
             {
                 this.context = context;
-                this.ScriptHash = context.GetScriptHash();
-
-                if (scriptIdMap.TryGetValue(this.ScriptHash, out var scriptHash))
-                {
-                    this.ScriptIdentifier = scriptHash;
-                }
-                else
-                {
-                    this.ScriptIdentifier = Neo.SmartContract.Helper.ToScriptHash(context.Script);
-                    scriptIdMap[this.ScriptHash] = this.ScriptIdentifier;
-                }
+                // this.ScriptHash = context.GetScriptHash();
+
+                // if (scriptIdMap.TryGetValue(context.GetScriptHash(), out var scriptHash))
+                // {
+                //     this.ScriptIdentifier = scriptHash;
+                // }
+                // else
+                // {
+                //     this.ScriptIdentifier = Neo.SmartContract.Helper.ToScriptHash(context.Script);
+                //     scriptIdMap[this.ScriptHash] = this.ScriptIdentifier;
+                // }
             }
 
             public Instruction CurrentInstruction => context.CurrentInstruction;
@@ -42,11 +42,13 @@ public ExecutionContextAdapter(ExecutionContext context, IDictionary<UInt160, UI
             public IReadOnlyList<StackItem> Arguments => Coalese(context.Arguments);
 
             public Script Script => context.Script;
-            public MethodToken[] Tokens => context.GetState<ExecutionContextState>()?.Contract.Nef.Tokens
+            public IReadOnlyList<MethodToken> Tokens => context.GetState<ExecutionContextState>()?.Contract?.Nef.Tokens
                 ?? Array.Empty<MethodToken>();
 
-            public UInt160 ScriptHash { get; }
-            public UInt160 ScriptIdentifier { get; }
+            public uint? NefChecksum => context.GetState<ExecutionContextState>()?.Contract?.Nef.CheckSum;
+
+            public UInt160 ScriptHash => context.GetScriptHash();
+            // public UInt160 ScriptIdentifier { get; }
 
             static IReadOnlyList<StackItem> Coalese(Neo.VM.Slot? slot) => (slot == null) ? Array.Empty<StackItem>() : slot;
         }

diff --git a/src/adapter3/DebugApplicationEngine.StorageContainer.cs b/src/adapter3/DebugApplicationEngine.StorageContainer.cs
@@ -16,8 +16,8 @@ private class StorageContainer : StorageContainerBase
             private readonly DataCache snapshot;
             private readonly int? contractId;
 
-            public StorageContainer(UInt160 scriptHash, DataCache snapshot, IReadOnlyList<StorageDef> storageDefs, byte addressVersion, StorageView storageView)
-                : base(storageDefs, addressVersion, storageView)
+            public StorageContainer(UInt160 scriptHash, DataCache snapshot, IReadOnlyList<StorageGroupDef>? storageGroupDefs, byte addressVersion, StorageView storageView)
+                : base(storageGroupDefs, addressVersion, storageView)
             {
                 this.snapshot = snapshot;
                 this.contractId = NativeContract.ContractManagement.GetContract(snapshot, scriptHash)?.Id;

diff --git a/src/adapter3/DebugApplicationEngine.cs b/src/adapter3/DebugApplicationEngine.cs
@@ -22,33 +22,17 @@ internal partial class DebugApplicationEngine : TestApplicationEngine, IApplicat
         private readonly IReadOnlyStore checkpointStore;
         private readonly InvocationStackAdapter invocationStackAdapter;
         private readonly IDictionary<UInt160, UInt160> scriptIdMap = new Dictionary<UInt160, UInt160>();
-        private readonly IReadOnlyDictionary<UInt160, ContractStorageSchema> schemaMap;
 
         public event EventHandler<(UInt160 scriptHash, string scriptName, string eventName, NeoArray state)>? DebugNotify;
         public event EventHandler<(UInt160 scriptHash, string scriptName, string message)>? DebugLog;
 
-        public DebugApplicationEngine(IVerifiable container, IReadOnlyStore checkpointStore, ImmutableDictionary<UInt160, ContractStorageSchema> schemaMap, ProtocolSettings settings, Block persistingBlock, Func<byte[], bool>? witnessChecker)
+        public DebugApplicationEngine(IVerifiable container, IReadOnlyStore checkpointStore, ProtocolSettings settings, Block persistingBlock, Func<byte[], bool>? witnessChecker)
             : base(TriggerType.Application, container, new SnapshotCache(checkpointStore), persistingBlock, settings, TestModeGas, witnessChecker)
         {
             this.Log += OnLog;
             this.Notify += OnNotify;
             this.checkpointStore = checkpointStore;
             invocationStackAdapter = new InvocationStackAdapter(this);
-
-            foreach (var contract in NativeContract.ContractManagement.ListContracts(Snapshot))
-            {
-                if (schemaMap.ContainsKey(contract.Hash)) continue;
-
-                var schemaJson = contract.Manifest.Extra?["storage-schema"];
-                if (schemaJson is not null)
-                {
-                    var schema = ContractStorageSchema.Parse(schemaJson);
-                    schemaMap = schemaMap.Add(contract.Hash, schema);
-                }
-            }
-
-            this.schemaMap = schemaMap;
-
         }
 
         public override void Dispose()
@@ -114,11 +98,9 @@ public bool TryGetContract(UInt160 scriptHash, [MaybeNullWhen(false)] out Script
             return false;
         }
 
-        public StorageContainerBase GetStorageContainer(UInt160 scriptHash, StorageView storageView)
+        public StorageContainerBase GetStorageContainer(UInt160 scriptHash, IReadOnlyList<StorageGroupDef>? storageGroups, StorageView storageView)
         {
-            var storageDefs = schemaMap.TryGetValue(scriptHash, out var schema)
-                ? schema.StorageDefs : Array.Empty<StorageDef>();
-            return new StorageContainer(scriptHash, Snapshot, storageDefs, AddressVersion, storageView);
+            return new StorageContainer(scriptHash, Snapshot, storageGroups, AddressVersion, storageView);
         }
 
         IReadOnlyCollection<IExecutionContext> IApplicationEngine.InvocationStack => invocationStackAdapter;