WIP - InteractionProfilerGrain

OrleansDashboard.Core/DashboardClient.cs

@@ -68,5 +68,10 @@ public async Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrai
  {
  return await dashboardGrain.TopGrainMethods().ConfigureAwait(false);
  }
+
+ public async Task<string> GetInteractionsGraph()
+ {
+ return await dashboardGrain.GetInteractionsGraph().ConfigureAwait(false);
+ }
  }
 }

OrleansDashboard.Core/IDashboardClient.cs

@@ -18,5 +18,6 @@ public interface IDashboardClient
  Task<Immutable<StatCounter[]>> GetCounters(string siloAddress);
  Task<Immutable<Dictionary<string, Dictionary<string, GrainTraceEntry>>>> GrainStats(string grainName);
  Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrainMethods();
+ Task<string> GetInteractionsGraph();
  }
 }

OrleansDashboard.Core/IDashboardGrain.cs

@@ -14,6 +14,9 @@ public interface IDashboardGrain : IGrainWithIntegerKey
 
  [OneWay]
  Task SubmitTracing(string siloAddress, Immutable<SiloGrainTraceEntry[]> grainCallTime);
+
+ [OneWay]
+ Task SubmitGrainInteraction(string interactionsGraph);
 
  Task<Immutable<DashboardCounters>> GetCounters();
 
@@ -24,5 +27,7 @@ public interface IDashboardGrain : IGrainWithIntegerKey
  Task<Immutable<Dictionary<string, GrainTraceEntry>>> GetSiloTracing(string address);
 
  Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrainMethods();
+
+ Task<string> GetInteractionsGraph();
  }
 }

OrleansDashboard.Core/Metrics/IGrainInteractionProfiler.cs

@@ -0,0 +1,15 @@
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Orleans;
+using Orleans.Concurrency;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Metrics
+{
+ public interface IInteractionProfiler : IGrainWithIntegerKey
+ {
+ [OneWay]
+ Task Track(GrainInteractionInfoEntry entry);
+ }
+}

OrleansDashboard.Core/Model/DashboardCounters.cs

@@ -26,5 +26,6 @@ public DashboardCounters()
  public SimpleGrainStatisticCounter[] SimpleGrainStats { get; set; }
  public int TotalActivationCount { get; set; }
  public ImmutableQueue<int> TotalActivationCountHistory { get; set; }
+ public string InteractionsGraph { get; set; }
  }
 }

OrleansDashboard.Core/Model/GrainInteractionInfoEntry.cs

@@ -0,0 +1,15 @@
+using System;
+
+namespace OrleansDashboard.Model
+{
+ [Serializable]
+ public class GrainInteractionInfoEntry
+ {
+ public string Grain { get; set; }
+ public string TargetGrain { get; set; }
+ public string Method { get; set; }
+ public uint Count { get; set; } = 1;
+
+ public string Key => Grain + ":" + (TargetGrain ?? string.Empty) + ":" + Method;
+ }
+}

OrleansDashboard/Implementation/GrainInteractionFilter.cs

@@ -0,0 +1,101 @@
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Orleans;
+using Orleans.Runtime;
+using OrleansDashboard.Metrics;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Implementation
+{
+ public sealed class GrainInteractionFilter : IIncomingGrainCallFilter, IOutgoingGrainCallFilter
+ {
+ private readonly IGrainFactory grainFactory;
+ private IInteractionProfiler grainInteractionProfiler;
+
+ public GrainInteractionFilter(IGrainFactory grainFactory)
+ {
+ this.grainFactory = grainFactory;
+ }
+
+ public async Task Invoke(IIncomingGrainCallContext context)
+ {
+ try
+ {
+ var call = GetGrainMethod(context);
+ TrackBeginInvoke(call.Grain, call.Method);
+ await context.Invoke();
+ }
+ finally
+ {
+ await TrackEndInvoke();
+ }
+ }
+
+ public async Task Invoke(IOutgoingGrainCallContext context)
+ {
+ try
+ {
+ var call = GetGrainMethod(context);
+ TrackBeginInvoke(call.Grain, call.Method);
+ await context.Invoke();
+ }
+ finally
+ {
+ await TrackEndInvoke();
+ }
+ }
+
+ private Stack<GrainInteractionInfoEntry> GetCallStack()
+ {
+ return RequestContext.Get(nameof(GrainInteractionFilter)) as Stack<GrainInteractionInfoEntry> ?? new Stack<GrainInteractionInfoEntry>();
+ }
+
+ private void SaveCallStack(Stack<GrainInteractionInfoEntry> stack)
+ {
+ if (stack.Count == 0)
+ {
+ RequestContext.Remove(nameof(GrainInteractionFilter));
+ }
+ else
+ {
+ RequestContext.Set(nameof(GrainInteractionFilter), stack);
+ }
+ }
+
+ private void TrackBeginInvoke(string grain, string method)
+ {
+ var stack = GetCallStack();
+ if (stack.TryPeek(out var info))
+ {
+ info.TargetGrain = grain;
+ }
+
+ stack.Push(new GrainInteractionInfoEntry
+ {
+ Grain = grain,
+ Method = method
+ });
+ SaveCallStack(stack);
+ }
+
+ private async Task TrackEndInvoke()
+ {
+ var stack = GetCallStack();
+ var info = stack.Pop();
+
+ grainInteractionProfiler ??= grainFactory.GetGrain<IInteractionProfiler>(0);
+ await grainInteractionProfiler.Track(info);
+ SaveCallStack(stack);
+ }
+
+ private (string Grain, string Method) GetGrainMethod(IIncomingGrainCallContext context)
+ {
+ return (context.InterfaceMethod.ReflectedType.Name, context.InterfaceMethod.Name);
+ }
+
+ private (string Grain, string Method) GetGrainMethod(IOutgoingGrainCallContext context)
+ {
+ return (context.InterfaceMethod.ReflectedType.Name, context.InterfaceMethod.Name);
+ }
+ }
+}

OrleansDashboard/Implementation/Grains/DashboardGrain.cs

@@ -113,7 +113,7 @@ public override Task OnActivateAsync()
 
  return base.OnActivateAsync();
  }
-
+ 
  public async Task<Immutable<DashboardCounters>> GetCounters()
  {
  await EnsureCountersAreUpToDate();
@@ -156,7 +156,12 @@ public async Task<Immutable<Dictionary<string, GrainMethodAggregate[]>>> TopGrai
  { "errors", values.Where(x => x.ExceptionCount > 0 && x.Count > 0).OrderByDescending(x => x.ExceptionCount / x.Count).Take(numberOfResultsToReturn).ToArray() },
  }.AsImmutable();
  }
-
+
+ public Task<string> GetInteractionsGraph()
+ {
+ return Task.FromResult(counters.InteractionsGraph);
+ }
+
  public Task Init()
  {
  // just used to activate the grain
@@ -169,5 +174,11 @@ public Task SubmitTracing(string siloAddress, Immutable<SiloGrainTraceEntry[]> g
 
  return Task.CompletedTask;
  }
+
+ public Task SubmitGrainInteraction(string interactionsGraph)
+ {
+ counters.InteractionsGraph = interactionsGraph;
+ return Task.CompletedTask;
+ }
  }
 }

OrleansDashboard/Implementation/Grains/InteractionProfilerGrain.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Options;
+using Orleans;
+using Orleans.Concurrency;
+using OrleansDashboard.Model;
+
+namespace OrleansDashboard.Metrics.Grains
+{
+ [Reentrant]
+ public class InteractionProfilerGrain : Grain, IInteractionProfiler
+ {
+ private const int DefaultTimerIntervalMs = 1000; // 1 second
+ private readonly Dictionary<string, Dictionary<string, GrainInteractionInfoEntry>> interaction = new();
+ private readonly DashboardOptions options;
+ private IDisposable timer;
+
+ public InteractionProfilerGrain(IOptions<DashboardOptions> options)
+ {
+ this.options = options.Value;
+ }
+
+ public Task Track(GrainInteractionInfoEntry entry)
+ {
+ if (interaction.TryGetValue(entry.Grain, out var existing))
+ {
+ if (existing.TryGetValue(entry.Key, out var existingEntry))
+ {
+ existingEntry.Count++;
+ }
+ else
+ {
+ existing[entry.Key] = entry;
+ }
+ }
+ else
+ {
+ interaction.Add(entry.Grain, new Dictionary<string, GrainInteractionInfoEntry>
+ {
+ [entry.Key] = entry
+ });
+ }
+
+ return Task.CompletedTask;
+ }
+
+ public override async Task OnActivateAsync()
+ {
+ var updateInterval = TimeSpan.FromMilliseconds(Math.Max(options.CounterUpdateIntervalMs, DefaultTimerIntervalMs));
+
+ try
+ {
+ timer = RegisterTimer(x => CollectStatistics((bool)x), true, updateInterval, updateInterval);
+ }
+ catch (InvalidOperationException)
+ {
+ Debug.WriteLine("Not running in Orleans runtime");
+ }
+
+ await base.OnActivateAsync();
+ }
+
+ private async Task CollectStatistics(bool canDeactivate)
+ {
+ var dashboardGrain = GrainFactory.GetGrain<IDashboardGrain>(0);
+ try
+ {
+ await dashboardGrain.SubmitGrainInteraction(BuildGraph());
+ }
+ catch (Exception)
+ {
+ // we can't get the silo stats, it's probably dead, so kill the grain
+ if (canDeactivate)
+ {
+ timer?.Dispose();
+ timer = null;
+
+ DeactivateOnIdle();
+ }
+ }
+ }
+
+ private string BuildGraph()
+ {
+ var content = string.Join("\n ", interaction.Values.SelectMany(s => s)
+ /*.Where(w=>!string.IsNullOrEmpty(w.To))*/
+ .Select(s => $"{s.Value.Grain} -> {s.Value.TargetGrain ?? s.Value.Grain+"_self"} [ label = \"{s.Value.Method}\", color=\"0.650 0.700 0.700\" ];"));
+
+ var colors = string.Join("\n", interaction.Values.SelectMany(s => s)
+ .Select(s => s.Value.Grain)
+ .Distinct()
+ .Select(s => $"{s} [color=\"0.628 0.227 1.000\"];"));
+
+ var graphCode = @$"
+digraph finite_state_machine {{
+rankdir=LR;
+ratio = fill;
+node [style=filled];
+ {content}
+
+{colors}
+}}";
+ return graphCode;
+ }
+ }
+}

OrleansDashboard/ServiceCollectionExtensions.cs

@@ -26,6 +26,8 @@ public static ISiloHostBuilder UseDashboard(this ISiloHostBuilder builder,
  {
  builder.ConfigureApplicationParts(parts => parts.AddDashboardParts());
  builder.ConfigureServices(services => services.AddDashboard(configurator));
+ builder.AddIncomingGrainCallFilter<GrainInteractionFilter>();
+ builder.AddOutgoingGrainCallFilter<GrainInteractionFilter>();
  builder.AddStartupTask<Dashboard>();
 
  return builder;