Azure Service Fabric Stateful Services timeout when called downstream - azure

Question

I have a strange issue with a stateful service timing out, only when called downstream by another service.
The idea of this project is to record the latency of hops between nodes, given n number of hops. The implementation is in 3 projects:
API - to call downstream to the worker
Worker Service - to decrement a count and call another WorkerService
Result Service - to record the results to be fetched by the API.
The solution throws an exception both on premise and in Azure cloud, but works perfectly locally.
Implementation:
internal sealed class WorkerService : StatelessService, IHopWorker
{
public WorkerService(StatelessServiceContext context)
: base(context)
{ }
protected override IEnumerable<ServiceInstanceListener> CreateServiceInstanceListeners()
{
return this.CreateServiceRemotingInstanceListeners();
}
public async Task Hop(HopCounter counter)
{
counter.HopCount = counter.HopCount - 1;
counter.JunctionIds.Add(Context.NodeContext.NodeId.ToString());
if (counter.HopCount < 1)
{
counter.Completed = DateTime.Now;
await new ServiceProxyFactory().CreateServiceProxy<IResultService>(new Uri("fabric:/AsfNetworkHops/ResultsService"), new ServicePartitionKey("Data")).AddResults(counter);
}
else
{
await new ServiceProxyFactory().CreateServiceProxy<IHopWorker>(new Uri("fabric:/AsfNetworkHops/WorkerService"), targetReplicaSelector: TargetReplicaSelector.RandomInstance).Hop(counter);
}
}
}
Result Service
internal sealed class ResultsService : StatefulService, IResultService
{
Task<IReliableDictionary<string, HopCounter>> Results => StateManager.GetOrAddAsync<IReliableDictionary<string, HopCounter>>("results");
public ResultsService(StatefulServiceContext context)
: base(context, new ReliableStateManager(context, new ReliableStateManagerConfiguration(
new ReliableStateManagerReplicatorSettings
{
MaxReplicationMessageSize = 1073741824
})))
{
}
protected override IEnumerable<ServiceReplicaListener> CreateServiceReplicaListeners()
{
return this.CreateServiceRemotingReplicaListeners();
}
public async Task AddResults(HopCounter counter)
{
using (var transcation = StateManager.CreateTransaction())
{
var results = await Results;
await results.AddAsync(transcation, Guid.NewGuid().ToString(), counter);
await transcation.CommitAsync();
}
}
public async Task<HopCounter[]> GetResults()
{
IList<HopCounter> results = new List<HopCounter>();
using (var tx = StateManager.CreateTransaction())
{
var res = await Results;
var enumerator = (await res.CreateEnumerableAsync(tx)).GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(new CancellationToken()))
{
results.Add(enumerator.Current.Value);
}
await tx.CommitAsync();
return results.ToArray();
}
}
}
API Layer
[Route("api/[controller]")]
public class HopController : Controller
{
[HttpGet]
[Route("health")]
public IActionResult Health()
{
return Ok("OK");
}
[HttpGet]
[Route("start")]
public async Task<IActionResult> Start(int hopCount = 10)
{
try
{
var counter = new HopCounter();
counter.HopCount = hopCount;
counter.Initiated = DateTime.Now;
IHopWorker worker = null;
try
{
worker = new ServiceProxyFactory().CreateServiceProxy<IHopWorker>(new Uri("fabric:/AsfNetworkHops/WorkerService"));
}
catch (Exception excep)
{
return Ok($"Error creating proxy {excep.Message} {excep.StackTrace}");
}
await worker.Hop(counter);
return Ok();
}
catch (Exception excep)
{
return Ok(excep);
}
}
[HttpGet]
[Route("results")]
public async Task<IActionResult> Results()
{
try
{
IResultService results = null;
try
{
results = new ServiceProxyFactory().CreateServiceProxy<IResultService>(
new Uri("fabric:/AsfNetworkHops/ResultsService"), new ServicePartitionKey("Data"));
}
catch (Exception excep)
{
return Ok($"Error creating proxy {excep.Message} {excep.StackTrace}");
}
return Ok(await results.GetResults());
}
catch (Exception excep)
{
return Ok(excep);
}
}
}
App Manifest:
<ApplicationManifest xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" ApplicationTypeName="AsfNetworkHopsType" ApplicationTypeVersion="1.0.0" xmlns="http://schemas.microsoft.com/2011/01/fabric">
<Parameters>
<Parameter Name="ResultsService_MinReplicaSetSize" DefaultValue="3" />
<Parameter Name="ResultsService_PartitionCount" DefaultValue="1" />
<Parameter Name="ResultsService_TargetReplicaSetSize" DefaultValue="3" />
<Parameter Name="NetworkHopsAPI_InstanceCount" DefaultValue="-1" />
<Parameter Name="WorkerService_InstanceCount" DefaultValue="-1" />
</Parameters>
<!-- Import the ServiceManifest from the ServicePackage. The ServiceManifestName and ServiceManifestVersion
should match the Name and Version attributes of the ServiceManifest element defined in the
ServiceManifest.xml file. -->
<ServiceManifestImport>
<ServiceManifestRef ServiceManifestName="ResultsServicePkg" ServiceManifestVersion="1.0.0" />
<ConfigOverrides />
</ServiceManifestImport>
<ServiceManifestImport>
<ServiceManifestRef ServiceManifestName="NetworkHopsAPIPkg" ServiceManifestVersion="1.0.0" />
<ConfigOverrides />
</ServiceManifestImport>
<ServiceManifestImport>
<ServiceManifestRef ServiceManifestName="WorkerServicePkg" ServiceManifestVersion="1.0.0" />
<ConfigOverrides />
</ServiceManifestImport>
<DefaultServices>
<!-- The section below creates instances of service types, when an instance of this
application type is created. You can also create one or more instances of service type using the
ServiceFabric PowerShell module.
The attribute ServiceTypeName below must match the name defined in the imported ServiceManifest.xml file. -->
<Service Name="ResultsService" ServicePackageActivationMode="ExclusiveProcess">
<StatefulService ServiceTypeName="ResultsServiceType" TargetReplicaSetSize="[ResultsService_TargetReplicaSetSize]" MinReplicaSetSize="[ResultsService_MinReplicaSetSize]">
<NamedPartition>
<Partition Name="Data" />
</NamedPartition>
</StatefulService>
</Service>
<Service Name="NetworkHopsAPI" ServicePackageActivationMode="ExclusiveProcess">
<StatelessService ServiceTypeName="NetworkHopsAPIType" InstanceCount="[NetworkHopsAPI_InstanceCount]">
<SingletonPartition />
</StatelessService>
</Service>
<Service Name="WorkerService" ServicePackageActivationMode="ExclusiveProcess">
<StatelessService ServiceTypeName="WorkerServiceType" InstanceCount="[WorkerService_InstanceCount]">
<SingletonPartition />
</StatelessService>
</Service>
</DefaultServices>
</ApplicationManifest>
DTO
[DataContract]
[Serializable]
public class HopCounter
{
[DataMember]
private DateTime _completed;
[DataMember]
public DateTime Initiated { get; set; }
[DataMember]
public DateTime Completed
{
get { return _completed; }
set
{
_completed = value;
MilisecondsTaken = (Completed - Initiated).Milliseconds;
}
}
[DataMember]
public int HopCount { get; set; }
[DataMember]
public List<string> JunctionIds { get; set; } = new List<string>();
[DataMember]
public bool WorkerOnly { get; set; }
[DataMember]
public double MilisecondsTaken { get; set; }
}
The timeout only occurs if the Results Service is called, otherwise it returns a HTTP 200
This happens in both .Net Core & .Net 4.6.1 so must be the pattern I am using but it seems like a pretty normal access pattern to me.
I have an example application here.
Can anyone help?
The error is:
The operation has timed out. at
Microsoft.ServiceFabric.Replicator.DynamicStateManager.d__188.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at
Microsoft.ServiceFabric.Replicator.DynamicStateManager.d__187.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at
Microsoft.ServiceFabric.Replicator.DynamicStateManager.d__170.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at
Microsoft.ServiceFabric.Replicator.DynamicStateManager.d__1091.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at
Microsoft.ServiceFabric.Data.ReliableStateManagerImpl.<Microsoft-ServiceFabric-Data-IReliableStateManager-GetOrAddAsync>d__231.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at
Microsoft.ServiceFabric.Data.ReliableStateManagerImpl.d__251.MoveNext()
--- End of stack trace from previous location where exception was thrown --- at
System.Runtime.ExceptionServices.ExceptionDispatchInfo.Throw() at
System.Runtime.CompilerServices.TaskAwaiter.HandleNonSuccessAndDebuggerNotification(Task
task) at System.Runtime.CompilerServices.TaskAwaiter1.GetResult()
at Results.Results.d__4.MoveNext() in
C:\dev\asf-latency\Results\Results.cs:line 42