ApacheIgnite stability with WebAPI 2 - iis

Question

I have a setup that runs ApacheIgnite 2.7.6 on multiple WebAPI services. I instantiate IIGnite within the Global.asax Application_Start method for each WebAPI service. Then within my controllers I access the cache from a public DataCache property in the WebApplication class. The services call each other and Ignite makes a common cache for all services. Now, when running unit tests continuously Ignite becomes unstable. The timing is random, sometimes 30 minutes, sometimes 20 hours. Looking for tips to make my code stable or tips to on how to recover after instability.
My setup code includes a loop for instantiating the IIgnite and looks something like this:
using Apache.Ignite.Core;
using Apache.Ignite.Core.Cache;
using Apache.Ignite.Core.Resource;
using Apache.Ignite.Linq;
namespace MyService
{
public class WebApiApplication : System.Web.HttpApplication
{
[InstanceResource]
private static IIgnite IgniteEngine; //{ get; set; } = null;
private static Apache.Ignite.Core.Cache.Configuration.QueryEntity queryEntity = new Apache.Ignite.Core.Cache.Configuration.QueryEntity( typeof( Tuple<string, byte> ), typeof( SomeClass ) );
public static ICache<Tuple<string, byte>, SomeClass> DataCache => IgniteEngine.GetOrCreateCache<Tuple<string, byte>, SomeClass>( new Apache.Ignite.Core.Cache.Configuration.CacheConfiguration( nameof( DataCache ), queryEntity ) );
protected void Application_Start( object sender, EventArgs e )
{
GlobalConfiguration.Configure(WebApiConfig.Register);
InitializeCache();
}
protected void Application_End( object sender, EventArgs e ){}
public static void InitializeCache()
{
try
{
short startCounter = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
//Get the Ignite name - give options for debug environment and production environment
string siteName = System.Web.Hosting.HostingEnvironment.SiteName ?? System.Reflection.Assembly.GetExecutingAssembly()?.GetName()?.Name ?? System.AppDomain.CurrentDomain?.FriendlyName;
#if ( DEBUG || UNIT_TEST )
//Ignite uses H2 Database internally. Cache data is represented as SQL tables based on the CacheConfiguration
//To open H2 debug console and examine the database at runtime, set the IGNITE_H2_DEBUG_CONSOLE environment variable to true before starting Ignite.
//A web browser window will open during Ignite startup and show the state of the H2 database on the current node. Wait for the caches to start and refresh the browser.
Environment.SetEnvironmentVariable( "IGNITE_H2_DEBUG_CONSOLE", "true" );
#endif
//The JVM takes time to start. It could take several minutes when debugging
do
{
startCounter++;
//Sleep a little between tries
#if ( DEBUG || UNIT_TEST )
if ( startCounter >= 1 ) System.Threading.Thread.Sleep( 2000 );
#endif
try
{//Try to get a pre-existing instance of Ignite
IgniteEngine = Ignition.TryGetIgnite( siteName );
if ( null == IgniteEngine )
{
Debug.WriteLine( "InitializeCache attempt {0} to get an existing ignite instance failed.".Format( startCounter ) );
}
else
{
Debug.WriteLine( "InitializeCache attempt {0} to get an existing ignite instance succeeded.".Format( startCounter ) );
}
}
catch ( Exception ex )
{
Debug.WriteLine( "InitializeCache error while trying to get existing ignite instance. {0}".Format( ex.Message ) );
}
if ( null == IgniteEngine )
{
try
{
//Start the engine
string path = System.Web.HttpContext.Current?.Server?.MapPath( #"~\bin\libs" ) ?? System.IO.Path.GetDirectoryName( System.Reflection.Assembly.GetExecutingAssembly().Location ) + #"\libs";
string workPath = #"C:\Dev\Somefolder";
IgniteEngine = Ignition.Start( new IgniteConfiguration
{
JvmClasspath = System.IO.Directory.GetFiles( path ).Aggregate( ( x, y ) => x + ";" + y ),
IgniteInstanceName = siteName,
//Logger = new LogHelperForApacheIgnite(),
WorkDirectory = workPath
} );
if ( null != IgniteEngine )
{
Debug.WriteLine( "InitializeCache success starting Apache Ignite after {0} attempts and {1} seconds".Format( startCounter, sw.Elapsed.TotalSeconds ) );
}
}
catch ( System.BadImageFormatException ex1 )
{
Debug.WriteLine( "InitializeCache failed while trying to start a new ignite instance. The Apache Ignite cache runs as a 64bit dll but an attempt was made to invoke it while running the unit test at 32bits. Fix this from the Test menu by selecting \"Test Settings -> Default Processor Architecture -> x64\". The full error message is: {0}".Format( ex1.Message() ) );
}
catch ( Exception ex2 )
{
if (ex2.InnerException != null && ex2.InnerException.Message.Contains( "ERROR_BAD_EXE_FORMAT" ))
{
Debug.WriteLine( "InitializeCache failed while trying to start a new ignite instance. The Apache Ignite cache runs as a 64bit dll but an attempt was made to invoke it while running the unit test at 32bits. Fix this from the Test menu by selecting \"Test Settings -> Default Processor Architecture -> x64\". The full error message is: {0}".Format( ex2.InnerException?.Message ) );
}
else
{
Debug.WriteLine( "InitializeCache error while trying to start a new Apache Ignite instance. {0}".Format( ex2.Message ) );
}
}
}
}
while ( null == IgniteEngine && sw.Elapsed.TotalMinutes <= 4 );//The JVM sometimes takes more than a minute
sw.Stop();
if ( IgniteEngine == null )
{
string message = "Apache Ignite failed to start. InitializeCache unable to start ignite after {0} tries and {1} seconds".Format( startCounter, sw.Elapsed.TotalSeconds );
Debug.WriteLine( message );
throw new Exception( message );
}
else
{
Debug.WriteLine( "InitializeCache: Ignite running fine. Proceeding to create cached structure(s)." );
}
}
catch ( Exception ex )
{
Debug.WriteLine( "InitializeCache: Exception thrown during Cache initialization {0}".Format( ex.Message ) );
}
}
}
}
I have seen two types of errors once Ignite becomes unstable:
(1)When calling DataCache.ContainsKey( new Tuple( somestring, somebyte ) ) I get:
```BinaryObjectException "Failed to serialize object [typeName=String]"
Failed to serialize object [typeName=String]
EXCEPTION
Failed to serialize object [typeName=String]
at Apache.Ignite.Core.Impl.PlatformJniTarget.InStreamOutLong[TR](Int32 type, Action`1 outAction, Func`3 inAction, Func`2 readErrorAction)
at Apache.Ignite.Core.Impl.PlatformTargetAdapter.DoOutInOpX(Int32 type, Action`1 outAction, Func`2 inErrorAction)
INNER EXCEPTION
{"class org.apache.ignite.binary.BinaryObjectException: Failed to serialize object [typeName=String]\r\n\tat org.apache.ignite.internal.binary.BinaryClassDescriptor.write(BinaryClassDescriptor.java:840)\r\n\```
(2)When calling DataCache.AsCacheQueryable().Where( x => x.Name == "Something" ).SingleOrDefault() or other queries I get
```Java exception occurred [class=java.lang.NullPointerException, message=]
Apache.Ignite.Core.Common.IgniteException, Apache.Ignite.Core, Version=2.7.6.40414, Culture=neutral, PublicKeyToken=a487a7ff0b2aaa4a
java.lang.NullPointerException
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.getConfiguration(IgniteCacheProxyImpl.java:254)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.getConfiguration(GatewayProtectedCacheProxy.java:121)```
For both errors I tried to recover by calling InitializeCache() to refresh IIGnite but I get new errors like java.lang.IllegalStateException: Grid is in invalid state to perform this operation. It either not started yet or has already being or have stopped [igniteInstanceName=Dydx.Logistics.Service, state=STOPPED] The issue here is not my appdomain restarting or other similar issues that I have seen in other StackOverflow posts. I run my unit tests continously until the cache fails.
Note that I don't host Ignite in IIS in PROD environments. But this code is easy for DEV environments. Hoping someone can point me in the right direction here...
Exception details:
EXCEPTION #1A - Occurs when calling DataCache.ContainsKey( new Tuple( somestring, somebyte ) )
Failed to serialize object [typeName=String]
at Apache.Ignite.Core.Impl.PlatformJniTarget.InStreamOutLong[TR](Int32 type, Action`1 outAction, Func`3 inAction, Func`2 readErrorAction)
at Apache.Ignite.Core.Impl.PlatformTargetAdapter.DoOutInOpX(Int32 type, Action`1 outAction, Func`2 inErrorAction)
INNER EXCEPTION
{"class org.apache.ignite.binary.BinaryObjectException: Failed to serialize object [typeName=String]\r\n\tat org.apache.ignite.internal.binary.BinaryClassDescriptor.write(BinaryClassDescriptor.java:840)\r\n\tat org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal0(BinaryWriterExImpl.java:223)\r\n\tat org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:164)\r\n\tat org.apache.ignite.internal.binary.BinaryWriterExImpl.marshal(BinaryWriterExImpl.java:151)\r\n\tat org.apache.ignite.internal.binary.BinaryWriterExImpl.writeObjectDetached(BinaryWriterExImpl.java:1506)\r\n\tat org.apache.ignite.internal.processors.platform.utils.PlatformUtils.writeError(PlatformUtils.java:647)\r\n\tat org.apache.ignite.internal.processors.platform.cache.PlatformCache.processInStreamOutLong(PlatformCache.java:771)\r\n\tat org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl.inStreamOutLong(PlatformTargetProxyImpl.java:67)\r\nCaused by: class org.apache.ignite.IgniteException: Failed to execute native callback because grid is stopping.\r\n\tat org.apache.ignite.internal.processors.platform.callback.PlatformCallbackGateway.enter(PlatformCallbackGateway.java:1200)\r\n\tat org.apache.ignite.internal.processors.platform.callback.PlatformCallbackGateway.memoryReallocate(PlatformCallbackGateway.java:837)\r\n\tat org.apache.ignite.internal.processors.platform.memory.PlatformExternalMemory.reallocate(PlatformExternalMemory.java:48)\r\n\tat org.apache.ignite.internal.processors.platform.memory.PlatformOutputStreamImpl.ensureCapacity(PlatformOutputStreamImpl.java:305)\r\n\tat org.apache.ignite.internal.processors.platform.memory.PlatformOutputStreamImpl.copyAndShift(PlatformOutputStreamImpl.java:331)\r\n\tat org.apache.ignite.internal.processors.platform.memory.PlatformOutputStreamImpl.writeByteArray(PlatformOutputStreamImpl.java:59)\r\n\tat org.apache.ignite.internal.binary.BinaryWriterExImpl.doWriteString(BinaryWriterExImpl.java:443)\r\n\tat org.apache.ignite.internal.binary.BinaryClassDescriptor.write(BinaryClassDescriptor.java:609)\r\n\t... 7 more\r\n"}
Inspecting the Cache object in the locals window just before where the exception is thrown shows that the Cache is already stopped. When I try to expand the data in the locals window I see:
EXCEPTION #1B - class org.apache.ignite.internal.processors.cache.CacheStoppedException: Failed to perform cache operation (cache is stopped): DataCache
at Apache.Ignite.Core.Impl.PlatformJniTarget.OutObjectInternal(Int32 type)
at Apache.Ignite.Core.Impl.Cache.CacheImpl`2.CreateEnumerator(Boolean loc, Int32 peekModes)
at Apache.Ignite.Core.Impl.Cache.CacheImpl`2.GetEnumerator()
at System.Linq.SystemCore_EnumerableDebugView`1.get_Items()
INNER EXCEPTION:
java.lang.IllegalStateException: class org.apache.ignite.internal.processors.cache.CacheStoppedException: Failed to perform cache operation (cache is stopped): DataCache
at org.apache.ignite.internal.processors.cache.GridCacheGateway.enter(GridCacheGateway.java:164)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.onEnter(GatewayProtectedCacheProxy.java:1555)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.iterator(GatewayProtectedCacheProxy.java:1326)
at org.apache.ignite.internal.processors.platform.cache.PlatformCache.processOutObject(PlatformCache.java:1046)
at org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl.outObject(PlatformTargetProxyImpl.java:105)
Caused by: class org.apache.ignite.internal.processors.cache.CacheStoppedException: Failed to perform cache operation (cache is stopped): DataCache
... 5 more
EXCEPTION #2 - Occurs when calling dataCache.AsCacheQueryable().Where( x => x.Name == "Something" ).SingleOrDefault();
Java exception occurred [class=java.lang.NullPointerException, message=]
Apache.Ignite.Core.Common.IgniteException, Apache.Ignite.Core, Version=2.7.6.40414, Culture=neutral, PublicKeyToken=a487a7ff0b2aaa4a
java.lang.NullPointerException
at org.apache.ignite.internal.processors.cache.IgniteCacheProxyImpl.getConfiguration(IgniteCacheProxyImpl.java:254)
at org.apache.ignite.internal.processors.cache.GatewayProtectedCacheProxy.getConfiguration(GatewayProtectedCacheProxy.java:121)
at org.apache.ignite.internal.processors.platform.cache.PlatformCache.processOutStream(PlatformCache.java:983)
at org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl.outStream(PlatformTargetProxyImpl.java:93)
Apache.Ignite.Core.Common.JavaException, Apache.Ignite.Core, Version=2.7.6.40414, Culture=neutral, PublicKeyToken=a487a7ff0b2aaa4a
Stack Trace
at Apache.Ignite.Core.Impl.Unmanaged.Jni.Env.ExceptionCheck()
at Apache.Ignite.Core.Impl.Unmanaged.UnmanagedUtils.TargetOutStream(GlobalRef target, Int32 opType, Int64 memPtr)
at Apache.Ignite.Core.Impl.PlatformJniTarget.OutStream[T](Int32 type, Func`2 readAction)
Again, if I inspect the dataCache just before the error point I see java.lang.IllegalStateException: Grid is in invalid state to perform this operation. It either not started yet or has already being or have stopped [igniteInstanceName=Dydx.Logistics.Service, state=STOPPED]
at org.apache.ignite.internal.GridKernalGatewayImpl.illegalState(GridKernalGatewayImpl.java:201)
at org.apache.ignite.internal.GridKernalGatewayImpl.readLock(GridKernalGatewayImpl.java:95)
at org.apache.ignite.internal.IgniteKernal.guard(IgniteKernal.java:3886)
at org.apache.ignite.internal.IgniteKernal.getOrCreateCache0(IgniteKernal.java:3002)
at org.apache.ignite.internal.IgniteKernal.getOrCreateCache(IgniteKernal.java:2992)
at org.apache.ignite.internal.processors.platform.PlatformProcessorImpl.processInStreamOutObject(PlatformProcessorImpl.java:566)
at org.apache.ignite.internal.processors.platform.PlatformTargetProxyImpl.inStreamOutObject(PlatformTargetProxyImpl.java:79)