MVC, Lucene, AzureDirectory, Application Architecture - azure

Question

I've just included Lucene search into my first application (ASP.NET MVC 5) but I'm having some issues indexing my database initially and in figuring out what the correct overall application architecture should be.
My current setup is:
Azure scheduler that calls into my web API to request indexing of all categories, and then separately all items. Another scheduler that runs once every 3 months or so to call Optimize (db won't change that often). The API grabs all entities from the DB and adds a new CloudQueueMessage to a CloudQueueClient.
A running WebJob pulls from the queue and calls back into my web API with an ID. The API grabs the item and adds or removes it from the Lucene index accordingly using the AzureDirectory project (https://azuredirectory.codeplex.com/). This all functions as expected, and if I manually post ID's to my API method, everything is great. When it runs through the WebJob, somehow my index becomes corrupt and any call, even to get the number of indexed items, returns a 404 not found looking for a file that doesn't exist in my directory (typically something like _1a.cfs).
I'm thinking it's some sort of a locking issue or that my objects aren't being disposed of properly, but I can't see where I'm going wrong and I'm not sure this is the best way to structure the application and the workflow. Latter half of the code pasted below, any help would be hugely appreciated!
WebJob:
public class Program
{
static void Main()
{
JobHost host = new JobHost();
host.RunAndBlock();
}
public static void IndexItemOrCategory([QueueTrigger("searchindexrequest")] string id)
{
string baseUri = ConfigurationSettings.AppSettings["BaseUri"];
using (var client = new HttpClient())
{
client.BaseAddress = new Uri(baseUri);
string response = client.GetStringAsync("api/search/indexitem/" + id).Result;
Console.Out.WriteLine(response);
}
}
}
API:
public HttpResponseMessage IndexItem(string id)
{
try
{
var item = db.Items.Find(dbId);
if (item != null && item.Active && !string.IsNullOrWhiteSpace(item.Name))
LuceneSearch.AddUpdateLuceneIndex(item);
else
{
LuceneSearch.RemoveLuceneIndexRecord<Item>(dbId);
removed = true;
}
}
catch (Exception exc)
{
Request.CreateErrorResponse(HttpStatusCode.InternalServerError, exc);
}
return Request.CreateResponse(HttpStatusCode.OK, id + (removed ? " removed " : " indexed ") + "successfully.");
}
public class LuceneSearch
{
private static CloudStorageAccount storageAccount = CloudStorageAccount.Parse(ConfigurationManager.ConnectionStrings["AzureConnection"].ToString());
private static AzureDirectory azureDirectory = new AzureDirectory(storageAccount, "lucene-search", new RAMDirectory());
public static void AddUpdateLuceneIndex(Item item)
{
AddToLuceneIndex(item);
}
private static void AddToLuceneIndex(Item toIndex)
{
using (StandardAnalyzer analyzer = new StandardAnalyzer(Lucene.Net.Util.Version.LUCENE_30))
{
using (IndexWriter writer = new IndexWriter(azureDirectory, analyzer, new IndexWriter.MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)))
{
string Id = "Item" + toIndex.ItemID.ToString();
//Remove any existing index entry
var searchQuery = new TermQuery(new Term("Id", Id));
writer.DeleteDocuments(searchQuery);
string name = string.IsNullOrWhiteSpace(toIndex.CommonName) ? toIndex.Name : toIndex.Name + ", " + toIndex.CommonName;
if (!string.IsNullOrWhiteSpace(name))
{
//Create new entry
Document doc = new Document();
doc.Add(new Field("Id", Id, Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("Name", toIndex.Name, Field.Store.YES, Field.Index.ANALYZED));
if (!string.IsNullOrWhiteSpace(toIndex.Description1))
doc.Add(new Field("Description", toIndex.Description1, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("CategoryName", toIndex.Category.Name, Field.Store.YES, Field.Index.ANALYZED));
doc.Add(new Field("SeoUrl", toIndex.SeoUrl, Field.Store.YES, Field.Index.NOT_ANALYZED));
if (!string.IsNullOrWhiteSpace(toIndex.Image1))
doc.Add(new Field("Image", toIndex.Image1, Field.Store.YES, Field.Index.NOT_ANALYZED));
doc.Add(new Field("CategorySeoUrl", toIndex.Category.SeoUrl, Field.Store.YES, Field.Index.NOT_ANALYZED));
writer.AddDocument(doc);
}
writer.Dispose();
}
}
}
...
}