I have an azure function that sends a message to the service bus queue. Since a recent deployment, I see an exception occurring frequently: The connection was inactive for more than the allowed 60000 milliseconds and is closed by container.
I looked into this GitHub post: https://github.com/Azure/azure-service-bus-java/issues/280 it says this is a warning. Is there a way to increase this timeout? Or any suggestions on how to resolve this? Here is my code:
namespace Repositories.ServiceBusQueue
{
public class MembershipServiceBusRepository : IMembershipServiceBusRepository
{
private readonly QueueClient _queueClient;
public MembershipServiceBusRepository(string serviceBusNamespacePrefix, string queueName)
{
var msiTokenProvider = TokenProvider.CreateManagedIdentityTokenProvider();
_queueClient = new QueueClient($"https://{serviceBusNamespacePrefix}.servicebus.windows.net", queueName, msiTokenProvider);
}
public async Task SendMembership(GroupMembership groupMembership, string sentFrom = "")
{
if (groupMembership.SyncJobPartitionKey == null) { throw new ArgumentNullException("SyncJobPartitionKey must be set."); }
if (groupMembership.SyncJobRowKey == null) { throw new ArgumentNullException("SyncJobRowKey must be set."); }
foreach (var message in groupMembership.Split().Select(x => new Message
{
Body = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(x)),
SessionId = groupMembership.RunId.ToString(),
ContentType = "application/json",
Label = sentFrom
}))
{
await _queueClient.SendAsync(message);
}
}
}
}
This could be due to deadlock in the thread pool, please check if you are calling an async method from a sync method.
Related
I'm saving a row to my db (class with teacher/students, time, date, etc), once I have the id I create a message on my Azure service bus where the unique id of the row from my db is used as the message body of the service bus message. I'm creating scheduled messages so I can notify the students before the class and after the class is over so they can rate/review their teacher.
QUESTION - I'd like to know how to roll back or an easy way to remove the db row by not allowing it to fully save if the message to the Azure service bus fails to save?
Currently I'm using a generic repository with UnitOfWork to save to my db and I'm catching the exception from my service bus service if it fails, then deleting the row that was just saved, but it's sloppy looking and I can see it will lead to problems.
Here is what I'm doing now in the controller.
[HttpPost("create")]
public async Task<IActionResult> OnCreating(OnCreatingEventDto onCreatingDto)
{
var userFromRepo = await _userManager.FindByEmailFromClaimsPrinciple(HttpContext.User);
if (userFromRepo == null)
return Unauthorized(new ApiResponse(401));
var newEvent = _mapper.Map<ClassEvent>(onCreatingDto);
_unitOfWork.Repository<ClassEvent>().Add(newEvent);
var success = await _unitOfWork.Complete();
if (success > 0) {
try {
var sequenceNUmber = await _serviceBusProducer.SendMessage(newEvent.Id.ToString(), newEvent.eventTime.addDays(1), queueName);
newEvent.ServiceBusSequenceNumber = sequenceNUmber;
_unitOfWork.Repository<ClassEvent>().Update(newEvent);
var secondSuccess = await _unitOfWork.Complete();
if (secondSuccess > 0) {
return Ok();
}
} catch(Exception ex) {
_logger.LogError("error saving to service bus");
_unitOfWork.Repository<ClassEvent>().Delete(newEvent);
var deleteSuccess = await _unitOfWork.Complete();
if (deleteSuccess > 0) {
}
return BadRequest(new ApiResponse(400, "Problem Creating Event"));
}
}
return BadRequest(new ApiResponse(400, "Problem Creating Event"));
}
Here is the method from my service that creates the message on the queue
public async Task<long> SendMessage(string messageBody, DateTimeOffset scheduledEnqueueTime, string queueName)
{
await using (ServiceBusClient client = new ServiceBusClient(_config["ServiceBus:Connection"]))
{
ServiceBusSender sender = client.CreateSender(_config["ServiceBus:" + queueName]);
ServiceBusMessage message = new ServiceBusMessage(messageBody);
var sequenceNumber =
await sender.ScheduleMessageAsync(message, scheduledEnqueueTime);
return sequenceNumber;
}
}
In the following sample program (using MassTransit, Azure ServiceBus), I am able to send messages to the queue, but my Receive Endpoint/Consumer does not seems to get the message. What am I doing wrong here? (Simple publish and a handler example given in this link(http://masstransit-project.com/MassTransit/quickstart.html) works fine!)
static async Task MainAsync(string[] args)
{
var bus = Bus.Factory.CreateUsingAzureServiceBus(cfg =>
{
var serviceUri = ServiceBusEnvironment.CreateServiceUri("sb", "{sb}", "{sb-name}");
var host = cfg.Host(serviceUri, h =>
{
h.OperationTimeout = TimeSpan.FromSeconds(5);
h.TokenProvider = TokenProvider.CreateSharedAccessSignatureTokenProvider(
"RootManageSharedAccessKey",
"{key}");
h.TransportType = TransportType.NetMessaging;
});
cfg.ReceiveEndpoint(host, "test_queue", ep =>
{
ep.Consumer<SayHelloCommandConsumer>();
});
});
bus.Start();
await SendAHello(bus);
Console.WriteLine("Press any key to exit");
Console.ReadKey();
bus.Stop();
}
private static async Task SendAHello(IBusControl bus)
{
var sendToUri = new Uri("queue-end-point-address");
var endPoint = await bus.GetSendEndpoint(sendToUri);
await endPoint.Send<ISayHello>( new
{
Message = "Hello there !"
});
}
}
public class SayHelloCommandConsumer : IConsumer<ISayHello>
{
public Task Consume(ConsumeContext<ISayHello> context)
{
var command = context.Message;
return Console.Out.WriteLineAsync($"Recieved a message {command}");
}
}
public interface ISayHello
{
string Message { get; set; }
}
}
The queue address looked suspect, and it seems like you've corrected it.
public static async Task DoMessage()
{
const int numberOfMessages = 10;
queueClient = new QueueClient(ConnectionString, QueueName);
await SendMessageAsync(numberOfMessages);
await queueClient.CloseAsync();
}
private static async Task SendMessageAsync(int numOfMessages)
{
try
{
for (var i = 0; i < numOfMessages; i++)
{
var messageBody = $"Message {i}";
var message = new Message(Encoding.UTF8.GetBytes(messageBody));
message.SessionId = i.ToString();
await queueClient.SendAsync(message);
}
}
catch (Exception e)
{
}
}
This is my sample code to send message to the service bus queue with session id.
My question is if I call DoMessage function 2 times: Let's name it as MessageSet1 and MessageSet2, respectively. Will the MessageSet2 be received and processed by the received azure function who dealing with the receiving ends of the message.
I want to handle in order like MessageSet1 then the MessageSet2 and never handle with MessageSet2 unless MessageSet1 finished.
There are a couple of issues with what you're doing.
First, Azure Functions do not currently support sessions. There's an issue for that you can track.
Second, the sessions you're creating are off. A session should be applied on a set of messages using the same SessionId. Meaning your for loop should be assigning the same SessionId to all the messages in the set. Something like this:
private static async Task SendMessageAsync(int numOfMessages, string sessionID)
{
try
{
var tasks = new List<Task>();
for (var i = 0; i < numOfMessages; i++)
{
var messageBody = $"Message {i}";
var message = new Message(Encoding.UTF8.GetBytes(messageBody));
message.SessionId = sessionId;
tasks.Add(queueClient.SendAsync(message));
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
catch (Exception e)
{
// handle exception
}
}
For ordered messages using Sessions, see documentation here.
I have this code I am implemented from this azure wcf relay
I am getting this exception when sending a message bigger than 64K (with smaller message it works OK):
System.ServiceModel.CommunicationException: 'The maximum message size quota for incoming messages has been exceeded for the remote channel. See the server logs for more details.
the quota is unlimited in NetTcpRelayBinding according to this quota web page
here is my code
class WCFRelay
{
[ServiceContract(Namespace = "urn:ps")]
interface IProblemSolver
{
[OperationContract]
int Test(byte[] bytes);
}
class ProblemSolver : IProblemSolver
{
public int Test(byte[] bytes)
{
return bytes.Length;
}
}
interface IProblemSolverChannel : IProblemSolver, IClientChannel { }
public static void CreateClient()
{
var cf = new ChannelFactory<IProblemSolverChannel>(
new NetTcpRelayBinding(),
new EndpointAddress(ServiceBusEnvironment.CreateServiceUri("sb", "...", "solver")));
cf.Endpoint.Behaviors.Add(new TransportClientEndpointBehavior
{ TokenProvider = TokenProvider.CreateSharedAccessSignatureTokenProvider("RootManageSharedAccessKey", "...") });
using (var ch = cf.CreateChannel())
{
// if its 50K its ok - if its 70K i get exception
Console.WriteLine(ch.Test(new byte[1000 * 70]));
}
}
public static void CreateServer()
{
ServiceHost sh = new ServiceHost(typeof(ProblemSolver));
sh.AddServiceEndpoint(
typeof(IProblemSolver), new NetTcpRelayBinding(),
ServiceBusEnvironment.CreateServiceUri("sb", "...", "solver"))
.Behaviors.Add(new TransportClientEndpointBehavior
{
TokenProvider = TokenProvider.CreateSharedAccessSignatureTokenProvider("RootManageSharedAccessKey", "...")
});
sh.Open();
while (true)
{
Thread.Sleep(1000);
}
Console.WriteLine("Press ENTER to close");
Console.ReadLine();
sh.Close();
}
}
According to your description,I checked this issue and found the cause. When you construct the NetTcpRelayBinding, the default value for MaxBufferSize and MaxReceivedMessageSize is 64K as follows:
You could specific the MaxBufferSize,MaxReceivedMessageSize,MaxBufferPoolSize to a larger value when constructing the NetTcpRelayBinding instance both in your server and client side.
Result:
When I declare a temporary reply queue to be exclusive (e.g. anonymous queue (exclusive=true, autodelete=true) in rpc-pattern), the response message cannot be posted to the specified reply queue (e.g. message.replyTo="amq.gen-Jg_tv8QYxtEQhq0tF30vAA") because RabbitMqProducer.PublishMessage() tries to redeclare the queue with different parameters (exclusive=false), which understandably results in an error.
Unfortunately, the erroneous call to channel.RegisterQueue(queueName) in RabbitMqProducer.PublishMessage() seems to nack the request message in the incoming queue so that, when ServiceStack.Messaging.MessageHandler.DefaultInExceptionHandler tries to acknowlege the request message (to remove it from the incoming queue), the message just stays on top of the incoming queue and gets processed all over again. This procedure repeats indefinitely and results in one dlq-message per iteration which slowly fills up the dlq.
I am wondering,
if ServiceStack handles the case, when ServiceStack.RabbitMq.RabbitMqProducer cannot declare the response queue, correctly
if ServiceStack.RabbitMq.RabbitMqProducer muss always declare the response queue before publishing the response
if it wouldn't be best to have some configuration flag to omit all exchange and queue declaration calls (outside of the first initialization). The RabbitMqProducer would just assume every queue/exchange to be properly set up and just publish the message.
(At the moment our client just declares its response queue to be exclusive=false and everything works fine. But I'd really like to use rabbitmq's built-in temporary queues.)
MQ-Client Code, requires simple "SayHello" service:
const string INQ_QUEUE_NAME = "mq:SayHello.inq";
const string EXCHANGE_NAME="mx.servicestack";
var factory = new ConnectionFactory() { HostName = "192.168.179.110" };
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
// Create temporary queue and setup bindings
// this works (because "mq:tmp:" stops RabbitMqProducer from redeclaring response queue)
string responseQueueName = "mq:tmp:SayHello_" + Guid.NewGuid().ToString() + ".inq";
channel.QueueDeclare(responseQueueName, false, false, true, null);
// this does NOT work (RabbitMqProducer tries to declare queue again => error):
//string responseQueueName = Guid.NewGuid().ToString() + ".inq";
//channel.QueueDeclare(responseQueueName, false, false, true, null);
// this does NOT work either (RabbitMqProducer tries to declare queue again => error)
//var responseQueueName = channel.QueueDeclare().QueueName;
// publish simple SayHello-Request to standard servicestack exchange ("mx.servicestack") with routing key "mq:SayHello.inq":
var props = channel.CreateBasicProperties();
props.ReplyTo = responseQueueName;
channel.BasicPublish(EXCHANGE_NAME, INQ_QUEUE_NAME, props, Encoding.UTF8.GetBytes("{\"ToName\": \"Chris\"}"));
// consume response from response queue
var consumer = new QueueingBasicConsumer(channel);
channel.BasicConsume(responseQueueName, true, consumer);
var ea = (BasicDeliverEventArgs)consumer.Queue.Dequeue();
// print result: should be "Hello, Chris!"
Console.WriteLine(Encoding.UTF8.GetString(ea.Body));
}
}
Everything seems to work fine when RabbitMqProducer does not try to declare the queues, like that:
public void PublishMessage(string exchange, string routingKey, IBasicProperties basicProperties, byte[] body)
{
const bool MustDeclareQueue = false; // new config parameter??
try
{
if (MustDeclareQueue && !Queues.Contains(routingKey))
{
Channel.RegisterQueueByName(routingKey);
Queues = new HashSet<string>(Queues) { routingKey };
}
Channel.BasicPublish(exchange, routingKey, basicProperties, body);
}
catch (OperationInterruptedException ex)
{
if (ex.Is404())
{
Channel.RegisterExchangeByName(exchange);
Channel.BasicPublish(exchange, routingKey, basicProperties, body);
}
throw;
}
}
The issue got adressed in servicestack's version v4.0.32 (fixed in this commit).
The RabbitMqProducer no longer tries to redeclare temporary queues and instead assumes that the reply queue already exist (which solves my problem.)
(The underlying cause of the infinite loop (wrong error handling while publishing response message) probably still exists.)
Edit: Example
The following basic mq-client (which does not use ServiceStackmq client and instead depends directly on rabbitmq's .net-library; it uses ServiceStack.Text for serialization though) can perform generic RPCs:
public class MqClient : IDisposable
{
ConnectionFactory factory = new ConnectionFactory()
{
HostName = "192.168.97.201",
UserName = "guest",
Password = "guest",
//VirtualHost = "test",
Port = AmqpTcpEndpoint.UseDefaultPort,
};
private IConnection connection;
private string exchangeName;
public MqClient(string defaultExchange)
{
this.exchangeName = defaultExchange;
this.connection = factory.CreateConnection();
}
public TResponse RpcCall<TResponse>(IReturn<TResponse> reqDto, string exchange = null)
{
using (var channel = connection.CreateModel())
{
string inq_queue_name = string.Format("mq:{0}.inq", reqDto.GetType().Name);
string responseQueueName = channel.QueueDeclare().QueueName;
var props = channel.CreateBasicProperties();
props.ReplyTo = responseQueueName;
var message = ServiceStack.Text.JsonSerializer.SerializeToString(reqDto);
channel.BasicPublish(exchange ?? this.exchangeName, inq_queue_name, props, UTF8Encoding.UTF8.GetBytes(message));
var consumer = new QueueingBasicConsumer(channel);
channel.BasicConsume(responseQueueName, true, consumer);
var ea = (BasicDeliverEventArgs)consumer.Queue.Dequeue();
//channel.BasicAck(ea.DeliveryTag, false);
string response = UTF8Encoding.UTF8.GetString(ea.Body);
string responseType = ea.BasicProperties.Type;
Console.WriteLine(" [x] New Message of Type '{1}' Received:{2}{0}", response, responseType, Environment.NewLine);
return ServiceStack.Text.JsonSerializer.DeserializeFromString<TResponse>(response);
}
}
~MqClient()
{
this.Dispose();
}
public void Dispose()
{
if (connection != null)
{
this.connection.Dispose();
this.connection = null;
}
}
}
Key points:
client declares anonymous queue (=with empty queue name) channel.QueueDeclare()
server generates queue and returns queue name (amq.gen*)
client adds queue name to message properties (props.ReplyTo = responseQueueName;)
ServiceStack automatically sends response to temporary queue
client picks up response and deserializes
It can be used like that:
using (var mqClient = new MqClient("mx.servicestack"))
{
var pingResponse = mqClient.RpcCall<PingResponse>(new Ping { });
}
Important: You've got to use servicestack version 4.0.32+.