BackgroundService disappears in Google App Engine - can run for days on debugger no problem

We have a few BackgroundService classes in our ASP.NET Core Web API running on Google App Engine (Flex) and we notice some fairly problematic behavior where our workers basically just flame out with no reported error of failure. This happens quite frequently in our production environment. But when we run the classes locally with Visual Studio and the debugger, the workers can run forever - even when connecting and reading from the same database -- why is that?

`private ILogger _logger; private readonly IManagedTracer _tracer;

    public TransactionBackgroundWorkerService(IBackgroundQueueService<TransactionProcessorRequest> queue,
                                              IServiceScopeFactory scopeFactory,
                                              ILogger<TransactionBackgroundWorkerService> logger,
                                              IManagedTracer tracer)
    {
        _queue = queue;
        _scopeFactory = scopeFactory;
        _logger = logger;
        _tracer = tracer;
    }

    protected override async Task ExecuteAsync(CancellationToken stoppingToken)
    {
        using (_tracer.StartSpan("TransactionBackgroundWorkerExecuteAsync"))
        {
            await BackgroundProcessing(stoppingToken);
        }
            
    }

    public override Task StopAsync(CancellationToken cancellationToken)
    {
        using (_tracer.StartSpan("TransactionBackgroundWorkerStopAsync"))
        {
            return base.StopAsync(cancellationToken);
        }
            
    }

    private async Task BackgroundProcessing(CancellationToken stoppingToken)
    {
        using (_tracer.StartSpan("TransactionBackgroundWorkerBackgroundProcessing"))
        {
            while (!stoppingToken.IsCancellationRequested)
            {
                var current_method_name = Utilities.GetActualAsyncMethodName();
                //Uncomment line below for detailed trace logging. Otherwise, this is not necessary
                //_logger.LogTrace(current_method_name + Utilities.INFO_LOG_INVOKE, DateTime.Now);

                try
                {
                    await Task.Delay(DELAY_BETWEEN_DEQUEUES, stoppingToken);
                    TransactionProcessorRequest transactionRequest = _queue.Dequeue();

                    if (transactionRequest == null) continue;


                    using (var scope = _scopeFactory.CreateScope())
                    {

                        try
                        {

                            IShipmentService shipmentService = scope.ServiceProvider.GetRequiredService<IShipmentService>();
                            
                            NShipment newShipment = await shipmentService.AddAsync(transactionRequest.Transaction.Shipment, transactionRequest.CompanyContext, transactionRequest.ServiceLevelId);

                            //add new transaction record to the DB
                            ITransactionService transactionService = scope.ServiceProvider.GetRequiredService<ITransactionService>();
                            var newTransactionRecord = new NTransaction()
                            {
                                ShipmentId = newShipment.Id,
                                ShipmentRateId = transactionRequest.Transaction.Shipment.NShipmentRates.FirstOrDefault().Id,

                                ShipmentDate = transactionRequest.Transaction.ShipmentDate,
                                PaidAmount = transactionRequest.Transaction.PaidAmount,
                                CurrencyId = transactionRequest.Transaction.CurrencyId,
                                FullShipmentPayment = transactionRequest.Transaction.FullShipmentPayment,
                                LabelUrl = transactionRequest.Transaction.LabelUrl,
                                                               };
                            NTransaction newTrasanction = await transactionService.AddAsync(newTransactionRecord, transactionRequest.CompanyContext);

                        }
                        catch (Exception ex)
                        {
                            

                            //_logger.LogError(current_method_name + CoreAPIServices.Common.Utilities.ERROR_LOG + ex.Message + ex.InnerException?.Message + @", Transaction Request:{1}", DateTime.Now, transactionRequestJson);
                            //required a pickup schedule. 
                            _logger.LogError("ERROR : {exception}, Method Name: {methodname}, BODY: {body}", ex.Message + ex.InnerException?.Message, current_method_name, transactionRequestJson);
                            throw new Exception(current_method_name + ":" + ex.Message + ex.InnerException?.Message);
                        }
                        finally
                        {
                            //Uncomment line below for detailed trace logging. Otherwise, this is not necessary
                            //_logger.LogTrace(current_method_name + Utilities.INFO_LOG_EXIT, DateTime.Now);
                        }

                    }
                }
                catch (Exception ex)
                {
                    _logger.LogError(current_method_name + Utilities.ERROR_LOG + ex.Message + ex.InnerException?.Message, DateTime.Now);
                }
            }`
How many English words
do you know?
Test your English vocabulary size, and measure
how many words do you know
Online Test
Powered by Examplum