.Net Core Queue Background Tasks

Just wanted to add some additional notes to @johnny5 answer. Right now you can use https://devblogs.microsoft.com/dotnet/an-introduction-to-system-threading-channels/ instead of ConcurrentQueue with Semaphore. The code will be something like this:

public class HostedService: BackgroundService
{
        private readonly ILogger _logger;
        private readonly ChannelReader<Stream> _channel;

        public HostedService(
            ILogger logger,
            ChannelReader<Stream> channel)
        {
            _logger = logger;
            _channel = channel;
        }

        protected override async Task ExecuteAsync(CancellationToken cancellationToken)
        {
            await foreach (var item in _channel.ReadAllAsync(cancellationToken))
            {
                try
                {
                    // do your work with data
                }
                catch (Exception e)
                {
                    _logger.Error(e, "An unhandled exception occured");
                }
            }
        }
}

[ApiController]
[Route("api/data/upload")]
public class UploadController : ControllerBase
{
    private readonly ChannelWriter<Stream> _channel;

    public UploadController (
        ChannelWriter<Stream> channel)
    {
        _channel = channel;
    }

    public async Task<IActionResult> Upload([FromForm] FileInfo fileInfo)
    {
        var ms = new MemoryStream();
        await fileInfo.FormFile.CopyToAsync(ms);
        await _channel.WriteAsync(ms);
        return Ok();
    }
}

.NET Core 2.1 has an IHostedService, which will safely run tasks in the background. I've found an example in the documentation for QueuedHostedService which I've modified to use the BackgroundService.

public class QueuedHostedService : BackgroundService
{
   
    private Task _backgroundTask;
    private readonly ILogger _logger;

    public QueuedHostedService(IBackgroundTaskQueue taskQueue, ILoggerFactory loggerFactory)
    {
        TaskQueue = taskQueue;
        _logger = loggerFactory.CreateLogger<QueuedHostedService>();
    }

    public IBackgroundTaskQueue TaskQueue { get; }

    protected async override Task ExecuteAsync(CancellationToken stoppingToken)
    {
        while (false == stoppingToken.IsCancellationRequested)
        {
            var workItem = await TaskQueue.DequeueAsync(stoppingToken);
            try
            {
                await workItem(stoppingToken);
            }
            catch (Exception ex)
            {
                this._logger.LogError(ex, $"Error occurred executing {nameof(workItem)}.");
            }
        }
    }
}

public interface IBackgroundTaskQueue
{
    void QueueBackgroundWorkItem(Func<CancellationToken, Task> workItem);

    Task<Func<CancellationToken, Task>> DequeueAsync(
        CancellationToken cancellationToken);
}

public class BackgroundTaskQueue : IBackgroundTaskQueue
{
    private ConcurrentQueue<Func<CancellationToken, Task>> _workItems =
        new ConcurrentQueue<Func<CancellationToken, Task>>();

    private SemaphoreSlim _signal = new SemaphoreSlim(0);

    public void QueueBackgroundWorkItem(Func<CancellationToken, Task> workItem)
    {
        if (workItem == null)
        {
            throw new ArgumentNullException(nameof(workItem));
        }

        _workItems.Enqueue(workItem);
        _signal.Release();
    }

    public async Task<Func<CancellationToken, Task>> DequeueAsync( CancellationToken cancellationToken)
    {
        await _signal.WaitAsync(cancellationToken);
        _workItems.TryDequeue(out var workItem);

        return workItem;
    }
}

Now we can safely queue up tasks in the background without affecting the time it takes to respond to a request.