StrangeTools/CompressWorker/Program.cs

// See https://aka.ms/new-console-template for more information

using System.Buffers;
using System.Diagnostics;
using System.IO.Compression;

Console.WriteLine("Hello, World!");

var getFrom = args[0];
var saveTo = args[1];

var isRunning = true;
Console.CancelKeyPress += delegate
{
    Console.WriteLine("Control-C");
    isRunning = false;
};

var client = new HttpClient();

//shared allocation
var bufDownload = new byte[1024 * 1024 * 1024];
var bufBrCompressed = new byte[1024 * 1024 * 1024];

while (isRunning)
{
    Console.WriteLine("Fetching task...");
    try
    {
        var r = await client.GetAsync(getFrom, HttpCompletionOption.ResponseHeadersRead);
        if (r.IsSuccessStatusCode)
        {
            r.Headers.TryGetValues("x-compress-task-name", out var values);
            var taskName = values?.FirstOrDefault();

            r.Headers.TryGetValues("x-compress-task-id", out values);
            var taskId = values?.FirstOrDefault();

            if (taskName == null || taskId == null || !(r.Content.Headers.ContentLength > 0))
            {
                Console.WriteLine("No task for worker.");
            }
            else
            {
                Console.WriteLine("Got task");
                Console.WriteLine($"      Name: {taskName}");
                Console.WriteLine($"        Id: {taskId}");
                Console.WriteLine($"    Length: {r.Content.Headers.ContentLength:N0}");

                var outDir = Path.Combine(saveTo, taskName);
                if (Directory.Exists(outDir) == false) Directory.CreateDirectory(outDir);

                var outFile = Path.Combine(outDir, $"{taskId}.br");

                if (File.Exists(outFile))
                {
                    if (new FileInfo(outFile).Length == 0) File.Delete(outFile);
                    else
                    {
                        Console.WriteLine("  Exist, Skip");
                        r.Dispose();
                        continue;
                    }
                }

                Console.WriteLine("  Downloading...");
                var msDownload = new MemoryStream(bufDownload);
                var downloadStream = await r.Content.ReadAsStreamAsync();
                await downloadStream.CopyToAsync(msDownload);
                var nbDownload = (int)msDownload.Position;

                try
                {
                    Console.WriteLine("   Creating outout file...");

                    await using var fs = File.Create(outFile);

                    using var br = new BrotliEncoder(11, 24);
                    Console.WriteLine("   Compressing...");
                    var sw = Stopwatch.StartNew();
                    var operationStatus = br.Compress(new ReadOnlySpan<byte>(bufDownload, 0, nbDownload), bufBrCompressed, out _, out var bytesWritten, true);
                    sw.Stop();
                    if (operationStatus == OperationStatus.Done)
                    {
                        fs.Write(bufBrCompressed, 0, bytesWritten);
                        fs.Close();
                        Console.WriteLine($"Done task in {sw.Elapsed.TotalSeconds:N2}s");
                    }
                    else
                    {
                        Console.WriteLine($"Fail:{operationStatus}");
                    }
                }
                catch (Exception e)
                {
                    Console.WriteLine($"Fail with err:{e.Message}");
                }
            }
        }
        else
        {
            Console.WriteLine("Failure to request:" + r.StatusCode);
        }
    }
    catch (Exception e)
    {
        Console.WriteLine($"Err:{e.Message}");
    }

    Console.WriteLine("Force GC...");
    GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true);

    Console.WriteLine("Delay 1 sec for next request");
    await Task.Delay(1000);
}