// See https://aka.ms/new-console-template for more information using System.Buffers; using System.Diagnostics; using System.IO.Compression; Console.WriteLine("Hello, World!"); var getFrom = args[0]; var saveTo = args[1]; var isRunning = true; Console.CancelKeyPress += delegate { Console.WriteLine("Control-C"); isRunning = false; }; var client = new HttpClient(); //shared allocation var bufDownload = new byte[1024 * 1024 * 1024]; var bufBrCompressed = new byte[1024 * 1024 * 1024]; while (isRunning) { Console.WriteLine("Fetching task..."); try { var r = await client.GetAsync(getFrom, HttpCompletionOption.ResponseHeadersRead); if (r.IsSuccessStatusCode) { r.Headers.TryGetValues("x-compress-task-name", out var values); var taskName = values?.FirstOrDefault(); r.Headers.TryGetValues("x-compress-task-id", out values); var taskId = values?.FirstOrDefault(); if (taskName == null || taskId == null || !(r.Content.Headers.ContentLength > 0)) { Console.WriteLine("No task for worker."); } else { Console.WriteLine("Got task"); Console.WriteLine($" Name: {taskName}"); Console.WriteLine($" Id: {taskId}"); Console.WriteLine($" Length: {r.Content.Headers.ContentLength:N0}"); var outDir = Path.Combine(saveTo, taskName); if (Directory.Exists(outDir) == false) Directory.CreateDirectory(outDir); var outFile = Path.Combine(outDir, $"{taskId}.br"); if (File.Exists(outFile)) { if (new FileInfo(outFile).Length == 0) File.Delete(outFile); else { Console.WriteLine(" Exist, Skip"); r.Dispose(); continue; } } Console.WriteLine(" Downloading..."); var msDownload = new MemoryStream(bufDownload); var downloadStream = await r.Content.ReadAsStreamAsync(); await downloadStream.CopyToAsync(msDownload); var nbDownload = (int)msDownload.Position; try { Console.WriteLine(" Creating outout file..."); await using var fs = File.Create(outFile); using var br = new BrotliEncoder(11, 24); Console.WriteLine(" Compressing..."); var sw = Stopwatch.StartNew(); var operationStatus = br.Compress(new ReadOnlySpan(bufDownload, 0, nbDownload), bufBrCompressed, out _, out var bytesWritten, true); sw.Stop(); if (operationStatus == OperationStatus.Done) { fs.Write(bufBrCompressed, 0, bytesWritten); fs.Close(); Console.WriteLine($"Done task in {sw.Elapsed.TotalSeconds:N2}s"); } else { Console.WriteLine($"Fail:{operationStatus}"); } } catch (Exception e) { Console.WriteLine($"Fail with err:{e.Message}"); } } } else { Console.WriteLine("Failure to request:" + r.StatusCode); } } catch (Exception e) { Console.WriteLine($"Err:{e.Message}"); } Console.WriteLine("Force GC..."); GC.Collect(GC.MaxGeneration, GCCollectionMode.Aggressive, true, true); Console.WriteLine("Delay 1 sec for next request"); await Task.Delay(1000); }