RawArchiver/Rac.Core/Tools/Requester.cs

using Rac.Models;
using System.Net;

namespace Rac.Tools
{
    // This class can change to instance class
    //  Implement different driver (WebRequest/WinNet/Sock/Etc(Pr0xy))

    internal static class Requester
    {
        public static Response GetHttp(string url, string referer = null)
        {
            var req = WebRequest.CreateHttp(url);
            req.UserAgent = "RawArchiveCrawler";
            if (referer != null) req.Referer = referer;

            req.AllowAutoRedirect = false; //record redirections

            try
            {
                var resp = (HttpWebResponse)req.GetResponse();
                return new Response
                {
                    StatusCode = resp.StatusCode,
                    StatusDescription = resp.StatusDescription,
                    Headers = resp.Headers,
                    Body = resp.GetResponseStream().ToBytes()
                };
            }
            catch (WebException e)
            {
                if (e.Status != WebExceptionStatus.ProtocolError) throw;
                var resp = (HttpWebResponse)e.Response;
                return new Response
                {
                    StatusCode = resp.StatusCode,
                    StatusDescription = resp.StatusDescription,
                    Headers = resp.Headers,
                    Body = resp.GetResponseStream().ToBytes()
                };
            }
        }
    }
}