referer support

Rac.Core/Crawler.cs

@@ -138,9 +138,12 @@ namespace Rac
 
                     LogTrace($"GET {uri}");
 
+                    var referer = _db.GetReferer(url);
+                    LogTrace($" -> Referer: {referer}");
+
                     try
                     {
-                        resp = Requester.GetHttp(url);
+                        resp = Requester.GetHttp(url, referer);
 
                         entry = new ArchiveEntry
                         {

Rac.Core/Tools/DataAccess.cs

@@ -191,6 +191,14 @@ namespace Rac.Tools
             return BulkOperation(items, AddPageLink);
         }
 
+        public string GetReferer(string url)
+        {
+            using (var conn = GetConnection())
+            {
+                return conn.Query<string>($"SELECT Resource FROM ResourceLinks WHERE {nameof(ResourceLinkEntry.Link)}=@url", new { url }).FirstOrDefault();
+            }
+        }
+
         // --- for offline server ---
 
         public ArchiveEntry GetEntry(string url)

Rac.Core/Tools/Requester.cs

@@ -8,10 +8,11 @@ namespace Rac.Tools
 
     internal static class Requester
     {
-        public static Response GetHttp(string url)
+        public static Response GetHttp(string url, string referer = null)
         {
             var req = WebRequest.CreateHttp(url);
             req.UserAgent = "RawArchiveCrawler";
+            if (referer != null) req.Referer = referer;
 
             req.AllowAutoRedirect = false; //record redirections