|
@@ -10,18 +10,6 @@ using System.Threading;
|
|
|
|
|
|
namespace Rac
|
|
namespace Rac
|
|
{
|
|
{
|
|
- //public enum CrawlTaskEventType
|
|
|
|
- //{
|
|
|
|
- // Started,
|
|
|
|
- // Stopped,
|
|
|
|
- //}
|
|
|
|
-
|
|
|
|
- //public class CrawlTaskEventArgs : EventArgs
|
|
|
|
- //{
|
|
|
|
- // public string Url { get; set; }
|
|
|
|
- // public CrawlTaskEventType Type { get; set; }
|
|
|
|
- //}
|
|
|
|
-
|
|
|
|
//TODO: CHECK AFFECT OF `Error loading type Typespec 0x1b000044 from due to Could not resolve typespec token 1b000044'
|
|
//TODO: CHECK AFFECT OF `Error loading type Typespec 0x1b000044 from due to Could not resolve typespec token 1b000044'
|
|
|
|
|
|
public class Crawler : BaseService
|
|
public class Crawler : BaseService
|
|
@@ -31,10 +19,9 @@ namespace Rac
|
|
|
|
|
|
public bool IsRunning { get; private set; }
|
|
public bool IsRunning { get; private set; }
|
|
|
|
|
|
- public Crawler(string dbFilename)
|
|
|
|
-
|
|
|
|
|
|
+ public Crawler(string dbFilePath)
|
|
{
|
|
{
|
|
- _db = new DataAccess(dbFilename);
|
|
|
|
|
|
+ _db = new DataAccess(dbFilePath);
|
|
}
|
|
}
|
|
|
|
|
|
public override void Start()
|
|
public override void Start()
|
|
@@ -142,7 +129,7 @@ namespace Rac
|
|
|
|
|
|
var hssExtractedLinks = new HashSet<string>();
|
|
var hssExtractedLinks = new HashSet<string>();
|
|
|
|
|
|
- ArchiveEntry PageProce(string url)
|
|
|
|
|
|
+ ArchiveEntry PageProc(string url)
|
|
{
|
|
{
|
|
ArchiveEntry entry;
|
|
ArchiveEntry entry;
|
|
Response resp;
|
|
Response resp;
|
|
@@ -208,7 +195,7 @@ namespace Rac
|
|
#if !DEBUG
|
|
#if !DEBUG
|
|
.AsParallel().WithDegreeOfParallelism(conf.Parallel)
|
|
.AsParallel().WithDegreeOfParallelism(conf.Parallel)
|
|
#endif
|
|
#endif
|
|
- .Select(PageProce)
|
|
|
|
|
|
+ .Select(PageProc)
|
|
.ToArray();
|
|
.ToArray();
|
|
|
|
|
|
LogInfo($"Saving extracted {hssExtractedLinks.Count} new link...");
|
|
LogInfo($"Saving extracted {hssExtractedLinks.Count} new link...");
|
|
@@ -224,4 +211,4 @@ namespace Rac
|
|
LogInfo("Finished!");
|
|
LogInfo("Finished!");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
-}
|
|
|
|
|
|
+}
|