fussmattenprofi/hsntsn-scraper
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add total records tracking and update progress logging in vehicle detail retrieval

Browse Source
This commit is contained in:
akinayturan
2026-03-04 23:06:50 +03:00
parent e5df149808
commit c7750ac4ca
1 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/HsnTsnScraper/Program.cs
+13 -5
View File
@@ -37,6 +37,7 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
var processed = 0; var processed = 0;
var failed = 0; var failed = 0;
var updated = 0; var updated = 0;
var totalRecords = Math.Max(0, File.ReadLines(inputCsvPath).Count() - 1);
await using var inputStream = File.OpenRead(inputCsvPath); await using var inputStream = File.OpenRead(inputCsvPath);
using var inputReader = new StreamReader(inputStream); using var inputReader = new StreamReader(inputStream);
@@ -69,7 +70,7 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
{ {
try try
{ {
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, record.SourceDetailUrl, record.HsnTsn); var detail = await GetVehicleDetailWithRetry(hsnTsnClient, record.SourceDetailUrl, record.HsnTsn, processed, totalRecords);
if (detail is not null) if (detail is not null)
{ {
if (record.YearFrom is null && detail.YearFrom is not null) if (record.YearFrom is null && detail.YearFrom is not null)
@@ -104,12 +105,12 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
if (processed % 250 == 0) if (processed % 250 == 0)
{ {
await csvWriter.FlushAsync(); await csvWriter.FlushAsync();
Console.Error.WriteLine($"[info] Repair progress: processed={processed}, updated={updated}, failed={failed}"); Console.Error.WriteLine($"[info] Repair progress: {processed}/{totalRecords}, updated={updated}, failed={failed}");
} }
} }
await csvWriter.FlushAsync(); await csvWriter.FlushAsync();
Console.Error.WriteLine($"[info] Repair finished. processed={processed}, updated={updated}, failed={failed}"); Console.Error.WriteLine($"[info] Repair finished. {processed}/{totalRecords}, updated={updated}, failed={failed}");
} }
async Task RunScrapeMode(HsnTsnClient hsnTsnClient, bool includeDetailPages) async Task RunScrapeMode(HsnTsnClient hsnTsnClient, bool includeDetailPages)
@@ -203,7 +204,7 @@ async Task RunScrapeMode(HsnTsnClient hsnTsnClient, bool includeDetailPages)
{ {
try try
{ {
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, vehicle.SourceDetailUrl, vehicle.HsnTsn); var detail = await GetVehicleDetailWithRetry(hsnTsnClient, vehicle.SourceDetailUrl, vehicle.HsnTsn, null, null);
if (detail is not null) if (detail is not null)
{ {
if (!string.IsNullOrWhiteSpace(detail.Brand)) if (!string.IsNullOrWhiteSpace(detail.Brand))
@@ -265,10 +266,17 @@ static string? GetOptionValue(string[] cliArgs, string optionName)
return null; return null;
} }
async Task<VehicleDetail?> GetVehicleDetailWithRetry(HsnTsnClient hsnTsnClient, string detailUrl, string hsnTsn) async Task<VehicleDetail?> GetVehicleDetailWithRetry(HsnTsnClient hsnTsnClient, string detailUrl, string hsnTsn, int? currentIndex, int? totalCount)
{ {
const int maxAttempts = 7; const int maxAttempts = 7;
if (currentIndex is not null && totalCount is not null && totalCount > 0)
{
Console.Error.WriteLine($"[info] [{currentIndex}/{totalCount}] Fetching detail for HSN/TSN: {hsnTsn}");
}
else
{
Console.Error.WriteLine($"[info] Fetching detail for HSN/TSN: {hsnTsn}"); Console.Error.WriteLine($"[info] Fetching detail for HSN/TSN: {hsnTsn}");
}
for (var attempt = 1; attempt <= maxAttempts; attempt++) for (var attempt = 1; attempt <= maxAttempts; attempt++)
{ {