fussmattenprofi/hsntsn-scraper
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add total records tracking and update progress logging in vehicle detail retrieval

Browse Source
This commit is contained in:
akinayturan
2026-03-04 23:06:50 +03:00
parent e5df149808
commit c7750ac4ca
1 changed files with 14 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/HsnTsnScraper/Program.cs
+14 -6
View File
@@ -37,6 +37,7 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
var processed = 0;
var failed = 0;
var updated = 0;
var totalRecords = Math.Max(0, File.ReadLines(inputCsvPath).Count() - 1);
await using var inputStream = File.OpenRead(inputCsvPath);
using var inputReader = new StreamReader(inputStream);
@@ -69,7 +70,7 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
{
try
{
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, record.SourceDetailUrl, record.HsnTsn);
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, record.SourceDetailUrl, record.HsnTsn, processed, totalRecords);
if (detail is not null)
{
if (record.YearFrom is null && detail.YearFrom is not null)
@@ -104,12 +105,12 @@ async Task RunRepairYearsMode(HsnTsnClient hsnTsnClient, string? inputPath, stri
if (processed % 250 == 0)
{
await csvWriter.FlushAsync();
Console.Error.WriteLine($"[info] Repair progress: processed={processed}, updated={updated}, failed={failed}");
Console.Error.WriteLine($"[info] Repair progress: {processed}/{totalRecords}, updated={updated}, failed={failed}");
}
}
await csvWriter.FlushAsync();
Console.Error.WriteLine($"[info] Repair finished. processed={processed}, updated={updated}, failed={failed}");
Console.Error.WriteLine($"[info] Repair finished. {processed}/{totalRecords}, updated={updated}, failed={failed}");
}
async Task RunScrapeMode(HsnTsnClient hsnTsnClient, bool includeDetailPages)
@@ -203,7 +204,7 @@ async Task RunScrapeMode(HsnTsnClient hsnTsnClient, bool includeDetailPages)
{
try
{
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, vehicle.SourceDetailUrl, vehicle.HsnTsn);
var detail = await GetVehicleDetailWithRetry(hsnTsnClient, vehicle.SourceDetailUrl, vehicle.HsnTsn, null, null);
if (detail is not null)
{
if (!string.IsNullOrWhiteSpace(detail.Brand))
@@ -265,10 +266,17 @@ static string? GetOptionValue(string[] cliArgs, string optionName)
return null;
}
async Task<VehicleDetail?> GetVehicleDetailWithRetry(HsnTsnClient hsnTsnClient, string detailUrl, string hsnTsn)
async Task<VehicleDetail?> GetVehicleDetailWithRetry(HsnTsnClient hsnTsnClient, string detailUrl, string hsnTsn, int? currentIndex, int? totalCount)
{
const int maxAttempts = 7;
Console.Error.WriteLine($"[info] Fetching detail for HSN/TSN: {hsnTsn}");
if (currentIndex is not null && totalCount is not null && totalCount > 0)
{
Console.Error.WriteLine($"[info] [{currentIndex}/{totalCount}] Fetching detail for HSN/TSN: {hsnTsn}");
}
else
{
Console.Error.WriteLine($"[info] Fetching detail for HSN/TSN: {hsnTsn}");
}
for (var attempt = 1; attempt <= maxAttempts; attempt++)
{