Add HsnTsnClient and related classes for vehicle data scraping
- Implement HsnTsnClient to fetch vehicle data from hsn-tsn.de - Create VehicleDetail and HsnTsnVehicle classes for data representation - Add CSV output functionality in Program.cs - Include necessary NuGet packages in project file - Add README with usage instructions
This commit is contained in:
@@ -0,0 +1,361 @@
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using HtmlAgilityPack;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Polly;
|
||||
using Polly.Extensions.Http;
|
||||
|
||||
namespace HsnTsnScraper;
|
||||
|
||||
public sealed class HsnTsnClient : IDisposable
|
||||
{
|
||||
private static readonly Regex HsnTsnRegex = new(@"(?<hsn>\d{4})\s*/\s*(?<tsn>[A-Z0-9]{3})", RegexOptions.Compiled);
|
||||
private static readonly Regex PsKwRegex = new(@"(?<ps>\d+)\s*PS\s*\((?<kw>\d+)\s*kW\)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex CcmRegex = new(@"(?<ccm>[\d\.\,]+)\s*ccm", RegexOptions.Compiled | RegexOptions.IgnoreCase);
|
||||
private static readonly Regex YearRangeRegex = new(@"(?<from>\d{4})(?:\D+(?<to>\d{4}))?", RegexOptions.Compiled);
|
||||
private static readonly HashSet<string> ExcludedLinks = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
"/impressum.php",
|
||||
"/datenschutz.php",
|
||||
"/datenschutz"
|
||||
};
|
||||
|
||||
private readonly HttpClient _client;
|
||||
|
||||
public HsnTsnClient(string baseUrl = "http://www.hsn-tsn.de/")
|
||||
{
|
||||
var retryPolicy = HttpPolicyExtensions
|
||||
.HandleTransientHttpError()
|
||||
.OrResult(x => x.StatusCode == HttpStatusCode.TooManyRequests)
|
||||
.WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)));
|
||||
|
||||
var socketHandler = new SocketsHttpHandler
|
||||
{
|
||||
PooledConnectionLifetime = TimeSpan.FromMinutes(5),
|
||||
UseCookies = false
|
||||
};
|
||||
|
||||
var pollyHandler = new PolicyHttpMessageHandler(retryPolicy)
|
||||
{
|
||||
InnerHandler = socketHandler
|
||||
};
|
||||
|
||||
_client = new HttpClient(pollyHandler)
|
||||
{
|
||||
BaseAddress = new Uri(baseUrl)
|
||||
};
|
||||
|
||||
_client.DefaultRequestHeaders.UserAgent.ParseAdd("Mozilla/5.0 (compatible; hsntsn-scraper/1.0)");
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<string>> GetBrandPageUrls(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var homeHtml = await GetStringAsync("/", cancellationToken);
|
||||
var doc = LoadDocument(homeHtml);
|
||||
var links = doc.DocumentNode.SelectNodes("//a[@href]") ?? new HtmlNodeCollection(null);
|
||||
|
||||
var result = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var link in links)
|
||||
{
|
||||
var href = link.GetAttributeValue("href", string.Empty).Trim();
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!href.EndsWith(".html", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ExcludedLinks.Contains(href))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (href.StartsWith("http://", StringComparison.OrdinalIgnoreCase) ||
|
||||
href.StartsWith("https://", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Add(href);
|
||||
}
|
||||
else
|
||||
{
|
||||
result.Add(new Uri(_client.BaseAddress!, href).ToString());
|
||||
}
|
||||
}
|
||||
|
||||
return result.OrderBy(x => x, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<HsnTsnVehicle>> GetVehiclesFromSearchAsync(string query, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var encoded = Uri.EscapeDataString(query);
|
||||
var url = $"/liste.php?string={encoded}";
|
||||
var html = await GetStringAsync(url, cancellationToken);
|
||||
return ParseVehiclesFromListPage(html, new Uri(_client.BaseAddress!, url).ToString(), query);
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<HsnTsnVehicle>> GetVehiclesFromBrandPageAsync(string absoluteUrl, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var html = await GetStringAsync(absoluteUrl, cancellationToken);
|
||||
return ParseVehiclesFromListPage(html, absoluteUrl, null);
|
||||
}
|
||||
|
||||
public async Task<VehicleDetail?> GetVehicleDetailAsync(string detailUrl, CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(detailUrl))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var html = await GetStringAsync(detailUrl, cancellationToken);
|
||||
var doc = LoadDocument(html);
|
||||
|
||||
var carNode = doc.DocumentNode.SelectSingleNode("//div[@id='car']");
|
||||
if (carNode is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var detail = new VehicleDetail
|
||||
{
|
||||
Hsn = carNode.GetAttributeValue("data-hsn", string.Empty).Trim(),
|
||||
Tsn = carNode.GetAttributeValue("data-tsn", string.Empty).Trim(),
|
||||
Brand = HtmlEntity.DeEntitize(doc.DocumentNode.SelectSingleNode("//span[@property='brand']//span[@property='name']")?.InnerText ?? string.Empty).Trim(),
|
||||
Model = HtmlEntity.DeEntitize(doc.DocumentNode.SelectSingleNode("//span[@property='model']")?.InnerText ?? string.Empty).Trim(),
|
||||
OfficialType = HtmlEntity.DeEntitize(doc.DocumentNode.SelectSingleNode("//span[@property='alternateName']")?.InnerText ?? string.Empty).Trim(),
|
||||
CanonicalUrl = doc.DocumentNode.SelectSingleNode("//meta[@property='url']")?.GetAttributeValue("content", string.Empty).Trim() ?? string.Empty
|
||||
};
|
||||
|
||||
var yearText = HtmlEntity.DeEntitize(doc.DocumentNode.SelectSingleNode("//small[@property='vehicleModelDate']")?.InnerText ?? string.Empty).Trim();
|
||||
ParseYearRange(yearText, out var fromYear, out var toYear);
|
||||
detail.YearFrom = fromYear;
|
||||
detail.YearTo = toYear;
|
||||
|
||||
return detail;
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
_client.Dispose();
|
||||
}
|
||||
|
||||
private async Task<string> GetStringAsync(string relativeOrAbsoluteUrl, CancellationToken cancellationToken)
|
||||
{
|
||||
using var response = await _client.GetAsync(relativeOrAbsoluteUrl, cancellationToken);
|
||||
response.EnsureSuccessStatusCode();
|
||||
return await response.Content.ReadAsStringAsync(cancellationToken);
|
||||
}
|
||||
|
||||
private IReadOnlyList<HsnTsnVehicle> ParseVehiclesFromListPage(string html, string sourceListUrl, string? sourceQuery)
|
||||
{
|
||||
var doc = LoadDocument(html);
|
||||
var rows = doc.DocumentNode.SelectNodes("//table//tr") ?? new HtmlNodeCollection(null);
|
||||
var pageHeading = Clean(doc.DocumentNode.SelectSingleNode("//h1")?.InnerText ?? string.Empty);
|
||||
var pageBrand = IsLikelyBrandHeading(pageHeading) ? pageHeading : string.Empty;
|
||||
var result = new List<HsnTsnVehicle>();
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
var cells = row.SelectNodes("./td");
|
||||
if (cells is null || cells.Count < 5)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hsnTsnText = Clean(cells[0].InnerText);
|
||||
var match = HsnTsnRegex.Match(hsnTsnText);
|
||||
if (!match.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var vehicleTypeRaw = Clean(cells[1].InnerText);
|
||||
var detailHref = cells[1].SelectSingleNode(".//a[@href]")?.GetAttributeValue("href", string.Empty) ?? string.Empty;
|
||||
var detailUrl = ToAbsoluteUrl(detailHref);
|
||||
|
||||
var powerText = Clean(cells[2].InnerText);
|
||||
ParsePower(powerText, out var ps, out var kw);
|
||||
|
||||
var displacementText = Clean(cells[3].InnerText);
|
||||
var displacementCcm = ParseDisplacement(displacementText);
|
||||
|
||||
var fuelType = Clean(cells[4].InnerText);
|
||||
var brand = !string.IsNullOrWhiteSpace(pageBrand) ? pageBrand : ExtractBrand(vehicleTypeRaw);
|
||||
|
||||
var vehicle = new HsnTsnVehicle
|
||||
{
|
||||
Hsn = match.Groups["hsn"].Value,
|
||||
Tsn = match.Groups["tsn"].Value,
|
||||
HsnTsn = $"{match.Groups["hsn"].Value}/{match.Groups["tsn"].Value}",
|
||||
Brand = brand,
|
||||
VehicleType = vehicleTypeRaw,
|
||||
PowerPs = ps,
|
||||
PowerKw = kw,
|
||||
DisplacementCcm = displacementCcm,
|
||||
FuelType = fuelType,
|
||||
SourceListUrl = sourceListUrl,
|
||||
SourceDetailUrl = detailUrl,
|
||||
SourceQuery = sourceQuery ?? string.Empty
|
||||
};
|
||||
|
||||
vehicle.MatchKey = BuildMatchKey(vehicle);
|
||||
result.Add(vehicle);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private string ToAbsoluteUrl(string href)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
if (Uri.TryCreate(href, UriKind.Absolute, out var absolute) &&
|
||||
(absolute.Scheme.Equals(Uri.UriSchemeHttp, StringComparison.OrdinalIgnoreCase) ||
|
||||
absolute.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return absolute.ToString();
|
||||
}
|
||||
|
||||
var normalized = href.StartsWith("/", StringComparison.Ordinal) ? href : $"/{href.TrimStart('/')}";
|
||||
return new Uri(_client.BaseAddress!, normalized).ToString();
|
||||
}
|
||||
|
||||
private static HtmlDocument LoadDocument(string html)
|
||||
{
|
||||
var doc = new HtmlDocument();
|
||||
doc.LoadHtml(html);
|
||||
return doc;
|
||||
}
|
||||
|
||||
private static string Clean(string value)
|
||||
{
|
||||
return HtmlEntity.DeEntitize(value).Replace('\u00A0', ' ').Trim();
|
||||
}
|
||||
|
||||
private static void ParsePower(string powerText, out int? ps, out int? kw)
|
||||
{
|
||||
ps = null;
|
||||
kw = null;
|
||||
|
||||
var match = PsKwRegex.Match(powerText);
|
||||
if (!match.Success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (int.TryParse(match.Groups["ps"].Value, out var psParsed))
|
||||
{
|
||||
ps = psParsed;
|
||||
}
|
||||
|
||||
if (int.TryParse(match.Groups["kw"].Value, out var kwParsed))
|
||||
{
|
||||
kw = kwParsed;
|
||||
}
|
||||
}
|
||||
|
||||
private static int? ParseDisplacement(string text)
|
||||
{
|
||||
var match = CcmRegex.Match(text);
|
||||
if (!match.Success)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var numeric = match.Groups["ccm"].Value.Replace(".", string.Empty).Replace(",", string.Empty);
|
||||
return int.TryParse(numeric, out var ccm) ? ccm : null;
|
||||
}
|
||||
|
||||
private static void ParseYearRange(string yearText, out int? fromYear, out int? toYear)
|
||||
{
|
||||
fromYear = null;
|
||||
toYear = null;
|
||||
|
||||
var match = YearRangeRegex.Match(yearText);
|
||||
if (!match.Success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (int.TryParse(match.Groups["from"].Value, out var from))
|
||||
{
|
||||
fromYear = from;
|
||||
}
|
||||
|
||||
if (int.TryParse(match.Groups["to"].Value, out var to))
|
||||
{
|
||||
toYear = to;
|
||||
}
|
||||
}
|
||||
|
||||
private static string ExtractBrand(string vehicleType)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(vehicleType))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var parts = vehicleType.Split(' ', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
return parts.Length == 0 ? string.Empty : parts[0];
|
||||
}
|
||||
|
||||
private static bool IsLikelyBrandHeading(string heading)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(heading))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return !Regex.IsMatch(heading, @"^\d{4}$");
|
||||
}
|
||||
|
||||
private static string BuildMatchKey(HsnTsnVehicle vehicle)
|
||||
{
|
||||
var raw = $"{vehicle.Brand} {vehicle.VehicleType} {vehicle.OfficialType} {vehicle.PowerKw} {vehicle.FuelType}".Trim();
|
||||
var normalized = raw.ToUpperInvariant()
|
||||
.Replace("Ä", "AE")
|
||||
.Replace("Ö", "OE")
|
||||
.Replace("Ü", "UE")
|
||||
.Replace("ß", "SS");
|
||||
|
||||
var cleaned = Regex.Replace(normalized, @"[^A-Z0-9]+", " ").Trim();
|
||||
return Regex.Replace(cleaned, @"\s+", " ");
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class VehicleDetail
|
||||
{
|
||||
public string Hsn { get; set; } = string.Empty;
|
||||
public string Tsn { get; set; } = string.Empty;
|
||||
public string Brand { get; set; } = string.Empty;
|
||||
public string Model { get; set; } = string.Empty;
|
||||
public string OfficialType { get; set; } = string.Empty;
|
||||
public int? YearFrom { get; set; }
|
||||
public int? YearTo { get; set; }
|
||||
public string CanonicalUrl { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
public sealed class HsnTsnVehicle
|
||||
{
|
||||
public string HsnTsn { get; set; } = string.Empty;
|
||||
public string Hsn { get; set; } = string.Empty;
|
||||
public string Tsn { get; set; } = string.Empty;
|
||||
public string Brand { get; set; } = string.Empty;
|
||||
public string VehicleType { get; set; } = string.Empty;
|
||||
public string Model { get; set; } = string.Empty;
|
||||
public string OfficialType { get; set; } = string.Empty;
|
||||
public int? YearFrom { get; set; }
|
||||
public int? YearTo { get; set; }
|
||||
public int? PowerPs { get; set; }
|
||||
public int? PowerKw { get; set; }
|
||||
public int? DisplacementCcm { get; set; }
|
||||
public string FuelType { get; set; } = string.Empty;
|
||||
public string MatchKey { get; set; } = string.Empty;
|
||||
public string SourceQuery { get; set; } = string.Empty;
|
||||
public string SourceListUrl { get; set; } = string.Empty;
|
||||
public string SourceDetailUrl { get; set; } = string.Empty;
|
||||
}
|
||||
Reference in New Issue
Block a user