Improve book description parsing
MODIFY
XRayBuilder.Core/src/DataSources/Amazon/AmazonInfoParser.cs
XRayBuilder.Core/src/DataSources/Amazon/AmazonInfoParser.cs
using XRayBuilder.Core.Libraries.Logging;
using XRayBuilder.Core.Model;
-namespace XRayBuilder.Core.DataSources.Amazon
-{
+namespace XRayBuilder.Core.DataSources.Amazon;
+
public class AmazonInfoParser : IAmazonInfoParser
{
private readonly ILogger _logger;
@@ -72,6 +72,7 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
CheckCaptcha(bookDoc);
#region Image URL
+
var bookImageLoc = bookDoc.DocumentNode.SelectSingleNode("//*[@id='imgBlkFront']")
?? bookDoc.DocumentNode.SelectSingleNode("//*[@id='imageBlock']")
?? bookDoc.DocumentNode.SelectSingleNode("//*[@class='series-detail-product-image']")
@@ -80,7 +81,9 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
?? bookDoc.DocumentNode.SelectSingleNode("//*[@id='main-image']");
if (bookImageLoc == null)
+ {
_logger.Log("Error finding book image.");
+ }
else
{
var imageUrl = Regex.Replace(bookImageLoc.GetAttributeValue("src", ""), @"_.*?_\.", string.Empty);
@@ -110,6 +113,7 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
#endregion
#region Description
+
var descNode = bookDoc.DocumentNode.SelectSingleNode("//*[@id='bookDescription_feature_div']/noscript")
?? bookDoc.DocumentNode.SelectSingleNode("//*[@id='bookDescription_feature_div']/div[@data-a-expander-name='book_description_expander']")
?? bookDoc.DocumentNode.SelectSingleNode("//*[@class='a-size-medium series-detail-description-text']");
@@ -119,37 +123,49 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
// Match all p nodes in the description node.
// Filter out any node starting and ending with a html style tag
// "Should" remove single bold, italic and heading lines to leave a cleaner description.
- // Then replace known entities with characters using HtmlEntity.DeEntitize and join strings to form description.
- var nodes = descNode.SelectNodes(".//p/text()[normalize-space(.) != '']") ?? descNode.SelectNodes(".//div/text()[normalize-space(.) != '']");
+ // Then replace known entities with characters using HtmlEntity.DeEntitize and join strings (with spaces if needed) to form description.
+ var nodes = descNode.SelectNodes(".//p/span[not(contains(@class,'bold'))]")
+ ?? descNode.SelectNodes(".//div/span[not(contains(@class,'bold'))]");
- string description;
+ var description = string.Empty;
if (nodes != null)
{
var filteredNodes = nodes
- .Where(node => !_dirtyParas.IsMatch(node.InnerHtml.Trim()))
+ //.Where(node => !_dirtyParas.IsMatch(node.InnerHtml.Trim()))
.Select(node => HtmlEntity.DeEntitize(node.InnerText.Trim()))
+ .Where(node => node.Length > 1)
.ToArray();
- description = filteredNodes.Any()
- ? string.Join(" ", filteredNodes)
- : descNode.InnerHtml.Clean();
+ foreach (var node in filteredNodes)
+ if (char.IsUpper(node[0])) // | (description.Length != 0 && char.IsLower(description[^1])))
+ description += " " + node;
+ else
+ description += node;
+
+ description = description.Clean();
+
+ //description = filteredNodes.Any()
+ // ? filteredNodes.Aggregate(description, (current, s) => current + (char.IsLower(s[0]) ? s : s + " ")).Trim()
+ // : descNode.InnerHtml.Clean();
}
else
- description = descNode.InnerHtml.Clean();
+ {
+ description = descNode.InnerText.Clean();
+ }
// Following the example of Amazon, cut off desc around 1000 characters.
// If conveniently trimmed at the end of the sentence, let it end with the punctuation.
// If the sentence continues, cut it off and replace the space with an ellipsis
- if (description.Length > 1000)
- {
- description = description.Substring(0, 1000);
- var lastPunc = description.LastIndexOfAny(new[] { '.', '!', '?' });
- var lastSpace = description.LastIndexOf(' ');
- if (lastPunc > lastSpace)
- description = description.Substring(0, lastPunc + 1);
- else
- description = $"{description.Substring(0, lastSpace - 1)}{'\u2026'}";
- }
+ //if (description.Length > 1000)
+ //{
+ // description = description.Substring(0, 1000);
+ // var lastPunc = description.LastIndexOfAny(new[] { '.', '!', '?' });
+ // var lastSpace = description.LastIndexOf(' ');
+ // if (lastPunc > lastSpace)
+ // description = description.Substring(0, lastPunc + 1);
+ // else
+ // description = $"{description.Substring(0, lastSpace - 1)}{'\u2026'}";
+ //}
// Used to test output from Amazon description parsing
//var descsCheckFile = AppDomain.CurrentDomain.BaseDirectory + @"\descs.txt";
@@ -157,9 +173,11 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
response.Description = description;
}
+
#endregion
#region Reviews
+
try
{
var ratingNode = bookDoc.DocumentNode.SelectSingleNode("//*[@id='acrPopover']")
@@ -182,6 +200,7 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
{
throw new AggregateException($"Error finding book ratings. If you want, you can report the book's Amazon URL to help with parsing.\r\nError: {ex.Message}\r\n{ex.StackTrace}", ex);
}
+
#endregion
#region Pages
@@ -222,4 +241,3 @@ public InfoResponse ParseAmazonDocument(HtmlDocument bookDoc)
return response;
}
}
\ No newline at end of file
-}

Improve book description parsing