Task/extend error messages in scraper configuration (#6322)

* Improve error handling

* Update changelog

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Ignored nested ETFs when fetching top holdings for ETF and mutual fund assets from _Yahoo Finance_
+- Improved the scraper configuration with more detailed error messages
 
 ### Fixed
 

apps/api/src/app/admin/admin.controller.ts

@@ -247,14 +247,17 @@ export class AdminController {
     @Param('symbol') symbol: string
   ): Promise<{ price: number }> {
     try {
-      const price = await this.manualService.test(data.scraperConfiguration);
+      const price = await this.manualService.test({
+        symbol,
+        scraperConfiguration: data.scraperConfiguration
+      });
 
       if (price) {
         return { price };
       }
 
       throw new Error(
-        `Could not parse the current market price for ${symbol} (${dataSource})`
+        `Could not parse the market price for ${symbol} (${dataSource})`
       );
     } catch (error) {
       Logger.error(error, 'AdminController');

apps/api/src/services/data-provider/manual/manual.service.ts

@@ -105,7 +105,10 @@ export class ManualService implements DataProviderInterface {
         return {};
       }
 
-      const value = await this.scrape(symbolProfile.scraperConfiguration);
+      const value = await this.scrape({
+        symbol,
+        scraperConfiguration: symbolProfile.scraperConfiguration
+      });
 
       return {
         [symbol]: {
@@ -170,7 +173,10 @@ export class ManualService implements DataProviderInterface {
         symbolProfilesWithScraperConfigurationAndInstantMode.map(
           async ({ scraperConfiguration, symbol }) => {
             try {
-              const marketPrice = await this.scrape(scraperConfiguration);
+              const marketPrice = await this.scrape({
+                scraperConfiguration,
+                symbol
+              });
               return { marketPrice, symbol };
             } catch (error) {
               Logger.error(
@@ -267,13 +273,23 @@ export class ManualService implements DataProviderInterface {
     };
   }
 
-  public async test(scraperConfiguration: ScraperConfiguration) {
-    return this.scrape(scraperConfiguration);
+  public async test({
+    scraperConfiguration,
+    symbol
+  }: {
+    scraperConfiguration: ScraperConfiguration;
+    symbol: string;
+  }) {
+    return this.scrape({ scraperConfiguration, symbol });
   }
 
-  private async scrape(
-    scraperConfiguration: ScraperConfiguration
-  ): Promise<number> {
+  private async scrape({
+    scraperConfiguration,
+    symbol
+  }: {
+    scraperConfiguration: ScraperConfiguration;
+    symbol: string;
+  }): Promise<number> {
     let locale = scraperConfiguration.locale;
 
     const response = await fetch(scraperConfiguration.url, {
@@ -283,6 +299,12 @@ export class ManualService implements DataProviderInterface {
       )
     });
 
+    if (!response.ok) {
+      throw new Error(
+        `Failed to scrape the market price for ${symbol} (${this.getName()}): ${response.status} ${response.statusText} at ${scraperConfiguration.url}`
+      );
+    }
+
     let value: string;
 
     if (response.headers.get('content-type')?.includes('application/json')) {