diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d3c55eb3..7c41c75bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Ignored nested ETFs when fetching top holdings for ETF and mutual fund assets from _Yahoo Finance_ +- Improved the scraper configuration with more detailed error messages ### Fixed diff --git a/apps/api/src/app/admin/admin.controller.ts b/apps/api/src/app/admin/admin.controller.ts index 24467c732..8a202a926 100644 --- a/apps/api/src/app/admin/admin.controller.ts +++ b/apps/api/src/app/admin/admin.controller.ts @@ -247,14 +247,17 @@ export class AdminController { @Param('symbol') symbol: string ): Promise<{ price: number }> { try { - const price = await this.manualService.test(data.scraperConfiguration); + const price = await this.manualService.test({ + symbol, + scraperConfiguration: data.scraperConfiguration + }); if (price) { return { price }; } throw new Error( - `Could not parse the current market price for ${symbol} (${dataSource})` + `Could not parse the market price for ${symbol} (${dataSource})` ); } catch (error) { Logger.error(error, 'AdminController'); diff --git a/apps/api/src/services/data-provider/manual/manual.service.ts b/apps/api/src/services/data-provider/manual/manual.service.ts index 7392f0914..51e65e631 100644 --- a/apps/api/src/services/data-provider/manual/manual.service.ts +++ b/apps/api/src/services/data-provider/manual/manual.service.ts @@ -105,7 +105,10 @@ export class ManualService implements DataProviderInterface { return {}; } - const value = await this.scrape(symbolProfile.scraperConfiguration); + const value = await this.scrape({ + symbol, + scraperConfiguration: symbolProfile.scraperConfiguration + }); return { [symbol]: { @@ -170,7 +173,10 @@ export class ManualService implements DataProviderInterface { symbolProfilesWithScraperConfigurationAndInstantMode.map( async ({ scraperConfiguration, symbol }) => { try { - const marketPrice = await this.scrape(scraperConfiguration); + const marketPrice = await this.scrape({ + scraperConfiguration, + symbol + }); return { marketPrice, symbol }; } catch (error) { Logger.error( @@ -267,13 +273,23 @@ export class ManualService implements DataProviderInterface { }; } - public async test(scraperConfiguration: ScraperConfiguration) { - return this.scrape(scraperConfiguration); + public async test({ + scraperConfiguration, + symbol + }: { + scraperConfiguration: ScraperConfiguration; + symbol: string; + }) { + return this.scrape({ scraperConfiguration, symbol }); } - private async scrape( - scraperConfiguration: ScraperConfiguration - ): Promise { + private async scrape({ + scraperConfiguration, + symbol + }: { + scraperConfiguration: ScraperConfiguration; + symbol: string; + }): Promise { let locale = scraperConfiguration.locale; const response = await fetch(scraperConfiguration.url, { @@ -283,6 +299,12 @@ export class ManualService implements DataProviderInterface { ) }); + if (!response.ok) { + throw new Error( + `Failed to scrape the market price for ${symbol} (${this.getName()}): ${response.status} ${response.statusText} at ${scraperConfiguration.url}` + ); + } + let value: string; if (response.headers.get('content-type')?.includes('application/json')) {