Feature/setup statistics gathering queue (#6696)
* Set up statistics gathering queue * Update changelogpull/6721/head
Thomas Kaul
3 months ago
committed by
GitHub
parent
5afceb802c
commit
cdef2afa48
@ -0,0 +1,36 @@
|
||||
import { ConfigurationModule } from '@ghostfolio/api/services/configuration/configuration.module';
|
||||
import { PropertyModule } from '@ghostfolio/api/services/property/property.module';
|
||||
import { STATISTICS_GATHERING_QUEUE } from '@ghostfolio/common/config';
|
||||
|
||||
import { BullAdapter } from '@bull-board/api/bullAdapter';
|
||||
import { BullBoardModule } from '@bull-board/nestjs';
|
||||
import { BullModule } from '@nestjs/bull';
|
||||
import { Module } from '@nestjs/common';
|
||||
|
||||
import { StatisticsGatheringProcessor } from './statistics-gathering.processor';
|
||||
import { StatisticsGatheringService } from './statistics-gathering.service';
|
||||
|
||||
@Module({
|
||||
exports: [BullModule, StatisticsGatheringService],
|
||||
imports: [
|
||||
...(process.env.ENABLE_FEATURE_BULL_BOARD === 'true'
|
||||
? [
|
||||
BullBoardModule.forFeature({
|
||||
adapter: BullAdapter,
|
||||
name: STATISTICS_GATHERING_QUEUE,
|
||||
options: {
|
||||
displayName: 'Statistics Gathering',
|
||||
readOnlyMode: process.env.BULL_BOARD_IS_READ_ONLY !== 'false'
|
||||
}
|
||||
})
|
||||
]
|
||||
: []),
|
||||
BullModule.registerQueue({
|
||||
name: STATISTICS_GATHERING_QUEUE
|
||||
}),
|
||||
ConfigurationModule,
|
||||
PropertyModule
|
||||
],
|
||||
providers: [StatisticsGatheringProcessor, StatisticsGatheringService]
|
||||
})
|
||||
export class StatisticsGatheringQueueModule {}
|
||||
@ -0,0 +1,212 @@
|
||||
import { ConfigurationService } from '@ghostfolio/api/services/configuration/configuration.service';
|
||||
import { PropertyService } from '@ghostfolio/api/services/property/property.service';
|
||||
import {
|
||||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,
|
||||
HEADER_KEY_TOKEN,
|
||||
PROPERTY_BETTER_UPTIME_MONITOR_ID,
|
||||
PROPERTY_DOCKER_HUB_PULLS,
|
||||
PROPERTY_GITHUB_CONTRIBUTORS,
|
||||
PROPERTY_GITHUB_STARGAZERS,
|
||||
PROPERTY_UPTIME,
|
||||
STATISTICS_GATHERING_QUEUE
|
||||
} from '@ghostfolio/common/config';
|
||||
import {
|
||||
DATE_FORMAT,
|
||||
extractNumberFromString
|
||||
} from '@ghostfolio/common/helper';
|
||||
|
||||
import { Process, Processor } from '@nestjs/bull';
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { format, subDays } from 'date-fns';
|
||||
|
||||
@Injectable()
|
||||
@Processor(STATISTICS_GATHERING_QUEUE)
|
||||
export class StatisticsGatheringProcessor {
|
||||
public constructor(
|
||||
private readonly configurationService: ConfigurationService,
|
||||
private readonly propertyService: PropertyService
|
||||
) {}
|
||||
|
||||
@Process(GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME)
|
||||
public async gatherDockerHubPullsStatistics() {
|
||||
Logger.log(
|
||||
'Docker Hub pulls statistics gathering has been started',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
|
||||
const dockerHubPulls = await this.countDockerHubPulls();
|
||||
|
||||
await this.propertyService.put({
|
||||
key: PROPERTY_DOCKER_HUB_PULLS,
|
||||
value: String(dockerHubPulls)
|
||||
});
|
||||
|
||||
Logger.log(
|
||||
'Docker Hub pulls statistics gathering has been completed',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
}
|
||||
|
||||
@Process(GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME)
|
||||
public async gatherGitHubContributorsStatistics() {
|
||||
Logger.log(
|
||||
'GitHub contributors statistics gathering has been started',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
|
||||
const gitHubContributors = await this.countGitHubContributors();
|
||||
|
||||
await this.propertyService.put({
|
||||
key: PROPERTY_GITHUB_CONTRIBUTORS,
|
||||
value: String(gitHubContributors)
|
||||
});
|
||||
|
||||
Logger.log(
|
||||
'GitHub contributors statistics gathering has been completed',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
}
|
||||
|
||||
@Process(GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME)
|
||||
public async gatherGitHubStargazersStatistics() {
|
||||
Logger.log(
|
||||
'GitHub stargazers statistics gathering has been started',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
|
||||
const gitHubStargazers = await this.countGitHubStargazers();
|
||||
|
||||
await this.propertyService.put({
|
||||
key: PROPERTY_GITHUB_STARGAZERS,
|
||||
value: String(gitHubStargazers)
|
||||
});
|
||||
|
||||
Logger.log(
|
||||
'GitHub stargazers statistics gathering has been completed',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
}
|
||||
|
||||
@Process(GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME)
|
||||
public async gatherUptimeStatistics() {
|
||||
Logger.log(
|
||||
'Uptime statistics gathering has been started',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
|
||||
const uptime = await this.getUptime();
|
||||
|
||||
await this.propertyService.put({
|
||||
key: PROPERTY_UPTIME,
|
||||
value: String(uptime)
|
||||
});
|
||||
|
||||
Logger.log(
|
||||
'Uptime statistics gathering has been completed',
|
||||
'StatisticsGatheringProcessor'
|
||||
);
|
||||
}
|
||||
|
||||
private async countDockerHubPulls(): Promise<number> {
|
||||
try {
|
||||
const { pull_count } = (await fetch(
|
||||
'https://hub.docker.com/v2/repositories/ghostfolio/ghostfolio',
|
||||
{
|
||||
headers: { 'User-Agent': 'request' },
|
||||
signal: AbortSignal.timeout(
|
||||
this.configurationService.get('REQUEST_TIMEOUT')
|
||||
)
|
||||
}
|
||||
).then((res) => res.json())) as { pull_count: number };
|
||||
|
||||
return pull_count;
|
||||
} catch (error) {
|
||||
Logger.error(error, 'StatisticsGatheringProcessor - DockerHub');
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async countGitHubContributors(): Promise<number> {
|
||||
try {
|
||||
const body = await fetch('https://github.com/ghostfolio/ghostfolio', {
|
||||
signal: AbortSignal.timeout(
|
||||
this.configurationService.get('REQUEST_TIMEOUT')
|
||||
)
|
||||
}).then((res) => res.text());
|
||||
|
||||
const $ = cheerio.load(body);
|
||||
|
||||
const value = $(
|
||||
'a[href="/ghostfolio/ghostfolio/graphs/contributors"] .Counter'
|
||||
).text();
|
||||
|
||||
if (!value) {
|
||||
throw new Error('Could not find the contributors count in the page');
|
||||
}
|
||||
|
||||
return extractNumberFromString({
|
||||
value
|
||||
});
|
||||
} catch (error) {
|
||||
Logger.error(error, 'StatisticsGatheringProcessor - GitHub');
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async countGitHubStargazers(): Promise<number> {
|
||||
try {
|
||||
const { stargazers_count } = (await fetch(
|
||||
'https://api.github.com/repos/ghostfolio/ghostfolio',
|
||||
{
|
||||
headers: { 'User-Agent': 'request' },
|
||||
signal: AbortSignal.timeout(
|
||||
this.configurationService.get('REQUEST_TIMEOUT')
|
||||
)
|
||||
}
|
||||
).then((res) => res.json())) as { stargazers_count: number };
|
||||
|
||||
return stargazers_count;
|
||||
} catch (error) {
|
||||
Logger.error(error, 'StatisticsGatheringProcessor - GitHub');
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
private async getUptime(): Promise<number> {
|
||||
try {
|
||||
const monitorId = await this.propertyService.getByKey<string>(
|
||||
PROPERTY_BETTER_UPTIME_MONITOR_ID
|
||||
);
|
||||
|
||||
const { data } = await fetch(
|
||||
`https://uptime.betterstack.com/api/v2/monitors/${monitorId}/sla?from=${format(
|
||||
subDays(new Date(), 90),
|
||||
DATE_FORMAT
|
||||
)}&to${format(new Date(), DATE_FORMAT)}`,
|
||||
{
|
||||
headers: {
|
||||
[HEADER_KEY_TOKEN]: `Bearer ${this.configurationService.get(
|
||||
'API_KEY_BETTER_UPTIME'
|
||||
)}`
|
||||
},
|
||||
signal: AbortSignal.timeout(
|
||||
this.configurationService.get('REQUEST_TIMEOUT')
|
||||
)
|
||||
}
|
||||
).then((res) => res.json());
|
||||
|
||||
return data.attributes.availability / 100;
|
||||
} catch (error) {
|
||||
Logger.error(error, 'StatisticsGatheringProcessor - Better Stack');
|
||||
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,40 @@
|
||||
import {
|
||||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_PROCESS_JOB_OPTIONS,
|
||||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME,
|
||||
STATISTICS_GATHERING_QUEUE
|
||||
} from '@ghostfolio/common/config';
|
||||
|
||||
import { InjectQueue } from '@nestjs/bull';
|
||||
import { Injectable } from '@nestjs/common';
|
||||
import { Queue } from 'bull';
|
||||
|
||||
@Injectable()
|
||||
export class StatisticsGatheringService {
|
||||
public constructor(
|
||||
@InjectQueue(STATISTICS_GATHERING_QUEUE)
|
||||
private readonly statisticsGatheringQueue: Queue
|
||||
) {}
|
||||
|
||||
public async addJobsToQueue() {
|
||||
return Promise.all(
|
||||
[
|
||||
GATHER_STATISTICS_DOCKER_HUB_PULLS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_CONTRIBUTORS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_GITHUB_STARGAZERS_PROCESS_JOB_NAME,
|
||||
GATHER_STATISTICS_UPTIME_PROCESS_JOB_NAME
|
||||
].map((jobName) => {
|
||||
return this.statisticsGatheringQueue.add(
|
||||
jobName,
|
||||
{},
|
||||
{
|
||||
...GATHER_STATISTICS_PROCESS_JOB_OPTIONS,
|
||||
jobId: jobName
|
||||
}
|
||||
);
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue