mirror of
https://github.com/intergalacticalvariable/reader.git
synced 2024-11-16 03:32:25 +08:00
fix the logger thingy
This commit is contained in:
parent
a3f222638e
commit
ddbf0030b4
3
.gitmodules
vendored
3
.gitmodules
vendored
|
@ -1,3 +0,0 @@
|
||||||
[submodule "thinapps-shared"]
|
|
||||||
path = thinapps-shared
|
|
||||||
url = git@github.com:jina-ai/thinapps-shared.git
|
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"projects": {
|
"projects": {
|
||||||
"default": "reader-6b7dc"
|
"default": "reader-clone"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -5,7 +5,7 @@ import {
|
||||||
AssertionFailureError, ParamValidationError, Defer,
|
AssertionFailureError, ParamValidationError, Defer,
|
||||||
} from 'civkit';
|
} from 'civkit';
|
||||||
import { singleton } from 'tsyringe';
|
import { singleton } from 'tsyringe';
|
||||||
import { AsyncContext, CloudHTTPv2, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect } from '../shared';
|
import { AsyncContext, CloudHTTPv2, FirebaseStorageBucketControl, Logger, OutputServerEventStream, RPCReflect } from '../shared/index';
|
||||||
import _ from 'lodash';
|
import _ from 'lodash';
|
||||||
import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer';
|
import { PageSnapshot, PuppeteerControl, ScrappingOptions } from '../services/puppeteer';
|
||||||
import { Request, Response } from 'express';
|
import { Request, Response } from 'express';
|
||||||
|
@ -24,6 +24,8 @@ import { JSDomControl } from '../services/jsdom';
|
||||||
|
|
||||||
const md5Hasher = new HashManager('md5', 'hex');
|
const md5Hasher = new HashManager('md5', 'hex');
|
||||||
|
|
||||||
|
// const logger = new Logger('Crawler');
|
||||||
|
|
||||||
export interface ExtraScrappingOptions extends ScrappingOptions {
|
export interface ExtraScrappingOptions extends ScrappingOptions {
|
||||||
withIframe?: boolean;
|
withIframe?: boolean;
|
||||||
targetSelector?: string | string[];
|
targetSelector?: string | string[];
|
||||||
|
@ -61,7 +63,7 @@ const indexProto = {
|
||||||
|
|
||||||
@singleton()
|
@singleton()
|
||||||
export class CrawlerHost extends RPCHost {
|
export class CrawlerHost extends RPCHost {
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('Crawler');
|
||||||
|
|
||||||
turnDownPlugins = [require('turndown-plugin-gfm').tables];
|
turnDownPlugins = [require('turndown-plugin-gfm').tables];
|
||||||
|
|
||||||
|
@ -71,7 +73,6 @@ export class CrawlerHost extends RPCHost {
|
||||||
abuseBlockMs = 1000 * 3600;
|
abuseBlockMs = 1000 * 3600;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
protected globalLogger: Logger,
|
|
||||||
protected puppeteerControl: PuppeteerControl,
|
protected puppeteerControl: PuppeteerControl,
|
||||||
protected jsdomControl: JSDomControl,
|
protected jsdomControl: JSDomControl,
|
||||||
// protected altTextService: AltTextService,
|
// protected altTextService: AltTextService,
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { container, singleton } from 'tsyringe';
|
||||||
import fsp from 'fs/promises';
|
import fsp from 'fs/promises';
|
||||||
import { CityResponse, Reader } from 'maxmind';
|
import { CityResponse, Reader } from 'maxmind';
|
||||||
import { AsyncService, AutoCastable, Prop, runOnce } from 'civkit';
|
import { AsyncService, AutoCastable, Prop, runOnce } from 'civkit';
|
||||||
import { Logger } from '../shared';
|
import { Logger } from '../shared/index';
|
||||||
import path from 'path';
|
import path from 'path';
|
||||||
|
|
||||||
export enum GEOIP_SUPPORTED_LANGUAGES {
|
export enum GEOIP_SUPPORTED_LANGUAGES {
|
||||||
|
@ -56,12 +56,11 @@ export class GeoIPCityResponse extends AutoCastable {
|
||||||
@singleton()
|
@singleton()
|
||||||
export class GeoIPService extends AsyncService {
|
export class GeoIPService extends AsyncService {
|
||||||
|
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('CHANGE_LOGGER_NAME')
|
||||||
|
|
||||||
mmdbCity!: Reader<CityResponse>;
|
mmdbCity!: Reader<CityResponse>;
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
protected globalLogger: Logger,
|
|
||||||
) {
|
) {
|
||||||
super(...arguments);
|
super(...arguments);
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import { container, singleton } from 'tsyringe';
|
import { container, singleton } from 'tsyringe';
|
||||||
import { AsyncService, marshalErrorLike } from 'civkit';
|
import { AsyncService, marshalErrorLike } from 'civkit';
|
||||||
import { Logger } from '../shared/logger';
|
import { Logger } from '../shared/index';
|
||||||
import { ExtendedSnapshot, PageSnapshot } from './puppeteer';
|
import { ExtendedSnapshot, PageSnapshot } from './puppeteer';
|
||||||
import { JSDOM, VirtualConsole } from 'jsdom';
|
import { JSDOM, VirtualConsole } from 'jsdom';
|
||||||
import { Readability } from '@mozilla/readability';
|
import { Readability } from '@mozilla/readability';
|
||||||
|
@ -12,10 +12,9 @@ virtualConsole.on('error', () => void 0);
|
||||||
@singleton()
|
@singleton()
|
||||||
export class JSDomControl extends AsyncService {
|
export class JSDomControl extends AsyncService {
|
||||||
|
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('CHANGE_LOGGER_NAME')
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
protected globalLogger: Logger,
|
|
||||||
) {
|
) {
|
||||||
super(...arguments);
|
super(...arguments);
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@ import os from 'os';
|
||||||
import fs from 'fs';
|
import fs from 'fs';
|
||||||
import { container, singleton } from 'tsyringe';
|
import { container, singleton } from 'tsyringe';
|
||||||
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit';
|
import { AsyncService, Defer, marshalErrorLike, AssertionFailureError, delay, maxConcurrency } from 'civkit';
|
||||||
import { Logger } from '../shared/logger';
|
import { Logger } from '../shared/index';
|
||||||
|
|
||||||
import type { Browser, CookieParam, Page } from 'puppeteer';
|
import type { Browser, CookieParam, Page } from 'puppeteer';
|
||||||
import puppeteer from 'puppeteer-extra';
|
import puppeteer from 'puppeteer-extra';
|
||||||
|
@ -11,7 +11,7 @@ import puppeteerBlockResources from 'puppeteer-extra-plugin-block-resources';
|
||||||
import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
|
import puppeteerPageProxy from 'puppeteer-extra-plugin-page-proxy';
|
||||||
import { SecurityCompromiseError, ServiceCrashedError } from '../shared/errors';
|
import { SecurityCompromiseError, ServiceCrashedError } from '../shared/errors';
|
||||||
import { TimeoutError } from 'puppeteer';
|
import { TimeoutError } from 'puppeteer';
|
||||||
const tldExtract = require('tld-extract');
|
import tldExtract from 'tld-extract';
|
||||||
|
|
||||||
const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
|
const READABILITY_JS = fs.readFileSync(require.resolve('@mozilla/readability/Readability.js'), 'utf-8');
|
||||||
|
|
||||||
|
@ -203,7 +203,7 @@ export class PuppeteerControl extends AsyncService {
|
||||||
|
|
||||||
_sn = 0;
|
_sn = 0;
|
||||||
browser!: Browser;
|
browser!: Browser;
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('CHANGE_LOGGER_NAME')
|
||||||
|
|
||||||
private __healthCheckInterval?: NodeJS.Timeout;
|
private __healthCheckInterval?: NodeJS.Timeout;
|
||||||
|
|
||||||
|
@ -217,7 +217,6 @@ export class PuppeteerControl extends AsyncService {
|
||||||
circuitBreakerHosts: Set<string> = new Set();
|
circuitBreakerHosts: Set<string> = new Set();
|
||||||
|
|
||||||
constructor(
|
constructor(
|
||||||
protected globalLogger: Logger,
|
|
||||||
) {
|
) {
|
||||||
super(...arguments);
|
super(...arguments);
|
||||||
this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95;
|
this.setMaxListeners(2 * Math.floor(os.totalmem() / (256 * 1024 * 1024)) + 1); 148 - 95;
|
||||||
|
|
|
@ -1,3 +1,6 @@
|
||||||
|
import { injectable } from 'tsyringe';
|
||||||
|
|
||||||
|
@injectable()
|
||||||
export class Logger {
|
export class Logger {
|
||||||
constructor(private name: string) {}
|
constructor(private name: string) {}
|
||||||
|
|
||||||
|
|
|
@ -4,9 +4,9 @@ import { Logger } from '../logger';
|
||||||
|
|
||||||
@singleton()
|
@singleton()
|
||||||
export class CanvasService extends AsyncService {
|
export class CanvasService extends AsyncService {
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('CHANGE_LOGGER_NAME')
|
||||||
|
|
||||||
constructor(protected globalLogger: Logger) {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -4,11 +4,11 @@ import { Logger } from '../logger';
|
||||||
|
|
||||||
@singleton()
|
@singleton()
|
||||||
export class SecretExposer extends AsyncService {
|
export class SecretExposer extends AsyncService {
|
||||||
logger = this.globalLogger.child({ service: this.constructor.name });
|
logger = new Logger('CHANGE_LOGGER_NAME')
|
||||||
|
|
||||||
BRAVE_SEARCH_API_KEY: string = 'mock_brave_search_api_key';
|
BRAVE_SEARCH_API_KEY: string = 'mock_brave_search_api_key';
|
||||||
|
|
||||||
constructor(protected globalLogger: Logger) {
|
constructor() {
|
||||||
super();
|
super();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1 +0,0 @@
|
||||||
Subproject commit fe71cc2433f60ada86622f1670a752da40806e4d
|
|
Loading…
Reference in New Issue
Block a user