[pull] master from cube-js:master

packages/cubejs-api-gateway/src/gateway.ts

@@ -13,6 +13,7 @@ import {
   parseUtcIntoLocalDate,
   QueryAlias,
   CacheMode,
+  LoggerFn,
 } from '@cubejs-backend/shared';
 import {
   ResultArrayWrapper,
@@ -189,7 +190,7 @@ class ApiGateway {
      * It actually returns a Promise<OrchestratorApi>
      */
     protected readonly adapterApi: (ctx: RequestContext) => Promise<any>,
-    protected readonly logger: any,
+    protected readonly logger: LoggerFn,
     protected readonly options: ApiGatewayOptions,
   ) {
     this.dataSourceStorage = options.dataSourceStorage;

packages/cubejs-backend-native/src/logger.rs

@@ -26,13 +26,18 @@ impl NodeBridgeLogger {
 #[derive(Debug, Serialize)]
 struct EventBox {
     event: HashMap<String, String>,
+    level: String,
 }
 
 impl LogReporter for NodeBridgeLogger {
-    fn log(&self, event: String, properties: HashMap<String, String>, _level: Level) {
+    fn log(&self, event: String, properties: HashMap<String, String>, level: Level) {
         let mut props = properties;
         props.insert("type".to_string(), event);
-        let extra = serde_json::to_string(&EventBox { event: props }).unwrap();
+        let extra = serde_json::to_string(&EventBox {
+            event: props,
+            level: level.to_string(),
+        })
+        .unwrap();
 
         let channel = self.channel.clone();
         // Safety: Safe, because on_track take is used only for dropping

packages/cubejs-backend-shared/src/env.ts

@@ -163,6 +163,7 @@ const variables: Record<string, (...args: any) => any> = {
   devMode: () => get('CUBEJS_DEV_MODE')
     .default('false')
     .asBoolStrict(),
+  logLevel: () => get('CUBEJS_LOG_LEVEL').asString(),
   port: () => asPortOrSocket(process.env.PORT || '4000', 'PORT'),
   tls: () => get('CUBEJS_ENABLE_TLS')
     .default('false')

packages/cubejs-backend-shared/src/index.ts

@@ -25,3 +25,4 @@ export * from './FileRepository';
 export * from './decorators';
 export * from './PerfTracker';
 export * from './disposedProxy';
+export * from './logger';

packages/cubejs-backend-shared/src/logger.ts

@@ -0,0 +1,18 @@
+export type LogLevel = 'trace' | 'info' | 'warn' | 'error';
+
+export type LoggerFnParams = {
+  // It's possible to fill timestamp at the place of logging, otherwise, it will be filled in automatically
+  timestamp?: string,
+  requestId?: string;
+  duration?: number;
+  query?: string | Record<string, any>;
+  values?: any[];
+  allSqlLines?: boolean;
+  showRestParams?: boolean;
+  error?: Error | string;
+  trace?: string,
+  warning?: string,
+  [key: string]: any,
+};
+
+export type LoggerFn = (msg: string, params: LoggerFnParams) => void;

packages/cubejs-base-driver/src/BaseDriver.ts

@@ -13,6 +13,7 @@ import {
   isFilePath,
   isSslKey,
   isSslCert,
+  LoggerFn,
 } from '@cubejs-backend/shared';
 import fs from 'fs';
 
@@ -152,7 +153,7 @@ const DbTypeValueMatcher: Record<string, ((v: any) => boolean)> = {
 export abstract class BaseDriver implements DriverInterface {
   private readonly testConnectionTimeoutValue: number = 10000;
 
-  protected logger: any;
+  protected logger?: LoggerFn;
 
   /**
    * Class constructor.
@@ -673,7 +674,7 @@ export abstract class BaseDriver implements DriverInterface {
     return cancelCombinator(fn);
   }
 
-  public setLogger(logger: any) {
+  public setLogger(logger: LoggerFn) {
     this.logger = logger;
   }
 

packages/cubejs-databricks-jdbc-driver/src/DatabricksDriver.ts

@@ -5,7 +5,7 @@
  */
 
 import fetch from 'node-fetch';
-import { assertDataSource, getEnv, } from '@cubejs-backend/shared';
+import { assertDataSource, getEnv, LoggerFn } from '@cubejs-backend/shared';
 import {
   DatabaseStructure,
   DriverCapabilities,
@@ -329,7 +329,7 @@ export class DatabricksDriver extends JDBCDriver {
     };
   }
 
-  public override setLogger(logger: any) {
+  public override setLogger(logger: LoggerFn) {
     super.setLogger(logger);
     this.showDeprecations();
   }
@@ -480,15 +480,15 @@ export class DatabricksDriver extends JDBCDriver {
         ?.split('=')[1];
 
       if (result) {
-        this.logger('PWD Parameter Deprecation in connection string', {
+        this.logger?.('PWD Parameter Deprecation in connection string', {
           warning:
             'PWD parameter is deprecated and will be ignored in future releases. ' +
             'Please migrate to the CUBEJS_DB_DATABRICKS_TOKEN environment variable.'
         });
       }
     }
     if (this.showSparkProtocolWarn) {
-      this.logger('jdbc:spark protocol deprecation', {
+      this.logger?.('jdbc:spark protocol deprecation', {
         warning:
           'The `jdbc:spark` protocol is deprecated and will be ignored in future releases. ' +
           'Please migrate your CUBEJS_DB_DATABRICKS_URL environment variable to the ' +

packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationLoader.ts

@@ -1,6 +1,6 @@
 import R from 'ramda';
 import crypto from 'crypto';
-import { getEnv, MaybeCancelablePromise } from '@cubejs-backend/shared';
+import { getEnv, MaybeCancelablePromise, LoggerFn } from '@cubejs-backend/shared';
 import {
   cancelCombinator,
   DownloadQueryResultsResult,
@@ -103,7 +103,7 @@ export class PreAggregationLoader {
 
   public constructor(
     private readonly driverFactory: DriverFactory,
-    private readonly logger: any,
+    private readonly logger: LoggerFn,
     private readonly queryCache: QueryCache,
     preAggregations: PreAggregations,
     preAggregation,

packages/cubejs-query-orchestrator/src/orchestrator/PreAggregationPartitionRangeLoader.ts

@@ -10,6 +10,7 @@ import {
   timeSeries,
   localTimestampToUtc,
   parseUtcIntoLocalDate,
+  LoggerFn,
 } from '@cubejs-backend/shared';
 import { InlineTable, TableStructure } from '@cubejs-backend/base-driver';
 import { DriverFactory } from './DriverFactory';
@@ -64,7 +65,7 @@ export class PreAggregationPartitionRangeLoader {
 
   public constructor(
     private readonly driverFactory: DriverFactory,
-    private readonly logger: any,
+    private readonly logger: LoggerFn,
     private readonly queryCache: QueryCache,
     private readonly preAggregations: PreAggregations,
     private readonly preAggregation: PreAggregationDescription,

packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts

@@ -1,6 +1,6 @@
 import R from 'ramda';
 import crypto from 'crypto';
-import { getEnv, } from '@cubejs-backend/shared';
+import { getEnv, LoggerFn } from '@cubejs-backend/shared';
 
 import { BaseDriver, InlineTable, } from '@cubejs-backend/base-driver';
 import { CubeStoreDriver } from '@cubejs-backend/cubestore-driver';
@@ -270,7 +270,7 @@ export class PreAggregations {
   public constructor(
     private readonly redisPrefix: string,
     private readonly driverFactory: DriverFactoryByDataSource,
-    private readonly logger: any,
+    private readonly logger: LoggerFn,
     private readonly queryCache: QueryCache,
     options,
   ) {

packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts

@@ -8,6 +8,7 @@ import {
   MaybeCancelablePromise,
   streamToArray,
   CacheMode,
+  LoggerFn,
 } from '@cubejs-backend/shared';
 import { CubeStoreCacheDriver, CubeStoreDriver } from '@cubejs-backend/cubestore-driver';
 import {
@@ -160,7 +161,7 @@ export class QueryCache {
   public constructor(
     protected readonly cachePrefix: string,
     protected readonly driverFactory: DriverFactoryByDataSource,
-    protected readonly logger: any,
+    protected readonly logger: LoggerFn,
     public readonly options: QueryCacheOptions
   ) {
     switch (options.cacheAndQueueDriver || 'memory') {

packages/cubejs-query-orchestrator/src/orchestrator/QueryOrchestrator.ts

@@ -1,6 +1,6 @@
 import * as stream from 'stream';
 import R from 'ramda';
-import { CacheMode, getEnv } from '@cubejs-backend/shared';
+import { CacheMode, getEnv, LoggerFn } from '@cubejs-backend/shared';
 import { CubeStoreDriver } from '@cubejs-backend/cubestore-driver';
 import {
   QuerySchemasResult,
@@ -71,7 +71,7 @@ export class QueryOrchestrator {
   public constructor(
     protected readonly redisPrefix: string,
     protected readonly driverFactory: DriverFactoryByDataSource,
-    protected readonly logger: any,
+    protected readonly logger: LoggerFn,
     options: QueryOrchestratorOptions = {}
   ) {
     this.rollupOnlyMode = options.rollupOnlyMode;

packages/cubejs-query-orchestrator/src/orchestrator/QueryQueue.ts

@@ -1,5 +1,5 @@
 import { EventEmitter } from 'events';
-import { getEnv, getProcessUid } from '@cubejs-backend/shared';
+import { getEnv, getProcessUid, LoggerFn } from '@cubejs-backend/shared';
 import {
   QueueDriverInterface,
   QueryKey,
@@ -88,7 +88,7 @@ export class QueryQueue {
 
   protected cancelHandlers: Record<string, CancelHandlerFn>;
 
-  protected logger: any;
+  protected logger: LoggerFn;
 
   protected processUid: string;
 

packages/cubejs-schema-compiler/src/compiler/ErrorReporter.ts

@@ -1,4 +1,5 @@
 import { codeFrameColumns, SourceLocation } from '@babel/code-frame';
+import { LoggerFn } from '@cubejs-backend/shared';
 
 import { UserError } from './UserError';
 import { CompileError } from './CompileError';
@@ -30,7 +31,7 @@ interface File {
 }
 
 export interface ErrorReporterOptions {
-  logger: (msg: string) => void
+  logger: LoggerFn
 }
 
 const NO_FILE_SPECIFIED = '_No-file-specified';
@@ -46,7 +47,7 @@ export class ErrorReporter {
     protected readonly parent: ErrorReporter | null = null,
     protected readonly context: any[] = [],
     protected readonly options: ErrorReporterOptions = {
-      logger: (msg) => console.log(msg),
+      logger: (msg, _params) => console.log(msg),
     },
   ) {
   }
@@ -82,9 +83,9 @@ export class ErrorReporter {
       });
 
       if (targetFileName) {
-        this.options.logger(`${targetFileName}:\n${message}`);
+        this.options.logger(`${targetFileName}:\n${message}`, {});
       } else {
-        this.options.logger(message);
+        this.options.logger(message, {});
       }
     } else {
       if (this.rootReporter().warnings.find(m => (m.message || m) === e.message)) {
@@ -97,9 +98,9 @@ export class ErrorReporter {
       });
 
       if (targetFileName) {
-        this.options.logger(`${targetFileName}:\n${e.message}`);
+        this.options.logger(`${targetFileName}:\n${e.message}`, {});
       } else {
-        this.options.logger(e.message);
+        this.options.logger(e.message, {});
       }
     }
   }

packages/cubejs-schema-compiler/test/unit/error-reporter.test.ts

@@ -5,7 +5,7 @@ describe('ErrorReporter', () => {
   it('should group and format errors and warnings from different files', () => {
     const logs: string[] = [];
     const reporter = new ErrorReporter(null, [], {
-      logger: (msg) => logs.push(msg)
+      logger: (msg, _params) => logs.push(msg)
     });
 
     // Test inFile and exitFile
@@ -106,7 +106,7 @@ describe('ErrorReporter', () => {
 
   it('should handle inContext correctly', () => {
     const reporter = new ErrorReporter(null, [], {
-      logger: () => { /* empty */ }
+      logger: (_msg, _params) => { /* empty */ }
     });
 
     const contextReporter = reporter.inContext('Processing Users cube');
@@ -117,7 +117,7 @@ describe('ErrorReporter', () => {
 
   it('should deduplicate identical errors and warnings', () => {
     const reporter = new ErrorReporter(null, [], {
-      logger: () => { /* empty */ }
+      logger: (_msg, _params) => { /* empty */ }
     });
 
     reporter.inFile({
@@ -159,7 +159,7 @@ describe('ErrorReporter', () => {
 
   it('should handle addErrors and addWarnings', () => {
     const reporter = new ErrorReporter(null, [], {
-      logger: () => { /* empty */ }
+      logger: (_msg, _params) => { /* empty */ }
     });
 
     // Test addErrors with fileName
@@ -183,7 +183,7 @@ describe('ErrorReporter', () => {
 
   it('should not throw if no errors', () => {
     const reporter = new ErrorReporter(null, [], {
-      logger: () => { /* empty */ }
+      logger: (_msg, _params) => { /* empty */ }
     });
 
     reporter.warning({ message: 'Just a warning' });
@@ -193,7 +193,7 @@ describe('ErrorReporter', () => {
 
   it('should handle errors without fileName at the end', () => {
     const reporter = new ErrorReporter(null, [], {
-      logger: () => { /* empty */ }
+      logger: (_msg, _params) => { /* empty */ }
     });
 
     reporter.error({ message: 'Error in file A' }, 'fileA.js');

packages/cubejs-schema-compiler/test/unit/schema.test.ts

@@ -214,10 +214,12 @@ describe('Schema Testing', () => {
 
       expect(logger.mock.calls.length).toEqual(2);
       expect(logger.mock.calls[0]).toEqual([
-        'You specified both buildRangeStart and refreshRangeStart, buildRangeStart will be used.'
+        'You specified both buildRangeStart and refreshRangeStart, buildRangeStart will be used.',
+        {},
       ]);
       expect(logger.mock.calls[1]).toEqual([
-        'You specified both buildRangeEnd and refreshRangeEnd, buildRangeEnd will be used.'
+        'You specified both buildRangeEnd and refreshRangeEnd, buildRangeEnd will be used.',
+        {},
       ]);
     });