From de9c55a13829dc00ef8e697b1c12f5adbd3f437f Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Tue, 1 Jul 2025 18:05:32 +0200 Subject: [PATCH 1/5] feat(tesseract): Rollup Join and Lambda support --- .../src/adapter/BaseQuery.js | 11 +- .../src/adapter/PreAggregations.ts | 8 + .../src/compiler/CubeEvaluator.ts | 9 + .../postgres/pre-aggregations.test.ts | 2 +- .../src/cube_bridge/evaluator.rs | 6 + .../pre_aggregation_description.rs | 3 + .../compiled_pre_aggregation.rs | 107 +------- .../optimizers/pre_aggregation/mod.rs | 2 + .../optimizers/pre_aggregation/optimizer.rs | 97 +++----- .../pre_aggregation/original_sql_optimizer.rs | 25 +- .../pre_aggregations_compiler.rs | 230 ++++++++++++++++++ .../src/logical_plan/pre_aggregation.rs | 12 +- .../src/physical_plan_builder/builder.rs | 5 +- .../cubesqlplanner/src/planner/base_query.rs | 11 +- 14 files changed, 338 insertions(+), 190 deletions(-) create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index 6c50fc2398971..a9c64703e3e2f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -331,12 +331,12 @@ export class BaseQuery { */ this.customSubQueryJoins = this.options.subqueryJoins ?? []; this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner'); - this.canUseNativeSqlPlannerPreAggregation = false; - if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) { + this.canUseNativeSqlPlannerPreAggregation = true; + /* if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) { const fullAggregateMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); this.canUseNativeSqlPlannerPreAggregation = fullAggregateMeasures.multiStageMembers.length > 0 || fullAggregateMeasures.cumulativeMeasures.length > 0; - } + } */ this.queryLevelJoinHints = this.options.joinHints ?? []; this.prebuildJoin(); @@ -710,6 +710,7 @@ export class BaseQuery { preAggForQuery = undefined; } } + console.log("!!!! ========="); if (preAggForQuery) { const { multipliedMeasures, @@ -718,6 +719,7 @@ export class BaseQuery { withQueries, multiStageMembers, } = this.fullKeyQueryAggregateMeasures(); + console.log("!!!! OOOO ", preAggForQuery); if (cumulativeMeasures.length === 0) { sql = this.preAggregations.rollupPreAggregation( @@ -736,6 +738,8 @@ export class BaseQuery { } else { sql = this.fullKeyQueryAggregate(); } + console.log("!!! sql", sql); + console.log("!!!! --------"); return this.options.totalQuery ? this.countAllQuery(sql) : sql; @@ -967,6 +971,7 @@ export class BaseQuery { } getPreAggregationByName(cube, preAggregationName) { + console.log("!!!!! paaaaa", this.preAggregations.getRollupPreAggregationByName(cube, preAggregationName)); return this.preAggregations.getRollupPreAggregationByName(cube, preAggregationName); } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0dd6fad25ec4f..b024b097eb88e 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -981,12 +981,14 @@ export class PreAggregations { // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) )); + console.log("!!!! exixtingJoins", existingJoins); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && existing.originalTo === target.originalTo && R.equals(existing.fromMembers, target.fromMembers) && R.equals(existing.toMembers, target.toMembers) )); + console.log("!!!! nonexixtingJoins", nonExistingJoins); if (!nonExistingJoins.length) { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } @@ -1020,6 +1022,7 @@ export class PreAggregations { private resolveJoinMembers(join) { return join.joins.map(j => { const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); + console.log("!!!! memb paths ", memberPaths); const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); if (invalidMembers.length) { throw new UserError(`Members ${invalidMembers.join(', ')} in join from '${j.originalFrom}' to '${j.originalTo}' doesn't reference join cubes`); @@ -1344,6 +1347,7 @@ export class PreAggregations { } private rollupLambdaUnion(preAggregationForQuery: PreAggregationForQuery, rollupGranularity: string): string { + console.log("!!!!! EEEEEE"); if (!preAggregationForQuery.referencedPreAggregations) { return this.preAggregationTableName( preAggregationForQuery.cube, @@ -1376,6 +1380,8 @@ export class PreAggregations { } ); + console.log("!!!! ref pre aggr", preAggregationForQuery.referencedPreAggregations); + const tables = preAggregationForQuery.referencedPreAggregations.map(preAggregation => { const dimensionsReferences = this.dimensionsRenderedReference(preAggregation); const timeDimensionsReferences = this.timeDimensionsRenderedReference(rollupGranularity, preAggregation); @@ -1403,6 +1409,7 @@ export class PreAggregations { let toJoin; // TODO granularity shouldn't be null? const rollupGranularity = preAggregationForQuery.references.timeDimensions[0]?.granularity || 'day'; + console.log("!!!! IIIIIII"); const sqlAndAlias = (preAgg) => ({ preAggregation: preAgg, @@ -1412,6 +1419,7 @@ export class PreAggregations { if (preAggregationForQuery.preAggregation.type === 'rollupJoin') { const join = preAggregationForQuery.rollupJoin; + console.log("!!!!! cccc ", join); toJoin = [ sqlAndAlias(join[0].fromPreAggObj), diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index d8ee7b6bd19b0..400f0047e6a9b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -565,6 +565,7 @@ export class CubeEvaluator extends CubeSymbols { } public preAggregationsForCubeAsArray(path: string) { + console.log("!!!! pre aggrs", this.cubeFromPath(path).preAggregations); return Object.entries(this.cubeFromPath(path).preAggregations || {}).map(([name, preAggregation]) => ({ name, ...(preAggregation as Record) @@ -785,6 +786,14 @@ export class CubeEvaluator extends CubeSymbols { return { cubeReferencesUsed, pathReferencesUsed, evaluatedSql }; } + /** + * Evaluates rollup references for retrieving rollupReference used in Tesseract. + * This is a temporary solution until Tesseract takes ownership of all pre-aggregations. + */ + public evaluateRollupReferences>(cube: string, rollupReferences: (...args: Array) => T) { + return this.evaluateReferences(cube, rollupReferences, { originalSorting: true }); + } + public evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinition): PreAggregationReferences { const timeDimensions: Array = []; diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index bf603e2995947..a24881471e392 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2136,7 +2136,7 @@ describe('PreAggregations', () => { }); }); - it('rollup join', async () => { + it('rollup join 11', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs index 14b3c3d239cb9..e0e1eaf51c152 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs @@ -64,4 +64,10 @@ pub trait CubeEvaluator { &self, cube_name: String, ) -> Result>, CubeError>; + #[nbridge(vec)] + fn evaluate_rollup_references( + &self, + cube: String, + sql: Rc, + ) -> Result, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs index 72ffd10662151..aaf650503f00b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs @@ -32,4 +32,7 @@ pub trait PreAggregationDescription { #[nbridge(field, optional)] fn time_dimension_reference(&self) -> Result>, CubeError>; + + #[nbridge(field, optional)] + fn rollup_references(&self) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs index e47d5bfd2805b..8955ba07e8e79 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs @@ -5,10 +5,16 @@ use crate::planner::sql_evaluator::MemberSymbol; use cubenativeutils::CubeError; use std::fmt::Debug; use std::rc::Rc; +#[derive(Clone)] +pub enum PreAggregationSource { + Table(String), +} + #[derive(Clone)] pub struct CompiledPreAggregation { pub cube_name: String, pub name: String, + pub source: PreAggregationSource, pub granularity: Option, pub external: Option, pub measures: Vec>, @@ -34,104 +40,3 @@ impl Debug for CompiledPreAggregation { .finish() } } - -impl CompiledPreAggregation { - pub fn try_new( - query_tools: Rc, - cube_name: &String, - description: Rc, - ) -> Result, CubeError> { - let static_data = description.static_data(); - let measures = if let Some(refs) = description.measure_references()? { - Self::symbols_from_ref(query_tools.clone(), cube_name, refs, Self::check_is_measure)? - } else { - Vec::new() - }; - let dimensions = if let Some(refs) = description.dimension_references()? { - Self::symbols_from_ref( - query_tools.clone(), - cube_name, - refs, - Self::check_is_dimension, - )? - } else { - Vec::new() - }; - let time_dimensions = if let Some(refs) = description.time_dimension_reference()? { - let dims = Self::symbols_from_ref( - query_tools.clone(), - cube_name, - refs, - Self::check_is_time_dimension, - )?; - /* if dims.len() != 1 { - return Err(CubeError::user(format!( - "Pre aggregation should contains only one time dimension" - ))); - } */ - vec![(dims[0].clone(), static_data.granularity.clone())] //TODO remove unwrap - } else { - Vec::new() - }; - let allow_non_strict_date_range_match = description - .static_data() - .allow_non_strict_date_range_match - .unwrap_or(false); - let res = Rc::new(Self { - name: static_data.name.clone(), - cube_name: cube_name.clone(), - granularity: static_data.granularity.clone(), - external: static_data.external, - measures, - dimensions, - time_dimensions, - allow_non_strict_date_range_match, - }); - Ok(res) - } - - fn symbols_from_ref Result<(), CubeError>>( - query_tools: Rc, - cube_name: &String, - ref_func: Rc, - check_type_fn: F, - ) -> Result>, CubeError> { - let evaluator_compiler_cell = query_tools.evaluator_compiler().clone(); - let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); - let sql_call = evaluator_compiler.compile_sql_call(cube_name, ref_func)?; - let mut res = Vec::new(); - for symbol in sql_call.get_dependencies().iter() { - check_type_fn(&symbol)?; - res.push(symbol.clone()); - } - Ok(res) - } - - fn check_is_measure(symbol: &MemberSymbol) -> Result<(), CubeError> { - symbol - .as_measure() - .map_err(|_| CubeError::user(format!("Pre-aggregation measure must be a measure")))?; - Ok(()) - } - - fn check_is_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { - symbol.as_dimension().map_err(|_| { - CubeError::user(format!("Pre-aggregation dimension must be a dimension")) - })?; - Ok(()) - } - - fn check_is_time_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { - let dimension = symbol.as_dimension().map_err(|_| { - CubeError::user(format!( - "Pre-aggregation time dimension must be a dimension" - )) - })?; - if dimension.dimension_type() != "time" { - return Err(CubeError::user(format!( - "Pre-aggregation time dimension must be a dimension" - ))); - } - Ok(()) - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs index a3fbe777d9fe0..51c2da6e499db 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs @@ -4,6 +4,7 @@ mod measure_matcher; mod optimizer; mod original_sql_collector; mod original_sql_optimizer; +mod pre_aggregations_compiler; pub use compiled_pre_aggregation::*; use dimension_matcher::*; @@ -11,3 +12,4 @@ use measure_matcher::*; pub use optimizer::*; pub use original_sql_collector::*; pub use original_sql_optimizer::*; +pub use pre_aggregations_compiler::*; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs index 1fff70496060f..c812c57e3cdca 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs @@ -1,3 +1,4 @@ +use super::PreAggregationsCompiler; use super::*; use crate::logical_plan::*; use crate::plan::FilterItem; @@ -43,22 +44,9 @@ impl PreAggregationOptimizer { let mut cube_names_collector = CubeNamesCollector::new(); cube_names_collector.collect(&plan)?; let cube_names = cube_names_collector.result(); + let mut compiler = PreAggregationsCompiler::try_new(self.query_tools.clone(), &cube_names)?; - let mut compiled_pre_aggregations = Vec::new(); - for cube_name in cube_names.iter() { - let pre_aggregations = self - .query_tools - .cube_evaluator() - .pre_aggregations_for_cube_as_array(cube_name.clone())?; - for pre_aggregation in pre_aggregations.iter() { - let compiled = CompiledPreAggregation::try_new( - self.query_tools.clone(), - cube_name, - pre_aggregation.clone(), - )?; - compiled_pre_aggregations.push(compiled); - } - } + let compiled_pre_aggregations = compiler.compile_all_pre_aggregations()?; for pre_aggregation in compiled_pre_aggregations.iter() { let new_query = self.try_rewrite_query(plan.clone(), pre_aggregation)?; @@ -413,51 +401,44 @@ impl PreAggregationOptimizer { &mut self, pre_aggregation: &Rc, ) -> Result, CubeError> { - let pre_aggregation_obj = self.query_tools.base_tools().get_pre_aggregation_by_name( + /* let pre_aggregation_obj = self.query_tools.base_tools().get_pre_aggregation_by_name( pre_aggregation.cube_name.clone(), pre_aggregation.name.clone(), - )?; - if let Some(table_name) = &pre_aggregation_obj.static_data().table_name { - let schema = LogicalSchema { - time_dimensions: vec![], - dimensions: pre_aggregation - .dimensions - .iter() - .cloned() - .chain( - pre_aggregation - .time_dimensions - .iter() - .map(|(d, _)| d.clone()), - ) - .collect(), - measures: pre_aggregation.measures.to_vec(), - multiplied_measures: HashSet::new(), - }; - let pre_aggregation = PreAggregation { - name: pre_aggregation.name.clone(), - time_dimensions: pre_aggregation.time_dimensions.clone(), - dimensions: pre_aggregation.dimensions.clone(), - measures: pre_aggregation.measures.clone(), - schema: Rc::new(schema), - external: pre_aggregation.external.unwrap_or_default(), - granularity: pre_aggregation.granularity.clone(), - table_name: table_name.clone(), - cube_name: pre_aggregation.cube_name.clone(), - pre_aggregation_obj, - }; - let result = Rc::new(pre_aggregation); - self.used_pre_aggregations.insert( - (result.cube_name.clone(), result.name.clone()), - result.clone(), - ); - Ok(result) - } else { - Err(CubeError::internal(format!( - "Cannot find pre aggregation object for cube {} and name {}", - pre_aggregation.cube_name, pre_aggregation.name - ))) - } + )?; */ + //if let Some(table_name) = &pre_aggregation_obj.static_data().table_name { + let schema = LogicalSchema { + time_dimensions: vec![], + dimensions: pre_aggregation + .dimensions + .iter() + .cloned() + .chain( + pre_aggregation + .time_dimensions + .iter() + .map(|(d, _)| d.clone()), + ) + .collect(), + measures: pre_aggregation.measures.to_vec(), + multiplied_measures: HashSet::new(), + }; + let pre_aggregation = PreAggregation { + name: pre_aggregation.name.clone(), + time_dimensions: pre_aggregation.time_dimensions.clone(), + dimensions: pre_aggregation.dimensions.clone(), + measures: pre_aggregation.measures.clone(), + schema: Rc::new(schema), + external: pre_aggregation.external.unwrap_or_default(), + granularity: pre_aggregation.granularity.clone(), + source: pre_aggregation.source.clone(), + cube_name: pre_aggregation.cube_name.clone(), + }; + let result = Rc::new(pre_aggregation); + self.used_pre_aggregations.insert( + (result.cube_name.clone(), result.name.clone()), + result.clone(), + ); + Ok(result) } fn is_schema_and_filters_match( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs index 4bd5c1ea078d3..4edb5beb33007 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs @@ -1,3 +1,4 @@ +use super::PreAggregationsCompiler; use super::*; use crate::logical_plan::*; use crate::planner::query_tools::QueryTools; @@ -352,25 +353,11 @@ impl OriginalSqlOptimizer { let res = if let Some(found_pre_aggregation) = self.foud_pre_aggregations.get(cube_name) { Some(found_pre_aggregation.clone()) } else { - let pre_aggregations = self - .query_tools - .cube_evaluator() - .pre_aggregations_for_cube_as_array(cube_name.clone())?; - if let Some(found_pre_aggregation) = pre_aggregations - .iter() - .find(|p| p.static_data().pre_aggregation_type == "originalSql") - { - let compiled = CompiledPreAggregation::try_new( - self.query_tools.clone(), - cube_name, - found_pre_aggregation.clone(), - )?; - self.foud_pre_aggregations - .insert(cube_name.clone(), compiled.clone()); - Some(compiled) - } else { - None - } + let mut compiler = PreAggregationsCompiler::try_new( + self.query_tools.clone(), + &vec![cube_name.clone()], + )?; + compiler.compile_origin_sql_pre_aggregation(&cube_name)? }; Ok(res) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs new file mode 100644 index 0000000000000..87e4d0b96a123 --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs @@ -0,0 +1,230 @@ +use super::CompiledPreAggregation; +use super::PreAggregationSource; +use crate::cube_bridge::member_sql::MemberSql; +use crate::cube_bridge::pre_aggregation_description::PreAggregationDescription; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::collectors::collect_cube_names_from_symbols; +use crate::planner::sql_evaluator::MemberSymbol; +use crate::planner::planners::JoinPlanner; +use cubenativeutils::CubeError; +use itertools::Itertools; +use std::collections::HashMap; +use std::fmt::Debug; +use std::rc::Rc; + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct PreAggregationFullName { + pub cube_name: String, + pub name: String, +} + +impl PreAggregationFullName { + pub fn new(cube_name: String, name: String) -> Self { + Self { cube_name, name } + } +} + +pub struct PreAggregationsCompiler { + query_tools: Rc, + descriptions: Rc)>>, + compiled_cache: HashMap>, +} + +impl PreAggregationsCompiler { + pub fn try_new( + query_tools: Rc, + cube_names: &Vec, + ) -> Result { + let mut descriptions = Vec::new(); + for cube_name in cube_names.iter() { + let pre_aggregations = query_tools + .cube_evaluator() + .pre_aggregations_for_cube_as_array(cube_name.clone())?; + for pre_aggregation in pre_aggregations.iter() { + let full_name = PreAggregationFullName::new( + cube_name.clone(), + pre_aggregation.static_data().name.clone(), + ); + descriptions.push((full_name, pre_aggregation.clone())); + } + } + Ok(Self { + query_tools, + descriptions: Rc::new(descriptions), + compiled_cache: HashMap::new(), + }) + } + + pub fn compile_pre_aggregation( + &mut self, + name: &PreAggregationFullName, + ) -> Result, CubeError> { + if let Some(compiled) = self.compiled_cache.get(&name) { + return Ok(compiled.clone()); + } + + if let Some((_, description)) = self.descriptions.clone().iter().find(|(n, _)| n == name) { + let static_data = description.static_data(); + let measures = if let Some(refs) = description.measure_references()? { + Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_measure, + )? + } else { + Vec::new() + }; + let dimensions = if let Some(refs) = description.dimension_references()? { + Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_dimension, + )? + } else { + Vec::new() + }; + let time_dimensions = if let Some(refs) = description.time_dimension_reference()? { + let dims = Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_time_dimension, + )?; + vec![(dims[0].clone(), static_data.granularity.clone())] + } else { + Vec::new() + }; + let allow_non_strict_date_range_match = description + .static_data() + .allow_non_strict_date_range_match + .unwrap_or(false); + //FIXME sqlAlias!!! + let table_name = self + .query_tools + .base_tools() + .pre_aggregation_table_name(name.cube_name.clone(), name.name.clone())?; + let rollups = if let Some(refs) = description.rollup_references()? { + let r = self + .query_tools + .cube_evaluator() + .evaluate_rollup_references(name.cube_name.clone(), refs)?; + r + } else { + Vec::new() + }; + + if static_data.pre_aggregation_type == "rollupJoin" { + self.build_join_source(&measures, &dimensions, &rollups)?; + } + + let res = Rc::new(CompiledPreAggregation { + name: static_data.name.clone(), + cube_name: name.cube_name.clone(), + source: PreAggregationSource::Table(table_name), + granularity: static_data.granularity.clone(), + external: static_data.external, + measures, + dimensions, + time_dimensions, + allow_non_strict_date_range_match, + }); + self.compiled_cache.insert(name.clone(), res.clone()); + Ok(res) + } else { + Err(CubeError::internal(format!( + "Undefined pre-aggregation {}.{}", + name.cube_name, name.name + ))) + } + } + + fn build_join_source( + &mut self, + measures: &Vec>, + dimensions: &Vec>, + rollups: &Vec, + ) -> Result<(), CubeError> { + println!("!!!!! build join source"); + let all_symbols = measures + .iter() + .cloned() + .chain(dimensions.iter().cloned()) + .collect_vec(); + let pre_aggr_cube_names = collect_cube_names_from_symbols(&all_symbols)?; + println!("!!!! pre aggr cube names {:?}", pre_aggr_cube_names); + + todo!() + } + + pub fn compile_all_pre_aggregations( + &mut self, + ) -> Result>, CubeError> { + let mut result = Vec::new(); + for (name, _) in self.descriptions.clone().iter() { + result.push(self.compile_pre_aggregation(&name)?); + } + Ok(result) + } + + pub fn compile_origin_sql_pre_aggregation( + &mut self, + cube_name: &String, + ) -> Result>, CubeError> { + let res = if let Some((name, _)) = self.descriptions.clone().iter().find(|(name, descr)| { + &name.cube_name == cube_name + && &descr.static_data().pre_aggregation_type == "originalSql" + }) { + Some(self.compile_pre_aggregation(name)?) + } else { + None + }; + Ok(res) + } + + fn symbols_from_ref Result<(), CubeError>>( + query_tools: Rc, + cube_name: &String, + ref_func: Rc, + check_type_fn: F, + ) -> Result>, CubeError> { + let evaluator_compiler_cell = query_tools.evaluator_compiler().clone(); + let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); + let sql_call = evaluator_compiler.compile_sql_call(cube_name, ref_func)?; + let mut res = Vec::new(); + for symbol in sql_call.get_dependencies().iter() { + check_type_fn(&symbol)?; + res.push(symbol.clone()); + } + Ok(res) + } + + fn check_is_measure(symbol: &MemberSymbol) -> Result<(), CubeError> { + symbol + .as_measure() + .map_err(|_| CubeError::user(format!("Pre-aggregation measure must be a measure")))?; + Ok(()) + } + + fn check_is_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { + symbol.as_dimension().map_err(|_| { + CubeError::user(format!("Pre-aggregation dimension must be a dimension")) + })?; + Ok(()) + } + + fn check_is_time_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { + let dimension = symbol.as_dimension().map_err(|_| { + CubeError::user(format!( + "Pre-aggregation time dimension must be a dimension" + )) + })?; + if dimension.dimension_type() != "time" { + return Err(CubeError::user(format!( + "Pre-aggregation time dimension must be a dimension" + ))); + } + Ok(()) + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs index 85ec32be2fbc4..e59a536ba79a8 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs @@ -1,3 +1,4 @@ +use super::pre_aggregation::PreAggregationSource; use super::*; use crate::cube_bridge::pre_aggregation_obj::PreAggregationObj; use crate::planner::sql_evaluator::MemberSymbol; @@ -12,9 +13,8 @@ pub struct PreAggregation { pub time_dimensions: Vec<(Rc, Option)>, pub external: bool, pub granularity: Option, - pub table_name: String, + pub source: PreAggregationSource, pub cube_name: String, - pub pre_aggregation_obj: Rc, } impl PrettyPrint for PreAggregation { @@ -23,7 +23,13 @@ impl PrettyPrint for PreAggregation { let state = state.new_level(); result.println(&format!("name: {}", self.name), &state); result.println(&format!("cube_name: {}", self.cube_name), &state); - result.println(&format!("table_name: {}", self.table_name), &state); + result.println(&format!("source:"), &state); + match &self.source { + PreAggregationSource::Table(name) => { + let state = state.new_level(); + result.println(&format!("table: {}", name), &state); + } + } result.println(&format!("external: {}", self.external), &state); result.println( &format!( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 2923330135c2b..61e481d7efa23 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -234,8 +234,11 @@ impl PhysicalPlanBuilder { ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(meas.full_name()))); } + let table_name = match &pre_aggregation.source { + PreAggregationSource::Table(name) => name.clone() + }; let from = From::new_from_table_reference( - pre_aggregation.table_name.clone(), + table_name, Rc::new(pre_aggregation_schema), Some(pre_aggregation_alias), ); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs index 681be99b46773..c719d09569ccf 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/planner/base_query.rs @@ -117,12 +117,15 @@ impl BaseQuery { let res = self.context.empty_array()?; res.set(0, result_sql.to_native(self.context.clone())?)?; res.set(1, params.to_native(self.context.clone())?)?; - if let Some(used_pre_aggregations) = used_pre_aggregations.first() { + if let Some(used_pre_aggregation) = used_pre_aggregations.first() { + //FIXME We should build this object in Rust + let pre_aggregation_obj = self.query_tools.base_tools().get_pre_aggregation_by_name( + used_pre_aggregation.cube_name.clone(), + used_pre_aggregation.name.clone(), + )?; res.set( 2, - used_pre_aggregations - .pre_aggregation_obj - .clone() + pre_aggregation_obj .as_any() .downcast::>() .unwrap() From fcfa77f60116b2e23f5a9fb8d281e10c65048da9 Mon Sep 17 00:00:00 2001 From: Alexandr Romanenko Date: Wed, 2 Jul 2025 17:48:53 +0200 Subject: [PATCH 2/5] rollup join support --- .../src/adapter/BaseQuery.js | 11 +- .../src/adapter/PreAggregations.ts | 8 - .../src/compiler/CubeEvaluator.ts | 1 - .../postgres/pre-aggregations.test.ts | 2 +- .../compiled_pre_aggregation.rs | 34 +++- .../pre_aggregations_compiler.rs | 164 ++++++++++++++++-- .../src/logical_plan/pre_aggregation.rs | 16 +- .../src/physical_plan_builder/builder.rs | 82 +++++++-- .../cubesqlplanner/src/plan/builder/join.rs | 8 + .../src/planner/planners/join_planner.rs | 84 +++++++++ .../src/planner/planners/mod.rs | 2 +- .../src/planner/sql_evaluator/sql_call.rs | 6 + .../sql_evaluator/symbols/member_symbol.rs | 16 ++ 13 files changed, 373 insertions(+), 61 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index a9c64703e3e2f..6c50fc2398971 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -331,12 +331,12 @@ export class BaseQuery { */ this.customSubQueryJoins = this.options.subqueryJoins ?? []; this.useNativeSqlPlanner = this.options.useNativeSqlPlanner ?? getEnv('nativeSqlPlanner'); - this.canUseNativeSqlPlannerPreAggregation = true; - /* if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) { + this.canUseNativeSqlPlannerPreAggregation = false; + if (this.useNativeSqlPlanner && !this.neverUseSqlPlannerPreaggregation()) { const fullAggregateMeasures = this.fullKeyQueryAggregateMeasures({ hasMultipliedForPreAggregation: true }); this.canUseNativeSqlPlannerPreAggregation = fullAggregateMeasures.multiStageMembers.length > 0 || fullAggregateMeasures.cumulativeMeasures.length > 0; - } */ + } this.queryLevelJoinHints = this.options.joinHints ?? []; this.prebuildJoin(); @@ -710,7 +710,6 @@ export class BaseQuery { preAggForQuery = undefined; } } - console.log("!!!! ========="); if (preAggForQuery) { const { multipliedMeasures, @@ -719,7 +718,6 @@ export class BaseQuery { withQueries, multiStageMembers, } = this.fullKeyQueryAggregateMeasures(); - console.log("!!!! OOOO ", preAggForQuery); if (cumulativeMeasures.length === 0) { sql = this.preAggregations.rollupPreAggregation( @@ -738,8 +736,6 @@ export class BaseQuery { } else { sql = this.fullKeyQueryAggregate(); } - console.log("!!! sql", sql); - console.log("!!!! --------"); return this.options.totalQuery ? this.countAllQuery(sql) : sql; @@ -971,7 +967,6 @@ export class BaseQuery { } getPreAggregationByName(cube, preAggregationName) { - console.log("!!!!! paaaaa", this.preAggregations.getRollupPreAggregationByName(cube, preAggregationName)); return this.preAggregations.getRollupPreAggregationByName(cube, preAggregationName); } diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index b024b097eb88e..0dd6fad25ec4f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -981,14 +981,12 @@ export class PreAggregations { // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) )); - console.log("!!!! exixtingJoins", existingJoins); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && existing.originalTo === target.originalTo && R.equals(existing.fromMembers, target.fromMembers) && R.equals(existing.toMembers, target.toMembers) )); - console.log("!!!! nonexixtingJoins", nonExistingJoins); if (!nonExistingJoins.length) { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } @@ -1022,7 +1020,6 @@ export class PreAggregations { private resolveJoinMembers(join) { return join.joins.map(j => { const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); - console.log("!!!! memb paths ", memberPaths); const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); if (invalidMembers.length) { throw new UserError(`Members ${invalidMembers.join(', ')} in join from '${j.originalFrom}' to '${j.originalTo}' doesn't reference join cubes`); @@ -1347,7 +1344,6 @@ export class PreAggregations { } private rollupLambdaUnion(preAggregationForQuery: PreAggregationForQuery, rollupGranularity: string): string { - console.log("!!!!! EEEEEE"); if (!preAggregationForQuery.referencedPreAggregations) { return this.preAggregationTableName( preAggregationForQuery.cube, @@ -1380,8 +1376,6 @@ export class PreAggregations { } ); - console.log("!!!! ref pre aggr", preAggregationForQuery.referencedPreAggregations); - const tables = preAggregationForQuery.referencedPreAggregations.map(preAggregation => { const dimensionsReferences = this.dimensionsRenderedReference(preAggregation); const timeDimensionsReferences = this.timeDimensionsRenderedReference(rollupGranularity, preAggregation); @@ -1409,7 +1403,6 @@ export class PreAggregations { let toJoin; // TODO granularity shouldn't be null? const rollupGranularity = preAggregationForQuery.references.timeDimensions[0]?.granularity || 'day'; - console.log("!!!! IIIIIII"); const sqlAndAlias = (preAgg) => ({ preAggregation: preAgg, @@ -1419,7 +1412,6 @@ export class PreAggregations { if (preAggregationForQuery.preAggregation.type === 'rollupJoin') { const join = preAggregationForQuery.rollupJoin; - console.log("!!!!! cccc ", join); toJoin = [ sqlAndAlias(join[0].fromPreAggObj), diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 400f0047e6a9b..0fe0a71463fd0 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -565,7 +565,6 @@ export class CubeEvaluator extends CubeSymbols { } public preAggregationsForCubeAsArray(path: string) { - console.log("!!!! pre aggrs", this.cubeFromPath(path).preAggregations); return Object.entries(this.cubeFromPath(path).preAggregations || {}).map(([name, preAggregation]) => ({ name, ...(preAggregation as Record) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index a24881471e392..bf603e2995947 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -2136,7 +2136,7 @@ describe('PreAggregations', () => { }); }); - it('rollup join 11', async () => { + it('rollup join', async () => { await compiler.compile(); const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs index 8955ba07e8e79..291b7e1b69ff1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs @@ -1,20 +1,40 @@ -use crate::cube_bridge::member_sql::MemberSql; -use crate::cube_bridge::pre_aggregation_description::PreAggregationDescription; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::MemberSymbol; -use cubenativeutils::CubeError; +use crate::planner::sql_evaluator::{MemberSymbol, SqlCall}; use std::fmt::Debug; use std::rc::Rc; + +#[derive(Clone)] +pub struct PreAggregationJoinItem { + pub from: PreAggregationTable, + pub to: PreAggregationTable, + pub from_members: Vec>, + pub to_members: Vec>, + pub on_sql: Rc, +} + +#[derive(Clone)] +pub struct PreAggregationJoin { + pub root: PreAggregationTable, + pub items: Vec, +} + +#[derive(Clone)] +pub struct PreAggregationTable { + pub cube_name: String, + pub name: String, + pub alias: Option, +} + #[derive(Clone)] pub enum PreAggregationSource { - Table(String), + Table(PreAggregationTable), + Join(PreAggregationJoin), } #[derive(Clone)] pub struct CompiledPreAggregation { pub cube_name: String, pub name: String, - pub source: PreAggregationSource, + pub source: Rc, pub granularity: Option, pub external: Option, pub measures: Vec>, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs index 87e4d0b96a123..d7a1fbbe20d38 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs @@ -1,11 +1,16 @@ use super::CompiledPreAggregation; use super::PreAggregationSource; +use crate::cube_bridge::join_hints::JoinHintItem; use crate::cube_bridge::member_sql::MemberSql; use crate::cube_bridge::pre_aggregation_description::PreAggregationDescription; +use crate::logical_plan::PreAggregationJoin; +use crate::logical_plan::PreAggregationJoinItem; +use crate::logical_plan::PreAggregationTable; +use crate::planner::planners::JoinPlanner; +use crate::planner::planners::ResolvedJoinItem; use crate::planner::query_tools::QueryTools; use crate::planner::sql_evaluator::collectors::collect_cube_names_from_symbols; use crate::planner::sql_evaluator::MemberSymbol; -use crate::planner::planners::JoinPlanner; use cubenativeutils::CubeError; use itertools::Itertools; use std::collections::HashMap; @@ -18,6 +23,23 @@ pub struct PreAggregationFullName { pub name: String, } +impl PreAggregationFullName { + pub fn from_string(name: &str) -> Result { + let parts = name.split('.').collect_vec(); + if parts.len() != 2 { + Err(CubeError::user(format!( + "Invalid pre-aggregation name: {}", + name + ))) + } else { + Ok(Self { + cube_name: parts[0].to_string(), + name: parts[1].to_string(), + }) + } + } +} + impl PreAggregationFullName { pub fn new(cube_name: String, name: String) -> Self { Self { cube_name, name } @@ -100,11 +122,6 @@ impl PreAggregationsCompiler { .static_data() .allow_non_strict_date_range_match .unwrap_or(false); - //FIXME sqlAlias!!! - let table_name = self - .query_tools - .base_tools() - .pre_aggregation_table_name(name.cube_name.clone(), name.name.clone())?; let rollups = if let Some(refs) = description.rollup_references()? { let r = self .query_tools @@ -115,14 +132,24 @@ impl PreAggregationsCompiler { Vec::new() }; - if static_data.pre_aggregation_type == "rollupJoin" { - self.build_join_source(&measures, &dimensions, &rollups)?; - } + let source = if static_data.pre_aggregation_type == "rollupJoin" { + PreAggregationSource::Join(self.build_join_source( + &measures, + &dimensions, + &rollups, + )?) + } else { + PreAggregationSource::Table(PreAggregationTable { + cube_name: name.cube_name.clone(), + name: name.name.clone(), + alias: static_data.sql_alias.clone(), + }) + }; let res = Rc::new(CompiledPreAggregation { name: static_data.name.clone(), cube_name: name.cube_name.clone(), - source: PreAggregationSource::Table(table_name), + source: Rc::new(source), granularity: static_data.granularity.clone(), external: static_data.external, measures, @@ -145,17 +172,124 @@ impl PreAggregationsCompiler { measures: &Vec>, dimensions: &Vec>, rollups: &Vec, - ) -> Result<(), CubeError> { - println!("!!!!! build join source"); + ) -> Result { let all_symbols = measures .iter() .cloned() .chain(dimensions.iter().cloned()) .collect_vec(); - let pre_aggr_cube_names = collect_cube_names_from_symbols(&all_symbols)?; - println!("!!!! pre aggr cube names {:?}", pre_aggr_cube_names); + let pre_aggr_join_hints = collect_cube_names_from_symbols(&all_symbols)? + .into_iter() + .map(|v| JoinHintItem::Single(v)) + .collect_vec(); + + let join_planner = JoinPlanner::new(self.query_tools.clone()); + let pre_aggrs_for_join = rollups + .iter() + .map(|item| -> Result<_, CubeError> { + self.compile_pre_aggregation(&PreAggregationFullName::from_string(item)?) + }) + .collect::, _>>()?; + + let target_joins = join_planner.resolve_join_members_by_hints(&pre_aggr_join_hints)?; + let mut existing_joins = vec![]; + for join_pre_aggr in pre_aggrs_for_join.iter() { + let all_symbols = join_pre_aggr + .measures + .iter() + .cloned() + .chain(join_pre_aggr.dimensions.iter().cloned()) + .collect_vec(); + let join_pre_aggr_join_hints = collect_cube_names_from_symbols(&all_symbols)? + .into_iter() + .map(|v| JoinHintItem::Single(v)) + .collect_vec(); + existing_joins.append( + &mut join_planner.resolve_join_members_by_hints(&join_pre_aggr_join_hints)?, + ); + } + + let not_existing_joins = target_joins + .into_iter() + .filter(|join| { + !existing_joins + .iter() + .any(|existing| existing.is_same_as(join)) + }) + .collect_vec(); + + if not_existing_joins.is_empty() { + return Err(CubeError::user(format!("Nothing to join in rollup join. Target joins are included in existing rollup joins"))); + } + + let items = not_existing_joins + .iter() + .map(|item| self.make_pre_aggregation_join_item(&pre_aggrs_for_join, item)) + .collect::, _>>()?; + let res = PreAggregationJoin { + root: items[0].from.clone(), + items, + }; + Ok(res) + } + + fn make_pre_aggregation_join_item( + &self, + pre_aggrs_for_join: &Vec>, + join_item: &ResolvedJoinItem, + ) -> Result { + let from_pre_aggr = + self.find_pre_aggregation_for_join(pre_aggrs_for_join, &join_item.from_members)?; + let to_pre_aggr = + self.find_pre_aggregation_for_join(pre_aggrs_for_join, &join_item.to_members)?; + + let from_table = match from_pre_aggr.source.as_ref() { + PreAggregationSource::Table(t) => t.clone(), + PreAggregationSource::Join(_) => { + return Err(CubeError::user(format!("Rollup join can't be nested"))); + } + }; + let to_table = match to_pre_aggr.source.as_ref() { + PreAggregationSource::Table(t) => t.clone(), + PreAggregationSource::Join(_) => { + return Err(CubeError::user(format!("Rollup join can't be nested"))); + } + }; + let res = PreAggregationJoinItem { + from: from_table, + to: to_table, + from_members: join_item.from_members.clone(), + to_members: join_item.to_members.clone(), + on_sql: join_item.on_sql.clone(), + }; + Ok(res) + } + + fn find_pre_aggregation_for_join( + &self, + pre_aggrs_for_join: &Vec>, + members: &Vec>, + ) -> Result, CubeError> { + let found_pre_aggr = pre_aggrs_for_join + .iter() + .filter(|pa| { + members + .iter() + .all(|m| pa.dimensions.iter().any(|pa_m| m == pa_m)) + }) + .collect_vec(); + if found_pre_aggr.is_empty() { + return Err(CubeError::user(format!( + "No rollups found that can be used for rollup join" + ))); + } + if found_pre_aggr.len() > 1 { + return Err(CubeError::user(format!( + "Multiple rollups found that can be used for rollup join" + ))); + } - todo!() + Ok(found_pre_aggr[0].clone()) } pub fn compile_all_pre_aggregations( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs index e59a536ba79a8..d61a8facc853d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs @@ -1,6 +1,5 @@ use super::pre_aggregation::PreAggregationSource; use super::*; -use crate::cube_bridge::pre_aggregation_obj::PreAggregationObj; use crate::planner::sql_evaluator::MemberSymbol; use itertools::Itertools; use std::rc::Rc; @@ -13,7 +12,7 @@ pub struct PreAggregation { pub time_dimensions: Vec<(Rc, Option)>, pub external: bool, pub granularity: Option, - pub source: PreAggregationSource, + pub source: Rc, pub cube_name: String, } @@ -24,10 +23,17 @@ impl PrettyPrint for PreAggregation { result.println(&format!("name: {}", self.name), &state); result.println(&format!("cube_name: {}", self.cube_name), &state); result.println(&format!("source:"), &state); - match &self.source { - PreAggregationSource::Table(name) => { + match self.source.as_ref() { + PreAggregationSource::Table(table) => { let state = state.new_level(); - result.println(&format!("table: {}", name), &state); + result.println( + &format!("table: {}.{}", table.cube_name, table.name), + &state, + ); + } + PreAggregationSource::Join(_) => { + let state = state.new_level(); + result.println(&format!("rollup join"), &state); } } result.println(&format!("external: {}", self.external), &state); diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 61e481d7efa23..53063e444e964 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -185,16 +185,11 @@ impl PhysicalPlanBuilder { fn process_pre_aggregation( &self, pre_aggregation: &Rc, - _context: &PhysicalPlanBuilderContext, + context: &PhysicalPlanBuilderContext, measure_references: &mut HashMap, dimensions_references: &mut HashMap, ) -> Result, CubeError> { let mut pre_aggregation_schema = Schema::empty(); - let pre_aggregation_alias = PlanSqlTemplates::memeber_alias_name( - &pre_aggregation.cube_name, - &pre_aggregation.name, - &None, - ); for dim in pre_aggregation.dimensions.iter() { let alias = BaseMemberHelper::default_alias( &dim.cube_name(), @@ -204,7 +199,7 @@ impl PhysicalPlanBuilder { )?; dimensions_references.insert( dim.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(dim.full_name()))); } @@ -217,7 +212,7 @@ impl PhysicalPlanBuilder { )?; dimensions_references.insert( dim.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(dim.full_name()))); } @@ -230,19 +225,76 @@ impl PhysicalPlanBuilder { )?; measure_references.insert( meas.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(meas.full_name()))); } - let table_name = match &pre_aggregation.source { - PreAggregationSource::Table(name) => name.clone() + let from = self.make_pre_aggregation_source( + &pre_aggregation.source, + context, + measure_references, + dimensions_references, + )?; + Ok(from) + } + + fn make_pre_aggregation_source( + &self, + source: &Rc, + _context: &PhysicalPlanBuilderContext, + _measure_references: &mut HashMap, + dimensions_references: &mut HashMap, + ) -> Result, CubeError> { + let from = match source.as_ref() { + PreAggregationSource::Table(table) => { + let table_source = self.make_pre_aggregation_table_source(table)?; + From::new(FromSource::Single(table_source)) + } + PreAggregationSource::Join(join) => { + let root_table_source = self.make_pre_aggregation_table_source(&join.root)?; + let mut join_builder = JoinBuilder::new(root_table_source); + for item in join.items.iter() { + let to_table_source = self.make_pre_aggregation_table_source(&item.to)?; + let condition = + SqlJoinCondition::try_new(self.query_tools.clone(), item.on_sql.clone())?; + let on = JoinCondition::new_base_join(condition); + join_builder.left_join_aliased_source(to_table_source, on); + for member in item.from_members.iter().chain(item.to_members.iter()) { + let alias = BaseMemberHelper::default_alias( + &member.cube_name(), + &member.name(), + &None, + self.query_tools.clone(), + )?; + dimensions_references.insert( + member.full_name(), + QualifiedColumnName::new(None, alias.clone()), + ); + } + } + let from = From::new_from_join(join_builder.build()); + from + } }; - let from = From::new_from_table_reference( + Ok(from) + } + + fn make_pre_aggregation_table_source( + &self, + table: &PreAggregationTable, + ) -> Result { + let name = table.alias.clone().unwrap_or_else(|| table.name.clone()); + let table_name = self + .query_tools + .base_tools() + .pre_aggregation_table_name(table.cube_name.clone(), name.clone())?; + let alias = PlanSqlTemplates::memeber_alias_name(&table.cube_name, &name, &None); + let res = SingleAliasedSource::new_from_table_reference( table_name, - Rc::new(pre_aggregation_schema), - Some(pre_aggregation_alias), + Rc::new(Schema::empty()), + Some(alias), ); - Ok(from) + Ok(res) } fn build_full_key_aggregate_query( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs index db8914b71a647..733888b9bc663 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs @@ -47,6 +47,14 @@ impl JoinBuilder { )) } + pub fn left_join_aliased_source(&mut self, source: SingleAliasedSource, on: JoinCondition) { + self.joins.push(JoinItem { + from: source, + on, + join_type: JoinType::Left, + }); + } + pub fn left_join_subselect(&mut self, subquery: Rc