From 05bb52091ad2d3ac09303c0fa50603cd24c99d61 Mon Sep 17 00:00:00 2001 From: waralexrom <108349432+waralexrom@users.noreply.github.com> Date: Wed, 9 Jul 2025 16:00:54 +0200 Subject: [PATCH 1/8] feat(tesseract): Rollup Join support (#9745) --- .../src/compiler/CubeEvaluator.ts | 24 ++ .../src/cube_bridge/evaluator.rs | 12 + .../pre_aggregation_description.rs | 3 + .../compiled_pre_aggregation.rs | 137 ++----- .../optimizers/pre_aggregation/mod.rs | 2 + .../optimizers/pre_aggregation/optimizer.rs | 97 ++--- .../pre_aggregation/original_sql_optimizer.rs | 25 +- .../pre_aggregations_compiler.rs | 370 ++++++++++++++++++ .../src/logical_plan/pre_aggregation.rs | 20 +- .../src/physical_plan_builder/builder.rs | 83 +++- .../cubesqlplanner/src/plan/builder/join.rs | 8 + .../cubesqlplanner/src/planner/base_query.rs | 11 +- .../src/planner/planners/join_planner.rs | 84 ++++ .../src/planner/planners/mod.rs | 2 +- .../src/planner/sql_evaluator/sql_call.rs | 6 + .../sql_evaluator/symbols/member_symbol.rs | 16 + 16 files changed, 694 insertions(+), 206 deletions(-) create mode 100644 rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index d8ee7b6bd19b0..b1321762061ff 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -571,6 +571,22 @@ export class CubeEvaluator extends CubeSymbols { })); } + public preAggregationDescriptionByName(cubeName: string, preAggName: string) { + const cube = this.cubeFromPath(cubeName); + const preAggregations = cube.preAggregations || {}; + + const preAgg = preAggregations[preAggName]; + + if (!preAgg) { + return undefined; + } + + return { + name: preAggName, + ...(preAgg as Record) + }; + } + /** * Returns pre-aggregations filtered by the specified selector. */ @@ -785,6 +801,14 @@ export class CubeEvaluator extends CubeSymbols { return { cubeReferencesUsed, pathReferencesUsed, evaluatedSql }; } + /** + * Evaluates rollup references for retrieving rollupReference used in Tesseract. + * This is a temporary solution until Tesseract takes ownership of all pre-aggregations. + */ + public evaluateRollupReferences>(cube: string, rollupReferences: (...args: Array) => T) { + return this.evaluateReferences(cube, rollupReferences, { originalSorting: true }); + } + public evaluatePreAggregationReferences(cube: string, aggregation: PreAggregationDefinition): PreAggregationReferences { const timeDimensions: Array = []; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs index 14b3c3d239cb9..9baa01e1bd35a 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/evaluator.rs @@ -64,4 +64,16 @@ pub trait CubeEvaluator { &self, cube_name: String, ) -> Result>, CubeError>; + #[nbridge(optional)] + fn pre_aggregation_description_by_name( + &self, + cube_name: String, + name: String, + ) -> Result>, CubeError>; + #[nbridge(vec)] + fn evaluate_rollup_references( + &self, + cube: String, + sql: Rc, + ) -> Result, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs index 72ffd10662151..aaf650503f00b 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/cube_bridge/pre_aggregation_description.rs @@ -32,4 +32,7 @@ pub trait PreAggregationDescription { #[nbridge(field, optional)] fn time_dimension_reference(&self) -> Result>, CubeError>; + + #[nbridge(field, optional)] + fn rollup_references(&self) -> Result>, CubeError>; } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs index e47d5bfd2805b..291b7e1b69ff1 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/compiled_pre_aggregation.rs @@ -1,14 +1,40 @@ -use crate::cube_bridge::member_sql::MemberSql; -use crate::cube_bridge::pre_aggregation_description::PreAggregationDescription; -use crate::planner::query_tools::QueryTools; -use crate::planner::sql_evaluator::MemberSymbol; -use cubenativeutils::CubeError; +use crate::planner::sql_evaluator::{MemberSymbol, SqlCall}; use std::fmt::Debug; use std::rc::Rc; + +#[derive(Clone)] +pub struct PreAggregationJoinItem { + pub from: PreAggregationTable, + pub to: PreAggregationTable, + pub from_members: Vec>, + pub to_members: Vec>, + pub on_sql: Rc, +} + +#[derive(Clone)] +pub struct PreAggregationJoin { + pub root: PreAggregationTable, + pub items: Vec, +} + +#[derive(Clone)] +pub struct PreAggregationTable { + pub cube_name: String, + pub name: String, + pub alias: Option, +} + +#[derive(Clone)] +pub enum PreAggregationSource { + Table(PreAggregationTable), + Join(PreAggregationJoin), +} + #[derive(Clone)] pub struct CompiledPreAggregation { pub cube_name: String, pub name: String, + pub source: Rc, pub granularity: Option, pub external: Option, pub measures: Vec>, @@ -34,104 +60,3 @@ impl Debug for CompiledPreAggregation { .finish() } } - -impl CompiledPreAggregation { - pub fn try_new( - query_tools: Rc, - cube_name: &String, - description: Rc, - ) -> Result, CubeError> { - let static_data = description.static_data(); - let measures = if let Some(refs) = description.measure_references()? { - Self::symbols_from_ref(query_tools.clone(), cube_name, refs, Self::check_is_measure)? - } else { - Vec::new() - }; - let dimensions = if let Some(refs) = description.dimension_references()? { - Self::symbols_from_ref( - query_tools.clone(), - cube_name, - refs, - Self::check_is_dimension, - )? - } else { - Vec::new() - }; - let time_dimensions = if let Some(refs) = description.time_dimension_reference()? { - let dims = Self::symbols_from_ref( - query_tools.clone(), - cube_name, - refs, - Self::check_is_time_dimension, - )?; - /* if dims.len() != 1 { - return Err(CubeError::user(format!( - "Pre aggregation should contains only one time dimension" - ))); - } */ - vec![(dims[0].clone(), static_data.granularity.clone())] //TODO remove unwrap - } else { - Vec::new() - }; - let allow_non_strict_date_range_match = description - .static_data() - .allow_non_strict_date_range_match - .unwrap_or(false); - let res = Rc::new(Self { - name: static_data.name.clone(), - cube_name: cube_name.clone(), - granularity: static_data.granularity.clone(), - external: static_data.external, - measures, - dimensions, - time_dimensions, - allow_non_strict_date_range_match, - }); - Ok(res) - } - - fn symbols_from_ref Result<(), CubeError>>( - query_tools: Rc, - cube_name: &String, - ref_func: Rc, - check_type_fn: F, - ) -> Result>, CubeError> { - let evaluator_compiler_cell = query_tools.evaluator_compiler().clone(); - let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); - let sql_call = evaluator_compiler.compile_sql_call(cube_name, ref_func)?; - let mut res = Vec::new(); - for symbol in sql_call.get_dependencies().iter() { - check_type_fn(&symbol)?; - res.push(symbol.clone()); - } - Ok(res) - } - - fn check_is_measure(symbol: &MemberSymbol) -> Result<(), CubeError> { - symbol - .as_measure() - .map_err(|_| CubeError::user(format!("Pre-aggregation measure must be a measure")))?; - Ok(()) - } - - fn check_is_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { - symbol.as_dimension().map_err(|_| { - CubeError::user(format!("Pre-aggregation dimension must be a dimension")) - })?; - Ok(()) - } - - fn check_is_time_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { - let dimension = symbol.as_dimension().map_err(|_| { - CubeError::user(format!( - "Pre-aggregation time dimension must be a dimension" - )) - })?; - if dimension.dimension_type() != "time" { - return Err(CubeError::user(format!( - "Pre-aggregation time dimension must be a dimension" - ))); - } - Ok(()) - } -} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs index a3fbe777d9fe0..51c2da6e499db 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/mod.rs @@ -4,6 +4,7 @@ mod measure_matcher; mod optimizer; mod original_sql_collector; mod original_sql_optimizer; +mod pre_aggregations_compiler; pub use compiled_pre_aggregation::*; use dimension_matcher::*; @@ -11,3 +12,4 @@ use measure_matcher::*; pub use optimizer::*; pub use original_sql_collector::*; pub use original_sql_optimizer::*; +pub use pre_aggregations_compiler::*; diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs index 1fff70496060f..c812c57e3cdca 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/optimizer.rs @@ -1,3 +1,4 @@ +use super::PreAggregationsCompiler; use super::*; use crate::logical_plan::*; use crate::plan::FilterItem; @@ -43,22 +44,9 @@ impl PreAggregationOptimizer { let mut cube_names_collector = CubeNamesCollector::new(); cube_names_collector.collect(&plan)?; let cube_names = cube_names_collector.result(); + let mut compiler = PreAggregationsCompiler::try_new(self.query_tools.clone(), &cube_names)?; - let mut compiled_pre_aggregations = Vec::new(); - for cube_name in cube_names.iter() { - let pre_aggregations = self - .query_tools - .cube_evaluator() - .pre_aggregations_for_cube_as_array(cube_name.clone())?; - for pre_aggregation in pre_aggregations.iter() { - let compiled = CompiledPreAggregation::try_new( - self.query_tools.clone(), - cube_name, - pre_aggregation.clone(), - )?; - compiled_pre_aggregations.push(compiled); - } - } + let compiled_pre_aggregations = compiler.compile_all_pre_aggregations()?; for pre_aggregation in compiled_pre_aggregations.iter() { let new_query = self.try_rewrite_query(plan.clone(), pre_aggregation)?; @@ -413,51 +401,44 @@ impl PreAggregationOptimizer { &mut self, pre_aggregation: &Rc, ) -> Result, CubeError> { - let pre_aggregation_obj = self.query_tools.base_tools().get_pre_aggregation_by_name( + /* let pre_aggregation_obj = self.query_tools.base_tools().get_pre_aggregation_by_name( pre_aggregation.cube_name.clone(), pre_aggregation.name.clone(), - )?; - if let Some(table_name) = &pre_aggregation_obj.static_data().table_name { - let schema = LogicalSchema { - time_dimensions: vec![], - dimensions: pre_aggregation - .dimensions - .iter() - .cloned() - .chain( - pre_aggregation - .time_dimensions - .iter() - .map(|(d, _)| d.clone()), - ) - .collect(), - measures: pre_aggregation.measures.to_vec(), - multiplied_measures: HashSet::new(), - }; - let pre_aggregation = PreAggregation { - name: pre_aggregation.name.clone(), - time_dimensions: pre_aggregation.time_dimensions.clone(), - dimensions: pre_aggregation.dimensions.clone(), - measures: pre_aggregation.measures.clone(), - schema: Rc::new(schema), - external: pre_aggregation.external.unwrap_or_default(), - granularity: pre_aggregation.granularity.clone(), - table_name: table_name.clone(), - cube_name: pre_aggregation.cube_name.clone(), - pre_aggregation_obj, - }; - let result = Rc::new(pre_aggregation); - self.used_pre_aggregations.insert( - (result.cube_name.clone(), result.name.clone()), - result.clone(), - ); - Ok(result) - } else { - Err(CubeError::internal(format!( - "Cannot find pre aggregation object for cube {} and name {}", - pre_aggregation.cube_name, pre_aggregation.name - ))) - } + )?; */ + //if let Some(table_name) = &pre_aggregation_obj.static_data().table_name { + let schema = LogicalSchema { + time_dimensions: vec![], + dimensions: pre_aggregation + .dimensions + .iter() + .cloned() + .chain( + pre_aggregation + .time_dimensions + .iter() + .map(|(d, _)| d.clone()), + ) + .collect(), + measures: pre_aggregation.measures.to_vec(), + multiplied_measures: HashSet::new(), + }; + let pre_aggregation = PreAggregation { + name: pre_aggregation.name.clone(), + time_dimensions: pre_aggregation.time_dimensions.clone(), + dimensions: pre_aggregation.dimensions.clone(), + measures: pre_aggregation.measures.clone(), + schema: Rc::new(schema), + external: pre_aggregation.external.unwrap_or_default(), + granularity: pre_aggregation.granularity.clone(), + source: pre_aggregation.source.clone(), + cube_name: pre_aggregation.cube_name.clone(), + }; + let result = Rc::new(pre_aggregation); + self.used_pre_aggregations.insert( + (result.cube_name.clone(), result.name.clone()), + result.clone(), + ); + Ok(result) } fn is_schema_and_filters_match( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs index 4bd5c1ea078d3..4edb5beb33007 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/original_sql_optimizer.rs @@ -1,3 +1,4 @@ +use super::PreAggregationsCompiler; use super::*; use crate::logical_plan::*; use crate::planner::query_tools::QueryTools; @@ -352,25 +353,11 @@ impl OriginalSqlOptimizer { let res = if let Some(found_pre_aggregation) = self.foud_pre_aggregations.get(cube_name) { Some(found_pre_aggregation.clone()) } else { - let pre_aggregations = self - .query_tools - .cube_evaluator() - .pre_aggregations_for_cube_as_array(cube_name.clone())?; - if let Some(found_pre_aggregation) = pre_aggregations - .iter() - .find(|p| p.static_data().pre_aggregation_type == "originalSql") - { - let compiled = CompiledPreAggregation::try_new( - self.query_tools.clone(), - cube_name, - found_pre_aggregation.clone(), - )?; - self.foud_pre_aggregations - .insert(cube_name.clone(), compiled.clone()); - Some(compiled) - } else { - None - } + let mut compiler = PreAggregationsCompiler::try_new( + self.query_tools.clone(), + &vec![cube_name.clone()], + )?; + compiler.compile_origin_sql_pre_aggregation(&cube_name)? }; Ok(res) } diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs new file mode 100644 index 0000000000000..b97b8a529907f --- /dev/null +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/optimizers/pre_aggregation/pre_aggregations_compiler.rs @@ -0,0 +1,370 @@ +use super::CompiledPreAggregation; +use super::PreAggregationSource; +use crate::cube_bridge::join_hints::JoinHintItem; +use crate::cube_bridge::member_sql::MemberSql; +use crate::cube_bridge::pre_aggregation_description::PreAggregationDescription; +use crate::logical_plan::PreAggregationJoin; +use crate::logical_plan::PreAggregationJoinItem; +use crate::logical_plan::PreAggregationTable; +use crate::planner::planners::JoinPlanner; +use crate::planner::planners::ResolvedJoinItem; +use crate::planner::query_tools::QueryTools; +use crate::planner::sql_evaluator::collectors::collect_cube_names_from_symbols; +use crate::planner::sql_evaluator::MemberSymbol; +use cubenativeutils::CubeError; +use itertools::Itertools; +use std::collections::HashMap; +use std::fmt::Debug; +use std::rc::Rc; + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct PreAggregationFullName { + pub cube_name: String, + pub name: String, +} + +impl PreAggregationFullName { + pub fn from_string(name: &str) -> Result { + let parts = name.split('.').collect_vec(); + if parts.len() != 2 { + Err(CubeError::user(format!( + "Invalid pre-aggregation name: {}", + name + ))) + } else { + Ok(Self { + cube_name: parts[0].to_string(), + name: parts[1].to_string(), + }) + } + } + + pub fn new(cube_name: String, name: String) -> Self { + Self { cube_name, name } + } +} + +pub struct PreAggregationsCompiler { + query_tools: Rc, + descriptions: Rc)>>, + compiled_cache: HashMap>, +} + +impl PreAggregationsCompiler { + pub fn try_new( + query_tools: Rc, + cube_names: &Vec, + ) -> Result { + let mut descriptions = Vec::new(); + for cube_name in cube_names.iter() { + let pre_aggregations = query_tools + .cube_evaluator() + .pre_aggregations_for_cube_as_array(cube_name.clone())?; + for pre_aggregation in pre_aggregations.iter() { + let full_name = PreAggregationFullName::new( + cube_name.clone(), + pre_aggregation.static_data().name.clone(), + ); + descriptions.push((full_name, pre_aggregation.clone())); + } + } + Ok(Self { + query_tools, + descriptions: Rc::new(descriptions), + compiled_cache: HashMap::new(), + }) + } + + pub fn compile_pre_aggregation( + &mut self, + name: &PreAggregationFullName, + ) -> Result, CubeError> { + if let Some(compiled) = self.compiled_cache.get(&name) { + return Ok(compiled.clone()); + } + + let description = if let Some((_, description)) = + self.descriptions.clone().iter().find(|(n, _)| n == name) + { + description.clone() + } else { + if let Some(descr) = self + .query_tools + .cube_evaluator() + .pre_aggregation_description_by_name(name.cube_name.clone(), name.name.clone())? + { + descr + } else { + return Err(CubeError::internal(format!( + "Undefined pre-aggregation {}.{}", + name.cube_name, name.name + ))); + } + }; + + let static_data = description.static_data(); + let measures = if let Some(refs) = description.measure_references()? { + Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_measure, + )? + } else { + Vec::new() + }; + let dimensions = if let Some(refs) = description.dimension_references()? { + Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_dimension, + )? + } else { + Vec::new() + }; + let time_dimensions = if let Some(refs) = description.time_dimension_reference()? { + let dims = Self::symbols_from_ref( + self.query_tools.clone(), + &name.cube_name, + refs, + Self::check_is_time_dimension, + )?; + vec![(dims[0].clone(), static_data.granularity.clone())] + } else { + Vec::new() + }; + let allow_non_strict_date_range_match = description + .static_data() + .allow_non_strict_date_range_match + .unwrap_or(false); + let rollups = if let Some(refs) = description.rollup_references()? { + let r = self + .query_tools + .cube_evaluator() + .evaluate_rollup_references(name.cube_name.clone(), refs)?; + r + } else { + Vec::new() + }; + + let source = if static_data.pre_aggregation_type == "rollupJoin" { + PreAggregationSource::Join(self.build_join_source(&measures, &dimensions, &rollups)?) + } else { + PreAggregationSource::Table(PreAggregationTable { + cube_name: name.cube_name.clone(), + name: name.name.clone(), + alias: static_data.sql_alias.clone(), + }) + }; + + let res = Rc::new(CompiledPreAggregation { + name: static_data.name.clone(), + cube_name: name.cube_name.clone(), + source: Rc::new(source), + granularity: static_data.granularity.clone(), + external: static_data.external, + measures, + dimensions, + time_dimensions, + allow_non_strict_date_range_match, + }); + self.compiled_cache.insert(name.clone(), res.clone()); + Ok(res) + } + + fn build_join_source( + &mut self, + measures: &Vec>, + dimensions: &Vec>, + rollups: &Vec, + ) -> Result { + let all_symbols = measures + .iter() + .cloned() + .chain(dimensions.iter().cloned()) + .collect_vec(); + let pre_aggr_join_hints = collect_cube_names_from_symbols(&all_symbols)? + .into_iter() + .map(|v| JoinHintItem::Single(v)) + .collect_vec(); + + let join_planner = JoinPlanner::new(self.query_tools.clone()); + let pre_aggrs_for_join = rollups + .iter() + .map(|item| -> Result<_, CubeError> { + self.compile_pre_aggregation(&PreAggregationFullName::from_string(item)?) + }) + .collect::, _>>()?; + + let target_joins = join_planner.resolve_join_members_by_hints(&pre_aggr_join_hints)?; + let mut existing_joins = vec![]; + for join_pre_aggr in pre_aggrs_for_join.iter() { + let all_symbols = join_pre_aggr + .measures + .iter() + .cloned() + .chain(join_pre_aggr.dimensions.iter().cloned()) + .collect_vec(); + let join_pre_aggr_join_hints = collect_cube_names_from_symbols(&all_symbols)? + .into_iter() + .map(|v| JoinHintItem::Single(v)) + .collect_vec(); + existing_joins.append( + &mut join_planner.resolve_join_members_by_hints(&join_pre_aggr_join_hints)?, + ); + } + + let not_existing_joins = target_joins + .into_iter() + .filter(|join| { + !existing_joins + .iter() + .any(|existing| existing.is_same_as(join)) + }) + .collect_vec(); + + if not_existing_joins.is_empty() { + return Err(CubeError::user(format!("Nothing to join in rollup join. Target joins are included in existing rollup joins"))); + } + + let items = not_existing_joins + .iter() + .map(|item| self.make_pre_aggregation_join_item(&pre_aggrs_for_join, item)) + .collect::, _>>()?; + let res = PreAggregationJoin { + root: items[0].from.clone(), + items, + }; + Ok(res) + } + + fn make_pre_aggregation_join_item( + &self, + pre_aggrs_for_join: &Vec>, + join_item: &ResolvedJoinItem, + ) -> Result { + let from_pre_aggr = + self.find_pre_aggregation_for_join(pre_aggrs_for_join, &join_item.from_members)?; + let to_pre_aggr = + self.find_pre_aggregation_for_join(pre_aggrs_for_join, &join_item.to_members)?; + + let from_table = match from_pre_aggr.source.as_ref() { + PreAggregationSource::Table(t) => t.clone(), + PreAggregationSource::Join(_) => { + return Err(CubeError::user(format!("Rollup join can't be nested"))); + } + }; + let to_table = match to_pre_aggr.source.as_ref() { + PreAggregationSource::Table(t) => t.clone(), + PreAggregationSource::Join(_) => { + return Err(CubeError::user(format!("Rollup join can't be nested"))); + } + }; + let res = PreAggregationJoinItem { + from: from_table, + to: to_table, + from_members: join_item.from_members.clone(), + to_members: join_item.to_members.clone(), + on_sql: join_item.on_sql.clone(), + }; + Ok(res) + } + + fn find_pre_aggregation_for_join( + &self, + pre_aggrs_for_join: &Vec>, + members: &Vec>, + ) -> Result, CubeError> { + let found_pre_aggr = pre_aggrs_for_join + .iter() + .filter(|pa| { + members + .iter() + .all(|m| pa.dimensions.iter().any(|pa_m| m == pa_m)) + }) + .collect_vec(); + if found_pre_aggr.is_empty() { + return Err(CubeError::user(format!( + "No rollups found that can be used for rollup join" + ))); + } + if found_pre_aggr.len() > 1 { + return Err(CubeError::user(format!( + "Multiple rollups found that can be used for rollup join" + ))); + } + + Ok(found_pre_aggr[0].clone()) + } + + pub fn compile_all_pre_aggregations( + &mut self, + ) -> Result>, CubeError> { + let mut result = Vec::new(); + for (name, _) in self.descriptions.clone().iter() { + result.push(self.compile_pre_aggregation(&name)?); + } + Ok(result) + } + + pub fn compile_origin_sql_pre_aggregation( + &mut self, + cube_name: &String, + ) -> Result>, CubeError> { + let res = if let Some((name, _)) = self.descriptions.clone().iter().find(|(name, descr)| { + &name.cube_name == cube_name + && &descr.static_data().pre_aggregation_type == "originalSql" + }) { + Some(self.compile_pre_aggregation(name)?) + } else { + None + }; + Ok(res) + } + + fn symbols_from_ref Result<(), CubeError>>( + query_tools: Rc, + cube_name: &String, + ref_func: Rc, + check_type_fn: F, + ) -> Result>, CubeError> { + let evaluator_compiler_cell = query_tools.evaluator_compiler().clone(); + let mut evaluator_compiler = evaluator_compiler_cell.borrow_mut(); + let sql_call = evaluator_compiler.compile_sql_call(cube_name, ref_func)?; + let mut res = Vec::new(); + for symbol in sql_call.get_dependencies().iter() { + check_type_fn(&symbol)?; + res.push(symbol.clone()); + } + Ok(res) + } + + fn check_is_measure(symbol: &MemberSymbol) -> Result<(), CubeError> { + symbol + .as_measure() + .map_err(|_| CubeError::user(format!("Pre-aggregation measure must be a measure")))?; + Ok(()) + } + + fn check_is_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { + symbol.as_dimension().map_err(|_| { + CubeError::user(format!("Pre-aggregation dimension must be a dimension")) + })?; + Ok(()) + } + + fn check_is_time_dimension(symbol: &MemberSymbol) -> Result<(), CubeError> { + let dimension = symbol.as_dimension().map_err(|_| { + CubeError::user(format!( + "Pre-aggregation time dimension must be a dimension" + )) + })?; + if dimension.dimension_type() != "time" { + return Err(CubeError::user(format!( + "Pre-aggregation time dimension must be a dimension" + ))); + } + Ok(()) + } +} diff --git a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs index 85ec32be2fbc4..d61a8facc853d 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/logical_plan/pre_aggregation.rs @@ -1,5 +1,5 @@ +use super::pre_aggregation::PreAggregationSource; use super::*; -use crate::cube_bridge::pre_aggregation_obj::PreAggregationObj; use crate::planner::sql_evaluator::MemberSymbol; use itertools::Itertools; use std::rc::Rc; @@ -12,9 +12,8 @@ pub struct PreAggregation { pub time_dimensions: Vec<(Rc, Option)>, pub external: bool, pub granularity: Option, - pub table_name: String, + pub source: Rc, pub cube_name: String, - pub pre_aggregation_obj: Rc, } impl PrettyPrint for PreAggregation { @@ -23,7 +22,20 @@ impl PrettyPrint for PreAggregation { let state = state.new_level(); result.println(&format!("name: {}", self.name), &state); result.println(&format!("cube_name: {}", self.cube_name), &state); - result.println(&format!("table_name: {}", self.table_name), &state); + result.println(&format!("source:"), &state); + match self.source.as_ref() { + PreAggregationSource::Table(table) => { + let state = state.new_level(); + result.println( + &format!("table: {}.{}", table.cube_name, table.name), + &state, + ); + } + PreAggregationSource::Join(_) => { + let state = state.new_level(); + result.println(&format!("rollup join"), &state); + } + } result.println(&format!("external: {}", self.external), &state); result.println( &format!( diff --git a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs index 2c66e0977b4a6..f68609e0c00a2 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/physical_plan_builder/builder.rs @@ -185,16 +185,11 @@ impl PhysicalPlanBuilder { fn process_pre_aggregation( &self, pre_aggregation: &Rc, - _context: &PhysicalPlanBuilderContext, + context: &PhysicalPlanBuilderContext, measure_references: &mut HashMap, dimensions_references: &mut HashMap, ) -> Result, CubeError> { let mut pre_aggregation_schema = Schema::empty(); - let pre_aggregation_alias = PlanSqlTemplates::memeber_alias_name( - &pre_aggregation.cube_name, - &pre_aggregation.name, - &None, - ); for dim in pre_aggregation.dimensions.iter() { let alias = BaseMemberHelper::default_alias( &dim.cube_name(), @@ -204,7 +199,7 @@ impl PhysicalPlanBuilder { )?; dimensions_references.insert( dim.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(dim.full_name()))); } @@ -217,7 +212,7 @@ impl PhysicalPlanBuilder { )?; dimensions_references.insert( dim.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(dim.full_name()))); } @@ -230,18 +225,78 @@ impl PhysicalPlanBuilder { )?; measure_references.insert( meas.full_name(), - QualifiedColumnName::new(Some(pre_aggregation_alias.clone()), alias.clone()), + QualifiedColumnName::new(None, alias.clone()), ); pre_aggregation_schema.add_column(SchemaColumn::new(alias, Some(meas.full_name()))); } - let from = From::new_from_table_reference( - pre_aggregation.table_name.clone(), - Rc::new(pre_aggregation_schema), - Some(pre_aggregation_alias), - ); + let from = self.make_pre_aggregation_source( + &pre_aggregation.source, + context, + measure_references, + dimensions_references, + )?; + Ok(from) + } + + fn make_pre_aggregation_source( + &self, + source: &Rc, + _context: &PhysicalPlanBuilderContext, + _measure_references: &mut HashMap, + dimensions_references: &mut HashMap, + ) -> Result, CubeError> { + let from = match source.as_ref() { + PreAggregationSource::Table(table) => { + let table_source = self.make_pre_aggregation_table_source(table)?; + From::new(FromSource::Single(table_source)) + } + PreAggregationSource::Join(join) => { + let root_table_source = self.make_pre_aggregation_table_source(&join.root)?; + let mut join_builder = JoinBuilder::new(root_table_source); + for item in join.items.iter() { + let to_table_source = self.make_pre_aggregation_table_source(&item.to)?; + let condition = + SqlJoinCondition::try_new(self.query_tools.clone(), item.on_sql.clone())?; + let on = JoinCondition::new_base_join(condition); + join_builder.left_join_aliased_source(to_table_source, on); + for member in item.from_members.iter().chain(item.to_members.iter()) { + let alias = BaseMemberHelper::default_alias( + &member.cube_name(), + &member.name(), + &None, + self.query_tools.clone(), + )?; + dimensions_references.insert( + member.full_name(), + QualifiedColumnName::new(None, alias.clone()), + ); + } + } + let from = From::new_from_join(join_builder.build()); + from + } + }; Ok(from) } + fn make_pre_aggregation_table_source( + &self, + table: &PreAggregationTable, + ) -> Result { + let name = table.alias.clone().unwrap_or_else(|| table.name.clone()); + let table_name = self + .query_tools + .base_tools() + .pre_aggregation_table_name(table.cube_name.clone(), name.clone())?; + let alias = PlanSqlTemplates::memeber_alias_name(&table.cube_name, &name, &None); + let res = SingleAliasedSource::new_from_table_reference( + table_name, + Rc::new(Schema::empty()), + Some(alias), + ); + Ok(res) + } + fn build_full_key_aggregate_query( &self, logical_plan: &FullKeyAggregateQuery, diff --git a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs index db8914b71a647..733888b9bc663 100644 --- a/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs +++ b/rust/cubesqlplanner/cubesqlplanner/src/plan/builder/join.rs @@ -47,6 +47,14 @@ impl JoinBuilder { )) } + pub fn left_join_aliased_source(&mut self, source: SingleAliasedSource, on: JoinCondition) { + self.joins.push(JoinItem { + from: source, + on, + join_type: JoinType::Left, + }); + } + pub fn left_join_subselect(&mut self, subquery: Rc