Skip to content
This repository was archived by the owner on Jun 14, 2024. It is now read-only.

Data Skipping Index Part 5: ValueListSketch #486

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

Data Skipping Index Part 5: ValueListSketch #486

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
78044f2
Data Skipping Index Part 4: BloomFilterSketch
Jul 27, 2021
f3b9581
Address review comments
Sep 13, 2021
8bd3bea
Change the signature of aggregateFunctions
Sep 13, 2021
99304dc
Refactor extractors
Sep 13, 2021
e6a4931
Data Skipping Index Part 5: ValueListSketch
Jul 27, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor extractors
  • Loading branch information
Chungmin Lee committed Sep 13, 2021
commit 99304dc7677b0caecbf9e9761fca84d0dc4a1d35
46 changes: 38 additions & 8 deletions src/main/scala/com/microsoft/hyperspace/index/dataskipping/expressions/extractors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,42 +19,72 @@ package com.microsoft.hyperspace.index.dataskipping.expressions
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.BooleanType

case class EqualToExtractor(left: ExpressionExtractor, right: ExpressionExtractor) {
def unapply(p: Expression): Option[(Expression, Expression)] =
trait BinaryExpressionExtractor {
def unapply(p: Expression): Option[(Expression, Expression)]
}

case class EqualToExtractor(left: ExpressionExtractor, right: ExpressionExtractor)
extends BinaryExpressionExtractor {
override def unapply(p: Expression): Option[(Expression, Expression)] =
p match {
case EqualTo(left(l), right(r)) => Some((l, r))
case EqualTo(right(r), left(l)) => Some((l, r))
case _ => None
}
}

case class EqualNullSafeExtractor(left: ExpressionExtractor, right: ExpressionExtractor) {
def unapply(p: Expression): Option[(Expression, Expression)] =
case class EqualNullSafeExtractor(left: ExpressionExtractor, right: ExpressionExtractor)
extends BinaryExpressionExtractor {
override def unapply(p: Expression): Option[(Expression, Expression)] =
p match {
case EqualNullSafe(left(l), right(r)) => Some((l, r))
case EqualNullSafe(right(r), left(l)) => Some((l, r))
case _ => None
}
}

case class LessThanExtractor(left: ExpressionExtractor, right: ExpressionExtractor) {
def unapply(p: Expression): Option[(Expression, Expression)] =
case class LessThanExtractor(left: ExpressionExtractor, right: ExpressionExtractor)
extends BinaryExpressionExtractor {
override def unapply(p: Expression): Option[(Expression, Expression)] =
p match {
case LessThan(left(l), right(r)) => Some((l, r))
case GreaterThan(right(r), left(l)) => Some((l, r))
case _ => None
}
}

case class LessThanOrEqualExtractor(left: ExpressionExtractor, right: ExpressionExtractor) {
def unapply(p: Expression): Option[(Expression, Expression)] =
case class LessThanOrEqualExtractor(left: ExpressionExtractor, right: ExpressionExtractor)
extends BinaryExpressionExtractor {
override def unapply(p: Expression): Option[(Expression, Expression)] =
p match {
case LessThanOrEqual(left(l), right(r)) => Some((l, r))
case GreaterThanOrEqual(right(r), left(l)) => Some((l, r))
case _ => None
}
}

case class SwitchLeftRight(extractor: BinaryExpressionExtractor)
extends BinaryExpressionExtractor {
override def unapply(p: Expression): Option[(Expression, Expression)] = {
extractor.unapply(p) match {
case Some((a, b)) => Some((b, a))
case None => None
}
}
}

object GreaterThanExtractor {
def apply(left: ExpressionExtractor, right: ExpressionExtractor): BinaryExpressionExtractor = {
SwitchLeftRight(LessThanExtractor(right, left))
}
}

object GreaterThanOrEqualExtractor {
def apply(left: ExpressionExtractor, right: ExpressionExtractor): BinaryExpressionExtractor = {
SwitchLeftRight(LessThanOrEqualExtractor(right, left))
}
}

case class IsNullExtractor(expr: ExpressionExtractor) {
def unapply(p: Expression): Option[Expression] =
p match {
Expand Down
51 changes: 25 additions & 26 deletions src/main/scala/com/microsoft/hyperspace/index/dataskipping/sketches/MinMaxSketch.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,33 +69,32 @@ case class MinMaxSketch(override val expr: String, override val dataType: Option
val ExprEqualNullSafe = EqualNullSafeExtractor(exprExtractor, valueExtractor)
val ExprLessThan = LessThanExtractor(exprExtractor, valueExtractor)
val ExprLessThanOrEqualTo = LessThanOrEqualExtractor(exprExtractor, valueExtractor)
val ExprGreaterThan = LessThanExtractor(valueExtractor, exprExtractor)
val ExprGreaterThanOrEqualTo = LessThanOrEqualExtractor(valueExtractor, exprExtractor)
val ExprGreaterThan = GreaterThanExtractor(exprExtractor, valueExtractor)
val ExprGreaterThanOrEqualTo = GreaterThanOrEqualExtractor(exprExtractor, valueExtractor)
val ExprIn = InExtractor(exprExtractor, valueExtractor)
val ExprInSet = InSetExtractor(exprExtractor)
Option(predicate)
.collect {
case ExprIsTrue(_) => max
case ExprIsFalse(_) => Not(min)
case ExprIsNotNull(_) => IsNotNull(min)
case ExprEqualTo(_, v) => And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v))
case ExprEqualNullSafe(_, v) =>
Or(IsNull(v), And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v)))
case ExprLessThan(_, v) => LessThan(min, v)
case ExprLessThanOrEqualTo(_, v) => LessThanOrEqual(min, v)
case ExprGreaterThan(v, _) => GreaterThan(max, v)
case ExprGreaterThanOrEqualTo(v, _) => GreaterThanOrEqual(max, v)
case ExprIn(_, vs) =>
vs.map(v => And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v))).reduceLeft(Or)
case ExprInSet(_, vs) =>
val sortedValues = Literal(
ArrayData.toArrayData(
ArrayUtils.toArray(
vs.filter(_ != null).toArray.sorted(TypeUtils.getInterpretedOrdering(dataType)),
dataType)),
ArrayType(dataType, containsNull = false))
LessThanOrEqual(ElementAt(sortedValues, SortedArrayLowerBound(sortedValues, min)), max)
// TODO: StartsWith, Like with constant prefix
}
Option(predicate).collect {
case ExprIsTrue(_) => max
case ExprIsFalse(_) => Not(min)
case ExprIsNotNull(_) => IsNotNull(min)
case ExprEqualTo(_, v) => And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v))
case ExprEqualNullSafe(_, v) =>
Or(IsNull(v), And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v)))
case ExprLessThan(_, v) => LessThan(min, v)
case ExprLessThanOrEqualTo(_, v) => LessThanOrEqual(min, v)
case ExprGreaterThan(_, v) => GreaterThan(max, v)
case ExprGreaterThanOrEqualTo(_, v) => GreaterThanOrEqual(max, v)
case ExprIn(_, vs) =>
vs.map(v => And(LessThanOrEqual(min, v), GreaterThanOrEqual(max, v))).reduceLeft(Or)
case ExprInSet(_, vs) =>
val sortedValues = Literal(
ArrayData.toArrayData(
ArrayUtils.toArray(
vs.filter(_ != null).toArray.sorted(TypeUtils.getInterpretedOrdering(dataType)),
dataType)),
ArrayType(dataType, containsNull = false))
LessThanOrEqual(ElementAt(sortedValues, SortedArrayLowerBound(sortedValues, min)), max)
// TODO: StartsWith, Like with constant prefix
}
}
}
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy