Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .git-blame-ignore-revs
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Scala Steward: Reformat with scalafmt 3.11.2
d3ee1e453824b8d3c2979e5ca360545052eb8953
2 changes: 1 addition & 1 deletion .scalafmt.conf
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
version = 3.11.1
version = 3.11.2
runner.dialect = scala213

align.openParenCallSite = true
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,12 @@ package org.apache.spark.sql

import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.{ Alias, CreateStruct }
import org.apache.spark.sql.catalyst.expressions.{ Expression, NamedExpression }
import org.apache.spark.sql.catalyst.expressions.{Alias, CreateStruct}
import org.apache.spark.sql.catalyst.expressions.{Expression, NamedExpression}
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
import org.apache.spark.sql.catalyst.plans.logical.{ LogicalPlan, Project }
import org.apache.spark.sql.catalyst.plans.logical.{LogicalPlan, Project}
import org.apache.spark.sql.execution.QueryExecution
import org.apache.spark.sql.types._
import org.apache.spark.sql.types.ObjectType
Expand All @@ -16,9 +16,8 @@ import scala.reflect.ClassTag
object FramelessInternals {

def objectTypeFor[A](
implicit
classTag: ClassTag[A]
): ObjectType = ObjectType(classTag.runtimeClass)
implicit classTag: ClassTag[A]
): ObjectType = ObjectType(classTag.runtimeClass)

def resolveExpr(ds: Dataset[_], colNames: Seq[String]): NamedExpression = {
ds.toDF()
Expand Down Expand Up @@ -49,11 +48,11 @@ object FramelessInternals {
ds.sqlContext.getConf(key, default)

def joinPlan(
ds: Dataset[_],
plan: LogicalPlan,
leftPlan: LogicalPlan,
rightPlan: LogicalPlan
): LogicalPlan = {
ds: Dataset[_],
plan: LogicalPlan,
leftPlan: LogicalPlan,
rightPlan: LogicalPlan
): LogicalPlan = {
val joined = executePlan(ds, plan)
val leftOutput = joined.analyzed.output.take(leftPlan.output.length)
val rightOutput = joined.analyzed.output.takeRight(rightPlan.output.length)
Expand All @@ -68,21 +67,21 @@ object FramelessInternals {
}

def mkDataset[T](
source: Dataset[_],
plan: LogicalPlan,
encoder: Encoder[T]
): Dataset[T] =
source: Dataset[_],
plan: LogicalPlan,
encoder: Encoder[T]
): Dataset[T] =
new Dataset(source.sparkSession, plan, encoder)

def ofRows(sparkSession: SparkSession, logicalPlan: LogicalPlan): DataFrame =
Dataset.ofRows(sparkSession, logicalPlan)

/** Builds an `ExpressionEncoder` from frameless' own serializer/deserializer expressions. */
def expressionEncoder[T](
objSerializer: Expression,
objDeserializer: Expression,
classTag: ClassTag[T]
): ExpressionEncoder[T] =
objSerializer: Expression,
objDeserializer: Expression,
classTag: ClassTag[T]
): ExpressionEncoder[T] =
new ExpressionEncoder[T](objSerializer, objDeserializer, classTag)

// because org.apache.spark.sql.types.UserDefinedType is private[spark]
Expand All @@ -104,8 +103,8 @@ object FramelessInternals {
tagged.genCode(ctx)

protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]
): Expression = copy(newChildren.head)
newChildren: IndexedSeq[Expression]
): Expression = copy(newChildren.head)
}

/** Expression to tag columns from the right hand side of join expression. */
Expand All @@ -121,7 +120,7 @@ object FramelessInternals {
tagged.genCode(ctx)

protected def withNewChildrenInternal(
newChildren: IndexedSeq[Expression]
): Expression = copy(newChildren.head)
newChildren: IndexedSeq[Expression]
): Expression = copy(newChildren.head)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ class FramelessLitPushDownTests extends SQLRulesSuite {
val optimizedPlan = ds.queryExecution.optimizedPlan.collect { case logical.Filter(condition, _) => condition }.flatMap(_.toList)

// check the optimized plan
optimizedPlan.collectFirst(planShouldContain) should not be (empty)
optimizedPlan.collectFirst(planShouldContain) should not be empty

// compare filters
actualPushDownFilters shouldBe expectedPushDownFilters
Expand Down
Loading