From 1011c2c57671411ca7e05d5fdadad54a247148e2 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Thu, 1 Aug 2019 14:03:08 -0700 Subject: [PATCH 1/9] [SPARK-27768] Infinity, -Infinity, NaN should be recognized in a case insensitive manner --- .../spark/sql/catalyst/expressions/Cast.scala | 112 ++++++++++++------ .../sql/catalyst/expressions/CastSuite.scala | 22 ++++ .../inputs/pgSQL/aggregates_part1.sql | 10 +- .../results/pgSQL/aggregates_part1.sql.out | 98 ++++++++++----- .../sql-tests/results/pgSQL/float4.sql.out | 10 +- .../sql-tests/results/pgSQL/float8.sql.out | 8 +- 6 files changed, 185 insertions(+), 75 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 10464dac8d55e..a8838a48020ff 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -563,7 +563,17 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToDouble(from: DataType): Any => Any = from match { case StringType => buildCast[UTF8String](_, s => try s.toString.toDouble catch { - case _: NumberFormatException => null + case _: NumberFormatException => + val str = s.trim.toString + if (str.equalsIgnoreCase("infinity")) { + Double.PositiveInfinity + } else if (str.equalsIgnoreCase("-infinity")) { + Double.NegativeInfinity + } else if (str.equalsIgnoreCase("nan")) { + Double.NaN + } else { + null + } }) case BooleanType => buildCast[Boolean](_, b => if (b) 1d else 0d) @@ -579,7 +589,17 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToFloat(from: DataType): Any => Any = from match { case StringType => buildCast[UTF8String](_, s => try s.toString.toFloat catch { - case _: NumberFormatException => null + case _: NumberFormatException => + val str = s.trim.toString + if (str.equalsIgnoreCase("infinity")) { + Float.PositiveInfinity + } else if (str.equalsIgnoreCase("-infinity")) { + Float.NegativeInfinity + } else if (str.equalsIgnoreCase("nan")) { + Float.NaN + } else { + null + } }) case BooleanType => buildCast[Boolean](_, b => if (b) 1f else 0f) @@ -717,9 +737,9 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String case ByteType => castToByteCode(from, ctx) case ShortType => castToShortCode(from, ctx) case IntegerType => castToIntCode(from, ctx) - case FloatType => castToFloatCode(from) + case FloatType => castToFloatCode(from, ctx) case LongType => castToLongCode(from, ctx) - case DoubleType => castToDoubleCode(from) + case DoubleType => castToDoubleCode(from, ctx) case array: ArrayType => castArrayCode(from.asInstanceOf[ArrayType].elementType, array.elementType, ctx) @@ -1259,48 +1279,72 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String (c, evPrim, evNull) => code"$evPrim = (long) $c;" } - private[this] def castToFloatCode(from: DataType): CastFunction = from match { - case StringType => - (c, evPrim, evNull) => - code""" + private[this] def castToFloatCode(from: DataType, ctx: CodegenContext): CastFunction = { + from match { + case StringType => + val str = ctx.freshVariable("str", StringType) + (c, evPrim, evNull) => + code""" try { $evPrim = Float.valueOf($c.toString()); } catch (java.lang.NumberFormatException e) { - $evNull = true; + final String $str = $c.toString().trim(); + if ($str.equalsIgnoreCase("infinity")) { + $evPrim = Float.POSITIVE_INFINITY; + } else if ($str.equalsIgnoreCase("-infinity")) { + $evPrim = Float.NEGATIVE_INFINITY; + } else if ($str.equalsIgnoreCase("nan")) { + $evPrim = Float.NaN; + } else { + $evNull = true; + } } """ - case BooleanType => - (c, evPrim, evNull) => code"$evPrim = $c ? 1.0f : 0.0f;" - case DateType => - (c, evPrim, evNull) => code"$evNull = true;" - case TimestampType => - (c, evPrim, evNull) => code"$evPrim = (float) (${timestampToDoubleCode(c)});" - case DecimalType() => - (c, evPrim, evNull) => code"$evPrim = $c.toFloat();" - case x: NumericType => - (c, evPrim, evNull) => code"$evPrim = (float) $c;" + case BooleanType => + (c, evPrim, evNull) => code"$evPrim = $c ? 1.0f : 0.0f;" + case DateType => + (c, evPrim, evNull) => code"$evNull = true;" + case TimestampType => + (c, evPrim, evNull) => code"$evPrim = (float) (${timestampToDoubleCode(c)});" + case DecimalType() => + (c, evPrim, evNull) => code"$evPrim = $c.toFloat();" + case x: NumericType => + (c, evPrim, evNull) => code"$evPrim = (float) $c;" + } } - private[this] def castToDoubleCode(from: DataType): CastFunction = from match { - case StringType => - (c, evPrim, evNull) => - code""" + private[this] def castToDoubleCode(from: DataType, ctx: CodegenContext): CastFunction = { + from match { + case StringType => + val str = ctx.freshVariable("str", StringType) + (c, evPrim, evNull) => + code""" try { $evPrim = Double.valueOf($c.toString()); } catch (java.lang.NumberFormatException e) { - $evNull = true; + final String $str = $c.toString().trim(); + if ($str.equalsIgnoreCase("infinity")) { + $evPrim = Double.POSITIVE_INFINITY; + } else if ($str.equalsIgnoreCase("-infinity")) { + $evPrim = Double.NEGATIVE_INFINITY; + } else if ($str.equalsIgnoreCase("nan")) { + $evPrim = Double.NaN; + } else { + $evNull = true; + } } """ - case BooleanType => - (c, evPrim, evNull) => code"$evPrim = $c ? 1.0d : 0.0d;" - case DateType => - (c, evPrim, evNull) => code"$evNull = true;" - case TimestampType => - (c, evPrim, evNull) => code"$evPrim = ${timestampToDoubleCode(c)};" - case DecimalType() => - (c, evPrim, evNull) => code"$evPrim = $c.toDouble();" - case x: NumericType => - (c, evPrim, evNull) => code"$evPrim = (double) $c;" + case BooleanType => + (c, evPrim, evNull) => code"$evPrim = $c ? 1.0d : 0.0d;" + case DateType => + (c, evPrim, evNull) => code"$evNull = true;" + case TimestampType => + (c, evPrim, evNull) => code"$evPrim = ${timestampToDoubleCode(c)};" + case DecimalType() => + (c, evPrim, evNull) => code"$evPrim = $c.toDouble();" + case x: NumericType => + (c, evPrim, evNull) => code"$evPrim = (double) $c;" + } } private[this] def castArrayCode( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index bbb3cb516b7d5..e44f6f01dfecf 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -1045,4 +1045,26 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { Cast(Literal(134.12), DecimalType(3, 2)), "cannot be represented") } } + + test("SPARK-27768 process infinity, -infinity , nan in case insensitive manner") { + + checkEvaluation(cast("infinity", FloatType), Float.PositiveInfinity) + checkEvaluation(cast("infiNity", FloatType), Float.PositiveInfinity) + checkEvaluation(cast(" infinity ", FloatType), Float.PositiveInfinity) + checkEvaluation(cast("-infinity", FloatType), Float.NegativeInfinity) + checkEvaluation(cast("-infiniTy", FloatType), Float.NegativeInfinity) + checkEvaluation(cast(" -infinity ", FloatType), Float.NegativeInfinity) + checkEvaluation(cast("infinity", DoubleType), Double.PositiveInfinity) + checkEvaluation(cast("infiNity", DoubleType), Double.PositiveInfinity) + checkEvaluation(cast(" infinity ", DoubleType), Double.PositiveInfinity) + checkEvaluation(cast("-infinity", DoubleType), Double.NegativeInfinity) + checkEvaluation(cast("-infiniTy", DoubleType), Double.NegativeInfinity) + checkEvaluation(cast(" -infinity ", DoubleType), Double.NegativeInfinity) + checkEvaluation(cast("nan", FloatType), Float.NaN) + checkEvaluation(cast("nAn", FloatType), Float.NaN) + checkEvaluation(cast(" nan ", FloatType), Float.NaN) + checkEvaluation(cast("nan", DoubleType), Double.NaN) + checkEvaluation(cast("nAn", DoubleType), Double.NaN) + checkEvaluation(cast(" nan ", DoubleType), Double.NaN) + } } diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql index 801a16cf41f54..d49d1906bb1ba 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql @@ -59,7 +59,6 @@ select avg(CAST(null AS DOUBLE)) from range(1,4); select sum(CAST('NaN' AS DOUBLE)) from range(1,4); select avg(CAST('NaN' AS DOUBLE)) from range(1,4); --- [SPARK-27768] verify correct results for infinite inputs SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('Infinity' AS DOUBLE))) v(x); SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) @@ -69,6 +68,15 @@ FROM (VALUES ('Infinity'), ('Infinity')) v(x); SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) FROM (VALUES ('-Infinity'), ('Infinity')) v(x); +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('infinity' AS DOUBLE))) v(x); +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES ('infinity'), ('1')) v(x); +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES ('infinity'), ('infinity')) v(x); +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES ('-infinity'), ('infinity')) v(x); + -- test accuracy with a large input offset SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out index 51ca1d558691c..f722a0d120bc0 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 44 +-- Number of queries: 48 -- !query 0 @@ -272,60 +272,96 @@ NaN NaN -- !query 33 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x) +FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('infinity' AS DOUBLE))) v(x) -- !query 33 schema struct -- !query 33 output -1.00000005E8 2.5 +Infinity NaN -- !query 34 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (7000000000005), (7000000000007)) v(x) +FROM (VALUES ('infinity'), ('1')) v(x) -- !query 34 schema struct -- !query 34 output -7.000000000006E12 1.0 +Infinity NaN -- !query 35 -SELECT covar_pop(b, a), covar_samp(b, a) FROM aggtest +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES ('infinity'), ('infinity')) v(x) -- !query 35 schema -struct +struct -- !query 35 output -653.6289553875104 871.5052738500139 +Infinity NaN -- !query 36 -SELECT corr(b, a) FROM aggtest +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES ('-infinity'), ('infinity')) v(x) -- !query 36 schema -struct +struct -- !query 36 output -0.1396345165178734 +NaN NaN -- !query 37 -SELECT count(four) AS cnt_1000 FROM onek +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x) -- !query 37 schema -struct +struct -- !query 37 output -1000 +1.00000005E8 2.5 -- !query 38 -SELECT count(DISTINCT four) AS cnt_4 FROM onek +SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) +FROM (VALUES (7000000000005), (7000000000007)) v(x) -- !query 38 schema -struct +struct -- !query 38 output -4 +7.000000000006E12 1.0 -- !query 39 +SELECT covar_pop(b, a), covar_samp(b, a) FROM aggtest +-- !query 39 schema +struct +-- !query 39 output +653.6289553875104 871.5052738500139 + + +-- !query 40 +SELECT corr(b, a) FROM aggtest +-- !query 40 schema +struct +-- !query 40 output +0.1396345165178734 + + +-- !query 41 +SELECT count(four) AS cnt_1000 FROM onek +-- !query 41 schema +struct +-- !query 41 output +1000 + + +-- !query 42 +SELECT count(DISTINCT four) AS cnt_4 FROM onek +-- !query 42 schema +struct +-- !query 42 output +4 + + +-- !query 43 select ten, count(*), sum(four) from onek group by ten order by ten --- !query 39 schema +-- !query 43 schema struct --- !query 39 output +-- !query 43 output 0 100 100 1 100 200 2 100 100 @@ -338,12 +374,12 @@ struct 9 100 200 --- !query 40 +-- !query 44 select ten, count(four), sum(DISTINCT four) from onek group by ten order by ten --- !query 40 schema +-- !query 44 schema struct --- !query 40 output +-- !query 44 output 0 100 2 1 100 4 2 100 2 @@ -356,13 +392,13 @@ struct 9 100 4 --- !query 41 +-- !query 45 select ten, sum(distinct four) from onek a group by ten having exists (select 1 from onek b where sum(distinct a.four) = b.four) --- !query 41 schema +-- !query 45 schema struct --- !query 41 output +-- !query 45 output 0 2 2 2 4 2 @@ -370,14 +406,14 @@ struct 8 2 --- !query 42 +-- !query 46 select ten, sum(distinct four) from onek a group by ten having exists (select 1 from onek b where sum(distinct a.four + b.four) = b.four) --- !query 42 schema +-- !query 46 schema struct<> --- !query 42 output +-- !query 46 output org.apache.spark.sql.AnalysisException Aggregate/Window/Generate expressions are not valid in where clause of the query. @@ -385,12 +421,12 @@ Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))]; --- !query 43 +-- !query 47 select (select max((select i.unique2 from tenk1 i where i.unique1 = o.unique1))) from tenk1 o --- !query 43 schema +-- !query 47 schema struct<> --- !query 43 output +-- !query 47 output org.apache.spark.sql.AnalysisException cannot resolve '`o.unique1`' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 63 diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/float4.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/float4.sql.out index 86d88007d8892..6e47cff91a7d5 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/float4.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/float4.sql.out @@ -63,7 +63,7 @@ SELECT float('nan') -- !query 7 schema struct -- !query 7 output -NULL +NaN -- !query 8 @@ -71,7 +71,7 @@ SELECT float(' NAN ') -- !query 8 schema struct -- !query 8 output -NULL +NaN -- !query 9 @@ -79,7 +79,7 @@ SELECT float('infinity') -- !query 9 schema struct -- !query 9 output -NULL +Infinity -- !query 10 @@ -87,7 +87,7 @@ SELECT float(' -INFINiTY ') -- !query 10 schema struct -- !query 10 output -NULL +-Infinity -- !query 11 @@ -135,7 +135,7 @@ SELECT float('nan') / float('nan') -- !query 16 schema struct<(CAST(CAST(nan AS FLOAT) AS DOUBLE) / CAST(CAST(nan AS FLOAT) AS DOUBLE)):double> -- !query 16 output -NULL +NaN -- !query 17 diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/float8.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/float8.sql.out index eb9e8aa6361a1..b4ea3c1ad1cab 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/float8.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/float8.sql.out @@ -95,7 +95,7 @@ SELECT double('nan') -- !query 11 schema struct -- !query 11 output -NULL +NaN -- !query 12 @@ -103,7 +103,7 @@ SELECT double(' NAN ') -- !query 12 schema struct -- !query 12 output -NULL +NaN -- !query 13 @@ -111,7 +111,7 @@ SELECT double('infinity') -- !query 13 schema struct -- !query 13 output -NULL +Infinity -- !query 14 @@ -119,7 +119,7 @@ SELECT double(' -INFINiTY ') -- !query 14 schema struct -- !query 14 output -NULL +-Infinity -- !query 15 From 19957ac2b730c0819cb48ef2e6db56bcd93a60ae Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Thu, 1 Aug 2019 14:52:15 -0700 Subject: [PATCH 2/9] Code review --- .../spark/sql/catalyst/expressions/Cast.scala | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index a8838a48020ff..397e20326e071 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -1288,16 +1288,16 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String try { $evPrim = Float.valueOf($c.toString()); } catch (java.lang.NumberFormatException e) { - final String $str = $c.toString().trim(); - if ($str.equalsIgnoreCase("infinity")) { - $evPrim = Float.POSITIVE_INFINITY; - } else if ($str.equalsIgnoreCase("-infinity")) { - $evPrim = Float.NEGATIVE_INFINITY; - } else if ($str.equalsIgnoreCase("nan")) { - $evPrim = Float.NaN; - } else { - $evNull = true; - } + final String $str = $c.toString().trim(); + if ($str.equalsIgnoreCase("infinity")) { + $evPrim = Float.POSITIVE_INFINITY; + } else if ($str.equalsIgnoreCase("-infinity")) { + $evPrim = Float.NEGATIVE_INFINITY; + } else if ($str.equalsIgnoreCase("nan")) { + $evPrim = Float.NaN; + } else { + $evNull = true; + } } """ case BooleanType => From 6ce5094b3587dde7ca9b148535377763c4f806a9 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Fri, 2 Aug 2019 00:15:57 -0700 Subject: [PATCH 3/9] Code review --- .../spark/sql/catalyst/expressions/Cast.scala | 97 ++++++++-------- .../sql/catalyst/expressions/CastSuite.scala | 10 +- .../inputs/pgSQL/aggregates_part1.sql | 10 -- .../results/pgSQL/aggregates_part1.sql.out | 106 ++++++------------ 4 files changed, 94 insertions(+), 129 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 397e20326e071..5cb7f964e27a7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -192,6 +192,32 @@ object Cast { } def resolvableNullability(from: Boolean, to: Boolean): Boolean = !from || to + + + def processFloatingPointSpecialLiterals(v: String, isFloat: Boolean): Any = { + val str = v.trim + if (str.equalsIgnoreCase("infinity") || str.equalsIgnoreCase("+infinity")) { + if (isFloat) { + Float.PositiveInfinity + } else { + Double.PositiveInfinity + } + } else if (str.equalsIgnoreCase("-infinity")) { + if (isFloat) { + Float.NegativeInfinity + } else { + Double.NegativeInfinity + } + } else if (str.equalsIgnoreCase("nan")) { + if (isFloat) { + Float.NaN + } else { + Double.NaN + } + } else { + null + } + } } /** @@ -559,21 +585,16 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String } } + // DoubleConverter private[this] def castToDouble(from: DataType): Any => Any = from match { case StringType => - buildCast[UTF8String](_, s => try s.toString.toDouble catch { - case _: NumberFormatException => - val str = s.trim.toString - if (str.equalsIgnoreCase("infinity")) { - Double.PositiveInfinity - } else if (str.equalsIgnoreCase("-infinity")) { - Double.NegativeInfinity - } else if (str.equalsIgnoreCase("nan")) { - Double.NaN - } else { - null - } + buildCast[UTF8String](_, s => { + val doubleStr = s.toString + try doubleStr.toDouble catch { + case _: NumberFormatException => + Cast.processFloatingPointSpecialLiterals(doubleStr, false) + } }) case BooleanType => buildCast[Boolean](_, b => if (b) 1d else 0d) @@ -588,18 +609,12 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String // FloatConverter private[this] def castToFloat(from: DataType): Any => Any = from match { case StringType => - buildCast[UTF8String](_, s => try s.toString.toFloat catch { - case _: NumberFormatException => - val str = s.trim.toString - if (str.equalsIgnoreCase("infinity")) { - Float.PositiveInfinity - } else if (str.equalsIgnoreCase("-infinity")) { - Float.NegativeInfinity - } else if (str.equalsIgnoreCase("nan")) { - Float.NaN - } else { - null - } + buildCast[UTF8String](_, s => { + val floatStr = s.toString + try floatStr.toFloat catch { + case _: NumberFormatException => + Cast.processFloatingPointSpecialLiterals(floatStr, true) + } }) case BooleanType => buildCast[Boolean](_, b => if (b) 1f else 0f) @@ -1282,21 +1297,18 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToFloatCode(from: DataType, ctx: CodegenContext): CastFunction = { from match { case StringType => - val str = ctx.freshVariable("str", StringType) + val floatStr = ctx.freshVariable("floatStr", StringType) (c, evPrim, evNull) => code""" + final String $floatStr = $c.toString(); try { - $evPrim = Float.valueOf($c.toString()); + $evPrim = Float.valueOf($floatStr); } catch (java.lang.NumberFormatException e) { - final String $str = $c.toString().trim(); - if ($str.equalsIgnoreCase("infinity")) { - $evPrim = Float.POSITIVE_INFINITY; - } else if ($str.equalsIgnoreCase("-infinity")) { - $evPrim = Float.NEGATIVE_INFINITY; - } else if ($str.equalsIgnoreCase("nan")) { - $evPrim = Float.NaN; - } else { + Float f = (Float) Cast.processFloatingPointSpecialLiterals($floatStr, true); + if (f == null) { $evNull = true; + } else { + $evPrim = f.floatValue(); } } """ @@ -1316,21 +1328,18 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String private[this] def castToDoubleCode(from: DataType, ctx: CodegenContext): CastFunction = { from match { case StringType => - val str = ctx.freshVariable("str", StringType) + val doubleStr = ctx.freshVariable("doubleStr", StringType) (c, evPrim, evNull) => code""" + final String $doubleStr = $c.toString(); try { - $evPrim = Double.valueOf($c.toString()); + $evPrim = Double.valueOf($doubleStr); } catch (java.lang.NumberFormatException e) { - final String $str = $c.toString().trim(); - if ($str.equalsIgnoreCase("infinity")) { - $evPrim = Double.POSITIVE_INFINITY; - } else if ($str.equalsIgnoreCase("-infinity")) { - $evPrim = Double.NEGATIVE_INFINITY; - } else if ($str.equalsIgnoreCase("nan")) { - $evPrim = Double.NaN; - } else { + Double d = (Double) Cast.processFloatingPointSpecialLiterals($doubleStr, false); + if (d == null) { $evNull = true; + } else { + $evPrim = d.doubleValue(); } } """ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index e44f6f01dfecf..a02e71ef1eda7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -1046,23 +1046,25 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } } - test("SPARK-27768 process infinity, -infinity , nan in case insensitive manner") { - + test("Process Infinity, -Infinity, NaN in case insensitive manner") { checkEvaluation(cast("infinity", FloatType), Float.PositiveInfinity) - checkEvaluation(cast("infiNity", FloatType), Float.PositiveInfinity) + checkEvaluation(cast("+infiNity", FloatType), Float.PositiveInfinity) checkEvaluation(cast(" infinity ", FloatType), Float.PositiveInfinity) checkEvaluation(cast("-infinity", FloatType), Float.NegativeInfinity) checkEvaluation(cast("-infiniTy", FloatType), Float.NegativeInfinity) checkEvaluation(cast(" -infinity ", FloatType), Float.NegativeInfinity) + checkEvaluation(cast("infinity", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast("infiNity", DoubleType), Double.PositiveInfinity) + checkEvaluation(cast("+infiNity", DoubleType), Double.PositiveInfinity) checkEvaluation(cast(" infinity ", DoubleType), Double.PositiveInfinity) checkEvaluation(cast("-infinity", DoubleType), Double.NegativeInfinity) checkEvaluation(cast("-infiniTy", DoubleType), Double.NegativeInfinity) checkEvaluation(cast(" -infinity ", DoubleType), Double.NegativeInfinity) + checkEvaluation(cast("nan", FloatType), Float.NaN) checkEvaluation(cast("nAn", FloatType), Float.NaN) checkEvaluation(cast(" nan ", FloatType), Float.NaN) + checkEvaluation(cast("nan", DoubleType), Double.NaN) checkEvaluation(cast("nAn", DoubleType), Double.NaN) checkEvaluation(cast(" nan ", DoubleType), Double.NaN) diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql index d49d1906bb1ba..5d54be9341148 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/aggregates_part1.sql @@ -59,15 +59,6 @@ select avg(CAST(null AS DOUBLE)) from range(1,4); select sum(CAST('NaN' AS DOUBLE)) from range(1,4); select avg(CAST('NaN' AS DOUBLE)) from range(1,4); -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('Infinity' AS DOUBLE))) v(x); -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('Infinity'), ('1')) v(x); -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('Infinity'), ('Infinity')) v(x); -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('-Infinity'), ('Infinity')) v(x); - SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('infinity' AS DOUBLE))) v(x); SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) @@ -77,7 +68,6 @@ FROM (VALUES ('infinity'), ('infinity')) v(x); SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) FROM (VALUES ('-infinity'), ('infinity')) v(x); - -- test accuracy with a large input offset SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x); diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out b/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out index f722a0d120bc0..29bafb42f579e 100644 --- a/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/aggregates_part1.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 48 +-- Number of queries: 44 -- !query 0 @@ -236,7 +236,7 @@ NaN -- !query 29 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('Infinity' AS DOUBLE))) v(x) +FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('infinity' AS DOUBLE))) v(x) -- !query 29 schema struct -- !query 29 output @@ -245,7 +245,7 @@ Infinity NaN -- !query 30 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('Infinity'), ('1')) v(x) +FROM (VALUES ('infinity'), ('1')) v(x) -- !query 30 schema struct -- !query 30 output @@ -254,7 +254,7 @@ Infinity NaN -- !query 31 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('Infinity'), ('Infinity')) v(x) +FROM (VALUES ('infinity'), ('infinity')) v(x) -- !query 31 schema struct -- !query 31 output @@ -263,7 +263,7 @@ Infinity NaN -- !query 32 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('-Infinity'), ('Infinity')) v(x) +FROM (VALUES ('-infinity'), ('infinity')) v(x) -- !query 32 schema struct -- !query 32 output @@ -272,96 +272,60 @@ NaN NaN -- !query 33 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (CAST('1' AS DOUBLE)), (CAST('infinity' AS DOUBLE))) v(x) +FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x) -- !query 33 schema struct -- !query 33 output -Infinity NaN +1.00000005E8 2.5 -- !query 34 SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('infinity'), ('1')) v(x) +FROM (VALUES (7000000000005), (7000000000007)) v(x) -- !query 34 schema struct -- !query 34 output -Infinity NaN - - --- !query 35 -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('infinity'), ('infinity')) v(x) --- !query 35 schema -struct --- !query 35 output -Infinity NaN - - --- !query 36 -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES ('-infinity'), ('infinity')) v(x) --- !query 36 schema -struct --- !query 36 output -NaN NaN - - --- !query 37 -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (100000003), (100000004), (100000006), (100000007)) v(x) --- !query 37 schema -struct --- !query 37 output -1.00000005E8 2.5 - - --- !query 38 -SELECT avg(CAST(x AS DOUBLE)), var_pop(CAST(x AS DOUBLE)) -FROM (VALUES (7000000000005), (7000000000007)) v(x) --- !query 38 schema -struct --- !query 38 output 7.000000000006E12 1.0 --- !query 39 +-- !query 35 SELECT covar_pop(b, a), covar_samp(b, a) FROM aggtest --- !query 39 schema +-- !query 35 schema struct --- !query 39 output +-- !query 35 output 653.6289553875104 871.5052738500139 --- !query 40 +-- !query 36 SELECT corr(b, a) FROM aggtest --- !query 40 schema +-- !query 36 schema struct --- !query 40 output +-- !query 36 output 0.1396345165178734 --- !query 41 +-- !query 37 SELECT count(four) AS cnt_1000 FROM onek --- !query 41 schema +-- !query 37 schema struct --- !query 41 output +-- !query 37 output 1000 --- !query 42 +-- !query 38 SELECT count(DISTINCT four) AS cnt_4 FROM onek --- !query 42 schema +-- !query 38 schema struct --- !query 42 output +-- !query 38 output 4 --- !query 43 +-- !query 39 select ten, count(*), sum(four) from onek group by ten order by ten --- !query 43 schema +-- !query 39 schema struct --- !query 43 output +-- !query 39 output 0 100 100 1 100 200 2 100 100 @@ -374,12 +338,12 @@ struct 9 100 200 --- !query 44 +-- !query 40 select ten, count(four), sum(DISTINCT four) from onek group by ten order by ten --- !query 44 schema +-- !query 40 schema struct --- !query 44 output +-- !query 40 output 0 100 2 1 100 4 2 100 2 @@ -392,13 +356,13 @@ struct 9 100 4 --- !query 45 +-- !query 41 select ten, sum(distinct four) from onek a group by ten having exists (select 1 from onek b where sum(distinct a.four) = b.four) --- !query 45 schema +-- !query 41 schema struct --- !query 45 output +-- !query 41 output 0 2 2 2 4 2 @@ -406,14 +370,14 @@ struct 8 2 --- !query 46 +-- !query 42 select ten, sum(distinct four) from onek a group by ten having exists (select 1 from onek b where sum(distinct a.four + b.four) = b.four) --- !query 46 schema +-- !query 42 schema struct<> --- !query 46 output +-- !query 42 output org.apache.spark.sql.AnalysisException Aggregate/Window/Generate expressions are not valid in where clause of the query. @@ -421,12 +385,12 @@ Expression in where clause: [(sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT)) Invalid expressions: [sum(DISTINCT CAST((outer() + b.`four`) AS BIGINT))]; --- !query 47 +-- !query 43 select (select max((select i.unique2 from tenk1 i where i.unique1 = o.unique1))) from tenk1 o --- !query 47 schema +-- !query 43 schema struct<> --- !query 47 output +-- !query 43 output org.apache.spark.sql.AnalysisException cannot resolve '`o.unique1`' given input columns: [i.even, i.fivethous, i.four, i.hundred, i.odd, i.string4, i.stringu1, i.stringu2, i.ten, i.tenthous, i.thousand, i.twenty, i.two, i.twothousand, i.unique1, i.unique2]; line 2 pos 63 From 5b3b734cf0d6c8ae3cd757e82fce528a1e212b48 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Fri, 2 Aug 2019 13:47:48 -0700 Subject: [PATCH 4/9] viirya comment --- .../scala/org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 5cb7f964e27a7..920eb88b13ab5 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -193,7 +193,6 @@ object Cast { def resolvableNullability(from: Boolean, to: Boolean): Boolean = !from || to - def processFloatingPointSpecialLiterals(v: String, isFloat: Boolean): Any = { val str = v.trim if (str.equalsIgnoreCase("infinity") || str.equalsIgnoreCase("+infinity")) { @@ -585,7 +584,6 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String } } - // DoubleConverter private[this] def castToDouble(from: DataType): Any => Any = from match { case StringType => From 48795b1d27d3393c5f346f591fcffe9ad078488a Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Wed, 7 Aug 2019 19:01:56 -0700 Subject: [PATCH 5/9] Code review --- .../spark/sql/catalyst/expressions/Cast.scala | 30 ++++++------------- .../sql/catalyst/expressions/CastSuite.scala | 6 ++++ 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 920eb88b13ab5..0d5f22e2e5eee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions import java.math.{BigDecimal => JavaBigDecimal} import java.time.ZoneId import java.util.concurrent.TimeUnit._ +import java.util.Locale import org.apache.spark.SparkException import org.apache.spark.sql.catalyst.{InternalRow, WalkedTypePath} @@ -194,27 +195,14 @@ object Cast { def resolvableNullability(from: Boolean, to: Boolean): Boolean = !from || to def processFloatingPointSpecialLiterals(v: String, isFloat: Boolean): Any = { - val str = v.trim - if (str.equalsIgnoreCase("infinity") || str.equalsIgnoreCase("+infinity")) { - if (isFloat) { - Float.PositiveInfinity - } else { - Double.PositiveInfinity - } - } else if (str.equalsIgnoreCase("-infinity")) { - if (isFloat) { - Float.NegativeInfinity - } else { - Double.NegativeInfinity - } - } else if (str.equalsIgnoreCase("nan")) { - if (isFloat) { - Float.NaN - } else { - Double.NaN - } - } else { - null + v.trim.toLowerCase(Locale.ROOT) match { + case "inf" | "+inf" | "infinity" | "+infinity" => + if (isFloat) Float.PositiveInfinity else Double.PositiveInfinity + case "-inf" | "-infinity" => + if (isFloat) Float.NegativeInfinity else Double.NegativeInfinity + case "nan" => + if (isFloat) Float.NaN else Double.NaN + case _ => null } } } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index a02e71ef1eda7..ab840707aa3d1 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -1047,19 +1047,25 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } test("Process Infinity, -Infinity, NaN in case insensitive manner") { + checkEvaluation(cast("inf", FloatType), Float.PositiveInfinity) + checkEvaluation(cast("+inf", FloatType), Float.PositiveInfinity) checkEvaluation(cast("infinity", FloatType), Float.PositiveInfinity) checkEvaluation(cast("+infiNity", FloatType), Float.PositiveInfinity) checkEvaluation(cast(" infinity ", FloatType), Float.PositiveInfinity) checkEvaluation(cast("-infinity", FloatType), Float.NegativeInfinity) checkEvaluation(cast("-infiniTy", FloatType), Float.NegativeInfinity) checkEvaluation(cast(" -infinity ", FloatType), Float.NegativeInfinity) + checkEvaluation(cast(" -inf ", FloatType), Float.NegativeInfinity) + checkEvaluation(cast("inf", DoubleType), Double.PositiveInfinity) + checkEvaluation(cast("+inf", DoubleType), Double.PositiveInfinity) checkEvaluation(cast("infinity", DoubleType), Double.PositiveInfinity) checkEvaluation(cast("+infiNity", DoubleType), Double.PositiveInfinity) checkEvaluation(cast(" infinity ", DoubleType), Double.PositiveInfinity) checkEvaluation(cast("-infinity", DoubleType), Double.NegativeInfinity) checkEvaluation(cast("-infiniTy", DoubleType), Double.NegativeInfinity) checkEvaluation(cast(" -infinity ", DoubleType), Double.NegativeInfinity) + checkEvaluation(cast("-inf", DoubleType), Double.NegativeInfinity) checkEvaluation(cast("nan", FloatType), Float.NaN) checkEvaluation(cast("nAn", FloatType), Float.NaN) From e901dc49b2ffb1be0d637dd507568f81c68f567f Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Thu, 8 Aug 2019 18:04:08 -0700 Subject: [PATCH 6/9] Locale fix --- .../scala/org/apache/spark/sql/catalyst/expressions/Cast.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 0d5f22e2e5eee..9b4587457b4e2 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -19,8 +19,8 @@ package org.apache.spark.sql.catalyst.expressions import java.math.{BigDecimal => JavaBigDecimal} import java.time.ZoneId -import java.util.concurrent.TimeUnit._ import java.util.Locale +import java.util.concurrent.TimeUnit._ import org.apache.spark.SparkException import org.apache.spark.sql.catalyst.{InternalRow, WalkedTypePath} From 41baaa0b88852dd751145a84155aeefcef57c311 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Fri, 9 Aug 2019 10:36:47 -0700 Subject: [PATCH 7/9] Add doc --- .../org/apache/spark/sql/catalyst/expressions/Cast.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index 9b4587457b4e2..c1eca26713f4b 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -194,6 +194,10 @@ object Cast { def resolvableNullability(from: Boolean, to: Boolean): Boolean = !from || to + /** + * We process literals such as 'Infinity', 'Inf', '-Infinity' and 'NaN' etc in case + * insensitive manner to be compatible with other database systems such as Postgres and DB2. + */ def processFloatingPointSpecialLiterals(v: String, isFloat: Boolean): Any = { v.trim.toLowerCase(Locale.ROOT) match { case "inf" | "+inf" | "infinity" | "+infinity" => From 429048b864d4fdda133b58e7f20442d12deb8a41 Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Mon, 12 Aug 2019 08:28:26 -0700 Subject: [PATCH 8/9] migration guide --- docs/sql-migration-guide-upgrade.md | 89 +++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md index b2bd8cefc3f96..c8eac6ed63583 100644 --- a/docs/sql-migration-guide-upgrade.md +++ b/docs/sql-migration-guide-upgrade.md @@ -161,6 +161,95 @@ license: | - Since Spark 3.0, Dataset query fails if it contains ambiguous column reference that is caused by self join. A typical example: `val df1 = ...; val df2 = df1.filter(...);`, then `df1.join(df2, df1("a") > df2("a"))` returns an empty result which is quite confusing. This is because Spark cannot resolve Dataset column references that point to tables being self joined, and `df1("a")` is exactly the same as `df2("a")` in Spark. To restore the behavior before Spark 3.0, you can set `spark.sql.analyzer.failAmbiguousSelfJoin` to `false`. + - Since Spark 3.0, `Cast` function processes string literals such as 'Infinity', '+Infinity', '-Infinity', 'NaN', 'Inf', '+Inf', '-Inf' in case insensitive manner when casting the literals to `Double` or `Float` type to ensure greater compatibility with other database systems. This behaviour change is illustrated in the table below: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Operation + + Result prior to spark 3.0 + + Result starting spark 3.0 +
+ CAST('infinity' TO DOUBLE)
+ CAST('+infinity' TO DOUBLE)
+ CAST('inf' TO DOUBLE)
+ CAST('+inf' TO DOUBLE)
+ 		+ NULL + + Double.PositiveInfinity +
+ CAST('-infinity' TO DOUBLE)
+ CAST('-inf' TO DOUBLE)
+ 		+ NULL + + Double.NegativeInfinity +
+ CAST('infinity' TO FLOAT)
+ CAST('+infinity' TO FLOAT)
+ CAST('inf' TO FLOAT)
+ CAST('+inf' TO FLOAT)
+ 		+ NULL + + Float.PositiveInfinity +
+ CAST('-infinity' TO FLOAT)
+ CAST('-inf' TO FLOAT)
+ 		+ NULL + + Float.NegativeInfinity +
+ CAST('nan' TO DOUBLE) + + NULL + + Double.NaN +
+ CAST('nan' TO FLOAT) + + NULL + + Float.NaN +
+ ## Upgrading from Spark SQL 2.4 to 2.4.1 - The value of `spark.executor.heartbeatInterval`, when specified without units like "30" rather than "30s", was From 1a5d978707d519fc6cea5eb7caa1b3fe1eb4933f Mon Sep 17 00:00:00 2001 From: Dilip Biswal Date: Mon, 12 Aug 2019 22:04:32 -0700 Subject: [PATCH 9/9] Code review --- docs/sql-migration-guide-upgrade.md | 32 ++++++------ .../spark/sql/catalyst/expressions/Cast.scala | 6 +-- .../sql/catalyst/expressions/CastSuite.scala | 50 +++++++++---------- .../sql-tests/inputs/pgSQL/float4.sql | 2 - .../sql-tests/inputs/pgSQL/float8.sql | 4 +- 5 files changed, 43 insertions(+), 51 deletions(-) diff --git a/docs/sql-migration-guide-upgrade.md b/docs/sql-migration-guide-upgrade.md index c8eac6ed63583..a643a843a5cb4 100644 --- a/docs/sql-migration-guide-upgrade.md +++ b/docs/sql-migration-guide-upgrade.md @@ -168,18 +168,18 @@ license: | Operation - Result prior to spark 3.0 + Result prior to Spark 3.0 - Result starting spark 3.0 + Result starting Spark 3.0 - CAST('infinity' TO DOUBLE)
- CAST('+infinity' TO DOUBLE)
- CAST('inf' TO DOUBLE)
- CAST('+inf' TO DOUBLE)
+ CAST('infinity' AS DOUBLE)
+ CAST('+infinity' AS DOUBLE)
+ CAST('inf' AS DOUBLE)
+ CAST('+inf' AS DOUBLE)
NULL @@ -190,8 +190,8 @@ license: | - CAST('-infinity' TO DOUBLE)
- CAST('-inf' TO DOUBLE)
+ CAST('-infinity' AS DOUBLE)
+ CAST('-inf' AS DOUBLE)
NULL @@ -202,10 +202,10 @@ license: | - CAST('infinity' TO FLOAT)
- CAST('+infinity' TO FLOAT)
- CAST('inf' TO FLOAT)
- CAST('+inf' TO FLOAT)
+ CAST('infinity' AS FLOAT)
+ CAST('+infinity' AS FLOAT)
+ CAST('inf' AS FLOAT)
+ CAST('+inf' AS FLOAT)
NULL @@ -216,8 +216,8 @@ license: | - CAST('-infinity' TO FLOAT)
- CAST('-inf' TO FLOAT)
+ CAST('-infinity' AS FLOAT)
+ CAST('-inf' AS FLOAT)
NULL @@ -228,7 +228,7 @@ license: | - CAST('nan' TO DOUBLE) + CAST('nan' AS DOUBLE) NULL @@ -239,7 +239,7 @@ license: | - CAST('nan' TO FLOAT) + CAST('nan' AS FLOAT) NULL diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala index c1eca26713f4b..7ba0910ac2157 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala @@ -196,7 +196,7 @@ object Cast { /** * We process literals such as 'Infinity', 'Inf', '-Infinity' and 'NaN' etc in case - * insensitive manner to be compatible with other database systems such as Postgres and DB2. + * insensitive manner to be compatible with other database systems such as PostgreSQL and DB2. */ def processFloatingPointSpecialLiterals(v: String, isFloat: Boolean): Any = { v.trim.toLowerCase(Locale.ROOT) match { @@ -1294,7 +1294,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String try { $evPrim = Float.valueOf($floatStr); } catch (java.lang.NumberFormatException e) { - Float f = (Float) Cast.processFloatingPointSpecialLiterals($floatStr, true); + final Float f = (Float) Cast.processFloatingPointSpecialLiterals($floatStr, true); if (f == null) { $evNull = true; } else { @@ -1325,7 +1325,7 @@ case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String try { $evPrim = Double.valueOf($doubleStr); } catch (java.lang.NumberFormatException e) { - Double d = (Double) Cast.processFloatingPointSpecialLiterals($doubleStr, false); + final Double d = (Double) Cast.processFloatingPointSpecialLiterals($doubleStr, false); if (d == null) { $evNull = true; } else { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala index ab840707aa3d1..861bfc92bbe66 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala @@ -1047,32 +1047,28 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper { } test("Process Infinity, -Infinity, NaN in case insensitive manner") { - checkEvaluation(cast("inf", FloatType), Float.PositiveInfinity) - checkEvaluation(cast("+inf", FloatType), Float.PositiveInfinity) - checkEvaluation(cast("infinity", FloatType), Float.PositiveInfinity) - checkEvaluation(cast("+infiNity", FloatType), Float.PositiveInfinity) - checkEvaluation(cast(" infinity ", FloatType), Float.PositiveInfinity) - checkEvaluation(cast("-infinity", FloatType), Float.NegativeInfinity) - checkEvaluation(cast("-infiniTy", FloatType), Float.NegativeInfinity) - checkEvaluation(cast(" -infinity ", FloatType), Float.NegativeInfinity) - checkEvaluation(cast(" -inf ", FloatType), Float.NegativeInfinity) - - checkEvaluation(cast("inf", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast("+inf", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast("infinity", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast("+infiNity", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast(" infinity ", DoubleType), Double.PositiveInfinity) - checkEvaluation(cast("-infinity", DoubleType), Double.NegativeInfinity) - checkEvaluation(cast("-infiniTy", DoubleType), Double.NegativeInfinity) - checkEvaluation(cast(" -infinity ", DoubleType), Double.NegativeInfinity) - checkEvaluation(cast("-inf", DoubleType), Double.NegativeInfinity) - - checkEvaluation(cast("nan", FloatType), Float.NaN) - checkEvaluation(cast("nAn", FloatType), Float.NaN) - checkEvaluation(cast(" nan ", FloatType), Float.NaN) - - checkEvaluation(cast("nan", DoubleType), Double.NaN) - checkEvaluation(cast("nAn", DoubleType), Double.NaN) - checkEvaluation(cast(" nan ", DoubleType), Double.NaN) + Seq("inf", "+inf", "infinity", "+infiNity", " infinity ").foreach { value => + checkEvaluation(cast(value, FloatType), Float.PositiveInfinity) + } + Seq("-infinity", "-infiniTy", " -infinity ", " -inf ").foreach { value => + checkEvaluation(cast(value, FloatType), Float.NegativeInfinity) + } + Seq("inf", "+inf", "infinity", "+infiNity", " infinity ").foreach { value => + checkEvaluation(cast(value, DoubleType), Double.PositiveInfinity) + } + Seq("-infinity", "-infiniTy", " -infinity ", " -inf ").foreach { value => + checkEvaluation(cast(value, DoubleType), Double.NegativeInfinity) + } + Seq("nan", "nAn", " nan ").foreach { value => + checkEvaluation(cast(value, FloatType), Float.NaN) + } + Seq("nan", "nAn", " nan ").foreach { value => + checkEvaluation(cast(value, DoubleType), Double.NaN) + } + + // Invalid literals when casted to double and float results in null. + Seq(DoubleType, FloatType).foreach { dataType => + checkEvaluation(cast("badvalue", dataType), null) + } } } diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float4.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float4.sql index 3dad5cd56ba02..058467695a608 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float4.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float4.sql @@ -38,7 +38,6 @@ INSERT INTO FLOAT4_TBL VALUES ('1.2345678901234e-20'); -- special inputs SELECT float('NaN'); --- [SPARK-28060] Float type can not accept some special inputs SELECT float('nan'); SELECT float(' NAN '); SELECT float('infinity'); @@ -49,7 +48,6 @@ SELECT float('N A N'); SELECT float('NaN x'); SELECT float(' INFINITY x'); --- [SPARK-28060] Float type can not accept some special inputs SELECT float('Infinity') + 100.0; SELECT float('Infinity') / float('Infinity'); SELECT float('nan') / float('nan'); diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float8.sql b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float8.sql index 6f8e3b596e60e..957dabdebab4e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float8.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/float8.sql @@ -37,7 +37,6 @@ SELECT double('-10e-400'); -- special inputs SELECT double('NaN'); --- [SPARK-28060] Double type can not accept some special inputs SELECT double('nan'); SELECT double(' NAN '); SELECT double('infinity'); @@ -49,7 +48,6 @@ SELECT double('NaN x'); SELECT double(' INFINITY x'); SELECT double('Infinity') + 100.0; --- [SPARK-27768] Infinity, -Infinity, NaN should be recognized in a case insensitive manner SELECT double('Infinity') / double('Infinity'); SELECT double('NaN') / double('NaN'); -- [SPARK-28315] Decimal can not accept NaN as input @@ -190,7 +188,7 @@ SELECT tanh(double('1')); SELECT asinh(double('1')); SELECT acosh(double('2')); SELECT atanh(double('0.5')); --- [SPARK-27768] Infinity, -Infinity, NaN should be recognized in a case insensitive manner + -- test Inf/NaN cases for hyperbolic functions SELECT sinh(double('Infinity')); SELECT sinh(double('-Infinity'));