diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index ba400a2c8201d..c7fe99c35e79b 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -113,6 +113,8 @@ object functions { * * @group normal_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def lit(literal: Any): Column = { literal match { @@ -135,6 +137,8 @@ object functions { * * @group normal_funcs * @since 2.2.0 + * @return + * Returns a column of the same type as the input. */ def typedLit[T: TypeTag](literal: T): Column = { typedlit(literal) @@ -154,6 +158,8 @@ object functions { * * @group normal_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def typedlit[T: TypeTag](literal: T): Column = { literal match { @@ -246,6 +252,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column): Column = approx_count_distinct(e) @@ -253,6 +261,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) @@ -260,6 +270,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) @@ -267,6 +279,8 @@ object functions { /** * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use approx_count_distinct", "2.1.0") def approxCountDistinct(columnName: String, rsd: Double): Column = { @@ -276,16 +290,24 @@ object functions { /** * Aggregate function: returns the approximate number of distinct items in a group. * + * @param e + * The column to count distinct values in. A column of any type. * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) /** * Aggregate function: returns the approximate number of distinct items in a group. * + * @param columnName + * The name of the column to count distinct values in. A column of any type. * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String): Column = approx_count_distinct( column(columnName)) @@ -294,10 +316,13 @@ object functions { * Aggregate function: returns the approximate number of distinct items in a group. * * @param rsd - * maximum relative standard deviation allowed (default = 0.05) + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(e: Column, rsd: Double): Column = { Column.fn("approx_count_distinct", e, lit(rsd)) @@ -307,10 +332,13 @@ object functions { * Aggregate function: returns the approximate number of distinct items in a group. * * @param rsd - * maximum relative standard deviation allowed (default = 0.05) + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. * * @group agg_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a long. */ def approx_count_distinct(columnName: String, rsd: Double): Column = { approx_count_distinct(Column(columnName), rsd) @@ -319,64 +347,88 @@ object functions { /** * Aggregate function: returns the average of the values in a group. * + * @param e + * The column to average. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(e: Column): Column = Column.fn("avg", e) /** * Aggregate function: returns the average of the values in a group. * + * @param columnName + * The name of the column to average. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric. */ def avg(columnName: String): Column = avg(Column(columnName)) /** * Aggregate function: returns a list of objects with duplicates. * + * @param e + * The column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(e: Column): Column = Column.fn("collect_list", e) /** * Aggregate function: returns a list of objects with duplicates. * + * @param columnName + * The name of the column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_list(columnName: String): Column = collect_list(Column(columnName)) /** * Aggregate function: returns a set of objects with duplicate elements eliminated. * + * @param e + * The column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(e: Column): Column = Column.fn("collect_set", e) /** * Aggregate function: returns a set of objects with duplicate elements eliminated. * + * @param columnName + * The name of the column to collect. A column of any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an array. */ def collect_set(columnName: String): Column = collect_set(Column(columnName)) @@ -386,8 +438,21 @@ object functions { * sketch is a probabilistic data structure used for cardinality estimation using sub-linear * space. * + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @param seed + * The random seed. A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = Column.fn("count_min_sketch", e, eps, confidence, seed) @@ -398,8 +463,19 @@ object functions { * sketch is a probabilistic data structure used for cardinality estimation using sub-linear * space. * + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a binary. */ def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) @@ -407,16 +483,28 @@ object functions { /** * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * + * @param column1 + * The first column. A column that evaluates to a numeric. + * @param column2 + * The second column. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) /** * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * + * @param columnName1 + * The name of the first column. A column that evaluates to a numeric. + * @param columnName2 + * The name of the second column. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def corr(columnName1: String, columnName2: String): Column = { corr(Column(columnName1), Column(columnName2)) @@ -425,8 +513,12 @@ object functions { /** * Aggregate function: returns the number of items in a group. * + * @param e + * The column to count. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(e: Column): Column = Column.fn("count", e) @@ -434,8 +526,12 @@ object functions { /** * Aggregate function: returns the number of items in a group. * + * @param columnName + * The name of the column to count. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ def count(columnName: String): TypedColumn[Any, Long] = count(Column(columnName)).as(PrimitiveLongEncoder) @@ -445,8 +541,14 @@ object functions { * * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * + * @param expr + * The first column. A column of any type. + * @param exprs + * Additional columns. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) @@ -456,8 +558,14 @@ object functions { * * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * + * @param columnName + * first column to compute on. A column of any type. + * @param columnNames + * additional columns to compute on. Columns of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def countDistinct(columnName: String, columnNames: String*): Column = @@ -466,8 +574,14 @@ object functions { /** * Aggregate function: returns the number of distinct items in a group. * + * @param expr + * first column to compute on. A column of any type. + * @param exprs + * additional columns to compute on. Columns of any type. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def count_distinct(expr: Column, exprs: Column*): Column = @@ -476,8 +590,14 @@ object functions { /** * Aggregate function: returns the population covariance for two columns. * + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(column1: Column, column2: Column): Column = Column.fn("covar_pop", column1, column2) @@ -485,8 +605,14 @@ object functions { /** * Aggregate function: returns the population covariance for two columns. * + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_pop(columnName1: String, columnName2: String): Column = { covar_pop(Column(columnName1), Column(columnName2)) @@ -495,8 +621,14 @@ object functions { /** * Aggregate function: returns the sample covariance for two columns. * + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(column1: Column, column2: Column): Column = Column.fn("covar_samp", column1, column2) @@ -504,8 +636,14 @@ object functions { /** * Aggregate function: returns the sample covariance for two columns. * + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a double. */ def covar_samp(columnName1: String, columnName2: String): Column = { covar_samp(Column(columnName1), Column(columnName2)) @@ -517,12 +655,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column, ignoreNulls: Boolean): Column = Column.fn("first", false, e, lit(ignoreNulls)) @@ -533,12 +678,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String, ignoreNulls: Boolean): Column = { first(Column(columnName), ignoreNulls) @@ -550,12 +702,16 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(e: Column): Column = first(e, ignoreNulls = false) @@ -565,24 +721,32 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def first(columnName: String): Column = first(Column(columnName)) /** * Aggregate function: returns the first value in a group. * + * @param e + * column to fetch the first value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column): Column = Column.fn("first_value", e) @@ -592,12 +756,19 @@ object functions { * The function by default returns the first values it sees. It will return the first non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def first_value(e: Column, ignoreNulls: Column): Column = Column.fn("first_value", e, ignoreNulls) @@ -606,8 +777,12 @@ object functions { * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or * not, returns 1 for aggregated or 0 for not aggregated in the result set. * + * @param e + * column to check if it is aggregated. A column of any type. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(e: Column): Column = Column.fn("grouping", e) @@ -615,8 +790,12 @@ object functions { * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or * not, returns 1 for aggregated or 0 for not aggregated in the result set. * + * @param columnName + * column to check if it is aggregated. A column of any type. * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ def grouping(columnName: String): Column = grouping(Column(columnName)) @@ -627,12 +806,16 @@ object functions { * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) * }}} * + * @param cols + * columns to check for. Columns of any type. * @note * The list of columns should match with grouping columns exactly, or empty (means all the * grouping columns). * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) @@ -644,11 +827,17 @@ object functions { * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) * }}} * + * @param colName + * the name of the first grouping column. A column of any type. + * @param colNames + * the names of the remaining grouping columns. Columns of any type. * @note * The list of columns should match with grouping columns exactly. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def grouping_id(colName: String, colNames: String*): Column = { @@ -659,8 +848,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Column): Column = Column.fn("hll_sketch_agg", e, lgConfigK) @@ -669,8 +866,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column, lgConfigK: Int): Column = Column.fn("hll_sketch_agg", e, lit(lgConfigK)) @@ -679,8 +884,16 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with lgConfigK arg. * + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { hll_sketch_agg(Column(columnName), lgConfigK) @@ -690,8 +903,13 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with default lgConfigK value. * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(e: Column): Column = Column.fn("hll_sketch_agg", e) @@ -700,8 +918,13 @@ object functions { * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch * configured with default lgConfigK value. * + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_sketch_agg(columnName: String): Column = { hll_sketch_agg(Column(columnName)) @@ -713,8 +936,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = Column.fn("hll_union_agg", e, allowDifferentLgConfigK) @@ -725,8 +956,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) @@ -737,8 +976,16 @@ object functions { * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values * and allowDifferentLgConfigK is set to false. * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { hll_union_agg(Column(columnName), allowDifferentLgConfigK) @@ -749,8 +996,13 @@ object functions { * HllSketch, generated by merging previously created Datasketches HllSketch instances via a * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(e: Column): Column = Column.fn("hll_union_agg", e) @@ -760,8 +1012,13 @@ object functions { * HllSketch, generated by merging previously created Datasketches HllSketch instances via a * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union_agg(columnName: String): Column = { hll_union_agg(Column(columnName)) @@ -770,16 +1027,24 @@ object functions { /** * Aggregate function: returns the kurtosis of the values in a group. * + * @param e + * the column to compute the kurtosis on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(e: Column): Column = Column.fn("kurtosis", e) /** * Aggregate function: returns the kurtosis of the values in a group. * + * @param columnName + * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) @@ -789,12 +1054,19 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column, ignoreNulls: Boolean): Column = Column.fn("last", false, e, lit(ignoreNulls)) @@ -805,12 +1077,19 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param columnName + * the name of the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String, ignoreNulls: Boolean): Column = { last(Column(columnName), ignoreNulls) @@ -822,12 +1101,16 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the last value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(e: Column): Column = last(e, ignoreNulls = false) @@ -843,18 +1126,24 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) /** * Aggregate function: returns the last value in a group. * + * @param e + * column to fetch the last value for. A column of any type. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column): Column = Column.fn("last_value", e) @@ -864,12 +1153,18 @@ object functions { * The function by default returns the last values it sees. It will return the last non-null * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * + * @param e + * column to fetch the last value for. A column of any type. + * @param ignoreNulls + * whether to skip null values. A column that evaluates to a boolean. * @note * The function is non-deterministic because its results depends on the order of the rows * which may be non-deterministic after a shuffle. * * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def last_value(e: Column, ignoreNulls: Column): Column = Column.fn("last_value", e, ignoreNulls) @@ -878,13 +1173,15 @@ object functions { * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * * @param hour - * the hour to represent, from 0 to 23 + * the hour to represent, from 0 to 23. A column that evaluates to an integer. * @param minute - * the minute to represent, from 0 to 59 + * the minute to represent, from 0 to 59. A column that evaluates to an integer. * @param second - * the second to represent, from 0 to 59.999999 + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a time. */ def make_time(hour: Column, minute: Column, second: Column): Column = { Column.fn("make_time", hour, minute, second) @@ -893,8 +1190,12 @@ object functions { /** * Aggregate function: returns the most frequent value in a group. * + * @param e + * target column to compute on. A column of any type. * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column): Column = Column.fn("mode", e) @@ -905,16 +1206,27 @@ object functions { * if deterministic is false or is not defined, or the lowest value is returned if deterministic * is true. * + * @param e + * target column to compute on. A column of any type. + * @param deterministic + * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a + * constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) /** * Aggregate function: returns the maximum value of the expression in a group. * + * @param e + * the target column on which the maximum value is computed. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(e: Column): Column = Column.fn("max", e) @@ -923,18 +1235,26 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def max(columnName: String): Column = max(Column(columnName)) /** * Aggregate function: returns the value associated with the maximum value of ord. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. * @note * The function is non-deterministic so the output order can be different for those associated * the same values of `e`. * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) @@ -944,6 +1264,12 @@ object functions { * The result array contains values in descending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. An integer. Must be a constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -953,6 +1279,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) @@ -962,6 +1290,12 @@ object functions { * The result array contains values in descending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. A column that evaluates to an integer. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -971,14 +1305,20 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) /** * Aggregate function: returns the average of the values in a group. Alias for avg. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(e: Column): Column = avg(e) @@ -987,42 +1327,63 @@ object functions { * * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def mean(columnName: String): Column = avg(columnName) /** * Aggregate function: returns the median of the values in a group. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a double. */ def median(e: Column): Column = Column.fn("median", e) /** * Aggregate function: returns the minimum value of the expression in a group. * + * @param e + * the target column on which the minimum value is computed. A column of any type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(e: Column): Column = Column.fn("min", e) /** * Aggregate function: returns the minimum value of the column in a group. * + * @param columnName + * the name of the column on which the minimum value is computed. A column of an orderable + * type. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def min(columnName: String): Column = min(Column(columnName)) /** * Aggregate function: returns the value associated with the minimum value of ord. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. * @note * The function is non-deterministic so the output order can be different for those associated * the same values of `e`. * * @group agg_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) @@ -1033,6 +1394,12 @@ object functions { * The result array contains values in ascending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. An integer. Must be a constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -1042,6 +1409,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) @@ -1052,6 +1421,13 @@ object functions { * The result array contains values in ascending order by their associated ordering values. * Returns null if there are no non-null ordering values. * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. A column that evaluates to an integral. Must be a + * constant. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle when there are ties in the @@ -1061,6 +1437,8 @@ object functions { * * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) @@ -1068,8 +1446,15 @@ object functions { * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given * percentage(s) with value range in [0.0, 1.0]. * + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) @@ -1077,8 +1462,18 @@ object functions { * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given * percentage(s) with value range in [0.0, 1.0]. * + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param frequency + * the positive frequency with which to weight each value. A column that evaluates to an + * integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def percentile(e: Column, percentage: Column, frequency: Column): Column = Column.fn("percentile", e, percentage, frequency) @@ -1095,8 +1490,19 @@ object functions { * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the * relative error of the approximation. * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = Column.fn("percentile_approx", e, percentage, accuracy) @@ -1113,8 +1519,19 @@ object functions { * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the * relative error of the approximation. * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { Column.fn("approx_percentile", e, percentage, accuracy) @@ -1123,96 +1540,149 @@ object functions { /** * Aggregate function: returns the product of all numerical elements in a group. * + * @param e + * the column to compute the product on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a double. */ def product(e: Column): Column = Column.internalFn("product", e) /** * Aggregate function: returns the skewness of the values in a group. * + * @param e + * the column to compute the skewness on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(e: Column): Column = Column.fn("skewness", e) /** * Aggregate function: returns the skewness of the values in a group. * + * @param columnName + * the name of the column to compute the skewness on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def skewness(columnName: String): Column = skewness(Column(columnName)) /** * Aggregate function: alias for `stddev_samp`. * + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def std(e: Column): Column = Column.fn("std", e) /** * Aggregate function: alias for `stddev_samp`. * + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(e: Column): Column = Column.fn("stddev", e) /** * Aggregate function: alias for `stddev_samp`. * + * @param columnName + * the name of the column to compute the standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev(columnName: String): Column = stddev(Column(columnName)) /** * Aggregate function: returns the sample standard deviation of the expression in a group. * + * @param e + * the column to compute the sample standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) /** * Aggregate function: returns the sample standard deviation of the expression in a group. * + * @param columnName + * Name of the column to compute the sample standard deviation on. A column that evaluates to + * a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) /** * Aggregate function: returns the population standard deviation of the expression in a group. * + * @param e + * The column to compute the population standard deviation on. A column that evaluates to a + * numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) /** * Aggregate function: returns the population standard deviation of the expression in a group. * + * @param columnName + * Name of the column to compute the population standard deviation on. A column that evaluates + * to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) /** * Aggregate function: returns the sum of all values in the expression. * + * @param e + * The column to sum. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(e: Column): Column = Column.fn("sum", e) /** * Aggregate function: returns the sum of all values in the given column. * + * @param columnName + * Name of the column to sum. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum(columnName: String): Column = sum(Column(columnName)) @@ -1221,6 +1691,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(e: Column): Column = sum_distinct(e) @@ -1230,6 +1702,8 @@ object functions { * * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ @deprecated("Use sum_distinct", "3.2.0") def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) @@ -1237,8 +1711,12 @@ object functions { /** * Aggregate function: returns the sum of distinct values in the expression. * + * @param e + * The column to sum distinct values of. A column that evaluates to a numeric or interval. * @group agg_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a numeric or interval. */ def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) @@ -1247,8 +1725,13 @@ object functions { * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input * column via a Datasketches Intersection instance. * + * @param e + * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(e: Column): Column = Column.fn("theta_intersection_agg", e) @@ -1258,8 +1741,13 @@ object functions { * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input * volumn via a Datasketches Intersection instance. * + * @param columnName + * Name of the column of Datasketches ThetaSketch instances to intersect. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection_agg(columnName: String): Column = theta_intersection_agg(Column(columnName)) @@ -1269,8 +1757,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_sketch_agg", e, lgNomEntries) @@ -1280,8 +1776,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) @@ -1291,8 +1795,16 @@ object functions { * built with the values in the input column and configured with the `lgNomEntries` nominal * entries. * + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = theta_sketch_agg(Column(columnName), lgNomEntries) @@ -1302,8 +1814,13 @@ object functions { * built with the values in the input column and configured with the default value of 12 for * `lgNomEntries`. * + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(e: Column): Column = Column.fn("theta_sketch_agg", e) @@ -1313,8 +1830,13 @@ object functions { * built with the values in the input column and configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_sketch_agg(columnName: String): Column = theta_sketch_agg(Column(columnName)) @@ -1325,8 +1847,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Column): Column = Column.fn("theta_union_agg", e, lgNomEntries) @@ -1337,8 +1867,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column, lgNomEntries: Int): Column = Column.fn("theta_union_agg", e, lit(lgNomEntries)) @@ -1349,8 +1887,16 @@ object functions { * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String, lgNomEntries: Int): Column = theta_union_agg(Column(columnName), lgNomEntries) @@ -1361,8 +1907,13 @@ object functions { * via a Datasketches Union instance. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(e: Column): Column = Column.fn("theta_union_agg", e) @@ -1373,8 +1924,13 @@ object functions { * via a Datasketches Union instance. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union_agg(columnName: String): Column = theta_union_agg(Column(columnName)) @@ -1385,8 +1941,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_double", e, mode) @@ -1397,8 +1961,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_double", e, lit(mode)) @@ -1409,8 +1981,16 @@ object functions { * in the input column via a Datasketches Intersection instance. The mode parameter specifies * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String, mode: String): Column = tuple_intersection_agg_double(Column(columnName), mode) @@ -1421,8 +2001,13 @@ object functions { * in the input column via a Datasketches Intersection instance. It is configured with the * default mode of 'sum'. * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(e: Column): Column = Column.fn("tuple_intersection_agg_double", e) @@ -1433,8 +2018,13 @@ object functions { * in the input column via a Datasketches Intersection instance. It is configured with the * default mode of 'sum'. * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_double(columnName: String): Column = tuple_intersection_agg_double(Column(columnName)) @@ -1446,8 +2036,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: Column): Column = Column.fn("tuple_intersection_agg_integer", e, mode) @@ -1459,8 +2057,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column, mode: String): Column = Column.fn("tuple_intersection_agg_integer", e, lit(mode)) @@ -1472,8 +2078,16 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String, mode: String): Column = tuple_intersection_agg_integer(Column(columnName), mode) @@ -1484,8 +2098,13 @@ object functions { * instances in the input column via a Datasketches Intersection instance. It is configured with * the default mode of 'sum'. * + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(e: Column): Column = Column.fn("tuple_intersection_agg_integer", e) @@ -1496,8 +2115,13 @@ object functions { * instances in the input column via a Datasketches Intersection instance. It is configured with * the default mode of 'sum'. * + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_agg_integer(columnName: String): Column = tuple_intersection_agg_integer(Column(columnName)) @@ -1508,8 +2132,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1524,8 +2162,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( key: Column, @@ -1540,8 +2192,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1555,8 +2221,19 @@ object functions { * with a double type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) @@ -1566,8 +2243,19 @@ object functions { * with a double type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double( keyColumnName: String, @@ -1580,8 +2268,16 @@ object functions { * with a double type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_double", key, summary) @@ -1591,8 +2287,16 @@ object functions { * with a double type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) @@ -1603,8 +2307,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1619,8 +2337,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( key: Column, @@ -1635,8 +2367,22 @@ object functions { * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1650,8 +2396,19 @@ object functions { * with an integer type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) @@ -1661,8 +2418,19 @@ object functions { * with an integer type summary built with the key and summary values in the input columns and * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer( keyColumnName: String, @@ -1675,8 +2443,16 @@ object functions { * with an integer type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(key: Column, summary: Column): Column = Column.fn("tuple_sketch_agg_integer", key, summary) @@ -1686,8 +2462,16 @@ object functions { * with an integer type summary built with the key and summary values in the input columns. It * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) @@ -1699,8 +2483,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * the column containing binary TupleSketch representations to union. A column that evaluates + * to a binary. + * @param lgNomEntries + * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column + * that evaluates to an integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) @@ -1712,8 +2507,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) @@ -1725,8 +2531,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_double(Column(columnName), lgNomEntries, mode) @@ -1737,8 +2554,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) @@ -1749,8 +2574,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_double(Column(columnName), lgNomEntries) @@ -1761,8 +2594,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(e: Column): Column = Column.fn("tuple_union_agg_double", e) @@ -1773,8 +2611,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_double(columnName: String): Column = tuple_union_agg_double(Column(columnName)) @@ -1786,8 +2629,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) @@ -1799,8 +2653,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) @@ -1812,8 +2677,19 @@ object functions { * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric * summaries (sum, min, max, alwaysone). * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) @@ -1824,8 +2700,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) @@ -1836,8 +2720,16 @@ object functions { * the input column via a Datasketches Union instance. It allows the configuration of * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = tuple_union_agg_integer(Column(columnName), lgNomEntries) @@ -1848,8 +2740,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(e: Column): Column = Column.fn("tuple_union_agg_integer", e) @@ -1860,8 +2757,13 @@ object functions { * the input column via a Datasketches Union instance. It is configured with the default values * of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @group agg_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_agg_integer(columnName: String): Column = tuple_union_agg_integer(Column(columnName)) @@ -1871,8 +2773,16 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_bigint", e, k) @@ -1882,8 +2792,16 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_bigint", e, lit(k)) @@ -1893,8 +2811,15 @@ object functions { * KllLongsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String, k: Int): Column = kll_sketch_agg_bigint(Column(columnName), k) @@ -1903,8 +2828,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllLongsSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing bigint values to aggregate. A column that evaluates to an integral. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(e: Column): Column = Column.fn("kll_sketch_agg_bigint", e) @@ -1913,8 +2842,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllLongsSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_bigint(columnName: String): Column = kll_sketch_agg_bigint(Column(columnName)) @@ -1924,8 +2857,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing float values to aggregate. A column that evaluates to a float. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_float", e, k) @@ -1935,8 +2875,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_float", e, lit(k)) @@ -1946,8 +2893,15 @@ object functions { * KllFloatsSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String, k: Int): Column = kll_sketch_agg_float(Column(columnName), k) @@ -1956,8 +2910,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllFloatsSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(e: Column): Column = Column.fn("kll_sketch_agg_float", e) @@ -1966,8 +2924,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllFloatsSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_float(columnName: String): Column = kll_sketch_agg_float(Column(columnName)) @@ -1977,8 +2939,16 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing double values to aggregate. A column that evaluates to a float or + * double. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Column): Column = Column.fn("kll_sketch_agg_double", e, k) @@ -1988,8 +2958,15 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column, k: Int): Column = Column.fn("kll_sketch_agg_double", e, lit(k)) @@ -1999,8 +2976,15 @@ object functions { * KllDoublesSketch built with the values in the input column. The optional k parameter controls * the size and accuracy of the sketch (default 200, range 8-65535). * + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String, k: Int): Column = kll_sketch_agg_double(Column(columnName), k) @@ -2009,8 +2993,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllDoublesSketch built with the values in the input column with default k value of 200. * + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(e: Column): Column = Column.fn("kll_sketch_agg_double", e) @@ -2019,8 +3007,12 @@ object functions { * Aggregate function: returns the compact binary representation of the Datasketches * KllDoublesSketch built with the values in the input column with default k value of 200. * + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_agg_double(columnName: String): Column = kll_sketch_agg_double(Column(columnName)) @@ -2031,8 +3023,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Column): Column = Column.fn("kll_merge_agg_bigint", e, k) @@ -2043,8 +3043,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column, k: Int): Column = Column.fn("kll_merge_agg_bigint", e, lit(k)) @@ -2055,8 +3063,16 @@ object functions { * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String, k: Int): Column = kll_merge_agg_bigint(Column(columnName), k) @@ -2066,8 +3082,13 @@ object functions { * sketch. If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param e + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(e: Column): Column = Column.fn("kll_merge_agg_bigint", e) @@ -2077,8 +3098,13 @@ object functions { * sketch. If k is not specified, the merged sketch adopts the k value from the first input * sketch. * + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_bigint(columnName: String): Column = kll_merge_agg_bigint(Column(columnName)) @@ -2088,8 +3114,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Column): Column = Column.fn("kll_merge_agg_float", e, k) @@ -2099,8 +3133,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column, k: Int): Column = Column.fn("kll_merge_agg_float", e, lit(k)) @@ -2110,8 +3152,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String, k: Int): Column = kll_merge_agg_float(Column(columnName), k) @@ -2120,8 +3170,13 @@ object functions { * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(e: Column): Column = Column.fn("kll_merge_agg_float", e) @@ -2130,8 +3185,13 @@ object functions { * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_float(columnName: String): Column = kll_merge_agg_float(Column(columnName)) @@ -2141,8 +3201,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Column): Column = Column.fn("kll_merge_agg_double", e, k) @@ -2152,8 +3220,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column, k: Int): Column = Column.fn("kll_merge_agg_double", e, lit(k)) @@ -2163,8 +3239,16 @@ object functions { * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String, k: Int): Column = kll_merge_agg_double(Column(columnName), k) @@ -2173,8 +3257,13 @@ object functions { * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(e: Column): Column = Column.fn("kll_merge_agg_double", e) @@ -2183,8 +3272,13 @@ object functions { * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. * If k is not specified, the merged sketch adopts the k value from the first input sketch. * + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. * @group agg_funcs * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ def kll_merge_agg_double(columnName: String): Column = kll_merge_agg_double(Column(columnName)) @@ -2192,8 +3286,12 @@ object functions { /** * Aggregate function: returns the concatenation of non-null input values. * + * @param e + * The target column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column): Column = Column.fn("listagg", e) @@ -2201,16 +3299,27 @@ object functions { * Aggregate function: returns the concatenation of non-null input values, separated by the * delimiter. * + * @param e + * The target column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * The delimiter used to separate the values. A column that evaluates to a string or binary. + * Must be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) /** * Aggregate function: returns the concatenation of distinct non-null input values. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) @@ -2218,8 +3327,15 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values, separated by * the delimiter. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def listagg_distinct(e: Column, delimiter: Column): Column = Column.fn("listagg", isDistinct = true, e, delimiter) @@ -2227,8 +3343,12 @@ object functions { /** * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column): Column = Column.fn("string_agg", e) @@ -2236,8 +3356,15 @@ object functions { * Aggregate function: returns the concatenation of non-null input values, separated by the * delimiter. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) @@ -2245,8 +3372,12 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values. Alias for * `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) @@ -2254,8 +3385,15 @@ object functions { * Aggregate function: returns the concatenation of distinct non-null input values, separated by * the delimiter. Alias for `listagg`. * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. * @group agg_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def string_agg_distinct(e: Column, delimiter: Column): Column = Column.fn("string_agg", isDistinct = true, e, delimiter) @@ -2263,8 +3401,12 @@ object functions { /** * Aggregate function: alias for `var_samp`. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(e: Column): Column = Column.fn("variance", e) @@ -2273,14 +3415,20 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def variance(columnName: String): Column = variance(Column(columnName)) /** * Aggregate function: returns the unbiased variance of the values in a group. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(e: Column): Column = Column.fn("var_samp", e) @@ -2289,14 +3437,20 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_samp(columnName: String): Column = var_samp(Column(columnName)) /** * Aggregate function: returns the population variance of the values in a group. * + * @param e + * the column to compute on. A column that evaluates to a numeric. * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(e: Column): Column = Column.fn("var_pop", e) @@ -2305,6 +3459,8 @@ object functions { * * @group agg_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def var_pop(columnName: String): Column = var_pop(Column(columnName)) @@ -2312,8 +3468,14 @@ object functions { * Aggregate function: returns the average of the independent variable for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) @@ -2321,8 +3483,14 @@ object functions { * Aggregate function: returns the average of the dependent variable for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) @@ -2330,8 +3498,14 @@ object functions { * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the * dependent variable and `x` is the independent variable. * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) @@ -2340,8 +3514,14 @@ object functions { * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent * variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) @@ -2349,8 +3529,14 @@ object functions { * Aggregate function: returns the coefficient of determination for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) @@ -2358,8 +3544,14 @@ object functions { * Aggregate function: returns the slope of the linear regression line for non-null pairs in a * group, where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) @@ -2367,8 +3559,14 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) @@ -2376,8 +3574,14 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) @@ -2385,16 +3589,26 @@ object functions { * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, * where `y` is the dependent variable and `x` is the independent variable. * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) /** * Aggregate function: returns some value of `e` for a group of rows. * + * @param e + * The column to return some value from. A column of any type. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column): Column = Column.fn("any_value", e) @@ -2402,8 +3616,15 @@ object functions { * Aggregate function: returns some value of `e` for a group of rows. If `ignoreNulls` is true, * returns only non-null values. * + * @param e + * The column to return some value from. A column of any type. + * @param ignoreNulls + * If true, returns only non-null values. A column that evaluates to a boolean. Must be a + * constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def any_value(e: Column, ignoreNulls: Column): Column = Column.fn("any_value", e, ignoreNulls) @@ -2411,8 +3632,12 @@ object functions { /** * Aggregate function: returns the number of `TRUE` values for the expression. * + * @param e + * The expression to count TRUE values of. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def count_if(e: Column): Column = Column.fn("count_if", e) @@ -2421,8 +3646,7 @@ object functions { * 6 fractional digits of seconds. * * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2435,10 +3659,9 @@ object functions { * * @param precision * An integer literal in the range [0..6], indicating how many fractional digits of seconds to - * include in the result. + * include in the result. A column that evaluates to an integer. Must be a constant. * @return - * A time. - * + * A time. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -2457,8 +3680,14 @@ object functions { * statistical computing packages. Note: the output type of the 'x' field in the return value is * propagated from the input value consumed in the aggregate function. * + * @param e + * The column to compute the histogram on. A column that evaluates to a numeric. + * @param nBins + * The number of histogram bins. A column that evaluates to an integral. Must be a constant. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def histogram_numeric(e: Column, nBins: Column): Column = Column.fn("histogram_numeric", e, nBins) @@ -2466,64 +3695,96 @@ object functions { /** * Aggregate function: returns true if all values of `e` are true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def every(e: Column): Column = Column.fn("every", e) /** * Aggregate function: returns true if all values of `e` are true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_and(e: Column): Column = Column.fn("bool_and", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def some(e: Column): Column = Column.fn("some", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * The expression to evaluate. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def any(e: Column): Column = Column.fn("any", e) /** * Aggregate function: returns true if at least one value of `e` is true. * + * @param e + * column to check if at least one value is true. A column that evaluates to a boolean. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def bool_or(e: Column): Column = Column.fn("bool_or", e) /** * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_and(e: Column): Column = Column.fn("bit_and", e) /** * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_or(e: Column): Column = Column.fn("bit_or", e) /** * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. * + * @param e + * target column to compute on. A column that evaluates to an integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def bit_xor(e: Column): Column = Column.fn("bit_xor", e) @@ -2550,8 +3811,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2580,8 +3841,8 @@ object functions { * * @return * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. - * + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. * @group window_funcs * @since 4.3.0 */ @@ -2599,6 +3860,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def cume_dist(): Column = Column.fn("cume_dist") @@ -2615,6 +3878,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def dense_rank(): Column = Column.fn("dense_rank") @@ -2625,8 +3890,14 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int): Column = lag(e, offset, null) @@ -2637,8 +3908,14 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) @@ -2649,8 +3926,16 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(columnName: String, offset: Int, defaultValue: Any): Column = { lag(Column(columnName), offset, defaultValue) @@ -2663,8 +3948,16 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any): Column = { lag(e, offset, defaultValue, false) @@ -2679,8 +3972,18 @@ object functions { * * This is equivalent to the LAG function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @param ignoreNulls + * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2692,8 +3995,14 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } @@ -2704,8 +4013,14 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } @@ -2716,8 +4031,16 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(columnName: String, offset: Int, defaultValue: Any): Column = { lead(Column(columnName), offset, defaultValue) @@ -2730,8 +4053,16 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any): Column = { lead(e, offset, defaultValue, false) @@ -2746,8 +4077,21 @@ object functions { * * This is equivalent to the LEAD function in SQL. * + * @param e + * The column to compute the lead value for. A column of any type. + * @param offset + * Number of rows after the current row to look ahead. A column that evaluates to an integral. + * Must be a constant. + * @param defaultValue + * Value to return when there are fewer than `offset` rows after the current row. A column of + * any type. Must be a constant. + * @param ignoreNulls + * Whether to skip null values when computing the result. A column that evaluates to a + * boolean. Must be a constant. * @group window_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) @@ -2761,8 +4105,18 @@ object functions { * * This is equivalent to the nth_value function in SQL. * + * @param e + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @param ignoreNulls + * Whether the nth value should skip nulls when determining which row to use. A column that + * evaluates to a boolean. Must be a constant. * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) @@ -2773,8 +4127,15 @@ object functions { * * This is equivalent to the nth_value function in SQL. * + * @param e + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. * @group window_funcs * @since 3.1.0 + * @return + * Returns a column of the same type as the input. */ def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) @@ -2785,8 +4146,13 @@ object functions { * * This is equivalent to the NTILE function in SQL. * + * @param n + * The number of groups to divide the window partition into. A column that evaluates to an + * integral. Must be a constant. * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def ntile(n: Int): Column = Column.fn("ntile", lit(n)) @@ -2803,6 +4169,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ def percent_rank(): Column = Column.fn("percent_rank") @@ -2819,6 +4187,8 @@ object functions { * * @group window_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an integer. */ def rank(): Column = Column.fn("rank") @@ -2827,6 +4197,8 @@ object functions { * * @group window_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def row_number(): Column = Column.fn("row_number") @@ -2837,8 +4209,13 @@ object functions { /** * Creates a new array column. The input columns must all have the same data type. * + * @param cols + * The columns to combine into an array. Each is a column of any type, and all must share the + * same data type. * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(cols: Column*): Column = Column.fn("array", cols: _*) @@ -2848,6 +4225,8 @@ object functions { * * @group array_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def array(colName: String, colNames: String*): Column = { @@ -2859,8 +4238,13 @@ object functions { * value1, key2, value2, ...). The key columns must all have the same data type, and can't be * null. The value columns must all have the same data type. * + * @param cols + * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column + * of any type; key columns must share a type and value columns must share a type. * @group map_funcs * @since 2.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map(cols: Column*): Column = Column.fn("map", cols: _*) @@ -2868,8 +4252,13 @@ object functions { /** * Creates a struct with the given field names and values. * + * @param cols + * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are + * columns that evaluate to a string; values are columns of any type. * @group struct_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) @@ -2878,8 +4267,15 @@ object functions { * Creates a new map column. The array in the first column is used for keys. The array in the * second column is used for values. All elements in the array for key should not be null. * + * @param keys + * The array of keys for the map; elements must not be null. A column that evaluates to an + * array. + * @param values + * The array of values for the map. A column that evaluates to an array. * @group map_funcs * @since 2.4 + * @return + * Returns a column that evaluates to a map. */ def map_from_arrays(keys: Column, values: Column): Column = Column.fn("map_from_arrays", keys, values) @@ -2888,8 +4284,18 @@ object functions { * Creates a map after splitting the text into key/value pairs using delimiters. Both * `pairDelim` and `keyValueDelim` are treated as regular expressions. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @param keyValueDelim + * Delimiter used to split key and value, treated as a regular expression. A column that + * evaluates to a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = Column.fn("str_to_map", text, pairDelim, keyValueDelim) @@ -2898,8 +4304,15 @@ object functions { * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` * is treated as regular expressions. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column, pairDelim: Column): Column = Column.fn("str_to_map", text, pairDelim) @@ -2907,8 +4320,12 @@ object functions { /** * Creates a map after splitting the text into key/value pairs using delimiters. * + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. * @group map_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a map. */ def str_to_map(text: Column): Column = Column.fn("str_to_map", text) @@ -2934,8 +4351,12 @@ object functions { * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is * not null, or c if both a and b are null but c is not null. * + * @param e + * the columns to work on. A column that evaluates to any type. * @group conditional_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) @@ -2945,22 +4366,32 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def input_file_name(): Column = Column.fn("input_file_name") /** * Return true iff the column is NaN. * + * @param e + * the column to check. A column that evaluates to a numeric. * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnan(e: Column): Column = e.isNaN /** * Return true iff the column is null. * + * @param e + * the column to check. A column that evaluates to any type. * @group predicate_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnull(e: Column): Column = e.isNull @@ -2981,6 +4412,8 @@ object functions { * * @group misc_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long. */ @deprecated("Use monotonically_increasing_id()", "2.0.0") def monotonicallyIncreasingId(): Column = monotonically_increasing_id() @@ -3002,6 +4435,8 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a long. */ def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") @@ -3010,8 +4445,14 @@ object functions { * * Both inputs should be floating point columns (DoubleType or FloatType). * + * @param col1 + * the first column to check. A column that evaluates to a numeric. + * @param col2 + * the column to return if the first is NaN. A column that evaluates to a numeric. * @group conditional_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the first input. */ def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) @@ -3026,8 +4467,12 @@ object functions { * df.select( negate(df.col("amount")) ); * }}} * + * @param e + * the column to negate. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def negate(e: Column): Column = -e @@ -3041,8 +4486,12 @@ object functions { * df.filter( not(df.col("isActive")) ); * }}} * + * @param e + * the column to invert. A column that evaluates to a boolean. * @group predicate_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def not(e: Column): Column = !e @@ -3050,11 +4499,15 @@ object functions { * Generate a random column with independent and identically distributed (i.i.d.) samples * uniformly distributed in [0.0, 1.0). * + * @param seed + * the seed for the random generator. * @note * The function is non-deterministic in general case. * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(seed: Long): Column = Column.fn("rand", lit(seed)) @@ -3067,6 +4520,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rand(): Column = rand(SparkClassUtils.random.nextLong) @@ -3074,11 +4529,15 @@ object functions { * Generate a column with independent and identically distributed (i.i.d.) samples from the * standard normal distribution. * + * @param seed + * the seed for the random generator. * @note * The function is non-deterministic in general case. * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(seed: Long): Column = Column.fn("randn", lit(seed)) @@ -3091,6 +4550,8 @@ object functions { * * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def randn(): Column = randn(SparkClassUtils.random.nextLong) @@ -3099,8 +4560,13 @@ object functions { * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant * two-byte or four-byte integer (SMALLINT or INT, respectively). * + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column): Column = randstr(length, lit(SparkClassUtils.random.nextLong)) @@ -3110,8 +4576,15 @@ object functions { * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). * + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. + * @param seed + * the random seed to use. A column that evaluates to an integral. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) @@ -3123,22 +4596,32 @@ object functions { * * @group misc_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. */ def spark_partition_id(): Column = Column.fn("spark_partition_id") /** * Computes the square root of the specified float value. * + * @param e + * the value to compute the square root of. A column that evaluates to a numeric. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(e: Column): Column = Column.fn("sqrt", e) /** * Computes the square root of the specified float value. * + * @param colName + * the name of a numeric column to compute the square root of. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def sqrt(colName: String): Column = sqrt(Column(colName)) @@ -3146,16 +4629,26 @@ object functions { * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable * input types are the same with the `+` operator. * + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) /** * Returns the mean calculated from values of a group and the result is null on overflow. * + * @param e + * the value to compute the mean of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def try_avg(e: Column): Column = Column.fn("try_avg", e) @@ -3163,16 +4656,28 @@ object functions { * Returns `dividend``/``divisor`. It always performs floating point division. Its result is * always null if `divisor` is 0. * + * @param left + * the dividend. A column that evaluates to a numeric or interval. + * @param right + * the divisor. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) /** * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. * + * @param left + * the dividend. A column that evaluates to a numeric. + * @param right + * the divisor. A column that evaluates to a numeric. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) @@ -3180,8 +4685,14 @@ object functions { * Returns `left``*``right` and the result is null on overflow. The acceptable input types are * the same with the `*` operator. * + * @param left + * the multiplicand. A column that evaluates to a numeric or interval. + * @param right + * the multiplier. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) @@ -3189,16 +4700,26 @@ object functions { * Returns `left``-``right` and the result is null on overflow. The acceptable input types are * the same with the `-` operator. * + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) /** * Returns the sum calculated from values of a group and the result is null on overflow. * + * @param e + * the value to compute the sum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a numeric. */ def try_sum(e: Column): Column = Column.fn("try_sum", e) @@ -3208,8 +4729,12 @@ object functions { * StructField's name, otherwise, the newly generated StructField's name would be auto generated * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... * + * @param cols + * the columns to contain in the output struct. A column of any type. * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(cols: Column*): Column = Column.fn("struct", cols: _*) @@ -3217,8 +4742,14 @@ object functions { /** * Creates a new struct column that composes multiple input columns. * + * @param colName + * the name of the first column to contain in the output struct. + * @param colNames + * the names of the remaining columns to contain in the output struct. * @group struct_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. */ @scala.annotation.varargs def struct(colName: String, colNames: String*): Column = { @@ -3243,8 +4774,14 @@ object functions { * .otherwise(2)) * }}} * + * @param condition + * the condition to evaluate. A column that evaluates to a boolean. + * @param value + * the value to return when the condition is true. A literal value, or a column expression. * @group conditional_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ def when(condition: Column, value: Any): Column = Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) @@ -3254,6 +4791,8 @@ object functions { * * @group bitwise_funcs * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use bitwise_not", "3.2.0") def bitwiseNOT(e: Column): Column = bitwise_not(e) @@ -3261,8 +4800,12 @@ object functions { /** * Computes bitwise NOT (~) of a number. * + * @param e + * the target column to compute on. A column that evaluates to an integral. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def bitwise_not(e: Column): Column = Column.fn("~", e) @@ -3270,8 +4813,12 @@ object functions { * Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, * or NULL if the argument is NULL. * + * @param e + * the target column to compute on. A column that evaluates to an integral or boolean. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_count(e: Column): Column = Column.fn("bit_count", e) @@ -3279,8 +4826,15 @@ object functions { * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered * from right to left, starting at zero. The position argument cannot be negative. * + * @param e + * the target column to compute on. A column that evaluates to an integral. + * @param pos + * the bit position, numbered from right to left starting at zero. A column that evaluates to + * an integer. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def bit_get(e: Column, pos: Column): Column = Column.fn("bit_get", e, pos) @@ -3288,8 +4842,15 @@ object functions { * Returns the value of the bit (0 or 1) at the specified position. The positions are numbered * from right to left, starting at zero. The position argument cannot be negative. * + * @param e + * the target column to compute on. A column that evaluates to an integral. + * @param pos + * the bit position, numbered from right to left starting at zero. A column that evaluates to + * an integer. * @group bitwise_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a byte. */ def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) @@ -3312,14 +4873,22 @@ object functions { /** * Computes the absolute value of a numeric value. * + * @param e + * the value to compute the absolute value of. A column that evaluates to a numeric or + * interval. * @group math_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ def abs(e: Column): Column = Column.fn("abs", e) /** + * @param e + * the value to compute the inverse cosine of. A column that evaluates to a double. * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos` + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3327,17 +4896,21 @@ object functions { def acos(e: Column): Column = Column.fn("acos", e) /** + * @param columnName + * the value to compute the inverse cosine of. * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos` - * + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def acos(columnName: String): Column = acos(Column(columnName)) /** + * @param e + * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. * @return - * inverse hyperbolic cosine of `e` + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3345,17 +4918,21 @@ object functions { def acosh(e: Column): Column = Column.fn("acosh", e) /** + * @param columnName + * the value to compute the inverse hyperbolic cosine of. * @return - * inverse hyperbolic cosine of `columnName` - * + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ def acosh(columnName: String): Column = acosh(Column(columnName)) /** + * @param e + * the value to compute the inverse sine of. A column that evaluates to a double. * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin` + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3363,17 +4940,21 @@ object functions { def asin(e: Column): Column = Column.fn("asin", e) /** + * @param columnName + * the value to compute the inverse sine of. * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin` - * + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def asin(columnName: String): Column = asin(Column(columnName)) /** + * @param e + * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. * @return - * inverse hyperbolic sine of `e` + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3381,17 +4962,21 @@ object functions { def asinh(e: Column): Column = Column.fn("asinh", e) /** + * @param columnName + * the value to compute the inverse hyperbolic sine of. * @return - * inverse hyperbolic sine of `columnName` - * + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ def asinh(columnName: String): Column = asinh(Column(columnName)) /** + * @param e + * the value to compute the inverse tangent of. A column that evaluates to a double. * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan` + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3399,9 +4984,11 @@ object functions { def atan(e: Column): Column = Column.fn("atan", e) /** + * @param columnName + * the value to compute the inverse tangent of. * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan` - * + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3409,13 +4996,13 @@ object functions { /** * @param y - * coordinate on y-axis + * coordinate on y-axis. A column that evaluates to a double. * @param x - * coordinate on x-axis + * coordinate on x-axis. A column that evaluates to a double. * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3430,8 +5017,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3445,8 +5031,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3460,8 +5045,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3476,8 +5060,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3491,8 +5074,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3506,8 +5088,7 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3521,16 +5102,17 @@ object functions { * @return * the theta component of the point (r, theta) in polar coordinates that * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2` - * + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) /** + * @param e + * target column to compute on. A column that evaluates to a numeric. * @return - * inverse hyperbolic tangent of `e` + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.1.0 @@ -3538,9 +5120,10 @@ object functions { def atanh(e: Column): Column = Column.fn("atanh", e) /** + * @param columnName + * target column to compute on. * @return - * inverse hyperbolic tangent of `columnName` - * + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. * @group math_funcs * @since 3.1.0 */ @@ -3550,8 +5133,12 @@ object functions { * An expression that returns the string representation of the binary value of the given long * column. For example, bin("12") returns "1100". * + * @param e + * target column to work on. A column that evaluates to an integral. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(e: Column): Column = Column.fn("bin", e) @@ -3559,81 +5146,128 @@ object functions { * An expression that returns the string representation of the binary value of the given long * column. For example, bin("12") returns "1100". * + * @param columnName + * target column to work on. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def bin(columnName: String): Column = bin(Column(columnName)) /** * Computes the cube-root of the given value. * + * @param e + * target column to compute on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(e: Column): Column = Column.fn("cbrt", e) /** * Computes the cube-root of the given column. * + * @param columnName + * target column to compute on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def cbrt(columnName: String): Column = cbrt(Column(columnName)) /** * Computes the ceiling of the given value of `e` to `scale` decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integral. Must be + * a constant. * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) /** * Computes the ceiling of the given value of `e` to 0 decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(e: Column): Column = Column.fn("ceil", e) /** * Computes the ceiling of the given value of `columnName` to 0 decimal places. * + * @param columnName + * the value to compute the ceiling on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceil(columnName: String): Column = ceil(Column(columnName)) /** * Computes the ceiling of the given value of `e` to `scale` decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integer. Must be + * a constant. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) /** * Computes the ceiling of the given value of `e` to 0 decimal places. * + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def ceiling(e: Column): Column = Column.fn("ceiling", e) /** * Convert a number in a string column from one base to another. * + * @param num + * a column to convert base for. A column that evaluates to a string. + * @param fromBase + * from base number. A column that evaluates to an integer. + * @param toBase + * to base number. A column that evaluates to an integer. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def conv(num: Column, fromBase: Int, toBase: Int): Column = Column.fn("conv", num, lit(fromBase), lit(toBase)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3642,10 +5276,10 @@ object functions { /** * @param columnName - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos` - * + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3653,9 +5287,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -3666,8 +5301,8 @@ object functions { * @param columnName * hyperbolic angle * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh` - * + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -3675,9 +5310,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cotangent of the angle + * cotangent of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3686,9 +5321,9 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * cosecant of the angle + * cosecant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -3700,70 +5335,107 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def e(): Column = Column.fn("e") /** * Computes the exponential of the given value. * + * @param e + * target column to compute on. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(e: Column): Column = Column.fn("exp", e) /** * Computes the exponential of the given column. * + * @param columnName + * target column to compute on. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def exp(columnName: String): Column = exp(Column(columnName)) /** * Computes the exponential of the given value minus one. * + * @param e + * column to calculate exponential for. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(e: Column): Column = Column.fn("expm1", e) /** * Computes the exponential of the given column minus one. * + * @param columnName + * column name to calculate exponential for. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def expm1(columnName: String): Column = expm1(Column(columnName)) /** * Computes the factorial of the given value. * + * @param e + * a column to calculate factorial for. A column that evaluates to an integral. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def factorial(e: Column): Column = Column.fn("factorial", e) /** * Computes the floor of the given value of `e` to `scale` decimal places. * + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to control the rounding behavior. A column that evaluates to + * an integral. * @group math_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) /** * Computes the floor of the given value of `e` to 0 decimal places. * + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(e: Column): Column = Column.fn("floor", e) /** * Computes the floor of the given column value to 0 decimal places. * + * @param columnName + * the target column name to compute the floor on. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a long or decimal. */ def floor(columnName: String): Column = floor(Column(columnName)) @@ -3771,8 +5443,12 @@ object functions { * Returns the greatest value of the list of values, skipping null values. This function takes * at least 2 parameters. It will return null iff all parameters are null. * + * @param exprs + * columns to check for greatest value. A column that evaluates to any type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) @@ -3781,8 +5457,14 @@ object functions { * Returns the greatest value of the list of column names, skipping null values. This function * takes at least 2 parameters. It will return null iff all parameters are null. * + * @param columnName + * the first column name to check for greatest value. A column of a comparable type. + * @param columnNames + * the remaining column names to check for greatest value. Columns of a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def greatest(columnName: String, columnNames: String*): Column = { @@ -3792,8 +5474,12 @@ object functions { /** * Computes hex value of the given column. * + * @param column + * target column to work on. A column that evaluates to an integral, string or binary. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def hex(column: Column): Column = Column.fn("hex", column) @@ -3801,40 +5487,68 @@ object functions { * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to * the byte representation of number. * + * @param column + * target column to work on. A column that evaluates to a string. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unhex(column: Column): Column = Column.fn("unhex", column) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, rightName: String): Column = hypot(Column(leftName), Column(rightName)) @@ -3842,32 +5556,56 @@ object functions { /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param leftName + * The a leg of the triangle. A column that evaluates to a numeric. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, r: Column): Column = hypot(lit(l), r) /** * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param rightName + * The b leg of the triangle. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) @@ -3875,8 +5613,12 @@ object functions { * Returns the least value of the list of values, skipping null values. This function takes at * least 2 parameters. It will return null iff all parameters are null. * + * @param exprs + * The values to be compared. Columns that evaluate to a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(exprs: Column*): Column = Column.fn("least", exprs: _*) @@ -3885,8 +5627,14 @@ object functions { * Returns the least value of the list of column names, skipping null values. This function * takes at least 2 parameters. It will return null iff all parameters are null. * + * @param columnName + * The name of the first column to be compared. A column of a comparable type. + * @param columnNames + * The names of the remaining columns to be compared. Columns of a comparable type. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def least(columnName: String, columnNames: String*): Column = { @@ -3896,96 +5644,152 @@ object functions { /** * Computes the natural logarithm of the given value. * + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def ln(e: Column): Column = Column.fn("ln", e) /** * Computes the natural logarithm of the given value. * + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(e: Column): Column = ln(e) /** * Computes the natural logarithm of the given column. * + * @param columnName + * The name of the column to compute the natural logarithm of. A column that evaluates to a + * numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(columnName: String): Column = log(Column(columnName)) /** * Returns the first argument-base logarithm of the second argument. * + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param a + * The value to compute the logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) /** * Returns the first argument-base logarithm of the second argument. * + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param columnName + * The name of the column to compute the logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log(base: Double, columnName: String): Column = log(base, Column(columnName)) /** * Computes the logarithm of the given value in base 10. * + * @param e + * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(e: Column): Column = Column.fn("log10", e) /** * Computes the logarithm of the given value in base 10. * + * @param columnName + * The name of the column to compute the base-10 logarithm of. A column that evaluates to a + * numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log10(columnName: String): Column = log10(Column(columnName)) /** * Computes the natural logarithm of the given value plus one. * + * @param e + * The value to compute the natural logarithm of the value plus one. A column that evaluates + * to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(e: Column): Column = Column.fn("log1p", e) /** * Computes the natural logarithm of the given column plus one. * + * @param columnName + * The name of the column to compute the natural logarithm of the value plus one. A column + * that evaluates to a numeric. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def log1p(columnName: String): Column = log1p(Column(columnName)) /** * Computes the logarithm of the given column in base 2. * + * @param expr + * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(expr: Column): Column = Column.fn("log2", expr) /** * Computes the logarithm of the given value in base 2. * + * @param columnName + * a column to calculate logarithm for. A column that evaluates to a double. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. */ def log2(columnName: String): Column = log2(Column(columnName)) /** * Returns the negated value. * + * @param e + * column to calculate negative value for. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def negative(e: Column): Column = Column.fn("negative", e) @@ -3994,94 +5798,162 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def pi(): Column = Column.fn("pi") /** * Returns the value. * + * @param e + * input value column. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def positive(e: Column): Column = Column.fn("positive", e) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Column): Column = Column.fn("power", l, r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param rightName + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. + * @param rightName + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Column, r: Double): Column = pow(l, lit(r)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param leftName + * the base number. + * @param r + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, r: Column): Column = pow(lit(l), r) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. + * @param rightName + * the exponent number. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) /** * Returns the value of the first argument raised to the power of the second argument. * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def power(l: Column, r: Column): Column = Column.fn("power", l, r) /** * Returns the positive value of dividend mod divisor. * + * @param dividend + * the column that contains dividend, or the specified dividend value. A column that evaluates + * to a numeric. + * @param divisor + * the column that contains divisor, or the specified divisor value. A column that evaluates + * to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) @@ -4089,8 +5961,12 @@ object functions { * Returns the double value that is closest in value to the argument and is equal to a * mathematical integer. * + * @param e + * target column to compute on. A column that evaluates to a double. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(e: Column): Column = Column.fn("rint", e) @@ -4098,16 +5974,24 @@ object functions { * Returns the double value that is closest in value to the argument and is equal to a * mathematical integer. * + * @param columnName + * the numeric column name to round to the closest integer. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def rint(columnName: String): Column = rint(Column(columnName)) /** * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. * + * @param e + * the value to round. A column that evaluates to a numeric. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column): Column = round(e, 0) @@ -4115,8 +5999,15 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) @@ -4124,16 +6015,27 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) /** * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. * + * @param e + * the value to round. A column that evaluates to a numeric. * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column): Column = bround(e, 0) @@ -4141,8 +6043,15 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) @@ -4150,16 +6059,23 @@ object functions { * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is * greater than or equal to 0 or at integral part when `scale` is less than 0. * + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * secant of the angle + * secant of the angle. Returns a column that evaluates to a double. * * @group math_funcs * @since 3.3.0 @@ -4172,6 +6088,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftleft", "3.2.0") def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) @@ -4180,8 +6098,15 @@ object functions { * Shift the given value numBits left. If the given value is a long value, this function will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift left. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) @@ -4191,6 +6116,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftright", "3.2.0") def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) @@ -4199,8 +6126,15 @@ object functions { * (Signed) shift the given value numBits right. If the given value is a long value, it will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) @@ -4210,6 +6144,8 @@ object functions { * * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @deprecated("Use shiftrightunsigned", "3.2.0") def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) @@ -4218,8 +6154,15 @@ object functions { * Unsigned shift the given value numBits right. If the given value is a long value, it will * return a long value else it will return an integer value. * + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. * @group bitwise_funcs * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ def shiftrightunsigned(e: Column, numBits: Int): Column = Column.fn("shiftrightunsigned", e, lit(numBits)) @@ -4227,32 +6170,45 @@ object functions { /** * Computes the signum of the given value. * + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def sign(e: Column): Column = Column.fn("sign", e) /** * Computes the signum of the given value. * + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(e: Column): Column = Column.fn("signum", e) /** * Computes the signum of the given column. * + * @param columnName + * column to compute the signum on. A column that evaluates to a numeric or interval. * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ def signum(columnName: String): Column = signum(Column(columnName)) /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * * @group math_funcs * @since 1.4.0 @@ -4261,10 +6217,10 @@ object functions { /** * @param columnName - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * sine of the angle, as if computed by `java.lang.Math.sin` - * + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * @group math_funcs * @since 1.4.0 */ @@ -4272,10 +6228,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4283,10 +6239,10 @@ object functions { /** * @param columnName - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh` - * + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4294,9 +6250,10 @@ object functions { /** * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 1.4.0 @@ -4305,10 +6262,10 @@ object functions { /** * @param columnName - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * tangent of the given value, as if computed by `java.lang.Math.tan` - * + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4316,10 +6273,10 @@ object functions { /** * @param e - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4327,10 +6284,10 @@ object functions { /** * @param columnName - * hyperbolic angle + * hyperbolic angle. A column that evaluates to a double. * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh` - * + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. * @group math_funcs * @since 1.4.0 */ @@ -4339,6 +6296,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(e: Column): Column = degrees(e) @@ -4346,6 +6305,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use degrees", "2.1.0") def toDegrees(columnName: String): Column = degrees(Column(columnName)) @@ -4355,9 +6316,10 @@ object functions { * degrees. * * @param e - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4369,10 +6331,10 @@ object functions { * degrees. * * @param columnName - * angle in radians + * angle in radians. A column that evaluates to a double. * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees` - * + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4381,6 +6343,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(e: Column): Column = radians(e) @@ -4388,6 +6352,8 @@ object functions { /** * @group math_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ @deprecated("Use radians", "2.1.0") def toRadians(columnName: String): Column = radians(Column(columnName)) @@ -4397,9 +6363,10 @@ object functions { * radians. * * @param e - * angle in degrees + * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * * @group math_funcs * @since 2.1.0 @@ -4411,10 +6378,10 @@ object functions { * radians. * * @param columnName - * angle in degrees + * angle in degrees. A column that evaluates to a double. * @return - * angle in radians, as if computed by `java.lang.Math.toRadians` - * + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. * @group math_funcs * @since 2.1.0 */ @@ -4426,15 +6393,17 @@ object functions { * method will return null. * * @param v - * value to compute a bucket number in the histogram + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. * @param min - * minimum value of the histogram + * minimum value of the histogram. A column that evaluates to a double or interval. * @param max - * maximum value of the histogram + * maximum value of the histogram. A column that evaluates to a double or interval. * @param numBucket - * the number of buckets + * the number of buckets. A column that evaluates to a long. * @return - * the bucket number into which the value would fall after being evaluated + * the bucket number into which the value would fall after being evaluated. Returns a column + * that evaluates to a long. * @group math_funcs * @since 3.5.0 */ @@ -4450,6 +6419,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_catalog(): Column = Column.fn("current_catalog") @@ -4458,6 +6429,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_database(): Column = Column.fn("current_database") @@ -4466,6 +6439,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_schema(): Column = Column.fn("current_schema") @@ -4474,6 +6449,8 @@ object functions { * * @group misc_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a string. */ def current_path(): Column = Column.fn("current_path") @@ -4482,6 +6459,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_user(): Column = Column.fn("current_user") @@ -4489,8 +6468,12 @@ object functions { * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex * string. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def md5(e: Column): Column = Column.fn("md5", e) @@ -4498,8 +6481,12 @@ object functions { * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex * string. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha1(e: Column): Column = Column.fn("sha1", e) @@ -4508,12 +6495,14 @@ object functions { * hex string. * * @param e - * column to compute SHA-2 on. + * column to compute SHA-2 on. A column that evaluates to a binary. * @param numBits - * one of 224, 256, 384, or 512. + * one of 224, 256, 384, or 512. A column that evaluates to an integer. * * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha2(e: Column, numBits: Int): Column = { require( @@ -4526,16 +6515,24 @@ object functions { * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value * as a bigint. * + * @param e + * target column to compute on. A column that evaluates to a binary. * @group hash_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def crc32(e: Column): Column = Column.fn("crc32", e) /** * Calculates the hash code of given columns, and returns the result as an int column. * + * @param cols + * one or more columns to compute on. A column of any type. * @group hash_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to an integer. */ @scala.annotation.varargs def hash(cols: Column*): Column = Column.fn("hash", cols: _*) @@ -4544,8 +6541,12 @@ object functions { * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, * and returns the result as a long column. The hash computation uses an initial seed of 42. * + * @param cols + * one or more columns to compute on. A column of any type. * @group hash_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a long. */ @scala.annotation.varargs def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) @@ -4553,24 +6554,38 @@ object functions { /** * Returns null if the condition is true, and throws an exception otherwise. * + * @param c + * The condition to check. A column that evaluates to a boolean. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column): Column = Column.fn("assert_true", c) /** * Returns null if the condition is true; throws an exception with the error message otherwise. * + * @param c + * The condition to check. A column that evaluates to a boolean. + * @param e + * The error message to throw. A column that evaluates to a string. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) /** * Throws an exception with the provided error message. * + * @param c + * The error message to throw. A column that evaluates to a string. * @group misc_funcs * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ def raise_error(c: Column): Column = Column.fn("raise_error", c) @@ -4579,6 +6594,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def user(): Column = Column.fn("user") @@ -4587,6 +6604,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def session_user(): Column = Column.fn("session_user") @@ -4596,6 +6615,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) @@ -4603,8 +6624,12 @@ object functions { * Returns an universally unique identifier (UUID) string. The value is returned as a canonical * UUID 36-character string. * + * @param seed + * The random number seed to use. A column that evaluates to an integral. Must be a constant. * @group misc_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def uuid(seed: Column): Column = Column.fn("uuid", seed) @@ -4619,25 +6644,29 @@ object functions { * GCM. * * @param input - * The binary value to encrypt. + * The binary value to encrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to encrypt the data. + * The passphrase to use to encrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param iv * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt( input: Column, @@ -4650,12 +6679,29 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv) @@ -4663,12 +6709,25 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding) @@ -4676,12 +6735,21 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_encrypt", input, key, mode) @@ -4689,12 +6757,18 @@ object functions { /** * Returns an encrypted value of `input`. * + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, * Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_encrypt(input: Column, key: Column): Column = Column.fn("aes_encrypt", input, key) @@ -4707,22 +6781,25 @@ object functions { * for decryption. The default mode is GCM. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt( input: Column, @@ -4735,11 +6812,24 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("aes_decrypt", input, key, mode, padding) @@ -4747,11 +6837,20 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("aes_decrypt", input, key, mode) @@ -4759,11 +6858,17 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def aes_decrypt(input: Column, key: Column): Column = Column.fn("aes_decrypt", input, key) @@ -4773,22 +6878,25 @@ object functions { * NULL value instead of raising an error if the decryption cannot be performed. * * @param input - * The binary value to decrypt. + * The binary value to decrypt. A column that evaluates to a binary. * @param key - * The passphrase to use to decrypt the data. + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @param mode * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. + * GCM, CBC. A column that evaluates to a string. * @param padding * Specifies how to pad messages whose length is not a multiple of the block size. Valid * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. + * for CBC. A column that evaluates to a string. * @param aad * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt( input: Column, @@ -4801,11 +6909,24 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = Column.fn("try_aes_decrypt", input, key, mode, padding) @@ -4813,11 +6934,20 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = Column.fn("try_aes_decrypt", input, key, mode) @@ -4825,11 +6955,17 @@ object functions { /** * Returns a decrypted value of `input`. * + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. * @see * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_aes_decrypt(input: Column, key: Column): Column = Column.fn("try_aes_decrypt", input, key) @@ -4837,8 +6973,12 @@ object functions { /** * Returns a sha1 hash value as a hex string of the `col`. * + * @param col + * The value to hash. A column that evaluates to a string or binary. * @group hash_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def sha(col: Column): Column = Column.fn("sha", col) @@ -4847,6 +6987,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_length(): Column = Column.fn("input_file_block_length") @@ -4855,6 +6997,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def input_file_block_start(): Column = Column.fn("input_file_block_start") @@ -4863,6 +7007,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) @@ -4872,6 +7018,8 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) @@ -4882,6 +7030,8 @@ object functions { * * @group misc_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) @@ -4892,14 +7042,20 @@ object functions { * * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def version(): Column = Column.fn("version") /** * Return DDL-formatted type string for the data type of the input. * + * @param col + * The value whose data type is returned. A column of any type. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def typeof(col: Column): Column = Column.fn("typeof", col) @@ -4907,8 +7063,13 @@ object functions { * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default * unless specified otherwise. * + * @param cols + * The first column must be a constant integer for the number of rows, and the remaining + * columns are the input elements to be separated into rows. * @group generator_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def stack(cols: Column*): Column = Column.fn("stack", cols: _*) @@ -4920,8 +7081,14 @@ object functions { * be an integer. Otherwise if one or both of these are floating-point numbers, then the result * will also be a floating-point number. * + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column): Column = uniform(min, max, lit(SparkClassUtils.random.nextLong)) @@ -4933,8 +7100,16 @@ object functions { * integers, then the result will also be an integer. Otherwise if one or both of these are * floating-point numbers, then the result will also be a floating-point number. * + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. * @group math_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def uniform(min: Column, max: Column, seed: Column): Column = Column.fn("uniform", min, max, seed) @@ -4943,8 +7118,12 @@ object functions { * Returns a random value with independent and identically distributed (i.i.d.) uniformly * distributed values in [0, 1). * + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(seed: Column): Column = Column.fn("random", seed) @@ -4954,14 +7133,20 @@ object functions { * * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def random(): Column = random(lit(SparkClassUtils.random.nextLong)) /** * Returns the bit position for the given input column. * + * @param col + * The input column. A column that evaluates to an integral. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bit_position(col: Column): Column = Column.fn("bitmap_bit_position", col) @@ -4969,8 +7154,12 @@ object functions { /** * Returns the bucket number for the given input column. * + * @param col + * The input column. A column that evaluates to an integral. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_bucket_number(col: Column): Column = Column.fn("bitmap_bucket_number", col) @@ -4979,8 +7168,13 @@ object functions { * Returns a bitmap with the positions of the bits set from all the values from the input * column. The input column will most likely be bitmap_bit_position(). * + * @param col + * The input column will most likely be bitmap_bit_position(). A column that evaluates to an + * integral. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_construct_agg(col: Column): Column = Column.fn("bitmap_construct_agg", col) @@ -4988,8 +7182,12 @@ object functions { /** * Returns the number of set bits in the input bitmap. * + * @param col + * The input bitmap. A column that evaluates to a binary. * @group misc_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) @@ -4997,8 +7195,13 @@ object functions { * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The * input column should be bitmaps created from bitmap_construct_agg(). * + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) @@ -5006,8 +7209,13 @@ object functions { * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The * input column should be bitmaps created from bitmap_construct_agg(). * + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. * @group agg_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) @@ -5019,8 +7227,12 @@ object functions { * Computes the numeric value of the first character of the string column, and returns the * result as an int column. * + * @param e + * The target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def ascii(e: Column): Column = Column.fn("ascii", e) @@ -5028,16 +7240,24 @@ object functions { * Computes the BASE64 encoding of a binary column and returns it as a string column. This is * the reverse of unbase64. * + * @param e + * The target column to work on. A column that evaluates to a binary. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def base64(e: Column): Column = Column.fn("base64", e) /** * Calculates the bit length for the specified string column. * + * @param e + * The source column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def bit_length(e: Column): Column = Column.fn("bit_length", e) @@ -5045,11 +7265,18 @@ object functions { * Concatenates multiple input string columns together into a single string column, using the * given separator. * + * @param sep + * The words separator. A column that evaluates to a string. Must be a constant. + * @param exprs + * The list of columns to work on. Each a column that evaluates to a string or an array of + * strings. * @note * Input strings which are null are skipped. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def concat_ws(sep: String, exprs: Column*): Column = @@ -5060,8 +7287,14 @@ object functions { * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either * argument is null, the result will also be null. * + * @param value + * The target column to work on. A column that evaluates to a binary. + * @param charset + * The charset to use to decode to. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def decode(value: Column, charset: String): Column = Column.fn("decode", value, lit(charset)) @@ -5071,8 +7304,14 @@ object functions { * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either * argument is null, the result will also be null. * + * @param value + * The target column to work on. A column that evaluates to a string. + * @param charset + * The charset to use to encode. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def encode(value: Column, charset: String): Column = Column.fn("encode", value, lit(charset)) @@ -5080,8 +7319,13 @@ object functions { /** * Returns true if the input is a valid UTF-8 string, otherwise returns false. * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_utf8(str: Column): Column = Column.fn("is_valid_utf8", str) @@ -5090,8 +7334,13 @@ object functions { * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the * Unicode replacement character (U+FFFD). * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def make_valid_utf8(str: Column): Column = Column.fn("make_valid_utf8", str) @@ -5100,8 +7349,13 @@ object functions { * Returns the input value if it corresponds to a valid UTF-8 string, or emits a * SparkIllegalArgumentException exception otherwise. * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def validate_utf8(str: Column): Column = Column.fn("validate_utf8", str) @@ -5109,8 +7363,12 @@ object functions { /** * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. * + * @param str + * the input value. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_validate_utf8(str: Column): Column = Column.fn("try_validate_utf8", str) @@ -5122,16 +7380,29 @@ object functions { * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the * result will be null. * + * @param x + * the numeric value to be formatted. A column that evaluates to a numeric. + * @param d + * the number of decimal places. A column that evaluates to an integral. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) /** * Formats the arguments in printf-style and returns the result as a string column. * + * @param format + * the format string that can contain embedded format tags. A column that evaluates to a + * string. Must be a constant. + * @param arguments + * the values to be used in formatting. Columns that evaluate to any type. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def format_string(format: String, arguments: Column*): Column = @@ -5143,8 +7414,12 @@ object functions { * * For example, "hello world" will become "Hello World". * + * @param e + * the target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def initcap(e: Column): Column = Column.fn("initcap", e) @@ -5152,12 +7427,18 @@ object functions { * Locate the position of the first occurrence of substr column in the given string. Returns * null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. Must be a constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: String): Column = instr(str, lit(substring)) @@ -5165,12 +7446,18 @@ object functions { * Locate the position of the first occurrence of substr column in the given string. Returns * null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) @@ -5178,6 +7465,13 @@ object functions { * Locate the position of the first occurrence of `substring` in `str`, starting the search from * position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5187,6 +7481,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int): Column = Column.fn("instr", str, substring, lit(start)) @@ -5195,6 +7491,12 @@ object functions { * Locate the position of the first occurrence of `substring` in `str`, starting the search from * position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5204,6 +7506,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column): Column = Column.fn("instr", str, substring, start) @@ -5212,6 +7516,16 @@ object functions { * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the * search from position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. Must + * be a constant. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5223,6 +7537,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = Column.fn("instr", str, substring, lit(start), lit(occurrence)) @@ -5231,6 +7547,14 @@ object functions { * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the * search from position `start`. Returns null if either of the arguments are null. * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. @@ -5242,6 +7566,8 @@ object functions { * * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an integer. */ def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = Column.fn("instr", str, substring, start, occurrence) @@ -5251,8 +7577,12 @@ object functions { * length of character strings include the trailing spaces. The length of binary strings * includes binary zeros. * + * @param e + * the target column to work on. A column that evaluates to a string or binary. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def length(e: Column): Column = Column.fn("length", e) @@ -5261,24 +7591,39 @@ object functions { * length of character strings include the trailing spaces. The length of binary strings * includes binary zeros. * + * @param e + * the target column to work on. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def len(e: Column): Column = Column.fn("len", e) /** * Converts a string column to lower case. * + * @param e + * the target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def lower(e: Column): Column = Column.fn("lower", e) /** * Computes the Levenshtein distance of the two given string columns if it's less than or equal * to a given threshold. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @param threshold + * the maximum distance to compute. A column that evaluates to an integral. Must be a + * constant. * @return - * result distance, or -1 + * result distance, or -1. Returns a column that evaluates to an integer. * @group string_funcs * @since 3.5.0 */ @@ -5287,16 +7632,28 @@ object functions { /** * Computes the Levenshtein distance of the two given string columns. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) /** * Computes the Jaro-Winkler similarity between the two given string columns. The result is a * double between 0.0 (no similarity) and 1.0 (identical). + * @param l + * A column that evaluates to a string. + * @param r + * A column that evaluates to a string. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a double. */ def jaro_winkler_similarity(l: Column, r: Column): Column = Column.fn("jaro_winkler_similarity", l, r) @@ -5304,24 +7661,38 @@ object functions { /** * Locate the position of the first occurrence of substr. * + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. * @note * The position is not zero based, but 1 based index. Returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) /** * Locate the position of the first occurrence of substr in a string column, after position pos. * + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. + * @param pos + * The starting position. A column that evaluates to an integer. * @note * The position is not zero based, but 1 based index. returns 0 if substr could not be found * in str. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def locate(substr: String, str: Column, pos: Int): Column = Column.fn("locate", lit(substr), str, lit(pos)) @@ -5330,8 +7701,16 @@ object functions { * Left-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) @@ -5339,8 +7718,16 @@ object functions { * Left-pad the binary column with pad to a byte length of len. If the binary column is longer * than len, the return value is shortened to len bytes. * + * @param str + * A column that evaluates to a binary. + * @param len + * The byte length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding bytes. A column that evaluates to a binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) @@ -5348,78 +7735,134 @@ object functions { * Left-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) /** * Trim the spaces from left end for the specified string value. * + * @param e + * A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column): Column = Column.fn("ltrim", e) /** * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trimString + * The trim string. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) /** * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trim + * The trim string. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) /** * Calculates the byte length for the specified string column. * + * @param e + * A column that evaluates to a string or binary. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to an integer. */ def octet_length(e: Column): Column = Column.fn("octet_length", e) /** * Marks a given column with specified collation. * + * @param e + * A column that evaluates to a string. + * @param collation + * The collation name. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) /** * Returns the collation name of a given column. * + * @param e + * A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def collation(e: Column): Column = Column.fn("collation", e) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) /** * Returns true if `str` matches `regexp`, or false otherwise. * + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) @@ -5427,8 +7870,14 @@ object functions { * Returns a count of the number of times that the regular expression pattern `regexp` is * matched in the string `str`. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) @@ -5438,8 +7887,16 @@ object functions { * the specified group index exceeds the group count of regex, an IllegalArgumentException will * be thrown. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param exp + * regex pattern to apply. A string. Must be a constant. + * @param groupIdx + * matched group id. An integer. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) @@ -5448,8 +7905,14 @@ object functions { * Extract all strings in the `str` that match the `regexp` expression and corresponding to the * first regex group index. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column): Column = Column.fn("regexp_extract_all", str, regexp) @@ -5458,8 +7921,16 @@ object functions { * Extract all strings in the `str` that match the `regexp` expression and corresponding to the * regex group index. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_extract_all", str, regexp, idx) @@ -5467,8 +7938,16 @@ object functions { /** * Replace all substrings of the specified string value that match regexp with rep. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String): Column = regexp_replace(e, lit(pattern), lit(replacement)) @@ -5477,8 +7956,18 @@ object functions { * Replace all substrings of the specified string value that match regexp with rep, starting at * the specified position `pos`. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @param pos + * position to start replacement. The first position is 1. An integer. Must be a constant. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) @@ -5486,8 +7975,16 @@ object functions { /** * Replace all substrings of the specified string value that match regexp with rep. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. * @group string_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = Column.fn("regexp_replace", e, pattern, replacement) @@ -5496,8 +7993,19 @@ object functions { * Replace all substrings of the specified string value that match regexp with rep, starting at * the specified position `pos`. * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @param pos + * position to start replacement. The first position is 1. A column that evaluates to an + * integer. * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = Column.fn("regexp_replace", e, pattern, replacement, pos) @@ -5506,8 +8014,14 @@ object functions { * Returns the substring that matches the regular expression `regexp` within the string `str`. * If the regular expression is not found, the result is null. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) @@ -5516,8 +8030,14 @@ object functions { * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match * is found, returns 0. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) @@ -5526,8 +8046,16 @@ object functions { * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match * is found, returns 0. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def regexp_instr(str: Column, regexp: Column, idx: Column): Column = Column.fn("regexp_instr", str, regexp, idx) @@ -5536,8 +8064,12 @@ object functions { * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse * of base64. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ def unbase64(e: Column): Column = Column.fn("unbase64", e) @@ -5545,8 +8077,16 @@ object functions { * Right-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param len + * length of the final string. An integer. Must be a constant. + * @param pad + * chars to append. A string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) @@ -5554,8 +8094,16 @@ object functions { * Right-pad the binary column with pad to a byte length of len. If the binary column is longer * than len, the return value is shortened to len bytes. * + * @param str + * target column to work on. A column that evaluates to a binary. + * @param len + * byte length of the final binary. An integer. Must be a constant. + * @param pad + * bytes to append. A binary. Must be a constant. * @group string_funcs * @since 3.3.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) @@ -5563,54 +8111,95 @@ object functions { * Right-pad the string column with pad to a length of len. If the string column is longer than * len, the return value is shortened to len characters. * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param len + * length of the final result. A column that evaluates to an integer. + * @param pad + * chars or bytes to append. A column that evaluates to a string or binary. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) /** * Repeats a string column n times, and returns it as a new string column. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. Must be a + * constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) /** * Repeats a string column n times, and returns it as a new string column. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) /** * Trim the spaces from right end for the specified string value. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column): Column = Column.fn("rtrim", e) /** * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trimString + * the trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) /** * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trim + * the trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) /** * Returns the soundex code for the specified expression. * + * @param e + * target column to work on. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def soundex(e: Column): Column = Column.fn("soundex", e) @@ -5618,13 +8207,15 @@ object functions { * Splits str around matches of the given pattern. * * @param str - * a string expression to split + * a string expression to split. A column that evaluates to a string. * @param pattern * a string representing a regular expression. The regex string should be a Java regular - * expression. + * expression. A column that evaluates to a string. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) @@ -5632,13 +8223,15 @@ object functions { * Splits str around matches of the given pattern. * * @param str - * a string expression to split + * a string expression to split. A column that evaluates to a string. * @param pattern * a column of string representing a regular expression. The regex string should be a Java - * regular expression. + * regular expression. A column that evaluates to a string. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) @@ -5646,19 +8239,21 @@ object functions { * Splits str around matches of the given pattern. * * @param str - * a string expression to split + * a string expression to split. A column that evaluates to a string. * @param pattern * a string representing a regular expression. The regex string should be a Java regular - * expression. + * expression. A column that evaluates to a string. * @param limit * an integer expression which controls the number of times the regex is applied. + * the resulting array can be of any size. A column that evaluates to an integer. * * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: String, limit: Int): Column = Column.fn("split", str, lit(pattern), lit(limit)) @@ -5667,19 +8262,22 @@ object functions { * Splits str around matches of the given pattern. * * @param str - * a string expression to split + * a string expression to split. A column that evaluates to a string. * @param pattern * a column of string representing a regular expression. The regex string should be a Java - * regular expression. + * regular expression. A column that evaluates to a string. * @param limit * a column of integer expression which controls the number of times the regex is applied. * + * possible, and the resulting array can be of any size. A column that evaluates to + * an integer. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def split(str: Column, pattern: Column, limit: Column): Column = Column.fn("split", str, pattern, limit) @@ -5688,11 +8286,19 @@ object functions { * Substring starts at `pos` and is of length `len` when str is String type or returns the slice * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. Must be a constant. + * @param len + * length of chars. A column that evaluates to an integral. Must be a constant. * @note * The position is not zero based, but 1 based index. * * @group string_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Int, len: Int): Column = Column.fn("substring", str, lit(pos), lit(len)) @@ -5701,11 +8307,19 @@ object functions { * Substring starts at `pos` and is of length `len` when str is String type or returns the slice * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. + * @param len + * length of chars. A column that evaluates to an integral. * @note * The position is not zero based, but 1 based index. * * @group string_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def substring(str: Column, pos: Column, len: Column): Column = Column.fn("substring", str, pos, len) @@ -5716,7 +8330,15 @@ object functions { * returned. If count is negative, every to the right of the final delimiter (counting from the * right) is returned. substring_index performs a case-sensitive match when searching for delim. * + * @param str + * target column to work on. A column that evaluates to a string. + * @param delim + * delimiter of values. A column that evaluates to a string. Must be a constant. + * @param count + * number of occurrences. A column that evaluates to an integral. Must be a constant. * @group string_funcs + * @return + * Returns a column that evaluates to a string. */ def substring_index(str: Column, delim: String, count: Int): Column = Column.fn("substring_index", str, lit(delim), lit(count)) @@ -5725,8 +8347,18 @@ object functions { * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of * `src` and proceeding for `len` bytes. * + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @param len + * the number of bytes to replace in src. A column that evaluates to an integral. * @group string_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the input. */ def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = Column.fn("overlay", src, replace, pos, len) @@ -5735,16 +8367,32 @@ object functions { * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of * `src`. * + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. * @group string_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def overlay(src: Column, replace: Column, pos: Column): Column = Column.fn("overlay", src, replace, pos) /** * Splits a string into arrays of sentences, where each sentence is an array of words. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @param country + * a country of the locale. A column that evaluates to a string. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column, country: Column): Column = Column.fn("sentences", string, language, country) @@ -5752,8 +8400,14 @@ object functions { /** * Splits a string into arrays of sentences, where each sentence is an array of words. The * default `country`('') is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column, language: Column): Column = Column.fn("sentences", string, language) @@ -5761,8 +8415,12 @@ object functions { /** * Splits a string into arrays of sentences, where each sentence is an array of words. The * default locale is used. + * @param string + * a string to be split. A column that evaluates to a string. * @group string_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. */ def sentences(string: Column): Column = Column.fn("sentences", string) @@ -5771,8 +8429,16 @@ object functions { * replaceString correspond to the characters in matchingString. The translate will happen when * any character in the string matches the character in the `matchingString`. * + * @param src + * source column to work on. A column that evaluates to a string. + * @param matchingString + * matching characters. A column that evaluates to a string. Must be a constant. + * @param replaceString + * characters for replacement. A column that evaluates to a string. Must be a constant. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def translate(src: Column, matchingString: String, replaceString: String): Column = Column.fn("translate", src, lit(matchingString), lit(replaceString)) @@ -5780,30 +8446,50 @@ object functions { /** * Trim the spaces from both ends for the specified string column. * + * @param e + * The string column to trim. A column that evaluates to a string. * @group string_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column): Column = Column.fn("trim", e) /** * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trimString + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) /** * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) /** * Converts a string column to upper case. * + * @param e + * The input column to convert to upper case. A column that evaluates to a string. * @group string_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a string. */ def upper(e: Column): Column = Column.fn("upper", e) @@ -5813,8 +8499,15 @@ object functions { * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at * least one of the input parameters is NULL. * + * @param e + * The input value to convert. A column that evaluates to a string. + * @param f + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) @@ -5822,8 +8515,12 @@ object functions { * Converts the input `e` to a binary value based on the default format "hex". The function * returns NULL if at least one of the input parameters is NULL. * + * @param e + * The input value to convert. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def to_binary(e: Column): Column = Column.fn("to_binary", e) @@ -5851,8 +8548,16 @@ object functions { * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input * binary is decoded to UTF-8 string. * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) @@ -5881,8 +8586,16 @@ object functions { * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input * binary is decoded to UTF-8 string. * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) @@ -5904,8 +8617,15 @@ object functions { * Only allowed at the end of the format string; specifies that 'expr' indicates a negative * number with wrapping angled brackets. * + * @param e + * The input string to convert to a number. A column that evaluates to a string. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) @@ -5913,15 +8633,18 @@ object functions { * Replaces all occurrences of `search` with `replace`. * * @param src - * A column of string to be replaced + * A column of strings to be replaced. A column that evaluates to a string. * @param search - * A column of string, If `search` is not found in `str`, `str` is returned unchanged. + * A column of strings. If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. * @param replace - * A column of string, If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. + * A column of strings. If `replace` is not specified or is an empty string, nothing replaces + * the string that is removed from `str`. A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column, replace: Column): Column = Column.fn("replace", src, search, replace) @@ -5930,12 +8653,15 @@ object functions { * Replaces all occurrences of `search` with `replace`. * * @param src - * A column of string to be replaced + * A column of strings to be replaced. A column that evaluates to a string. * @param search - * A column of string, If `search` is not found in `src`, `src` is returned unchanged. + * A column of strings. If `search` is not found in `src`, `src` is returned unchanged. A + * column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) @@ -5945,8 +8671,16 @@ object functions { * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. * + * @param str + * A column of strings to be split. A column that evaluates to a string. + * @param delimiter + * The delimiter used for split. A column that evaluates to a string. + * @param partNum + * The requested part of the split (1-based). A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def split_part(str: Column, delimiter: Column, partNum: Column): Column = Column.fn("split_part", str, delimiter, partNum) @@ -5955,8 +8689,16 @@ object functions { * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of * byte array that starts at `pos` and is of length `len`. * + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @param len + * The length of the substring. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column, len: Column): Column = Column.fn("substr", str, pos, len) @@ -5965,16 +8707,30 @@ object functions { * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts * at `pos`. * + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("try_parse_url", url, partToExtract, key) @@ -5982,8 +8738,14 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_parse_url(url: Column, partToExtract: Column): Column = Column.fn("try_parse_url", url, partToExtract) @@ -5991,8 +8753,16 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column, key: Column): Column = Column.fn("parse_url", url, partToExtract, key) @@ -6000,8 +8770,14 @@ object functions { /** * Extracts a part from a URL. * + * @param url + * A column representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def parse_url(url: Column, partToExtract: Column): Column = Column.fn("parse_url", url, partToExtract) @@ -6009,8 +8785,14 @@ object functions { /** * Formats the arguments in printf-style and returns the result as a string column. * + * @param format + * A format string that can contain embedded format tags. A column that evaluates to a string. + * @param arguments + * The values to be used in formatting. Columns that evaluate to any type. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def printf(format: Column, arguments: Column*): Column = @@ -6020,8 +8802,12 @@ object functions { * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding * scheme. * + * @param str + * A URL-encoded string. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_decode(str: Column): Column = Column.fn("url_decode", str) @@ -6029,8 +8815,12 @@ object functions { * This is a special version of `url_decode` that performs the same operation, but returns a * NULL value instead of raising an error if the decoding cannot be performed. * + * @param str + * A URL-encoded string. A column that evaluates to a string. * @group url_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) @@ -6038,8 +8828,12 @@ object functions { * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding * scheme. * + * @param str + * A string to encode. A column that evaluates to a string. * @group url_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def url_encode(str: Column): Column = Column.fn("url_encode", str) @@ -6047,8 +8841,16 @@ object functions { * Returns the position of the first occurrence of `substr` in `str` after position `start`. The * given `start` and return value are 1-based. * + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @param start + * The 1-based position to start the search from. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column, start: Column): Column = Column.fn("position", substr, str, start) @@ -6057,8 +8859,14 @@ object functions { * Returns the position of the first occurrence of `substr` in `str` after position `1`. The * return value are 1-based. * + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def position(substr: Column, str: Column): Column = Column.fn("position", substr, str) @@ -6068,8 +8876,14 @@ object functions { * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY * type. * + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param suffix + * The suffix to test for. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def endswith(str: Column, suffix: Column): Column = Column.fn("endswith", str, suffix) @@ -6079,8 +8893,14 @@ object functions { * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY * type. * + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param prefix + * The prefix to test for. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def startswith(str: Column, prefix: Column): Column = Column.fn("startswith", str, prefix) @@ -6089,24 +8909,38 @@ object functions { * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the * result is equivalent to char(n % 256) * + * @param n + * The code point value. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def char(n: Column): Column = Column.fn("char", n) /** * Removes the leading and trailing space characters from `str`. * + * @param str + * The string to trim. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column): Column = Column.fn("btrim", str) /** * Remove the leading and trailing `trim` characters from `str`. * + * @param str + * The string to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) @@ -6114,8 +8948,15 @@ object functions { * This is a special version of `to_binary` that performs the same operation, but returns a NULL * value instead of raising an error if the conversion cannot be performed. * + * @param e + * The string to convert. A column that evaluates to a string. + * @param f + * The format to use for the conversion. A column that evaluates to a string. Must be a + * constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) @@ -6123,8 +8964,12 @@ object functions { * This is a special version of `to_binary` that performs the same operation, but returns a NULL * value instead of raising an error if the conversion cannot be performed. * + * @param e + * The string to convert. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) @@ -6133,8 +8978,15 @@ object functions { * string `e` does not match the expected format. The format follows the same semantics as the * to_number function. * + * @param e + * The string to convert. A column that evaluates to a string. + * @param format + * The format used to convert the string to a number. A column that evaluates to a string. + * Must be a constant. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a decimal. */ def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) @@ -6142,8 +8994,12 @@ object functions { * Returns the character length of string data or number of bytes of binary data. The length of * string data includes the trailing spaces. The length of binary data includes binary zeros. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def char_length(str: Column): Column = Column.fn("char_length", str) @@ -6151,8 +9007,12 @@ object functions { * Returns the character length of string data or number of bytes of binary data. The length of * string data includes the trailing spaces. The length of binary data includes binary zeros. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def character_length(str: Column): Column = Column.fn("character_length", str) @@ -6160,8 +9020,12 @@ object functions { * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the * result is equivalent to chr(n % 256) * + * @param n + * The code point. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def chr(n: Column): Column = Column.fn("chr", n) @@ -6170,8 +9034,14 @@ object functions { * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or * BINARY type. * + * @param left + * The input to check, may be NULL. A column that evaluates to a string or binary. + * @param right + * The input to find, may be NULL. A column that evaluates to a string or binary. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) @@ -6181,8 +9051,13 @@ object functions { * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid * indices. * + * @param inputs + * The index followed by the inputs to select from. Columns where the first evaluates to an + * integral and the rest evaluate to strings or binaries. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) @@ -6192,8 +9067,14 @@ object functions { * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains * a comma. * + * @param str + * The given string to be found. A column that evaluates to a string. + * @param strArray + * The comma-delimited list. A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) @@ -6201,8 +9082,16 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, * false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("like", str, pattern, escapeChar) @@ -6211,8 +9100,14 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, * false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) @@ -6220,8 +9115,16 @@ object functions { * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any * arguments are null, false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column, escapeChar: Column): Column = Column.fn("ilike", str, pattern, escapeChar) @@ -6230,24 +9133,38 @@ object functions { * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any * arguments are null, false otherwise. * + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) /** * Returns `str` with all characters changed to lowercase. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def lcase(str: Column): Column = Column.fn("lcase", str) /** * Returns `str` with all characters changed to uppercase. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def ucase(str: Column): Column = Column.fn("ucase", str) @@ -6255,8 +9172,14 @@ object functions { * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if * `len` is less or equal than 0 the result is an empty string. * + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @param len + * The number of leftmost characters. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def left(str: Column, len: Column): Column = Column.fn("left", str, len) @@ -6264,8 +9187,14 @@ object functions { * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if * `len` is less or equal than 0 the result is an empty string. * + * @param str + * Input column or strings. A column that evaluates to a string. + * @param len + * The number of rightmost characters. A column that evaluates to an integral. * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def right(str: Column, len: Column): Column = Column.fn("right", str, len) @@ -6273,8 +9202,12 @@ object functions { * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded * by a backslash. * + * @param str + * A column that evaluates to a string. * @group string_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def quote(str: Column): Column = Column.fn("quote", str) @@ -6286,8 +9219,12 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches HllSketch. * + * @param c + * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) @@ -6295,8 +9232,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches HllSketch. * + * @param columnName + * Name of the column containing the binary representation of a Datasketches HllSketch. A + * column that evaluates to a binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def hll_sketch_estimate(columnName: String): Column = { hll_sketch_estimate(Column(columnName)) @@ -6306,8 +9248,16 @@ object functions { * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches * Union object. Throws an exception if sketches have different lgConfigK values. * + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column): Column = Column.fn("hll_union", c1, c2) @@ -6316,8 +9266,16 @@ object functions { * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches * Union object. Throws an exception if sketches have different lgConfigK values. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(columnName1: String, columnName2: String): Column = { hll_union(Column(columnName1), Column(columnName2)) @@ -6328,8 +9286,19 @@ object functions { * Union object. Throws an exception if sketches have different lgConfigK values and * allowDifferentLgConfigK is set to false. * + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) @@ -6339,8 +9308,19 @@ object functions { * Union object. Throws an exception if sketches have different lgConfigK values and * allowDifferentLgConfigK is set to false. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. * @group sketch_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ def hll_union( columnName1: String, @@ -6353,8 +9333,16 @@ object functions { * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns * using a Datasketches AnotB object * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(c1: Column, c2: Column): Column = Column.fn("theta_difference", c1, c2) @@ -6363,8 +9351,16 @@ object functions { * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns * using a Datasketches AnotB object * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_difference(columnName1: String, columnName2: String): Column = { theta_difference(Column(columnName1), Column(columnName2)) @@ -6374,8 +9370,16 @@ object functions { * Intersects two binary representations of Datasketches ThetaSketch objects in the input * columns using a Datasketches Intersection object * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(c1: Column, c2: Column): Column = Column.fn("theta_intersection", c1, c2) @@ -6384,8 +9388,16 @@ object functions { * Intersects two binary representations of Datasketches ThetaSketch objects in the input * columns using a Datasketches Intersection object * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_intersection(columnName1: String, columnName2: String): Column = { theta_intersection(Column(columnName1), Column(columnName2)) @@ -6395,8 +9407,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches ThetaSketch. * + * @param c + * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) @@ -6404,8 +9421,12 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches ThetaSketch. * + * @param columnName + * Name of the column containing the binary representation of a Datasketches ThetaSketch. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def theta_sketch_estimate(columnName: String): Column = { theta_sketch_estimate(Column(columnName)) @@ -6416,8 +9437,16 @@ object functions { * using a Datasketches Union object. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column): Column = Column.fn("theta_union", c1, c2) @@ -6427,8 +9456,16 @@ object functions { * using a Datasketches Union object. It is configured with the default value of 12 for * `lgNomEntries`. * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String): Column = { theta_union(Column(columnName1), Column(columnName2)) @@ -6439,8 +9476,19 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("theta_union", c1, c2, lit(lgNomEntries)) @@ -6450,8 +9498,17 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param columnName1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param columnName2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { theta_union(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6462,8 +9519,17 @@ object functions { * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal * entries for the union buffer. * + * @param c1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param c2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = Column.fn("theta_union", c1, c2, lgNomEntries) @@ -6473,8 +9539,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_double", c1, c2) @@ -6484,8 +9556,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_double(columnName1: String, columnName2: String): Column = tuple_difference_double(Column(columnName1), Column(columnName2)) @@ -6495,8 +9573,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_integer", c1, c2) @@ -6506,8 +9590,14 @@ object functions { * data type in the input columns using a Datasketches AnotB object. Returns elements in the * first sketch that are not in the second sketch. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_integer(columnName1: String, columnName2: String): Column = tuple_difference_integer(Column(columnName1), Column(columnName2)) @@ -6518,8 +9608,14 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_double", c1, c2) @@ -6530,8 +9626,14 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2)) @@ -6542,8 +9644,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_double", c1, c2, lit(mode)) @@ -6554,8 +9665,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_double(Column(columnName1), Column(columnName2), mode) @@ -6566,8 +9686,17 @@ object functions { * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, * alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_double", c1, c2, mode) @@ -6578,8 +9707,14 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_integer", c1, c2) @@ -6590,8 +9725,14 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2)) @@ -6602,8 +9743,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) @@ -6614,8 +9764,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) @@ -6626,8 +9785,17 @@ object functions { * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, * max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_integer", c1, c2, mode) @@ -6636,8 +9804,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with double summary data type. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(c: Column): Column = Column.fn("tuple_sketch_estimate_double", c) @@ -6646,8 +9819,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with double summary data type. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_double(columnName: String): Column = tuple_sketch_estimate_double(Column(columnName)) @@ -6656,8 +9834,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with integer summary data type. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(c: Column): Column = Column.fn("tuple_sketch_estimate_integer", c) @@ -6666,8 +9849,13 @@ object functions { * Returns the estimated number of unique values given the binary representation of a * Datasketches TupleSketch with integer summary data type. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_estimate_integer(columnName: String): Column = tuple_sketch_estimate_integer(Column(columnName)) @@ -6677,8 +9865,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column): Column = Column.fn("tuple_sketch_summary_double", c) @@ -6688,8 +9881,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String): Column = tuple_sketch_summary_double(Column(columnName)) @@ -6698,8 +9896,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_double", c, lit(mode)) @@ -6708,8 +9914,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(columnName: String, mode: String): Column = tuple_sketch_summary_double(Column(columnName), mode) @@ -6718,8 +9932,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_summary_double(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_double", c, mode) @@ -6729,8 +9951,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column): Column = Column.fn("tuple_sketch_summary_integer", c) @@ -6740,8 +9967,13 @@ object functions { * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is * configured with the default mode of 'sum'. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String): Column = tuple_sketch_summary_integer(Column(columnName)) @@ -6750,8 +9982,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: String): Column = Column.fn("tuple_sketch_summary_integer", c, lit(mode)) @@ -6760,8 +10000,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(columnName: String, mode: String): Column = tuple_sketch_summary_integer(Column(columnName), mode) @@ -6770,8 +10018,16 @@ object functions { * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def tuple_sketch_summary_integer(c: Column, mode: Column): Column = Column.fn("tuple_sketch_summary_integer", c, mode) @@ -6781,8 +10037,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(c: Column): Column = Column.fn("tuple_sketch_theta_double", c) @@ -6792,8 +10053,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_double(columnName: String): Column = tuple_sketch_theta_double(Column(columnName)) @@ -6803,8 +10069,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(c: Column): Column = Column.fn("tuple_sketch_theta_integer", c) @@ -6814,8 +10085,13 @@ object functions { * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 * and 1.0. * + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ def tuple_sketch_theta_integer(columnName: String): Column = tuple_sketch_theta_integer(Column(columnName)) @@ -6825,8 +10101,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_double", c1, c2) @@ -6836,8 +10118,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String): Column = tuple_union_double(Column(columnName1), Column(columnName2)) @@ -6848,8 +10136,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) @@ -6860,8 +10157,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6872,8 +10178,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6884,8 +10202,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double( columnName1: String, @@ -6900,8 +10230,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) @@ -6911,8 +10253,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_integer", c1, c2) @@ -6922,8 +10270,14 @@ object functions { * data type in the input columns using a Datasketches Union object. It is configured with the * default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String): Column = tuple_union_integer(Column(columnName1), Column(columnName2)) @@ -6934,8 +10288,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) @@ -6946,8 +10309,17 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of * 'sum'. * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) @@ -6958,8 +10330,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -6970,8 +10354,20 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer( columnName1: String, @@ -6986,8 +10382,18 @@ object functions { * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for * numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) @@ -6997,8 +10403,14 @@ object functions { * double summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_double", c1, c2) @@ -7008,8 +10420,14 @@ object functions { * double summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = tuple_difference_theta_double(Column(columnName1), Column(columnName2)) @@ -7019,8 +10437,14 @@ object functions { * integer summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_difference_theta_integer", c1, c2) @@ -7030,8 +10454,14 @@ object functions { * integer summary data type in the input columns using a Datasketches AnotB object. Returns * elements in the TupleSketch that are not in the ThetaSketch. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) @@ -7042,8 +10472,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2) @@ -7054,8 +10490,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) @@ -7066,8 +10508,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) @@ -7078,8 +10529,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double( columnName1: String, @@ -7093,8 +10553,16 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_double", c1, c2, mode) @@ -7105,8 +10573,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2) @@ -7117,8 +10591,14 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) @@ -7129,8 +10609,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) @@ -7141,8 +10630,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer( columnName1: String, @@ -7156,8 +10654,17 @@ object functions { * object. The mode parameter specifies the aggregation mode for numeric summaries during * intersection (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = Column.fn("tuple_intersection_theta_integer", c1, c2, mode) @@ -7167,8 +10674,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_double", c1, c2) @@ -7178,8 +10691,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(columnName1: String, columnName2: String): Column = tuple_union_theta_double(Column(columnName1), Column(columnName2)) @@ -7190,8 +10709,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) @@ -7202,8 +10730,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7217,8 +10754,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7229,8 +10778,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( columnName1: String, @@ -7245,8 +10806,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_double( c1: Column, @@ -7260,8 +10833,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column): Column = Column.fn("tuple_union_theta_integer", c1, c2) @@ -7271,8 +10850,14 @@ object functions { * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = tuple_union_theta_integer(Column(columnName1), Column(columnName2)) @@ -7283,8 +10868,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) @@ -7295,8 +10889,17 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses * the default mode of 'sum'. * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7310,8 +10913,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) @@ -7322,8 +10937,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( columnName1: String, @@ -7338,8 +10965,20 @@ object functions { * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the * aggregation mode for numeric summaries (sum, min, max, alwaysone). * + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. * @group sketch_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def tuple_union_theta_integer( c1: Column, @@ -7351,8 +10990,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL bigint sketch. * + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_bigint(e: Column): Column = Column.fn("kll_sketch_to_string_bigint", e) @@ -7360,8 +11003,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL float sketch. * + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_float(e: Column): Column = Column.fn("kll_sketch_to_string_float", e) @@ -7369,8 +11016,12 @@ object functions { /** * Returns a string with human readable summary information about the KLL double sketch. * + * @param e + * The KLL double sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ def kll_sketch_to_string_double(e: Column): Column = Column.fn("kll_sketch_to_string_double", e) @@ -7378,8 +11029,12 @@ object functions { /** * Returns the number of items collected in the KLL bigint sketch. * + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_bigint(e: Column): Column = Column.fn("kll_sketch_get_n_bigint", e) @@ -7387,8 +11042,12 @@ object functions { /** * Returns the number of items collected in the KLL float sketch. * + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_float(e: Column): Column = Column.fn("kll_sketch_get_n_float", e) @@ -7396,8 +11055,12 @@ object functions { /** * Returns the number of items collected in the KLL double sketch. * + * @param e + * The KLL double sketch binary representation. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ def kll_sketch_get_n_double(e: Column): Column = Column.fn("kll_sketch_get_n_double", e) @@ -7405,8 +11068,14 @@ object functions { /** * Merges two KLL bigint sketch buffers together into one. * + * @param left + * The first KLL bigint sketch. A column that evaluates to a binary. + * @param right + * The second KLL bigint sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_bigint(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_bigint", left, right) @@ -7414,8 +11083,14 @@ object functions { /** * Merges two KLL float sketch buffers together into one. * + * @param left + * The first KLL float sketch. A column that evaluates to a binary. + * @param right + * The second KLL float sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_float(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_float", left, right) @@ -7423,8 +11098,14 @@ object functions { /** * Merges two KLL double sketch buffers together into one. * + * @param left + * The first KLL double sketch. A column that evaluates to a binary. + * @param right + * The second KLL double sketch. A column that evaluates to a binary. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def kll_sketch_merge_double(left: Column, right: Column): Column = Column.fn("kll_sketch_merge_double", left, right) @@ -7433,8 +11114,15 @@ object functions { * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. Must be a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a long, or an array of longs when `rank` is an array. */ def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) @@ -7443,8 +11131,15 @@ object functions { * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a float, or an array of floats when `rank` is an array. */ def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_float", sketch, rank) @@ -7453,8 +11148,16 @@ object functions { * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be * a single value or an array. * + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double, or an array of doubles when `rank` is an + * array. */ def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = Column.fn("kll_sketch_get_quantile_double", sketch, rank) @@ -7463,8 +11166,15 @@ object functions { * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile * can be a single value or an array. * + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to an integral or an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) @@ -7473,8 +11183,15 @@ object functions { * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can * be a single value or an array. * + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_float", sketch, quantile) @@ -7483,8 +11200,16 @@ object functions { * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile * can be a single value or an array. * + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be + * a constant. * @group sketch_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_double", sketch, quantile) @@ -7498,11 +11223,14 @@ object functions { * * @param startDate * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7514,11 +11242,14 @@ object functions { * * @param startDate * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param numMonths - * A column of the number of months to add to `startDate`, can be negative to subtract months + * A column of the number of months to add to `startDate`, can be negative to subtract months. + * A column that evaluates to an integer. * @return - * A date, or null if `startDate` was a string that could not be cast to a date + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7531,6 +11262,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def curdate(): Column = Column.fn("curdate") @@ -7540,6 +11273,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def current_date(): Column = Column.fn("current_date") @@ -7548,6 +11283,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def current_timezone(): Column = Column.fn("current_timezone") @@ -7557,6 +11294,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def current_timestamp(): Column = Column.fn("current_timestamp") @@ -7565,6 +11304,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def now(): Column = Column.fn("now") @@ -7575,6 +11316,8 @@ object functions { * * @group datetime_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def localtimestamp(): Column = Column.fn("localtimestamp") @@ -7587,11 +11330,14 @@ object functions { * * @param dateExpr * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993` + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns + * a column that evaluates to a string. * @note * Use specialized functions like [[year]] whenever possible as they benefit from a * specialized implementation. @@ -7608,11 +11354,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to add to `start`, can be negative to subtract days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7623,11 +11372,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * A column of the number of days to add to `start`, can be negative to subtract days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7638,11 +11390,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * A column of the number of days to add to `start`, can be negative to subtract days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.5.0 */ @@ -7653,11 +11408,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * The number of days to subtract from `start`, can be negative to add days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7668,11 +11426,14 @@ object functions { * * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param days - * A column of the number of days to subtract from `start`, can be negative to add days + * A column of the number of days to subtract from `start`, can be negative to add days. A + * column that evaluates to an integer, short, or byte. * @return - * A date, or null if `start` was a string that could not be cast to a date + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. * @group datetime_funcs * @since 3.0.0 */ @@ -7690,13 +11451,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7713,13 +11476,15 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param start * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start` + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7728,15 +11493,23 @@ object functions { /** * Create date from the number of `days` since 1970-01-01. * + * @param days + * The number of days since 1970-01-01. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) /** * Extracts the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the year from. A column that evaluates to a date, + * timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7744,8 +11517,12 @@ object functions { /** * Extracts the quarter as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the quarter from. A column that evaluates to a + * date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7753,8 +11530,12 @@ object functions { /** * Extracts the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the month from. A column that evaluates to a date, + * timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7763,8 +11544,12 @@ object functions { /** * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 * for a Sunday through to 7 for a Saturday + * @param e + * The date, timestamp or string to extract the day of the week from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 2.3.0 */ @@ -7772,8 +11557,12 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7781,8 +11570,12 @@ object functions { /** * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 3.5.0 */ @@ -7790,8 +11583,12 @@ object functions { /** * Extracts the day of the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the year from. A column that evaluates + * to a date, timestamp or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7801,8 +11598,12 @@ object functions { * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the hours from. A column that evaluates to a date, time, timestamp or + * string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7814,9 +11615,10 @@ object functions { * @param field * selects which part of the source should be extracted. * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * a date, time, timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7831,9 +11633,10 @@ object functions { * selects which part of the source should be extracted, and supported string values are as * same as the fields of the equivalent function `extract`. * @param source - * a date/timestamp or interval column from where `field` should be extracted. + * a date/timestamp or time or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7850,7 +11653,8 @@ object functions { * @param source * a date/timestamp or interval column from where `field` should be extracted. * @return - * a part of the date/timestamp or interval source + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. * @group datetime_funcs * @since 3.5.0 */ @@ -7864,9 +11668,11 @@ object functions { * * @param e * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * A date, or null if the input was a string that could not be cast to a date + * A date, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7876,8 +11682,12 @@ object functions { * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the minutes from. A column that evaluates to a date, time, timestamp + * or string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7886,14 +11696,25 @@ object functions { /** * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). * + * @param e + * The column to extract the day of the week from. A column that evaluates to a date, + * timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def weekday(e: Column): Column = Column.fn("weekday", e) /** + * @param year + * The year to build the date. A column that evaluates to an integral. + * @param month + * The month to build the date. A column that evaluates to an integral. + * @param day + * The day to build the date. A column that evaluates to an integral. * @return - * A date created from year, month and day fields. + * A date created from year, month and day fields. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.3.0 */ @@ -7916,13 +11737,16 @@ object functions { * * @param end * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param start * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. * @return * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start` + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a + * double. * @group datetime_funcs * @since 1.5.0 */ @@ -7932,8 +11756,21 @@ object functions { /** * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the * result is rounded off to 8 digits; it is not rounded otherwise. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. + * @param roundOff + * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be + * a constant. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a double. */ def months_between(end: Column, start: Column, roundOff: Boolean): Column = Column.fn("months_between", end, start, lit(roundOff)) @@ -7947,12 +11784,14 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun" + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -7967,13 +11806,14 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param dayOfWeek * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", - * "Fri", "Sat", "Sun" + * "Fri", "Sat", "Sun". A column that evaluates to a string. * @return * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value + * was an invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 3.2.0 */ @@ -7984,8 +11824,12 @@ object functions { * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the seconds from. A column that evaluates to a date, time, timestamp + * or string. * @return - * An integer, or null if the input was a string that could not be cast to a timestamp + * An integer, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -7997,8 +11841,12 @@ object functions { * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, * as defined by ISO 8601 * + * @param e + * The column to extract the week number from. A column that evaluates to a date, timestamp or + * string. * @return - * An integer, or null if the input was a string that could not be cast to a date + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. * @group datetime_funcs * @since 1.5.0 */ @@ -8013,7 +11861,8 @@ object functions { * A number of a type that is castable to a long, such as string or integer. Can be negative * for timestamps before the unix epoch * @return - * A string, or null if the input was a string that could not be cast to a long + * A string, or null if the input was a string that could not be cast to a long. Returns a + * column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8034,7 +11883,7 @@ object functions { * A date time pattern that the input will be formatted to * @return * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern + * invalid date time pattern. Returns a column that evaluates to a string. * @group datetime_funcs * @since 1.5.0 */ @@ -8050,6 +11899,8 @@ object functions { * * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_timestamp(): Column = unix_timestamp(current_timestamp()) @@ -8061,7 +11912,8 @@ object functions { * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` * format * @return - * A long, or null if the input was a string not of the correct format + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8075,12 +11927,14 @@ object functions { * * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. * @param p - * A date time pattern detailing the format of `s` when `s` is a string + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format + * format. Returns a column that evaluates to a long. * @group datetime_funcs * @since 1.5.0 */ @@ -8091,9 +11945,10 @@ object functions { * Parses a string value to a time value. * * @param str - * A string to be parsed to time. + * A string to be parsed to time. A column that evaluates to a string. * @return - * A time, or raises an error if the input is malformed. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * * @group datetime_funcs * @since 4.1.0 @@ -8111,10 +11966,10 @@ object functions { * @param str * A string to be parsed to time. * @param format - * A time format pattern to follow. + * A time format pattern to follow. A column that evaluates to a string. * @return - * A time, or raises an error if the input is malformed. - * + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. * @group datetime_funcs * @since 4.1.0 */ @@ -8127,9 +11982,11 @@ object functions { * * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp + * A timestamp, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8143,12 +12000,14 @@ object functions { * * @param s * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @param fmt - * A date time pattern detailing the format of `s` when `s` is a string + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format + * an invalid format. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.2.0 */ @@ -8158,9 +12017,9 @@ object functions { * Parses a string value to a time value. * * @param str - * A string to be parsed to time. + * A string to be parsed to time. A column that evaluates to a string. * @return - * A time, or null if the input is malformed. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * * @group datetime_funcs * @since 4.1.0 @@ -8178,10 +12037,9 @@ object functions { * @param str * A string to be parsed to time. * @param format - * A time format pattern to follow. + * A time format pattern to follow. A column that evaluates to a string. * @return - * A time, or null if the input is malformed. - * + * A time, or null if the input is malformed. Returns a column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 */ @@ -8194,8 +12052,14 @@ object functions { * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with * the value of configuration `spark.sql.timestampType`. * + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @param format + * Format to use to convert timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column, format: Column): Column = Column.fn("try_to_timestamp", s, format) @@ -8205,16 +12069,24 @@ object functions { * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result * data type is consistent with the value of configuration `spark.sql.timestampType`. * + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) /** * Converts the column into `DateType` by casting rules to `DateType`. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. * @group datetime_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ def to_date(e: Column): Column = Column.fn("to_date", e) @@ -8226,12 +12098,14 @@ object functions { * * @param e * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. * @param fmt - * A date time pattern detailing the format of `e` when `e`is a string + * A date time pattern detailing the format of `e` when `e`is a string. A column that + * evaluates to a string. * @return * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format + * invalid format. Returns a column that evaluates to a date. * @group datetime_funcs * @since 2.2.0 */ @@ -8241,8 +12115,12 @@ object functions { * This is a special version of `to_date` that performs the same operation, but returns a NULL * value instead of raising an error if date cannot be created. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column): Column = Column.fn("try_to_date", e) @@ -8250,24 +12128,39 @@ object functions { * This is a special version of `to_date` that performs the same operation, but returns a NULL * value instead of raising an error if date cannot be created. * + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @param fmt + * Format to use to convert date values. A column that evaluates to a string. Must be a + * constant. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) /** * Returns the number of days since 1970-01-01. * + * @param e + * Input column of values to convert. A column that evaluates to a date. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def unix_date(e: Column): Column = Column.fn("unix_date", e) /** * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. * + * @param e + * Input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_micros(e: Column): Column = Column.fn("unix_micros", e) @@ -8276,8 +12169,13 @@ object functions { * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a * lossless `DECIMAL(21, 0)`. * + * @param e + * input column of nanosecond-precision timestamp values to convert. A column that evaluates + * to a timestamp. * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a decimal. */ def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) @@ -8285,8 +12183,12 @@ object functions { * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of * precision. * + * @param e + * input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_millis(e: Column): Column = Column.fn("unix_millis", e) @@ -8294,8 +12196,12 @@ object functions { * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of * precision. * + * @param e + * input column of values to convert. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) @@ -8306,14 +12212,15 @@ object functions { * * @param date * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @param format: * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter' + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * * @return * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value + * invalid value. Returns a column that evaluates to a date. * @group datetime_funcs * @since 1.5.0 */ @@ -8327,13 +12234,14 @@ object functions { * @param format: * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter' + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. * @param timestamp * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value + * `format` was an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 2.3.0 */ @@ -8347,16 +12255,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8366,8 +12276,17 @@ object functions { * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 * 03:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def from_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("from_utc_timestamp", ts, tz) @@ -8379,16 +12298,18 @@ object functions { * * @param ts * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @param tz * A string detailing the time zone ID that the input should be adjusted to. It should be in * the format of either region-based zone IDs or zone offsets. Region IDs must have the form * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value + * an invalid value. Returns a column that evaluates to a timestamp. * @group datetime_funcs * @since 1.5.0 */ @@ -8398,8 +12319,17 @@ object functions { * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield * '2017-07-14 01:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. * @group datetime_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) @@ -8429,24 +12359,29 @@ object functions { * * @param timeColumn * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. * @param windowDuration * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. * @param slideDuration * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. * @param startTime * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. * For example, in order to have hourly tumbling windows that start 15 minutes past the hour, - * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. + * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. A column that + * evaluates to a string. * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window( timeColumn: Column, @@ -8482,20 +12417,24 @@ object functions { * * @param timeColumn * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. * @param windowDuration * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. * @param slideDuration * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { window(timeColumn, windowDuration, slideDuration, "0 second") @@ -8527,13 +12466,16 @@ object functions { * * @param timeColumn * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. * @param windowDuration * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. * * @group datetime_funcs * @since 2.0.0 + * @return + * Returns a column that evaluates to a struct. */ def window(timeColumn: Column, windowDuration: String): Column = { window(timeColumn, windowDuration, windowDuration, "0 second") @@ -8548,10 +12490,12 @@ object functions { * * @param windowColumn * The window column (typically produced by window aggregation) of type StructType { start: - * Timestamp, end: Timestamp } + * Timestamp, end: Timestamp }. A column that evaluates to a struct. * * @group datetime_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a timestamp. */ def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) @@ -8571,13 +12515,16 @@ object functions { * * @param timeColumn * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. * @param gapDuration * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: String): Column = session_window(timeColumn, lit(gapDuration)) @@ -8606,38 +12553,52 @@ object functions { * * @param timeColumn * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. * @param gapDuration * A column specifying the timeout of the session. It could be static value, e.g. `10 * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on - * the input row. + * the input row. A column that evaluates to a string or interval. * * @group datetime_funcs * @since 3.2.0 + * @return + * Returns a column that evaluates to a struct. */ def session_window(timeColumn: Column, gapDuration: Column): Column = Column.fn("session_window", timeColumn, gapDuration) /** * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. + * @param e + * unix time values. A column that evaluates to a numeric. * @group datetime_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) /** * Creates timestamp from the number of milliseconds since UTC epoch. * + * @param e + * unix time values. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) /** * Creates timestamp from the number of microseconds since UTC epoch. * + * @param e + * unix time values. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) @@ -8645,8 +12606,12 @@ object functions { * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from * the number of nanoseconds since UTC epoch. * + * @param e + * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. * @group datetime_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a timestamp. */ def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) @@ -8654,8 +12619,19 @@ object functions { * Gets the difference between the timestamps in the specified units by truncating the fraction * part. * + * @param unit + * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', + * 'HOUR'. A column that evaluates to a string. Must be a constant. + * @param start + * A timestamp which the expression subtracts from `end`. A column that evaluates to a + * timestamp. + * @param end + * A timestamp from which the expression subtracts `start`. A column that evaluates to a + * timestamp. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a long. */ def timestamp_diff(unit: String, start: Column, end: Column): Column = Column.internalFn("timestampdiff", lit(unit), start, end) @@ -8663,8 +12639,17 @@ object functions { /** * Adds the specified number of units to the given timestamp. * + * @param unit + * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates + * to a string. Must be a constant. + * @param quantity + * the number of units of time to add. A column that evaluates to an integral. + * @param ts + * A timestamp to which to add. A column that evaluates to a timestamp. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def timestamp_add(unit: String, quantity: Column, ts: Column): Column = Column.internalFn("timestampadd", lit(unit), quantity, ts) @@ -8681,6 +12666,8 @@ object functions { * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column): Column = Column.fn("time_bucket", bucketSize, ts) @@ -8699,6 +12686,8 @@ object functions { * Alignment anchor. Must be the same type as `ts` and must be foldable. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column of the same type as the input. */ def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = Column.fn("time_bucket", bucketSize, ts, origin) @@ -8709,13 +12698,15 @@ object functions { * * @param unit * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param start - * A starting TIME. + * A starting TIME. A column that evaluates to a time. * @param end - * An ending TIME. + * An ending TIME. A column that evaluates to a time. * @return - * The difference between `end` and `start` times, measured in specified units. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @group datetime_funcs @@ -8730,11 +12721,12 @@ object functions { * * @param unit * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. * @param time - * A TIME to truncate. + * A TIME to truncate. A column that evaluates to a time. * @return - * A TIME truncated to the specified unit. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. * @note * If any of the inputs is `NULL`, the result is `NULL`. * @throws IllegalArgumentException @@ -8749,24 +12741,36 @@ object functions { /** * Creates a TIME from the number of seconds since midnight. * + * @param e + * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) /** * Creates a TIME from the number of milliseconds since midnight. * + * @param e + * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) /** * Creates a TIME from the number of microseconds since midnight. * + * @param e + * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a time. */ def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) @@ -8774,24 +12778,36 @@ object functions { * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a * DECIMAL(14,6) to preserve microsecond precision. * + * @param e + * TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a decimal. */ def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) /** * Extracts the number of milliseconds since midnight from a TIME value. * + * @param e + * the TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) /** * Extracts the number of microseconds since midnight from a TIME value. * + * @param e + * the TIME value to convert. A column that evaluates to a time. * @group datetime_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) @@ -8799,8 +12815,14 @@ object functions { * Parses the `timestamp` expression with the `format` expression to a timestamp with local time * zone. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ltz", timestamp, format) @@ -8810,8 +12832,12 @@ object functions { * zone. The default format follows casting rules to a timestamp. Returns null with invalid * input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ltz(timestamp: Column): Column = Column.fn("to_timestamp_ltz", timestamp) @@ -8820,8 +12846,14 @@ object functions { * Parses the `timestamp_str` expression with the `format` expression to a timestamp without * time zone. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column, format: Column): Column = Column.fn("to_timestamp_ntz", timestamp, format) @@ -8830,8 +12862,12 @@ object functions { * Parses the `timestamp` expression with the default format to a timestamp without time zone. * The default format follows casting rules to a timestamp. Returns null with invalid input. * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def to_timestamp_ntz(timestamp: Column): Column = Column.fn("to_timestamp_ntz", timestamp) @@ -8839,8 +12875,14 @@ object functions { /** * Returns the UNIX timestamp of the given time. * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to convert the time values. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column, format: Column): Column = Column.fn("to_unix_timestamp", timeExp, format) @@ -8848,8 +12890,12 @@ object functions { /** * Returns the UNIX timestamp of the given time. * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def to_unix_timestamp(timeExp: Column): Column = Column.fn("to_unix_timestamp", timeExp) @@ -8857,8 +12903,13 @@ object functions { /** * Extracts the three-letter abbreviated month name from a given date/timestamp/string. * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def monthname(timeExp: Column): Column = Column.fn("monthname", timeExp) @@ -8866,8 +12917,13 @@ object functions { /** * Extracts the three-letter abbreviated day name from a given date/timestamp/string. * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def dayname(timeExp: Column): Column = Column.fn("dayname", timeExp) @@ -8878,8 +12934,15 @@ object functions { /** * Returns null if the array is null, true if the array contains `value`, and false otherwise. + * @param column + * the target column containing the arrays. A column that evaluates to an array. + * @param value + * the value to check for in the array. A column that evaluates to a value matching the + * array's element type. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def array_contains(column: Column, value: Any): Column = Column.fn("array_contains", column, lit(value)) @@ -8888,8 +12951,15 @@ object functions { * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. * The new element/column is located at end of the ARRAY. * + * @param column + * the source column containing the array. A column that evaluates to an array. + * @param element + * the value to append to the array. A column that evaluates to a value matching the array's + * element type. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_append(column: Column, element: Any): Column = Column.fn("array_append", column, lit(element)) @@ -8898,8 +12968,14 @@ object functions { * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns * `false` otherwise. + * @param a1 + * the first input array. A column that evaluates to an array. + * @param a2 + * the second input array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a boolean. */ def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) @@ -8908,14 +12984,16 @@ object functions { * end if `start` is negative) with the specified `length`. * * @param x - * the array column to be sliced + * the array column to be sliced. A column that evaluates to an array. * @param start - * the starting index + * the starting index. A column that evaluates to an integer. * @param length - * the length of the slice + * the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Int, length: Int): Column = slice(x, lit(start), lit(length)) @@ -8925,14 +13003,16 @@ object functions { * end if `start` is negative) with the specified `length`. * * @param x - * the array column to be sliced + * the array column to be sliced. A column that evaluates to an array. * @param start - * the starting index + * the starting index. A column that evaluates to an integer. * @param length - * the length of the slice + * the length of the slice. A column that evaluates to an integer. * * @group array_funcs * @since 3.1.0 + * @return + * Returns a column that evaluates to an array. */ def slice(x: Column, start: Column, length: Column): Column = Column.fn("slice", x, start, length) @@ -8940,16 +13020,30 @@ object functions { /** * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with * `nullReplacement`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @param nullReplacement + * the string used to replace null values. A column that evaluates to a string. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String, nullReplacement: String): Column = Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) /** * Concatenates the elements of `column` using the `delimiter`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def array_join(column: Column, delimiter: String): Column = Column.fn("array_join", column, lit(delimiter)) @@ -8958,11 +13052,15 @@ object functions { * Concatenates multiple input columns together into a single column. The function works with * strings, binary and compatible array columns. * + * @param exprs + * Input columns to concatenate. A column that evaluates to a string, binary or an array. * @note * Returns null if any of the input columns are null. * * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ @scala.annotation.varargs def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) @@ -8971,12 +13069,18 @@ object functions { * Locates the position of the first occurrence of the value in the given array as long. Returns * null if either of the arguments are null. * + * @param column + * The array to search. A column that evaluates to an array. + * @param value + * The value to locate. A column. * @note * The position is not zero based, but 1 based index. Returns 0 if value could not be found in * array. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a long. */ def array_position(column: Column, value: Any): Column = Column.fn("array_position", column, lit(value)) @@ -8985,8 +13089,15 @@ object functions { * Returns element of array at given index in value if column is array. Returns value for the * given key in value if column is map. * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input + * map. */ def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) @@ -8998,8 +13109,15 @@ object functions { * (map, key) - Returns value for given key. The function always returns NULL if the key is not * contained in the map. * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the element type of the input array, or the value type of the input + * map. */ def try_element_at(column: Column, value: Column): Column = Column.fn("try_element_at", column, value) @@ -9008,8 +13126,14 @@ object functions { * Returns element of array at given (0-based) index. If the index points outside of the array * boundaries, then this function returns NULL. * + * @param column + * The array to extract from. A column that evaluates to an array. + * @param index + * The 0-based index. A column that evaluates to an integral. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column of the element type of the input array. */ def get(column: Column, index: Column): Column = Column.fn("get", column, index) @@ -9022,8 +13146,12 @@ object functions { * fields in the struct must be orderable. If the default comparator does not support the input * type, you can specify a custom comparator. * + * @param e + * The array to sort. A column that evaluates to an array. * @group collection_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column): Column = Column.fn("array_sort", e) @@ -9033,8 +13161,15 @@ object functions { * positive integer as the first element is less than, equal to, or greater than the second * element. If the comparator function returns null, the function will fail and raise an error. * + * @param e + * The array to sort. A column that evaluates to an array. + * @param comparator + * A binary comparator function that returns a negative integer, 0, or a positive integer as + * the first element is less than, equal to, or greater than the second element. * @group collection_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_sort(e: Column, comparator: (Column, Column) => Column): Column = Column.fn("array_sort", e, createLambda(comparator)) @@ -9042,8 +13177,14 @@ object functions { /** * Remove all elements that equal to element from the given array. * + * @param column + * The array to remove from. A column that evaluates to an array. + * @param element + * The element to remove. A column. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_remove(column: Column, element: Any): Column = Column.fn("array_remove", column, lit(element)) @@ -9051,8 +13192,12 @@ object functions { /** * Remove all null elements from the given array. * + * @param column + * The array to compact. A column that evaluates to an array. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_compact(column: Column): Column = Column.fn("array_compact", column) @@ -9060,16 +13205,26 @@ object functions { * Returns an array containing value as well as all elements from array. The new element is * positioned at the beginning of the array. * + * @param column + * The array to prepend to. A column that evaluates to an array. + * @param element + * The element to prepend. A column. * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_prepend(column: Column, element: Any): Column = Column.fn("array_prepend", column, lit(element)) /** * Removes duplicate values from the array. + * @param e + * The array to deduplicate. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_distinct(e: Column): Column = Column.fn("array_distinct", e) @@ -9077,8 +13232,14 @@ object functions { * Returns an array of the elements in the intersection of the given two arrays, without * duplicates. * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_intersect(col1: Column, col2: Column): Column = Column.fn("array_intersect", col1, col2) @@ -9086,8 +13247,17 @@ object functions { /** * Adds an item into a given array at a specified position * + * @param arr + * The array to insert into. A column that evaluates to an array. + * @param pos + * The 1-based position at which to insert (negative counts from the end). A column that + * evaluates to an integral. + * @param value + * The value to insert. A column. * @group array_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_insert(arr: Column, pos: Column, value: Column): Column = Column.fn("array_insert", arr, pos, value) @@ -9095,8 +13265,14 @@ object functions { /** * Returns an array of the elements in the union of the given two arrays, without duplicates. * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_union(col1: Column, col2: Column): Column = Column.fn("array_union", col1, col2) @@ -9105,8 +13281,14 @@ object functions { * Returns an array of the elements in the first array but not in the second array, without * duplicates. The order of elements in the result is not determined * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_except(col1: Column, col2: Column): Column = Column.fn("array_except", col1, col2) @@ -9140,12 +13322,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => transformed_col, the lambda function to transform the input column + * col => transformed_col, the lambda function to transform the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: Column => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9158,13 +13342,15 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f * (col, index) => transformed_col, the lambda function to transform the input column given * the index. Indices start at 0. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def transform(column: Column, f: (Column, Column) => Column): Column = Column.fn("transform", column, createLambda(f)) @@ -9176,12 +13362,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def exists(column: Column, f: Column => Column): Column = Column.fn("exists", column, createLambda(f)) @@ -9193,12 +13381,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to check the input column + * col => predicate, the Boolean predicate to check the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def forall(column: Column, f: Column => Column): Column = Column.fn("forall", column, createLambda(f)) @@ -9210,12 +13400,14 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f - * col => predicate, the Boolean predicate to filter the input column + * col => predicate, the Boolean predicate to filter the input column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: Column => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9227,13 +13419,15 @@ object functions { * }}} * * @param column - * the input array column + * the input array column. A column that evaluates to an array. * @param f * (col, index) => predicate, the Boolean predicate to filter the input column given the * index. Indices start at 0. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def filter(column: Column, f: (Column, Column) => Column): Column = Column.fn("filter", column, createLambda(f)) @@ -9247,18 +13441,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate( expr: Column, @@ -9275,14 +13471,16 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value * to the combined_value * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column of the same type as the initial value. */ def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = aggregate(expr, initialValue, merge, c => c) @@ -9296,18 +13494,20 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value + * to the combined_value. * @param finish * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result + * inputs to final result. * * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce( expr: Column, @@ -9324,14 +13524,16 @@ object functions { * }}} * * @param expr - * the input array column + * the input array column. A column that evaluates to an array. * @param initialValue - * the initial value + * the initial value. A column of any type. * @param merge * (combined_value, input_value) => combined_value, the merge function to merge an input value * to the combined_value * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the initial value. */ def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = reduce(expr, initialValue, merge, c => c) @@ -9345,14 +13547,16 @@ object functions { * }}} * * @param left - * the left input array column + * the left input array column. A column that evaluates to an array. * @param right - * the right input array column + * the right input array column. A column that evaluates to an array. * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = Column.fn("zip_with", left, right, createLambda(f)) @@ -9365,12 +13569,14 @@ object functions { * }}} * * @param expr - * the input map column + * the input map column. A column that evaluates to a map. * @param f * (key, value) => new_key, the lambda function to transform the key of input map column * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_keys(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_keys", expr, createLambda(f)) @@ -9383,12 +13589,14 @@ object functions { * }}} * * @param expr - * the input map column + * the input map column. A column that evaluates to a map. * @param f * (key, value) => new_value, the lambda function to transform the value of input map column * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def transform_values(expr: Column, f: (Column, Column) => Column): Column = Column.fn("transform_values", expr, createLambda(f)) @@ -9400,12 +13608,14 @@ object functions { * }}} * * @param expr - * the input map column + * the input map column. A column that evaluates to a map. * @param f * (key, value) => predicate, the Boolean predicate to filter the input map column * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_filter(expr: Column, f: (Column, Column) => Column): Column = Column.fn("map_filter", expr, createLambda(f)) @@ -9417,14 +13627,16 @@ object functions { * }}} * * @param left - * the left input map column + * the left input map column. A column that evaluates to a map. * @param right - * the right input map column + * the right input map column. A column that evaluates to a map. * @param f * (key, value1, value2) => new_value, the lambda function to merge the map values * * @group collection_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a map. */ def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = Column.fn("map_zip_with", left, right, createLambda(f)) @@ -9434,8 +13646,13 @@ object functions { * name `col` for elements in the array and `key` and `value` for elements in the map unless * specified otherwise. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 1.3.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ def explode(e: Column): Column = Column.fn("explode", e) @@ -9444,8 +13661,13 @@ object functions { * name `col` for elements in the array and `key` and `value` for elements in the map unless * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.2.0 + * @return + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ def explode_outer(e: Column): Column = Column.fn("explode_outer", e) @@ -9454,8 +13676,13 @@ object functions { * default column name `pos` for position, and `col` for elements in the array and `key` and * `value` for elements in the map unless specified otherwise. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.1.0 + * @return + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ def posexplode(e: Column): Column = Column.fn("posexplode", e) @@ -9465,16 +13692,25 @@ object functions { * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the * array/map is null or empty then the row (null, null) is produced. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. * @group generator_funcs * @since 2.2.0 + * @return + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e) /** * Creates a new row for each element in the given array of structs. * + * @param e + * the target column to explode. A column that evaluates to an array of structs. * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline(e: Column): Column = Column.fn("inline", e) @@ -9482,8 +13718,12 @@ object functions { * Creates a new row for each element in the given array of structs. Unlike inline, if the array * is null or empty then null is produced for each nested column. * + * @param e + * the target column to explode. A column that evaluates to an array of structs. * @group generator_funcs * @since 3.4.0 + * @return + * Returns a column that evaluates to a struct. */ def inline_outer(e: Column): Column = Column.fn("inline_outer", e) @@ -9491,8 +13731,14 @@ object functions { * Extracts json object from a json string based on json path specified, and returns json string * of the extracted json object. It will return null if the input json string is invalid. * + * @param e + * the JSON string column. A column that evaluates to a string. + * @param path + * the JSON path to extract. A column that evaluates to a string. Must be a constant. * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ def get_json_object(e: Column, path: String): Column = Column.fn("get_json_object", e, lit(path)) @@ -9500,8 +13746,14 @@ object functions { /** * Creates a new row for a json column according to the given field names. * + * @param json + * the JSON string column. A column that evaluates to a string. + * @param fields + * the field names to extract. A column that evaluates to a string. Must be a constant. * @group json_funcs * @since 1.6.0 + * @return + * Returns a column that evaluates to a string. */ @scala.annotation.varargs def json_tuple(json: Column, fields: String*): Column = { @@ -9515,17 +13767,20 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the json is parsed. Accepts the same options as the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -9538,17 +13793,20 @@ object functions { * case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { @@ -9561,17 +13819,20 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -9584,17 +13845,20 @@ object functions { * case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ // scalastyle:on line.size.limit def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { @@ -9606,12 +13870,15 @@ object functions { * Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a struct. */ def from_json(e: Column, schema: StructType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9622,12 +13889,15 @@ object functions { * unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. * * @group json_funcs * @since 2.2.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ def from_json(e: Column, schema: DataType): Column = from_json(e, schema, Map.empty[String, String]) @@ -9639,17 +13909,19 @@ object functions { * case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema as a DDL-formatted string. + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -9663,17 +13935,19 @@ object functions { * case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema as a DDL-formatted string. + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.3.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ // scalastyle:on line.size.limit def from_json(e: Column, schema: String, options: Map[String, String]): Column = { @@ -9686,12 +13960,14 @@ object functions { * `null`, in the case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A column that evaluates to a string. * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ def from_json(e: Column, schema: Column): Column = { from_json(e, schema, Map.empty[String, String].asJava) @@ -9704,17 +13980,19 @@ object functions { * `null`, in the case of an unparseable string. * * @param e - * a string column containing JSON data. + * a string column containing JSON data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the json string + * the schema to use when parsing the json string. A column that evaluates to a string. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column of the type given by the schema (a struct, array, or map). */ // scalastyle:on line.size.limit def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { @@ -9733,10 +14011,12 @@ object functions { * a valid JSON value. * * @param json - * a string column that contains JSON data. + * a string column that contains JSON data. A column that evaluates to a string. * * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) @@ -9744,9 +14024,11 @@ object functions { * Parses a JSON string and constructs a Variant value. * * @param json - * a string column that contains JSON data. + * a string column that contains JSON data. A column that evaluates to a string. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def parse_json(json: Column): Column = Column.fn("parse_json", json) @@ -9756,9 +14038,12 @@ object functions { * can only have string keys. * * @param col - * a column with a nested schema or column name. + * a column with a nested schema or column name. A column that evaluates to a struct, array, + * map, or variant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) @@ -9767,9 +14052,11 @@ object functions { * variant null and false otherwise (including in the case of SQL NULL). * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) @@ -9778,9 +14065,11 @@ object functions { * malformed, and NULL if the input is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @group variant_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a boolean. */ def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) @@ -9789,15 +14078,18 @@ object functions { * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the first JSONPath string. A valid path should start with `$` and is * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. + * path `$` is not allowed. A column that evaluates to a string. * @param paths - * additional JSONPath arguments, applied after `path` in order. + * additional JSONPath arguments, applied after `path` in order. A column that evaluates to a + * string. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: Column, paths: Column*): Column = @@ -9808,15 +14100,17 @@ object functions { * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the first JSONPath identifying a deletion target. A valid path should start with `$` and is * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. + * path `$` is not allowed. A string. Must be a constant. * @param paths - * additional JSONPath strings, applied after `path` in order. + * additional JSONPath strings, applied after `path` in order. A string. Must be a constant. * @group variant_funcs * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs def variant_delete(v: Column, path: String, paths: String*): Column = @@ -9829,15 +14123,18 @@ object functions { * a value of an incompatible type. Returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the JSONPath string identifying the insertion target. A valid path * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. * @param value * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: Column, value: Column): Column = Column.fn("variant_insert", v, path, value) @@ -9849,15 +14146,17 @@ object functions { * a value of an incompatible type. Returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the JSONPath identifying the insertion target. A valid path should start with `$` and is * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. + * path `$` is not allowed. A string. Must be a constant. * @param value * the value to insert. Any expression castable to variant. * @group variant_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. */ def variant_insert(v: Column, path: String, value: Column): Column = Column.fn("variant_insert", v, lit(path), value) @@ -9869,11 +14168,12 @@ object functions { * hits a value of an incompatible type, or if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the JSONPath string identifying the insertion target. A valid path * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. * @param value * the value to insert. Any expression castable to variant. * @group variant_funcs @@ -9889,11 +14189,11 @@ object functions { * hits a value of an incompatible type, or if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the JSONPath identifying the insertion target. A valid path should start with `$` and is * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. + * path `$` is not allowed. A string. Must be a constant. * @param value * the value to insert. Any expression castable to variant. * @group variant_funcs @@ -9909,11 +14209,11 @@ object functions { * Returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the JSONPath string identifying the set target. A valid path should * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. * @param value * the value to set. Any expression castable to variant. * @group variant_funcs @@ -9929,11 +14229,11 @@ object functions { * Returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the JSONPath identifying the set target. A valid path should start with `$` and is followed * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. + * is not allowed. A string. Must be a constant. * @param value * the value to set. Any expression castable to variant. * @group variant_funcs @@ -9950,15 +14250,16 @@ object functions { * NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the JSONPath string identifying the set target. A valid path should * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. * @param value * the value to set. Any expression castable to variant. * @param createIfMissing - * whether to create missing keys or out-of-range array indices. + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. * @group variant_funcs * @since 4.3.0 */ @@ -9973,15 +14274,16 @@ object functions { * NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the JSONPath identifying the set target. A valid path should start with `$` and is followed * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. + * is not allowed. A string. Must be a constant. * @param value * the value to set. Any expression castable to variant. * @param createIfMissing - * whether to create missing keys or out-of-range array indices. + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. * @group variant_funcs * @since 4.3.0 */ @@ -9994,11 +14296,11 @@ object functions { * an incompatible type or the target is not an array, and returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the JSONPath string identifying the target array. A valid path should * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. + * or `["name"]`. A column that evaluates to a string. * @param value * the value to append. Any expression castable to variant. * @group variant_funcs @@ -10013,10 +14315,11 @@ object functions { * an incompatible type or the target is not an array, and returns NULL if any argument is NULL. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the JSONPath identifying the target array. A valid path should start with `$` and is - * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. A + * string. Must be a constant. * @param value * the value to append. Any expression castable to variant. * @group variant_funcs @@ -10030,14 +14333,16 @@ object functions { * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. * @param targetType - * the target data type to cast into, in a DDL-formatted string. + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: String, targetType: String): Column = Column.fn("variant_get", v, lit(path), lit(targetType)) @@ -10047,15 +14352,17 @@ object functions { * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the extraction path strings. A valid path string should start with * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. + * `["name"]`. A column that evaluates to a string. * @param targetType - * the target data type to cast into, in a DDL-formatted string. + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("variant_get", v, path, lit(targetType)) @@ -10065,14 +14372,16 @@ object functions { * `targetType`. Returns null if the path does not exist or the cast fails.. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. * @param targetType - * the target data type to cast into, in a DDL-formatted string. + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: String, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -10082,15 +14391,17 @@ object functions { * `targetType`. Returns null if the path does not exist or the cast fails.. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @param path * the column containing the extraction path strings. A valid path string should start with * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. + * `["name"]`. A column that evaluates to a string. * @param targetType - * the target data type to cast into, in a DDL-formatted string. + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column of the type specified by the `targetType` argument. */ def try_variant_get(v: Column, path: Column, targetType: String): Column = Column.fn("try_variant_get", v, lit(path), lit(targetType)) @@ -10099,9 +14410,11 @@ object functions { * Returns schema in the SQL format of a variant. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) @@ -10109,9 +14422,11 @@ object functions { * Returns the merged schema in the SQL format of a variant column. * * @param v - * a variant column. + * a variant column. A column that evaluates to a variant. * @group variant_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) @@ -10119,10 +14434,12 @@ object functions { * Parses a JSON string and infers its schema in DDL format. * * @param json - * a JSON string. + * a JSON string. A string. Must be a constant. * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: String): Column = schema_of_json(lit(json)) @@ -10130,10 +14447,12 @@ object functions { * Parses a JSON string and infers its schema in DDL format. * * @param json - * a foldable string column containing a JSON string. + * a foldable string column containing a JSON string. A column that evaluates to a string. * * @group json_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) @@ -10142,14 +14461,15 @@ object functions { * Parses a JSON string and infers its schema in DDL format using options. * * @param json - * a foldable string column containing JSON data. + * a foldable string column containing JSON data. A column that evaluates to a string. * @param options * options to control how the json is parsed. accepts the same options and the json data * source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * * @group json_funcs * @since 3.0.0 @@ -10162,8 +14482,12 @@ object functions { * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any * other valid JSON string, `NULL` or an invalid JSON. * + * @param e + * the JSON array string column. A column that evaluates to a string. * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def json_array_length(e: Column): Column = Column.fn("json_array_length", e) @@ -10172,8 +14496,12 @@ object functions { * given, all the keys of the outermost object will be returned as an array. If it is any other * valid JSON string, an invalid JSON string or an empty string, the function returns null. * + * @param e + * the JSON object string column. A column that evaluates to a string. * @group json_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) @@ -10184,16 +14512,19 @@ object functions { * type. * * @param e - * a column containing a struct, an array or a map. + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. * @param options * options to control how the struct column is converted into a json string. accepts the same * options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. + * option which enables pretty JSON generation. A map of string options. Must be a constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_json(e: Column, options: Map[String, String]): Column = @@ -10206,16 +14537,19 @@ object functions { * type. * * @param e - * a column containing a struct, an array or a map. + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. * @param options * options to control how the struct column is converted into a json string. accepts the same * options and the json data source. See Data * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. + * option which enables pretty JSON generation. A map of string options. Must be a constant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_json(e: Column, options: java.util.Map[String, String]): Column = @@ -10226,10 +14560,13 @@ object functions { * with the specified schema. Throws an exception, in the case of an unsupported type. * * @param e - * a column containing a struct, an array or a map. + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. * * @group json_funcs * @since 2.1.0 + * @return + * Returns a column that evaluates to a string. */ def to_json(e: Column): Column = to_json(e, Map.empty[String, String]) @@ -10240,10 +14577,13 @@ object functions { * removed. * * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column): Column = Column.fn("mask", input) @@ -10253,12 +14593,16 @@ object functions { * creating copies of tables with sensitive information removed. * * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. * @param upperChar * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column): Column = Column.fn("mask", input, upperChar) @@ -10269,14 +14613,19 @@ object functions { * copies of tables with sensitive information removed. * * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. * @param upperChar * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * @param lowerChar * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column, lowerChar: Column): Column = Column.fn("mask", input, upperChar, lowerChar) @@ -10287,16 +14636,22 @@ object functions { * tables with sensitive information removed. * * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. * @param upperChar * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * @param lowerChar * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = Column.fn("mask", input, upperChar, lowerChar, digitChar) @@ -10306,18 +14661,25 @@ object functions { * information removed. * * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. * @param upperChar * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * @param lowerChar * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. * @param otherChar * character to replace all other characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * * @group string_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def mask( input: Column, @@ -10334,8 +14696,12 @@ object functions { * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the * default settings, the function returns null for null input. * + * @param e + * the target column. A column that evaluates to an array or a map. * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an integer. */ def size(e: Column): Column = Column.fn("size", e) @@ -10346,8 +14712,12 @@ object functions { * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the * default settings, the function returns null for null input. * + * @param e + * the target column. A column that evaluates to an array or a map. * @group collection_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def cardinality(e: Column): Column = Column.fn("cardinality", e) @@ -10356,8 +14726,12 @@ object functions { * ordering of the array elements. Null elements will be placed at the beginning of the returned * array. * + * @param e + * the array column to sort. A column that evaluates to an array. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def sort_array(e: Column): Column = sort_array(e, asc = true) @@ -10367,8 +14741,15 @@ object functions { * double/float type. Null elements will be placed at the beginning of the returned array in * ascending order or at the end of the returned array in descending order. * + * @param e + * the array column to sort. A column that evaluates to an array. + * @param asc + * whether to sort in ascending order. A column that evaluates to a boolean. Must be a + * constant. * @group array_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to an array. */ def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) @@ -10376,8 +14757,12 @@ object functions { * Returns the minimum value in the array. NaN is greater than any non-NaN elements for * double/float type. NULL elements are skipped. * + * @param e + * the array column. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array. */ def array_min(e: Column): Column = Column.fn("array_min", e) @@ -10385,72 +14770,114 @@ object functions { * Returns the maximum value in the array. NaN is greater than any non-NaN elements for * double/float type. NULL elements are skipped. * + * @param e + * the input column. A column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column of the element type of the input array. */ def array_max(e: Column): Column = Column.fn("array_max", e) /** * Returns the total number of elements in the array. The function returns null for null input. * + * @param e + * the input column. A column that evaluates to an array. * @group array_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def array_size(e: Column): Column = Column.fn("array_size", e) /** * Aggregate function: returns a list of objects with duplicates. * + * @param e + * the input column. A column that evaluates to any type. * @note * The function is non-deterministic because the order of collected results depends on the * order of the rows which may be non-deterministic after a shuffle. * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def array_agg(e: Column): Column = Column.fn("array_agg", e) /** * Returns a random permutation of the given array. * + * @param e + * the input column. A column that evaluates to an array. * @note * The function is non-deterministic. * * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) /** * Returns a random permutation of the given array. * + * @param e + * the input column. A column that evaluates to an array. + * @param seed + * the seed for the random generator. A column that evaluates to an integral. Must be a + * constant. * @note * The function is non-deterministic. * * @group array_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an array. */ def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) /** * Returns a reversed string or an array with reverse order of elements. + * @param e + * the input column. A column that evaluates to a string, a binary, or an array. * @group collection_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ def reverse(e: Column): Column = Column.fn("reverse", e) /** * Creates a single array from an array of arrays. If a structure of nested arrays is deeper * than two levels, only one level of nesting is removed. + * @param e + * the input column. A column that evaluates to an array of arrays. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def flatten(e: Column): Column = Column.fn("flatten", e) /** * Generate a sequence of integers from start to stop, incrementing by step. * + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. + * @param step + * the value to add to the current element to get the next element. A column that evaluates to + * an integral or interval. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def sequence(start: Column, stop: Column, step: Column): Column = Column.fn("sequence", start, stop, step) @@ -10459,8 +14886,16 @@ object functions { * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than * or equal to stop, otherwise -1. * + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) @@ -10468,8 +14903,14 @@ object functions { * Creates an array containing the left argument repeated the number of times given by the right * argument. * + * @param left + * the value to repeat. A column that evaluates to any type. + * @param right + * the number of times to repeat the value. A column that evaluates to an integral. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) @@ -10477,60 +14918,98 @@ object functions { * Creates an array containing the left argument repeated the number of times given by the right * argument. * + * @param e + * the value to repeat. A column that evaluates to any type. + * @param count + * the number of times to repeat the value. A column that evaluates to an integral. Must be a + * constant. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) /** * Returns true if the map contains the key. + * @param column + * the input column. A column that evaluates to a map. + * @param key + * the key to check for. A column that evaluates to the map's key type. Must be a constant. * @group map_funcs * @since 3.3.0 + * @return + * Returns a column that evaluates to a boolean. */ def map_contains_key(column: Column, key: Any): Column = Column.fn("map_contains_key", column, lit(key)) /** * Returns an unordered array containing the keys of the map. + * @param e + * the input column. A column that evaluates to a map. * @group map_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to an array. */ def map_keys(e: Column): Column = Column.fn("map_keys", e) /** * Returns an unordered array containing the values of the map. + * @param e + * the input column. A column that evaluates to a map. * @group map_funcs * @since 2.3.0 + * @return + * Returns a column that evaluates to an array. */ def map_values(e: Column): Column = Column.fn("map_values", e) /** * Returns an unordered array of all entries in the given map. + * @param e + * the input column. A column that evaluates to a map. * @group map_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to an array. */ def map_entries(e: Column): Column = Column.fn("map_entries", e) /** * Returns a map created from the given array of entries. + * @param e + * the array of entries to convert. A column that evaluates to an array of structs, each with + * a key and value field. * @group map_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a map. */ def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) /** * Returns a merged array of structs in which the N-th struct contains all N-th values of input * arrays. + * @param e + * the columns of arrays to be merged. Each is a column that evaluates to an array. * @group array_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ @scala.annotation.varargs def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) /** * Returns the union of all the given maps. + * @param cols + * the maps to merge. Each is a column that evaluates to a map. * @group map_funcs * @since 2.4.0 + * @return + * Returns a column that evaluates to a map. */ @scala.annotation.varargs def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) @@ -10541,17 +15020,20 @@ object functions { * Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing CSV data. + * a string column containing CSV data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the CSV string + * the schema to use when parsing the CSV string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the CSV is parsed. accepts the same options and the CSV data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = @@ -10563,17 +15045,19 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing CSV data. + * a string column containing CSV data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the CSV string + * the schema to use when parsing the CSV string. A column that evaluates to a string. * @param options * options to control how the CSV is parsed. accepts the same options and the CSV data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -10586,10 +15070,12 @@ object functions { * Parses a CSV string and infers its schema in DDL format. * * @param csv - * a CSV string. + * a CSV string. A string. Must be a constant. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) @@ -10597,10 +15083,12 @@ object functions { * Parses a CSV string and infers its schema in DDL format. * * @param csv - * a foldable string column containing a CSV string. + * a foldable string column containing a CSV string. A column that evaluates to a string. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) @@ -10609,15 +15097,15 @@ object functions { * Parses a CSV string and infers its schema in DDL format using options. * * @param csv - * a foldable string column containing a CSV string. + * a foldable string column containing a CSV string. A column that evaluates to a string. * @param options * options to control how the CSV is parsed. accepts the same options and the CSV data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * a column with string literal containing schema in DDL format. - * + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * @group csv_funcs * @since 3.0.0 */ @@ -10631,15 +15119,17 @@ object functions { * specified schema. Throws an exception, in the case of an unsupported type. * * @param e - * a column containing a struct. + * a column containing a struct. A column that evaluates to a string. * @param options * options to control how the struct column is converted into a CSV string. It accepts the * same options and the CSV data source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_csv(e: Column, options: java.util.Map[String, String]): Column = @@ -10650,10 +15140,12 @@ object functions { * Throws an exception, in the case of an unsupported type. * * @param e - * a column containing a struct. + * a column containing a struct. A column that evaluates to a string. * * @group csv_funcs * @since 3.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) @@ -10663,16 +15155,19 @@ object functions { * schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing XML data. + * a string column containing XML data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the XML string + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. * @param options * options to control how the XML is parsed. accepts the same options and the XML data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = @@ -10684,16 +15179,18 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing XML data. + * a string column containing XML data. A column that evaluates to a string. * @param schema - * the schema as a DDL-formatted string. + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. * @param options * options to control how the XML is parsed. accepts the same options and the xml data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { @@ -10706,11 +15203,13 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing XML data. + * a string column containing XML data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the XML string + * the schema to use when parsing the XML string. A column that evaluates to a string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column): Column = { @@ -10723,16 +15222,18 @@ object functions { * specified schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing XML data. + * a string column containing XML data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the XML string + * the schema to use when parsing the XML string. A column that evaluates to a string. * @param options * options to control how the XML is parsed. accepts the same options and the XML data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ // scalastyle:on line.size.limit def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = @@ -10743,12 +15244,15 @@ object functions { * schema. Returns `null`, in the case of an unparseable string. * * @param e - * a string column containing XML data. + * a string column containing XML data. A column that evaluates to a string. * @param schema - * the schema to use when parsing the XML string + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. * * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a struct. */ def from_xml(e: Column, schema: StructType): Column = from_xml(e, schema, Map.empty[String, String].asJava) @@ -10761,9 +15265,11 @@ object functions { * Parses a XML string and infers its schema in DDL format. * * @param xml - * a XML string. + * a XML string. A string. Must be a constant. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) @@ -10771,9 +15277,11 @@ object functions { * Parses a XML string and infers its schema in DDL format. * * @param xml - * a foldable string column containing a XML string. + * a foldable string column containing a XML string. A column that evaluates to a string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) @@ -10783,14 +15291,15 @@ object functions { * Parses a XML string and infers its schema in DDL format using options. * * @param xml - * a foldable string column containing XML data. + * a foldable string column containing XML data. A column that evaluates to a string. * @param options * options to control how the xml is parsed. accepts the same options and the XML data source. * See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * a column with string literal containing schema in DDL format. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. * @group xml_funcs * @since 4.0.0 */ @@ -10805,14 +15314,16 @@ object functions { * specified schema. Throws an exception, in the case of an unsupported type. * * @param e - * a column containing a struct. + * a column containing a struct. A column that evaluates to a string. * @param options * options to control how the struct column is converted into a XML string. It accepts the * same options as the XML data source. See Data - * Source Option in the version you use. + * Source Option in the version you use. A map of string options. Must be a constant. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ // scalastyle:on line.size.limit def to_xml(e: Column, options: java.util.Map[String, String]): Column = @@ -10823,15 +15334,19 @@ object functions { * Throws an exception, in the case of an unsupported type. * * @param e - * a column containing a struct. + * a column containing a struct. A column that evaluates to a string. * @group xml_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) /** * (Java-specific) A transform for timestamps and dates to partition data into years. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10840,6 +15355,8 @@ object functions { /** * (Java-specific) A transform for timestamps and dates to partition data into months. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10848,6 +15365,8 @@ object functions { /** * (Java-specific) A transform for timestamps and dates to partition data into days. * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10856,8 +15375,14 @@ object functions { /** * Returns a string array of values within the nodes of xml that match the XPath expression. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an array. */ def xpath(xml: Column, path: Column): Column = Column.fn("xpath", xml, path) @@ -10865,8 +15390,14 @@ object functions { /** * Returns true if the XPath expression evaluates to true, or if a matching node is found. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def xpath_boolean(xml: Column, path: Column): Column = Column.fn("xpath_boolean", xml, path) @@ -10875,8 +15406,14 @@ object functions { * Returns a double value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_double(xml: Column, path: Column): Column = Column.fn("xpath_double", xml, path) @@ -10885,8 +15422,14 @@ object functions { * Returns a double value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ def xpath_number(xml: Column, path: Column): Column = Column.fn("xpath_number", xml, path) @@ -10895,8 +15438,14 @@ object functions { * Returns a float value, the value zero if no match is found, or NaN if a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a float. */ def xpath_float(xml: Column, path: Column): Column = Column.fn("xpath_float", xml, path) @@ -10905,8 +15454,14 @@ object functions { * Returns an integer value, or the value zero if no match is found, or a match is found but the * value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ def xpath_int(xml: Column, path: Column): Column = Column.fn("xpath_int", xml, path) @@ -10915,8 +15470,14 @@ object functions { * Returns a long integer value, or the value zero if no match is found, or a match is found but * the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ def xpath_long(xml: Column, path: Column): Column = Column.fn("xpath_long", xml, path) @@ -10925,8 +15486,14 @@ object functions { * Returns a short integer value, or the value zero if no match is found, or a match is found * but the value is non-numeric. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a short. */ def xpath_short(xml: Column, path: Column): Column = Column.fn("xpath_short", xml, path) @@ -10934,8 +15501,14 @@ object functions { /** * Returns the text contents of the first xml node that matches the XPath expression. * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. * @group xml_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ def xpath_string(xml: Column, path: Column): Column = Column.fn("xpath_string", xml, path) @@ -10943,6 +15516,8 @@ object functions { /** * (Java-specific) A transform for timestamps to partition data into hours. * + * @param e + * target date or timestamp column to work on. A column that evaluates to a date or timestamp. * @group partition_transforms * @since 3.0.0 */ @@ -10954,13 +15529,16 @@ object functions { * * @param sourceTz * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. + * used as the source time zone. A column that evaluates to a string. * @param targetTz - * the time zone to which the input timestamp should be converted. + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. * @param sourceTs - * a timestamp without time zone. + * a timestamp without time zone. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) @@ -10969,11 +15547,14 @@ object functions { * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`. * * @param targetTz - * the time zone to which the input timestamp should be converted. + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. * @param sourceTs - * a timestamp without time zone. + * a timestamp without time zone. A column that evaluates to a timestamp. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def convert_timezone(targetTz: Column, sourceTs: Column): Column = Column.fn("convert_timezone", targetTz, sourceTs) @@ -10981,8 +15562,19 @@ object functions { /** * Make DayTimeIntervalType duration from days, hours, mins and secs. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = Column.fn("make_dt_interval", days, hours, mins, secs) @@ -10990,8 +15582,16 @@ object functions { /** * Make DayTimeIntervalType duration from days, hours and mins. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column, mins: Column): Column = Column.fn("make_dt_interval", days, hours, mins) @@ -10999,8 +15599,14 @@ object functions { /** * Make DayTimeIntervalType duration from days and hours. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column, hours: Column): Column = Column.fn("make_dt_interval", days, hours) @@ -11008,8 +15614,12 @@ object functions { /** * Make DayTimeIntervalType duration from days. * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(days: Column): Column = Column.fn("make_dt_interval", days) @@ -11019,6 +15629,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_dt_interval(): Column = Column.fn("make_dt_interval") @@ -11027,8 +15639,25 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -11043,8 +15672,25 @@ object functions { /** * Make interval from years, months, weeks, days, hours, mins and secs. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11060,8 +15706,22 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -11075,8 +15735,22 @@ object functions { /** * Make interval from years, months, weeks, days, hours and mins. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11091,8 +15765,20 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval( years: Column, @@ -11105,8 +15791,20 @@ object functions { /** * Make interval from years, months, weeks, days and hours. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval( years: Column, @@ -11120,8 +15818,18 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("try_make_interval", years, months, weeks, days) @@ -11129,8 +15837,18 @@ object functions { /** * Make interval from years, months, weeks and days. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = Column.fn("make_interval", years, months, weeks, days) @@ -11139,8 +15857,16 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("try_make_interval", years, months, weeks) @@ -11148,8 +15874,16 @@ object functions { /** * Make interval from years, months and weeks. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * The number of weeks, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column, weeks: Column): Column = Column.fn("make_interval", years, months, weeks) @@ -11158,8 +15892,14 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column, months: Column): Column = Column.fn("try_make_interval", years, months) @@ -11167,8 +15907,14 @@ object functions { /** * Make interval from years and months. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column, months: Column): Column = Column.fn("make_interval", years, months) @@ -11177,8 +15923,12 @@ object functions { * This is a special version of `make_interval` that performs the same operation, but returns a * NULL value instead of raising an error if interval cannot be created. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to an interval. */ def try_make_interval(years: Column): Column = Column.fn("try_make_interval", years) @@ -11186,8 +15936,12 @@ object functions { /** * Make interval from years. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(years: Column): Column = Column.fn("make_interval", years) @@ -11197,6 +15951,8 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_interval(): Column = Column.fn("make_interval") @@ -11207,8 +15963,25 @@ object functions { * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. * Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11226,8 +15999,23 @@ object functions { * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it * will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp( years: Column, @@ -11241,8 +16029,16 @@ object functions { /** * Create a local date-time from date, time, and timezone fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("make_timestamp", date, time, timezone) @@ -11250,8 +16046,14 @@ object functions { /** * Create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp(date: Column, time: Column): Column = Column.fn("make_timestamp", date, time) @@ -11261,8 +16063,25 @@ object functions { * The result data type is consistent with the value of configuration `spark.sql.timestampType`. * The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11279,8 +16098,23 @@ object functions { * data type is consistent with the value of configuration `spark.sql.timestampType`. The * function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp( years: Column, @@ -11294,8 +16128,16 @@ object functions { /** * Try to create a local date-time from date, time, and timezone fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = Column.fn("try_make_timestamp", date, time, timezone) @@ -11303,8 +16145,14 @@ object functions { /** * Try to create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp(date: Column, time: Column): Column = Column.fn("try_make_timestamp", date, time) @@ -11314,8 +16162,25 @@ object functions { * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function * returns NULL on invalid inputs. Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11332,8 +16197,24 @@ object functions { * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns * NULL on invalid inputs. Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ltz( years: Column, @@ -11348,8 +16229,26 @@ object functions { * Try to create the current timestamp with local time zone from years, months, days, hours, * mins, secs and timezone fields. The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11365,8 +16264,24 @@ object functions { * Try to create the current timestamp with local time zone from years, months, days, hours, * mins and secs fields. The function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ltz( years: Column, @@ -11382,8 +16297,24 @@ object functions { * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. * Otherwise, it will throw an error instead. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz( years: Column, @@ -11397,8 +16328,14 @@ object functions { /** * Create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("make_timestamp_ntz", date, time) @@ -11407,8 +16344,24 @@ object functions { * Try to create a local date-time from years, months, days, hours, mins, secs fields. The * function returns NULL on invalid inputs. * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. * @group datetime_funcs * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz( years: Column, @@ -11422,8 +16375,14 @@ object functions { /** * Try to create a local date-time from date and time fields. * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. * @group datetime_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. */ def try_make_timestamp_ntz(date: Column, time: Column): Column = Column.fn("try_make_timestamp_ntz", date, time) @@ -11431,8 +16390,14 @@ object functions { /** * Make year-month interval from years, months. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column, months: Column): Column = Column.fn("make_ym_interval", years, months) @@ -11440,8 +16405,12 @@ object functions { /** * Make year-month interval from years. * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) @@ -11450,12 +16419,18 @@ object functions { * * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ def make_ym_interval(): Column = Column.fn("make_ym_interval") /** * (Java-specific) A transform for any type that partitions by a hash of the input column. * + * @param numBuckets + * The number of buckets. A column that evaluates to an integral. Must be a constant. + * @param e + * The input column to partition. A column of any type. * @group partition_transforms * @since 3.0.0 */ @@ -11464,6 +16439,10 @@ object functions { /** * (Java-specific) A transform for any type that partitions by a hash of the input column. * + * @param numBuckets + * The number of buckets. Must be a constant. + * @param e + * The input column to partition. A column of any type. * @group partition_transforms * @since 3.0.0 */ @@ -11476,16 +16455,26 @@ object functions { /** * Returns `col2` if `col1` is null, or `col1` otherwise. * + * @param col1 + * The column to test for null. A column of any type. + * @param col2 + * The column to return when col1 is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) /** * Returns true if `col` is not null, or false otherwise. * + * @param col + * The column to check. A column of any type. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def isnotnull(col: Column): Column = Column.fn("isnotnull", col) @@ -11493,48 +16482,82 @@ object functions { * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both * are null, false if one of the them is null. * + * @param col1 + * The first column to compare. A column of any type. + * @param col2 + * The second column to compare. A column of any type. * @group predicate_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) /** * Returns null if `col1` equals to `col2`, or `col1` otherwise. * + * @param col1 + * The value to return if it is not equal to `col2`. A column of any type. + * @param col2 + * The value compared with `col1`. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) /** * Returns null if `col` is equal to zero, or `col` otherwise. * + * @param col + * The input value. A column that evaluates to a numeric. * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def nullifzero(col: Column): Column = Column.fn("nullifzero", col) /** * Returns `col2` if `col1` is null, or `col1` otherwise. * + * @param col1 + * The value to return if it is not null. A column of any type. + * @param col2 + * The value to return if `col1` is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) /** * Returns `col2` if `col1` is not null, or `col3` otherwise. * + * @param col1 + * The value that determines which branch to return. A column of any type. + * @param col2 + * The value to return if `col1` is not null. A column of any type. + * @param col3 + * The value to return if `col1` is null. A column of any type. * @group conditional_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) /** * Returns zero if `col` is null, or `col` otherwise. * + * @param col + * The input value. A column that evaluates to a numeric. * @group conditional_funcs * @since 4.0.0 + * @return + * Returns a column of the same type as the input. */ def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) @@ -11589,8 +16612,13 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column): Column = Column.fn("st_asbinary", geo) @@ -11598,8 +16626,16 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format using the specified endianness. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param endianness + * The endianness of the output WKB, 'NDR' for little-endian or 'XDR' for big-endian. A column + * that evaluates to a string. * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: Column): Column = Column.fn("st_asbinary", geo, endianness) @@ -11607,8 +16643,16 @@ object functions { /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format using the specified endianness. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param endianness + * The endianness of the output WKB, 'NDR' for little-endian or 'XDR' for big-endian. A column + * that evaluates to a string. * @group st_funcs * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ def st_asbinary(geo: Column, endianness: String): Column = Column.fn("st_asbinary", geo, lit(endianness)) @@ -11616,6 +16660,8 @@ object functions { /** * Parses the WKB description of a geography and returns the corresponding GEOGRAPHY value. * + * @param wkb + * A value in WKB format, representing a GEOGRAPHY value. A column that evaluates to a binary. * @group st_funcs * @since 4.1.0 */ @@ -11625,6 +16671,8 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. * @group st_funcs * @since 4.1.0 */ @@ -11634,6 +16682,10 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. + * @param srid + * The SRID value of the geometry. A column that evaluates to an integer. * @group st_funcs * @since 4.2.0 */ @@ -11643,6 +16695,10 @@ object functions { /** * Parses the WKB description of a geometry and returns the corresponding GEOMETRY value. * + * @param wkb + * A value in WKB format, representing a GEOMETRY value. A column that evaluates to a binary. + * @param srid + * The SRID value of the geometry. A column that evaluates to an integer. * @group st_funcs * @since 4.2.0 */ @@ -11652,6 +16708,11 @@ object functions { /** * Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param srid + * The new SRID of the geospatial value. A column that evaluates to an integer. * @group st_funcs * @since 4.1.0 */ @@ -11661,6 +16722,11 @@ object functions { /** * Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @param srid + * The new SRID of the geospatial value. A column that evaluates to an integer. * @group st_funcs * @since 4.1.0 */ @@ -11670,8 +16736,13 @@ object functions { /** * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. * @group st_funcs * @since 4.1.0 + * @return + * Returns a column that evaluates to an integer. */ def st_srid(geo: Column): Column = Column.fn("st_srid", geo) @@ -12248,6 +17319,8 @@ object functions { /** * Unwrap UDT data type column into its underlying type. + * @param column + * the UDT column to unwrap. A column that evaluates to a user-defined type. * @group udf_funcs * @since 3.4.0 */ @@ -12257,71 +17330,117 @@ object functions { /** * Returns the cosine similarity between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_cosine_similarity(left: Column, right: Column): Column = Column.fn("vector_cosine_similarity", left, right) /** * Returns the inner product (dot product) between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_inner_product(left: Column, right: Column): Column = Column.fn("vector_inner_product", left, right) /** * Returns the Euclidean (L2) distance between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_l2_distance(left: Column, right: Column): Column = Column.fn("vector_l2_distance", left, right) /** * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column, degree: Column): Column = Column.fn("vector_norm", vector, degree) /** * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. */ def vector_norm(vector: Column): Column = Column.fn("vector_norm", vector) /** * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column, degree: Column): Column = Column.fn("vector_normalize", vector, degree) /** * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_normalize(vector: Column): Column = Column.fn("vector_normalize", vector) /** * Aggregate function: returns the element-wise mean of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_avg(col: Column): Column = Column.fn("vector_avg", col) /** * Aggregate function: returns the element-wise sum of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. * @group vector_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. */ def vector_sum(col: Column): Column = Column.fn("vector_sum", col)