diff --git a/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java b/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java index 475a1486a64..2d3ee1a52c8 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java +++ b/core/src/main/java/org/opensearch/sql/analysis/AnalysisContext.java @@ -6,7 +6,11 @@ package org.opensearch.sql.analysis; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; +import lombok.Getter; +import org.opensearch.sql.expression.NamedExpression; /** * The context used for Analyzer. @@ -16,13 +20,16 @@ public class AnalysisContext { * Environment stack for symbol scope management. */ private TypeEnvironment environment; + @Getter + private final List namedParseExpressions; public AnalysisContext() { - this.environment = new TypeEnvironment(null); + this(new TypeEnvironment(null)); } public AnalysisContext(TypeEnvironment environment) { this.environment = environment; + this.namedParseExpressions = new ArrayList<>(); } /** diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 664f5212ce5..52216aefdae 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -37,6 +37,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.ast.tree.Relation; @@ -47,11 +48,13 @@ import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.data.model.ExprMissingValue; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.Aggregator; import org.opensearch.sql.expression.aggregation.NamedAggregator; @@ -70,6 +73,7 @@ import org.opensearch.sql.planner.logical.LogicalValues; import org.opensearch.sql.storage.StorageEngine; import org.opensearch.sql.storage.Table; +import org.opensearch.sql.utils.ParseUtils; /** * Analyze the {@link UnresolvedPlan} in the {@link AnalysisContext} to construct the {@link @@ -286,7 +290,8 @@ public LogicalPlan visitProject(Project node, AnalysisContext context) { TypeEnvironment newEnv = context.peek(); namedExpressions.forEach(expr -> newEnv.define(new Symbol(Namespace.FIELD_NAME, expr.getNameOrAlias()), expr.type())); - return new LogicalProject(child, namedExpressions); + List namedParseExpressions = context.getNamedParseExpressions(); + return new LogicalProject(child, namedExpressions, namedParseExpressions); } /** @@ -308,6 +313,25 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { return new LogicalEval(child, expressionsBuilder.build()); } + /** + * Build {@link ParseExpression} to context and skip to child nodes. + */ + @Override + public LogicalPlan visitParse(Parse node, AnalysisContext context) { + LogicalPlan child = node.getChild().get(0).accept(this, context); + Expression expression = expressionAnalyzer.analyze(node.getExpression(), context); + String pattern = (String) node.getPattern().getValue(); + Expression patternExpression = DSL.literal(pattern); + + TypeEnvironment curEnv = context.peek(); + ParseUtils.getNamedGroupCandidates(pattern).forEach(group -> { + curEnv.define(new Symbol(Namespace.FIELD_NAME, group), ExprCoreType.STRING); + context.getNamedParseExpressions().add(new NamedExpression(group, + new ParseExpression(expression, patternExpression, DSL.literal(group)))); + }); + return child; + } + /** * Build {@link LogicalSort}. */ diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index d420467a9ae..bb419ec5164 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -47,6 +47,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.NamedArgumentExpression; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.AggregationState; import org.opensearch.sql.expression.aggregation.Aggregator; @@ -276,6 +277,13 @@ public Expression visitUnresolvedArgument(UnresolvedArgument node, AnalysisConte } private Expression visitIdentifier(String ident, AnalysisContext context) { + // ParseExpression will always override ReferenceExpression when ident conflicts + for (NamedExpression expr : context.getNamedParseExpressions()) { + if (expr.getNameOrAlias().equals(ident) && expr.getDelegated() instanceof ParseExpression) { + return expr.getDelegated(); + } + } + TypeEnvironment typeEnv = context.peek(); ReferenceExpression ref = DSL.ref(ident, typeEnv.resolve(new Symbol(Namespace.FIELD_NAME, ident))); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index aa04aa5ccef..86f5a6ebc8a 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -38,6 +38,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.ast.tree.Relation; @@ -175,6 +176,10 @@ public T visitEval(Eval node, C context) { return visitChildren(node, context); } + public T visitParse(Parse node, C context) { + return visitChildren(node, context); + } + public T visitLet(Let node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index cda673eb920..65f060a9214 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -44,6 +44,7 @@ import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; import org.opensearch.sql.ast.tree.Limit; +import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.ast.tree.RareTopN.CommandType; @@ -407,4 +408,9 @@ public static RareTopN rareTopN(UnresolvedPlan input, CommandType commandType, public static Limit limit(UnresolvedPlan input, Integer limit, Integer offset) { return new Limit(limit, offset).attach(input); } + + public static Parse parse(UnresolvedPlan input, UnresolvedExpression expression, + Literal pattern) { + return new Parse(expression, pattern, input); + } } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/Parse.java b/core/src/main/java/org/opensearch/sql/ast/tree/Parse.java new file mode 100644 index 00000000000..2eddc32c195 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/Parse.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.AllArgsConstructor; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Literal; +import org.opensearch.sql.ast.expression.UnresolvedExpression; + +/** + * AST node represent Parse operation. + */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +@AllArgsConstructor +public class Parse extends UnresolvedPlan { + /** + * Field. + */ + private final UnresolvedExpression expression; + + /** + * Pattern. + */ + private final Literal pattern; + + /** + * Child Plan. + */ + private UnresolvedPlan child; + + @Override + public Parse attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitParse(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/expression/DSL.java b/core/src/main/java/org/opensearch/sql/expression/DSL.java index fb92f6bc580..8ae94d3e115 100644 --- a/core/src/main/java/org/opensearch/sql/expression/DSL.java +++ b/core/src/main/java/org/opensearch/sql/expression/DSL.java @@ -96,6 +96,10 @@ public static NamedExpression named(Expression expression) { if (expression instanceof NamedExpression) { return (NamedExpression) expression; } + if (expression instanceof ParseExpression) { + return named(((ParseExpression) expression).getIdentifier().valueOf(null).stringValue(), + expression); + } return named(expression.toString(), expression); } @@ -115,6 +119,11 @@ public NamedArgumentExpression namedArgument(String argName, Expression value) { return new NamedArgumentExpression(argName, value); } + public static ParseExpression parsed(Expression expression, Expression pattern, + Expression identifier) { + return new ParseExpression(expression, pattern, identifier); + } + public static SpanExpression span(Expression field, Expression value, String unit) { return new SpanExpression(field, value, SpanUnit.of(unit)); } @@ -254,7 +263,7 @@ public FunctionExpression subtract(Expression... expressions) { public FunctionExpression multiply(Expression... expressions) { return function(BuiltinFunctionName.MULTIPLY, expressions); } - + public FunctionExpression adddate(Expression... expressions) { return function(BuiltinFunctionName.ADDDATE, expressions); } @@ -366,7 +375,7 @@ public FunctionExpression module(Expression... expressions) { public FunctionExpression substr(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } - + public FunctionExpression substring(Expression... expressions) { return function(BuiltinFunctionName.SUBSTR, expressions); } diff --git a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java index 81beeb941c1..fbee8305472 100644 --- a/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/expression/ExpressionNodeVisitor.java @@ -60,6 +60,10 @@ public T visitReference(ReferenceExpression node, C context) { return visitNode(node, context); } + public T visitParse(ParseExpression node, C context) { + return visitNode(node, context); + } + public T visitFunction(FunctionExpression node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/expression/ParseExpression.java b/core/src/main/java/org/opensearch/sql/expression/ParseExpression.java new file mode 100644 index 00000000000..67290f65bea --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/expression/ParseExpression.java @@ -0,0 +1,72 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.expression; + +import com.google.common.collect.ImmutableList; +import java.util.regex.Pattern; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.ToString; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.type.ExprCoreType; +import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.exception.SemanticCheckException; +import org.opensearch.sql.expression.env.Environment; +import org.opensearch.sql.expression.function.FunctionName; +import org.opensearch.sql.utils.ParseUtils; + +/** + * ParseExpression with regex and named capture group. + */ +@EqualsAndHashCode +@ToString +public class ParseExpression extends FunctionExpression { + @Getter + private final Expression expression; + private final Expression rawPattern; + @Getter + private final Expression identifier; + @Getter + @EqualsAndHashCode.Exclude + private final Pattern pattern; + + /** + * ParseExpression. + * + * @param expression text field + * @param rawPattern regex + * @param identifier named capture group to extract + */ + public ParseExpression(Expression expression, Expression rawPattern, Expression identifier) { + super(FunctionName.of("parse"), ImmutableList.of(expression, rawPattern, identifier)); + this.expression = expression; + this.rawPattern = rawPattern; + this.identifier = identifier; + this.pattern = Pattern.compile(rawPattern.valueOf(null).stringValue()); + } + + @Override + public ExprValue valueOf(Environment valueEnv) { + ExprValue value = valueEnv.resolve(expression); + try { + return ParseUtils.parseValue(value, pattern, identifier.valueOf(null).stringValue()); + } catch (ExpressionEvaluationException e) { + throw new SemanticCheckException( + String.format("failed to parse field \"%s\" with type [%s]", expression, value.type())); + } + } + + @Override + public ExprType type() { + return ExprCoreType.STRING; + } + + @Override + public T accept(ExpressionNodeVisitor visitor, C context) { + return visitor.visitParse(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java index 0ca280ff48b..9f2c2c5fa88 100644 --- a/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java +++ b/core/src/main/java/org/opensearch/sql/planner/DefaultImplementor.java @@ -70,7 +70,8 @@ public PhysicalPlan visitDedupe(LogicalDedupe node, C context) { @Override public PhysicalPlan visitProject(LogicalProject node, C context) { - return new ProjectOperator(visitChild(node, context), node.getProjectList()); + return new ProjectOperator(visitChild(node, context), node.getProjectList(), + node.getNamedParseExpressions()); } @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java index 660b00009ec..d5fef43c0db 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalPlanDSL.java @@ -6,6 +6,7 @@ package org.opensearch.sql.planner.logical; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.List; @@ -17,6 +18,7 @@ import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.LiteralExpression; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.expression.window.WindowDefinition; @@ -46,7 +48,12 @@ public static LogicalPlan rename( } public static LogicalPlan project(LogicalPlan input, NamedExpression... fields) { - return new LogicalProject(input, Arrays.asList(fields)); + return new LogicalProject(input, Arrays.asList(fields), ImmutableList.of()); + } + + public static LogicalPlan project(LogicalPlan input, List fields, + List namedParseExpressions) { + return new LogicalProject(input, fields, namedParseExpressions); } public LogicalPlan window(LogicalPlan input, @@ -81,14 +88,14 @@ public static LogicalPlan dedupe( return new LogicalDedupe( input, Arrays.asList(fields), allowedDuplication, keepEmpty, consecutive); } - + public static LogicalPlan rareTopN(LogicalPlan input, CommandType commandType, - List groupByList, Expression... fields) { + List groupByList, Expression... fields) { return rareTopN(input, commandType, 10, groupByList, fields); } public static LogicalPlan rareTopN(LogicalPlan input, CommandType commandType, int noOfResults, - List groupByList, Expression... fields) { + List groupByList, Expression... fields) { return new LogicalRareTopN(input, commandType, noOfResults, Arrays.asList(fields), groupByList); } diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalProject.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalProject.java index 7aa0341e9bd..427ccffc62d 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalProject.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalProject.java @@ -22,15 +22,19 @@ public class LogicalProject extends LogicalPlan { @Getter private final List projectList; + @Getter + private final List namedParseExpressions; /** * Constructor of LogicalProject. */ public LogicalProject( LogicalPlan child, - List projectList) { + List projectList, + List namedParseExpressions) { super(Collections.singletonList(child)); this.projectList = projectList; + this.namedParseExpressions = namedParseExpressions; } @Override diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java index 051aace5a67..938a4c532c8 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/PhysicalPlanDSL.java @@ -6,6 +6,7 @@ package org.opensearch.sql.planner.physical; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.util.Arrays; import java.util.List; @@ -42,7 +43,12 @@ public static RenameOperator rename( } public static ProjectOperator project(PhysicalPlan input, NamedExpression... fields) { - return new ProjectOperator(input, Arrays.asList(fields)); + return new ProjectOperator(input, Arrays.asList(fields), ImmutableList.of()); + } + + public static ProjectOperator project(PhysicalPlan input, List fields, + List namedParseExpressions) { + return new ProjectOperator(input, fields, namedParseExpressions); } public static RemoveOperator remove(PhysicalPlan input, ReferenceExpression... fields) { @@ -80,13 +86,13 @@ public WindowOperator window(PhysicalPlan input, } public static RareTopNOperator rareTopN(PhysicalPlan input, CommandType commandType, - List groups, Expression... expressions) { + List groups, Expression... expressions) { return new RareTopNOperator(input, commandType, Arrays.asList(expressions), groups); } public static RareTopNOperator rareTopN(PhysicalPlan input, CommandType commandType, - int noOfResults, - List groups, Expression... expressions) { + int noOfResults, + List groups, Expression... expressions) { return new RareTopNOperator(input, commandType, noOfResults, Arrays.asList(expressions), groups); } diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java index f9f69f3d92c..7c74f90ba01 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/ProjectOperator.java @@ -17,8 +17,10 @@ import lombok.ToString; import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.executor.ExecutionEngine; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ParseExpression; /** * Project the fields specified in {@link ProjectOperator#projectList} from input. @@ -31,6 +33,8 @@ public class ProjectOperator extends PhysicalPlan { private final PhysicalPlan input; @Getter private final List projectList; + @Getter + private final List namedParseExpressions; @Override public R accept(PhysicalPlanNodeVisitor visitor, C context) { @@ -53,7 +57,27 @@ public ExprValue next() { ImmutableMap.Builder mapBuilder = new Builder<>(); for (NamedExpression expr : projectList) { ExprValue exprValue = expr.valueOf(inputValue.bindingTuples()); - mapBuilder.put(expr.getNameOrAlias(), exprValue); + if (namedParseExpressions.stream() + .noneMatch(parsed -> parsed.getNameOrAlias().equals(expr.getNameOrAlias()))) { + mapBuilder.put(expr.getNameOrAlias(), exprValue); + } + } + // ParseExpression will always override NamedExpression when identifier conflicts + // TODO needs a better implementation, see https://github.com/opensearch-project/sql/issues/458 + for (NamedExpression expr : namedParseExpressions) { + ExprValue value = inputValue.bindingTuples() + .resolve(((ParseExpression) expr.getDelegated()).getExpression()); + if (value.isMissing()) { + // value will be missing after stats command, read from inputValue if it exists + // otherwise do nothing since it should not appear as a field + ExprValue exprValue = ExprValueUtils.getTupleValue(inputValue).get(expr.getNameOrAlias()); + if (exprValue != null) { + mapBuilder.put(expr.getNameOrAlias(), exprValue); + } + } else { + ExprValue parsedValue = expr.valueOf(inputValue.bindingTuples()); + mapBuilder.put(expr.getNameOrAlias(), parsedValue); + } } return ExprTupleValue.fromExprValueMap(mapBuilder.build()); } diff --git a/core/src/main/java/org/opensearch/sql/utils/ParseUtils.java b/core/src/main/java/org/opensearch/sql/utils/ParseUtils.java new file mode 100644 index 00000000000..2a8dac11805 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/utils/ParseUtils.java @@ -0,0 +1,67 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.utils; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import lombok.experimental.UtilityClass; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.sql.data.model.ExprStringValue; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.data.model.ExprValueUtils; +import org.opensearch.sql.exception.ExpressionEvaluationException; +import org.opensearch.sql.expression.ParseExpression; + +/** + * Utils for {@link ParseExpression}. + */ +@UtilityClass +public class ParseUtils { + private static final Logger log = LogManager.getLogger(ParseUtils.class); + private static final Pattern GROUP_PATTERN = Pattern.compile("\\(\\?<([a-zA-Z][a-zA-Z0-9]*)>"); + + /** + * Get matched group value, throws ExpressionEvaluationException if value type is not string. + * + * @param value text field + * @param pattern regex pattern + * @param identifier named capture group + * @return matched group value, empty string if pattern does not match + */ + public static ExprValue parseValue(ExprValue value, Pattern pattern, String identifier) + throws ExpressionEvaluationException { + if (value.isNull() || value.isMissing()) { + return ExprValueUtils.nullValue(); + } + + String rawString = value.stringValue(); + Matcher matcher = pattern.matcher(rawString); + if (matcher.matches()) { + return new ExprStringValue(matcher.group(identifier)); + } + log.warn("failed to extract pattern {} from input {}", pattern.pattern(), rawString); + return new ExprStringValue(""); + } + + /** + * Get capture groups from regex pattern. + * + * @param pattern regex pattern + * @return list of named capture groups in regex pattern + */ + public static List getNamedGroupCandidates(String pattern) { + ImmutableList.Builder namedGroups = ImmutableList.builder(); + Matcher m = GROUP_PATTERN.matcher(pattern); + while (m.find()) { + namedGroups.add(m.group(1)); + } + return namedGroups.build(); + } +} diff --git a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java index 8b733a78de7..63ee4f827a0 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/AnalyzerTest.java @@ -50,12 +50,14 @@ import org.opensearch.sql.expression.window.WindowDefinition; import org.opensearch.sql.planner.logical.LogicalPlanDSL; import org.springframework.context.annotation.Configuration; +import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; @Configuration @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {ExpressionConfig.class, AnalyzerTest.class}) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD) class AnalyzerTest extends AnalyzerTestBase { @Test @@ -669,4 +671,23 @@ public void ppl_stats_by_fieldAndSpan() { alias("span", span(field("long_value"), intLiteral(10), SpanUnit.NONE)), emptyList())); } + + @Test + public void parse_relation() { + assertAnalyzeEqual( + LogicalPlanDSL.project( + LogicalPlanDSL.relation("schema"), + ImmutableList.of(DSL.named("string_value", DSL.ref("string_value", STRING))), + ImmutableList.of(DSL.named("group", + DSL.parsed(DSL.ref("string_value", STRING), DSL.literal("(?.*)"), + DSL.literal("group")))) + ), + AstDSL.project( + AstDSL.parse( + AstDSL.relation("schema"), + AstDSL.field("string_value"), + AstDSL.stringLiteral("(?.*)")), + AstDSL.alias("string_value", qualifiedName("string_value")) + )); + } } diff --git a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java index ea1c1008a24..6cb59f918c6 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/ExpressionAnalyzerTest.java @@ -18,7 +18,9 @@ import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; +import static org.opensearch.sql.expression.DSL.ref; import java.util.Collections; import org.junit.jupiter.api.Test; @@ -30,6 +32,7 @@ import org.opensearch.sql.ast.expression.DataType; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; +import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprValueUtils; import org.opensearch.sql.exception.SemanticCheckException; @@ -309,6 +312,30 @@ public void named_argument() { ); } + @Test + public void named_parse_expression() { + analysisContext.push(); + analysisContext.peek().define(new Symbol(Namespace.FIELD_NAME, "string_field"), STRING); + analysisContext.getNamedParseExpressions() + .add(DSL.named("group", + DSL.parsed(ref("string_field", STRING), DSL.literal("(?\\d+)"), + DSL.literal("group")))); + assertAnalyzeEqual( + DSL.parsed(ref("string_field", STRING), DSL.literal("(?\\d+)"), + DSL.literal("group")), + qualifiedName("group") + ); + } + + @Test + public void named_non_parse_expression() { + analysisContext.push(); + analysisContext.peek().define(new Symbol(Namespace.FIELD_NAME, "string_field"), STRING); + analysisContext.getNamedParseExpressions() + .add(DSL.named("string_field", DSL.literal("123"))); + assertAnalyzeEqual(DSL.ref("string_field", STRING), qualifiedName("string_field")); + } + @Test void visit_span() { assertAnalyzeEqual( @@ -340,4 +367,9 @@ protected void assertAnalyzeEqual(Expression expected, UnresolvedExpression unresolvedExpression) { assertEquals(expected, analyze(unresolvedExpression)); } + + protected void assertAnalyzeEqual(Expression expected, + UnresolvedPlan unresolvedPlan) { + assertEquals(expected, analyze(unresolvedPlan)); + } } diff --git a/core/src/test/java/org/opensearch/sql/config/TestConfig.java b/core/src/test/java/org/opensearch/sql/config/TestConfig.java index 8bd58f693d9..ab78109aa26 100644 --- a/core/src/test/java/org/opensearch/sql/config/TestConfig.java +++ b/core/src/test/java/org/opensearch/sql/config/TestConfig.java @@ -35,8 +35,8 @@ public class TestConfig { public static final String DOUBLE_TYPE_MISSING_VALUE_FIELD = "double_missing_value"; public static final String BOOL_TYPE_NULL_VALUE_FIELD = "null_value_boolean"; public static final String BOOL_TYPE_MISSING_VALUE_FIELD = "missing_value_boolean"; - public static final String STRING_TYPE_NULL_VALUE_FILED = "string_null_value"; - public static final String STRING_TYPE_MISSING_VALUE_FILED = "string_missing_value"; + public static final String STRING_TYPE_NULL_VALUE_FIELD = "string_null_value"; + public static final String STRING_TYPE_MISSING_VALUE_FIELD = "string_missing_value"; public static Map typeMapping = new ImmutableMap.Builder() .put("integer_value", ExprCoreType.INTEGER) @@ -51,8 +51,8 @@ public class TestConfig { .put(BOOL_TYPE_NULL_VALUE_FIELD, ExprCoreType.BOOLEAN) .put(BOOL_TYPE_MISSING_VALUE_FIELD, ExprCoreType.BOOLEAN) .put("string_value", ExprCoreType.STRING) - .put(STRING_TYPE_NULL_VALUE_FILED, ExprCoreType.STRING) - .put(STRING_TYPE_MISSING_VALUE_FILED, ExprCoreType.STRING) + .put(STRING_TYPE_NULL_VALUE_FIELD, ExprCoreType.STRING) + .put(STRING_TYPE_MISSING_VALUE_FIELD, ExprCoreType.STRING) .put("struct_value", ExprCoreType.STRUCT) .put("array_value", ExprCoreType.ARRAY) .build(); diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java index 9641e26d338..caf11064ae8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionNodeVisitorTest.java @@ -45,15 +45,22 @@ void should_return_null_by_default() { assertNull(new WhenClause(literal("test"), literal(10)).accept(visitor, null)); assertNull(dsl.namedArgument("field", literal("message")).accept(visitor, null)); assertNull(DSL.span(ref("age", INTEGER), literal(1), "").accept(visitor, null)); + assertNull(DSL.parsed(ref("name", STRING), DSL.literal("(?\\d+)"), DSL.literal("group")) + .accept(visitor, null)); } @Test void can_visit_all_types_of_expression_node() { Expression expr = - dsl.sum( - dsl.add( - ref("balance", INTEGER), - literal(10))); + DSL.parsed( + dsl.castString( + dsl.sum( + dsl.add( + ref("balance", INTEGER), + literal(10)) + )), + DSL.literal("(?\\d+)"), + DSL.literal("group")); Expression actual = expr.accept(new ExpressionNodeVisitor() { @Override @@ -66,6 +73,11 @@ public Expression visitReference(ReferenceExpression node, Object context) { return node; } + @Override + public Expression visitParse(ParseExpression node, Object context) { + return node; + } + @Override public Expression visitFunction(FunctionExpression node, Object context) { return dsl.add(visitArguments(node.getArguments(), context)); diff --git a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java index 6e677cf11f6..fea985042a8 100644 --- a/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java +++ b/core/src/test/java/org/opensearch/sql/expression/ExpressionTestBase.java @@ -12,8 +12,8 @@ import static org.opensearch.sql.config.TestConfig.DOUBLE_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FIELD; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; import static org.opensearch.sql.data.model.ExprValueUtils.collectionValue; import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; @@ -77,12 +77,12 @@ protected static Environment valueEnv() { case BOOL_TYPE_NULL_VALUE_FIELD: case INT_TYPE_NULL_VALUE_FIELD: case DOUBLE_TYPE_NULL_VALUE_FIELD: - case STRING_TYPE_NULL_VALUE_FILED: + case STRING_TYPE_NULL_VALUE_FIELD: return nullValue(); case INT_TYPE_MISSING_VALUE_FIELD: case BOOL_TYPE_MISSING_VALUE_FIELD: case DOUBLE_TYPE_MISSING_VALUE_FIELD: - case STRING_TYPE_MISSING_VALUE_FILED: + case STRING_TYPE_MISSING_VALUE_FIELD: return missingValue(); default: throw new IllegalArgumentException("undefined reference"); diff --git a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java index 5363dfe3f2d..1957a24c2bc 100644 --- a/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/NamedExpressionTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import org.junit.jupiter.api.DisplayNameGeneration; import org.junit.jupiter.api.DisplayNameGenerator; @@ -50,4 +51,14 @@ void name_a_span_expression() { assertEquals(span, named.getDelegated()); } + @Test + void name_a_parse_expression() { + ParseExpression parse = + DSL.parsed(DSL.ref("string_value", STRING), DSL.literal("(?\\w{2})\\w"), + DSL.literal("group")); + NamedExpression named = DSL.named(parse); + assertEquals(parse, named.getDelegated()); + assertEquals(parse.getIdentifier().valueOf(null).stringValue(), named.getName()); + } + } diff --git a/core/src/test/java/org/opensearch/sql/expression/ParseExpressionTest.java b/core/src/test/java/org/opensearch/sql/expression/ParseExpressionTest.java new file mode 100644 index 00000000000..ce6dce63126 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/expression/ParseExpressionTest.java @@ -0,0 +1,57 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + +package org.opensearch.sql.expression; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FIELD; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FIELD; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; + +import org.junit.jupiter.api.DisplayNameGeneration; +import org.junit.jupiter.api.DisplayNameGenerator; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.exception.SemanticCheckException; + +@DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) +class ParseExpressionTest extends ExpressionTestBase { + + @Test + public void resolve_value() { + assertEquals(stringValue("st"), + DSL.parsed(DSL.ref("string_value", STRING), DSL.literal("(?\\w{2})\\w"), + DSL.literal("group")) + .valueOf(valueEnv())); + assertEquals(LITERAL_NULL, + DSL.parsed(DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING), + DSL.literal("(?\\w{2})\\w"), DSL.literal("group")) + .valueOf(valueEnv())); + assertEquals(LITERAL_NULL, + DSL.parsed(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), + DSL.literal("(?\\w{2})\\w"), DSL.literal("group")) + .valueOf(valueEnv())); + } + + @Test + public void resolve_type() { + assertEquals(STRING, + DSL.parsed(DSL.ref("string_value", STRING), DSL.literal("(?\\w{2})\\w"), + DSL.literal("group")).type()); + } + + @Test + public void throws_semantic_exception_if_value_type_is_not_string() { + assertThrows( + SemanticCheckException.class, + () -> DSL.parsed(DSL.ref("boolean_value", BOOLEAN), DSL.literal("(?\\w{2})\\w"), + DSL.literal("group")) + .valueOf(valueEnv())); + } +} diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java index 3c65c387e55..4ce975c7d2b 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/arthmetic/MathematicalFunctionTest.java @@ -16,8 +16,8 @@ import static org.opensearch.sql.config.TestConfig.DOUBLE_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FIELD; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.data.model.ExprValueUtils.getDoubleValue; import static org.opensearch.sql.data.model.ExprValueUtils.getFloatValue; import static org.opensearch.sql.data.type.ExprCoreType.BYTE; @@ -385,7 +385,7 @@ public void conv_to_decimal(Integer value) { @Test public void conv_null_value() { FunctionExpression conv = dsl.conv( - DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), DSL.literal(10), DSL.literal(2)); + DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING), DSL.literal(10), DSL.literal(2)); assertEquals(STRING, conv.type()); assertTrue(conv.valueOf(valueEnv()).isNull()); @@ -406,7 +406,7 @@ public void conv_null_value() { @Test public void conv_missing_value() { FunctionExpression conv = dsl.conv( - DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), DSL.literal(10), DSL.literal(2)); + DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), DSL.literal(10), DSL.literal(2)); assertEquals(STRING, conv.type()); assertTrue(conv.valueOf(valueEnv()).isMissing()); @@ -426,7 +426,7 @@ public void conv_missing_value() { */ @Test public void conv_null_missing() { - FunctionExpression conv = dsl.conv(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), + FunctionExpression conv = dsl.conv(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), DSL.ref(INT_TYPE_MISSING_VALUE_FIELD, INTEGER), DSL.literal(2)); assertEquals(STRING, conv.type()); assertTrue(conv.valueOf(valueEnv()).isMissing()); @@ -452,7 +452,7 @@ public void crc32_string_value(String value) { */ @Test public void crc32_null_value() { - FunctionExpression crc = dsl.crc32(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + FunctionExpression crc = dsl.crc32(DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING)); assertEquals(LONG, crc.type()); assertTrue(crc.valueOf(valueEnv()).isNull()); } @@ -462,7 +462,7 @@ public void crc32_null_value() { */ @Test public void crc32_missing_value() { - FunctionExpression crc = dsl.crc32(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + FunctionExpression crc = dsl.crc32(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING)); assertEquals(LONG, crc.type()); assertTrue(crc.valueOf(valueEnv()).isMissing()); } diff --git a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java index 3ace82c4ebc..aae76af8de6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/operator/predicate/BinaryPredicateOperatorTest.java @@ -13,8 +13,8 @@ import static org.opensearch.sql.config.TestConfig.BOOL_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_MISSING_VALUE_FIELD; import static org.opensearch.sql.config.TestConfig.INT_TYPE_NULL_VALUE_FIELD; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FILED; -import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FILED; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_MISSING_VALUE_FIELD; +import static org.opensearch.sql.config.TestConfig.STRING_TYPE_NULL_VALUE_FIELD; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_FALSE; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; @@ -716,16 +716,16 @@ public void test_like(ExprValue v1, ExprValue v2) { @Test public void test_like_null() { FunctionExpression like = - dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); - like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), DSL.literal("str")); + like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING), DSL.literal("str")); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); - like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), - DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING), + DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_NULL, like.valueOf(valueEnv())); } @@ -733,29 +733,29 @@ public void test_like_null() { @Test public void test_like_missing() { FunctionExpression like = - dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + dsl.like(DSL.literal("str"), DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); - like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), DSL.literal("str")); + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), DSL.literal("str")); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); - like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), - DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), + DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); } @Test public void test_null_like_missing() { - FunctionExpression like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING), - DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING)); + FunctionExpression like = dsl.like(DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING), + DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); - like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FILED, STRING), - DSL.ref(STRING_TYPE_NULL_VALUE_FILED, STRING)); + like = dsl.like(DSL.ref(STRING_TYPE_MISSING_VALUE_FIELD, STRING), + DSL.ref(STRING_TYPE_NULL_VALUE_FIELD, STRING)); assertEquals(BOOLEAN, like.type()); assertEquals(LITERAL_MISSING, like.valueOf(valueEnv())); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java index bdebfcd4052..2f56cd6c381 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/ProjectOperatorTest.java @@ -18,6 +18,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.planner.physical.PhysicalPlanDSL.project; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.List; import org.junit.jupiter.api.Test; @@ -102,4 +103,57 @@ public void project_schema() { new ExecutionEngine.Schema.Column("action", "act", STRING) )); } + + @Test + public void project_fields_with_parse_expressions() { + when(inputPlan.hasNext()).thenReturn(true, false); + when(inputPlan.next()) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("response", "GET 200"))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("action", DSL.ref("action", STRING))), + ImmutableList.of(DSL.named("action", + DSL.parsed(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("action"))), DSL.named("response", + DSL.parsed(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("response")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(1), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200"))))); + } + + @Test + public void project_parse_missing_will_fallback() { + when(inputPlan.hasNext()).thenReturn(true, true, false); + when(inputPlan.next()) + .thenReturn( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "GET 200"))) + .thenReturn(ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))); + PhysicalPlan plan = + project(inputPlan, ImmutableList.of(DSL.named("action", DSL.ref("action", STRING))), + ImmutableList.of(DSL.named("action", + DSL.parsed(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("action"))), DSL.named("response", + DSL.parsed(DSL.ref("response", STRING), + DSL.literal("(?\\w+) (?\\d+)"), + DSL.literal("response")))) + ); + List result = execute(plan); + + assertThat( + result, + allOf( + iterableWithSize(2), + hasItems( + ExprValueUtils.tupleValue(ImmutableMap.of("action", "GET", "response", "200")), + ExprValueUtils.tupleValue(ImmutableMap.of("action", "POST"))))); + } } diff --git a/core/src/test/java/org/opensearch/sql/utils/ParseUtilsTest.java b/core/src/test/java/org/opensearch/sql/utils/ParseUtilsTest.java new file mode 100644 index 00000000000..d42a7c71935 --- /dev/null +++ b/core/src/test/java/org/opensearch/sql/utils/ParseUtilsTest.java @@ -0,0 +1,61 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_MISSING; +import static org.opensearch.sql.data.model.ExprValueUtils.LITERAL_NULL; +import static org.opensearch.sql.data.model.ExprValueUtils.longValue; +import static org.opensearch.sql.data.model.ExprValueUtils.stringValue; + +import com.google.common.collect.ImmutableMap; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.data.model.ExprValue; +import org.opensearch.sql.exception.ExpressionEvaluationException; + +public class ParseUtilsTest { + @Test + void test_parse_group_and_value() { + ExprValue exprValue = stringValue( + "130.246.123.197 - - [2018-07-22T03:26:21.326Z] \"GET /beats/metricbeat_1 HTTP/1.1\" " + + "200 6850 \"-\" \"Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 " + + "Firefox/6.0a1\""); + String rawPattern = + "(?(\\d{1,3}\\.){3}\\d{1,3}) - - \\[(?\\d{4}-[01]\\d-[0-3]\\dT[0-2]\\d:" + + "[0-5]\\d:[0-5]\\d\\.\\d+([+-][0-2]\\d:[0-5]\\d|Z))] \"(?[^\"]+)\" " + + "(?\\d+) (?\\d+) \"-\" \"(?[^\"]+)\""; + Pattern pattern = Pattern.compile(rawPattern); + Map expected = + ImmutableMap.of("ip", "130.246.123.197", "date", "2018-07-22T03:26:21.326Z", "request", + "GET /beats/metricbeat_1 HTTP/1.1", "status", "200", "bytes", "6850", "userAgent", + "Mozilla/5.0 (X11; Linux x86_64; rv:6.0a1) Gecko/20110421 Firefox/6.0a1"); + List identifiers = new ArrayList<>(expected.keySet()); + assertEquals(identifiers, ParseUtils.getNamedGroupCandidates(rawPattern)); + identifiers.forEach(identifier -> assertEquals(stringValue(expected.get(identifier)), + ParseUtils.parseValue(exprValue, pattern, identifier))); + } + + @Test + void test_null_missing_non_match() { + Pattern pattern = Pattern.compile("(?\\d+)"); + assertEquals(LITERAL_NULL, ParseUtils.parseValue(LITERAL_NULL, pattern, "group")); + assertEquals(LITERAL_NULL, ParseUtils.parseValue(LITERAL_MISSING, pattern, "group")); + assertEquals(stringValue(""), + ParseUtils.parseValue(stringValue("non match"), pattern, "group")); + } + + @Test + void test_non_text_field_should_throw_expression_evaluation_exception() { + assertThrows(ExpressionEvaluationException.class, + () -> ParseUtils.parseValue(longValue(10000L), Pattern.compile("(?\\d+)"), "group")); + } +} diff --git a/docs/category.json b/docs/category.json index b23fbdd8b2d..bcd6318a463 100644 --- a/docs/category.json +++ b/docs/category.json @@ -10,6 +10,7 @@ "user/ppl/cmd/dedup.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/fields.rst", + "user/ppl/cmd/parse.rst", "user/ppl/cmd/rename.rst", "user/ppl/cmd/search.rst", "user/ppl/cmd/sort.rst", diff --git a/docs/user/ppl/cmd/parse.rst b/docs/user/ppl/cmd/parse.rst new file mode 100644 index 00000000000..6eeb5025943 --- /dev/null +++ b/docs/user/ppl/cmd/parse.rst @@ -0,0 +1,110 @@ +============= +parse +============= + +.. rubric:: Table of contents + +.. contents:: + :local: + :depth: 2 + + +Description +============ +| The ``parse`` command parses a text field using a regular expression and append the result to the search result. + + +Syntax +============ +parse + +* field: mandatory. The field must be a text field. +* regular-expression: mandatory. The regular expression used to extract new fields from given text field. If a new field name already exists, it will replace the original field. + +Regular Expression +================== + +The regular expression is used to match the whole text field of each document with Java regex engine. Each named capture group in the expression will become a new ``STRING`` field. + +Example 1: Create the new field +=============================== + +The example shows how to create new field ``host`` for each document. ``host`` will be the host name after ``@`` in ``email`` field. Parsing a null field will return an empty string. + +PPL query:: + + os> source=accounts | parse email '.+@(?.+)' | fields email, host ; + fetched rows / total rows = 4/4 + +-----------------------+------------+ + | email | host | + |-----------------------+------------| + | amberduke@pyrami.com | pyrami.com | + | hattiebond@netagy.com | netagy.com | + | null | | + | daleadams@boink.com | boink.com | + +-----------------------+------------+ + + +Example 2: Override the existing field +====================================== + +The example shows how to override the existing ``address`` field with street number removed. + +PPL query:: + + os> source=accounts | parse address '\d+ (?
.+)' | fields address ; + fetched rows / total rows = 4/4 + +------------------+ + | address | + |------------------| + | Holmes Lane | + | Bristol Street | + | Madison Street | + | Hutchinson Court | + +------------------+ + +Example 3: Filter and sort by casted parsed field +================================================= + +The example shows how to sort street numbers that are higher than 500 in ``address`` field. + +PPL query:: + + os> source=accounts | parse address '(?\d+) (?.+)' | where cast(streetNumber as int) > 500 | sort num(streetNumber) | fields streetNumber, street ; + fetched rows / total rows = 3/3 + +----------------+----------------+ + | streetNumber | street | + |----------------+----------------| + | 671 | Bristol Street | + | 789 | Madison Street | + | 880 | Holmes Lane | + +----------------+----------------+ + +Limitation +========== + +There are a few limitations with parse command: + +- Fields defined by parse cannot be parsed again. + + The following command will not work:: + + source=accounts | parse address '\d+ (?.+)' | parse street '\w+ (?\w+)' ; + +- Fields defined by parse cannot be overridden with other commands. + + ``where`` will not match any documents since ``street`` cannot be overridden:: + + source=accounts | parse address '\d+ (?.+)' | eval street='1' | where street='1' ; + +- The text field used by parse cannot be overridden. + + ``street`` will not be successfully parsed since ``address`` is overridden:: + + source=accounts | parse address '\d+ (?.+)' | eval address='1' ; + +- Fields defined by parse cannot be filtered/sorted after using them in ``stats`` command. + + ``where`` in the following command will not work:: + + source=accounts | parse email '.+@(?.+)' | stats avg(age) by host | where host=pyrami.com ; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java index a286737cc4c..aec8800944e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/protector/OpenSearchExecutionProtector.java @@ -70,7 +70,8 @@ public PhysicalPlan visitTableScan(TableScanOperator node, Object context) { @Override public PhysicalPlan visitProject(ProjectOperator node, Object context) { - return new ProjectOperator(visitInput(node.getInput(), context), node.getProjectList()); + return new ProjectOperator(visitInput(node.getInput(), context), node.getProjectList(), + node.getNamedParseExpressions()); } @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndIndexScan.java index e4fe29d5a67..43714282fbb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndIndexScan.java @@ -57,6 +57,7 @@ public LogicalPlan apply(LogicalProject project, Captures captures) { OpenSearchLogicalIndexScan indexScan = captures.get(indexScanCapture); indexScan.setProjectList(pushDownProjects); - return new LogicalProject(indexScan, project.getProjectList()); + return new LogicalProject(indexScan, project.getProjectList(), + project.getNamedParseExpressions()); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndRelation.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndRelation.java index 078356cb35b..a29a1df4664 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndRelation.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/logical/rule/PushProjectAndRelation.java @@ -60,7 +60,8 @@ public LogicalPlan apply(LogicalProject project, .relationName(relation.getRelationName()) .projectList(findReferenceExpressions(project.getProjectList())) .build(), - project.getProjectList() + project.getProjectList(), + project.getNamedParseExpressions() ); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java index 2df67a0a1ba..116d196fc39 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java @@ -25,6 +25,7 @@ import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.Expression; import org.opensearch.sql.expression.ExpressionNodeVisitor; +import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.env.Environment; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; @@ -92,6 +93,12 @@ public Object visitReference(ReferenceExpression node, Set context.add(node); return null; } + + @Override + public Object visitParse(ParseExpression node, Set context) { + node.getExpression().accept(this, context); + return null; + } }, fields); return fields; } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java index eecf7a893b3..5da9f5e01ad 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/ExpressionAggregationScriptTest.java @@ -94,6 +94,15 @@ void can_execute_expression_with_missing_field() { .shouldMatch(null); } + @Test + void can_execute_parse_expression() { + assertThat() + .docValues("age_string", "age: 30") + .evaluate(DSL.parsed(DSL.ref("age_string", STRING), DSL.literal("age: (?\\d+)"), + DSL.literal("age"))) + .shouldMatch("30"); + } + private ExprScriptAssertion assertThat() { return new ExprScriptAssertion(lookup, leafLookup, context); } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java index 24bc3170e55..4abdb6e0765 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/BucketAggregationBuilderTest.java @@ -7,8 +7,10 @@ package org.opensearch.sql.opensearch.storage.script.aggregation.dsl; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.when; import static org.opensearch.common.xcontent.ToXContent.EMPTY_PARAMS; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.named; import static org.opensearch.sql.expression.DSL.ref; import static org.opensearch.sql.opensearch.data.type.OpenSearchDataType.OPENSEARCH_TEXT_KEYWORD; @@ -30,7 +32,9 @@ import org.opensearch.common.xcontent.XContentType; import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.sort.SortOrder; +import org.opensearch.sql.expression.DSL; import org.opensearch.sql.expression.NamedExpression; +import org.opensearch.sql.expression.ParseExpression; import org.opensearch.sql.opensearch.storage.serialization.ExpressionSerializer; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) @@ -77,6 +81,27 @@ void should_build_bucket_with_keyword_field() { asc(named("name", ref("name", OPENSEARCH_TEXT_KEYWORD)))))); } + @Test + void should_build_bucket_with_parse_expression() { + ParseExpression parseExpression = + DSL.parsed(ref("name.keyword", STRING), DSL.literal("(?\\w+)"), DSL.literal("name")); + when(serializer.serialize(parseExpression)).thenReturn("mock-serialize"); + assertEquals( + "{\n" + + " \"terms\" : {\n" + + " \"script\" : {\n" + + " \"source\" : \"mock-serialize\",\n" + + " \"lang\" : \"opensearch_query_expression\"\n" + + " },\n" + + " \"missing_bucket\" : true,\n" + + " \"order\" : \"asc\"\n" + + " }\n" + + "}", + buildQuery( + Arrays.asList( + asc(named("name", parseExpression))))); + } + @SneakyThrows private String buildQuery(List> groupByExpressions) { XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java index aea9460b7e9..edd3fbeb961 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/ExpressionFilterScriptTest.java @@ -133,6 +133,16 @@ void can_execute_expression_with_empty_doc_value() { .shouldNotMatch(); } + @Test + void can_execute_parse_expression() { + assertThat() + .docValues("age_string", "age: 30") + .filterBy(dsl.equal( + DSL.parsed(DSL.ref("age_string", STRING), literal("age: (?\\d+)"), literal("age")), + literal("30"))) + .shouldMatch(); + } + @Test void cannot_execute_non_predicate_expression() { assertThrow(IllegalStateException.class, diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 109e349a152..eee8fe46a66 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -22,6 +22,7 @@ EVAL: 'EVAL'; HEAD: 'HEAD'; TOP: 'TOP'; RARE: 'RARE'; +PARSE: 'PARSE'; // COMMAND ASSIST KEYWORDS AS: 'AS'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index be0af8f7ab9..5cec6d4f569 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -20,7 +20,7 @@ pplStatement /** commands */ commands : whereCommand | fieldsCommand | renameCommand | statsCommand | dedupCommand | sortCommand | evalCommand | headCommand - | topCommand | rareCommand; + | topCommand | rareCommand | parseCommand; searchCommand : (SEARCH)? fromClause #searchFrom @@ -84,6 +84,10 @@ rareCommand (byClause)? ; +parseCommand + : PARSE expression pattern + ; + /** clauses */ fromClause : SOURCE EQUAL tableSource (COMMA tableSource)* @@ -349,6 +353,10 @@ booleanLiteral : TRUE | FALSE ; +pattern + : stringLiteral + ; + intervalUnit : MICROSECOND | SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR | SECOND_MICROSECOND | MINUTE_MICROSECOND | MINUTE_SECOND | HOUR_MICROSECOND | HOUR_SECOND | HOUR_MINUTE | DAY_MICROSECOND diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 918c8836a60..ab1509129b9 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -34,6 +34,7 @@ import org.opensearch.sql.ast.expression.Alias; import org.opensearch.sql.ast.expression.Field; import org.opensearch.sql.ast.expression.Let; +import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Map; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.tree.Aggregation; @@ -41,6 +42,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Head; +import org.opensearch.sql.ast.tree.Parse; import org.opensearch.sql.ast.tree.Project; import org.opensearch.sql.ast.tree.RareTopN; import org.opensearch.sql.ast.tree.RareTopN.CommandType; @@ -89,12 +91,14 @@ public UnresolvedPlan visitSearchFrom(SearchFromContext ctx) { @Override public UnresolvedPlan visitSearchFromFilter(SearchFromFilterContext ctx) { - return new Filter(visitExpression(ctx.logicalExpression())).attach(visit(ctx.fromClause())); + return new Filter(internalVisitExpression(ctx.logicalExpression())).attach( + visit(ctx.fromClause())); } @Override public UnresolvedPlan visitSearchFilterFrom(SearchFilterFromContext ctx) { - return new Filter(visitExpression(ctx.logicalExpression())).attach(visit(ctx.fromClause())); + return new Filter(internalVisitExpression(ctx.logicalExpression())).attach( + visit(ctx.fromClause())); } /** @@ -102,7 +106,7 @@ public UnresolvedPlan visitSearchFilterFrom(SearchFilterFromContext ctx) { */ @Override public UnresolvedPlan visitWhereCommand(WhereCommandContext ctx) { - return new Filter(visitExpression(ctx.logicalExpression())); + return new Filter(internalVisitExpression(ctx.logicalExpression())); } /** @@ -114,7 +118,7 @@ public UnresolvedPlan visitFieldsCommand(FieldsCommandContext ctx) { ctx.fieldList() .fieldExpression() .stream() - .map(this::visitExpression) + .map(this::internalVisitExpression) .collect(Collectors.toList()), ArgumentFactory.getArgumentList(ctx) ); @@ -128,7 +132,8 @@ public UnresolvedPlan visitRenameCommand(RenameCommandContext ctx) { return new Rename( ctx.renameClasue() .stream() - .map(ct -> new Map(visitExpression(ct.orignalField), visitExpression(ct.renamedField))) + .map(ct -> new Map(internalVisitExpression(ct.orignalField), + internalVisitExpression(ct.renamedField))) .collect(Collectors.toList()) ); } @@ -140,7 +145,7 @@ public UnresolvedPlan visitRenameCommand(RenameCommandContext ctx) { public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { ImmutableList.Builder aggListBuilder = new ImmutableList.Builder<>(); for (OpenSearchPPLParser.StatsAggTermContext aggCtx : ctx.statsAggTerm()) { - UnresolvedExpression aggExpression = visitExpression(aggCtx.statsFunction()); + UnresolvedExpression aggExpression = internalVisitExpression(aggCtx.statsFunction()); String name = aggCtx.alias == null ? getTextInQuery(aggCtx) : StringUtils .unquoteIdentifier(aggCtx.alias.getText()); Alias alias = new Alias(name, aggExpression); @@ -153,14 +158,14 @@ public UnresolvedPlan visitStatsCommand(StatsCommandContext ctx) { .map(expr -> expr.fieldExpression().stream() .map(groupCtx -> (UnresolvedExpression) new Alias(getTextInQuery(groupCtx), - visitExpression(groupCtx))) + internalVisitExpression(groupCtx))) .collect(Collectors.toList())) .orElse(Collections.emptyList()); UnresolvedExpression span = Optional.ofNullable(ctx.statsByClause()) .map(OpenSearchPPLParser.StatsByClauseContext::bySpanClause) - .map(this::visitExpression) + .map(this::internalVisitExpression) .orElse(null); Aggregation aggregation = new Aggregation( @@ -202,7 +207,7 @@ public UnresolvedPlan visitSortCommand(SortCommandContext ctx) { ctx.sortbyClause() .sortField() .stream() - .map(sort -> (Field) visitExpression(sort)) + .map(sort -> (Field) internalVisitExpression(sort)) .collect(Collectors.toList()) ); } @@ -215,20 +220,20 @@ public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { return new Eval( ctx.evalClause() .stream() - .map(ct -> (Let) visitExpression(ct)) + .map(ct -> (Let) internalVisitExpression(ct)) .collect(Collectors.toList()) ); } private List getGroupByList(ByClauseContext ctx) { - return ctx.fieldList().fieldExpression().stream().map(this::visitExpression) + return ctx.fieldList().fieldExpression().stream().map(this::internalVisitExpression) .collect(Collectors.toList()); } private List getFieldList(FieldListContext ctx) { return ctx.fieldExpression() .stream() - .map(field -> (Field) visitExpression(field)) + .map(field -> (Field) internalVisitExpression(field)) .collect(Collectors.toList()); } @@ -247,6 +252,14 @@ public UnresolvedPlan visitRareCommand(RareCommandContext ctx) { ); } + @Override + public UnresolvedPlan visitParseCommand(OpenSearchPPLParser.ParseCommandContext ctx) { + UnresolvedExpression expression = internalVisitExpression(ctx.expression()); + Literal pattern = (Literal) internalVisitExpression(ctx.pattern()); + + return new Parse(expression, pattern); + } + /** * Top command. */ @@ -268,14 +281,14 @@ public UnresolvedPlan visitTopCommand(TopCommandContext ctx) { @Override public UnresolvedPlan visitFromClause(FromClauseContext ctx) { return new Relation(ctx.tableSource() - .stream().map(this::visitExpression) + .stream().map(this::internalVisitExpression) .collect(Collectors.toList())); } /** * Navigate to & build AST expression. */ - private UnresolvedExpression visitExpression(ParseTree tree) { + private UnresolvedExpression internalVisitExpression(ParseTree tree) { return expressionBuilder.visit(tree); } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5738bbed4c..8f337b76602 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -29,6 +29,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.let; import static org.opensearch.sql.ast.dsl.AstDSL.map; import static org.opensearch.sql.ast.dsl.AstDSL.nullLiteral; +import static org.opensearch.sql.ast.dsl.AstDSL.parse; import static org.opensearch.sql.ast.dsl.AstDSL.projectWithArg; import static org.opensearch.sql.ast.dsl.AstDSL.qualifiedName; import static org.opensearch.sql.ast.dsl.AstDSL.rareTopN; @@ -564,6 +565,16 @@ public void testTopCommandWithMultipleFields() { )); } + @Test + public void testParseCommand() { + assertEqual("source=t | parse raw \"pattern\"", + parse( + relation("t"), + field("raw"), + stringLiteral("pattern") + )); + } + protected void assertEqual(String query, Node expectedPlan) { Node actualPlan = plan(query); assertEquals(expectedPlan, actualPlan);