[KYUUBI #6577] Add spark sql engine plugin module and define a sql stringify plugin

extensions/spark/kyuubi-spark-sql-engine-plugin/README.md

@@ -0,0 +1,92 @@
+<!--
+- Licensed to the Apache Software Foundation (ASF) under one or more
+- contributor license agreements.  See the NOTICE file distributed with
+- this work for additional information regarding copyright ownership.
+- The ASF licenses this file to You under the Apache License, Version 2.0
+- (the "License"); you may not use this file except in compliance with
+- the License.  You may obtain a copy of the License at
+-
+-   http://www.apache.org/licenses/LICENSE-2.0
+-
+- Unless required by applicable law or agreed to in writing, software
+- distributed under the License is distributed on an "AS IS" BASIS,
+- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+- See the License for the specific language governing permissions and
+- limitations under the License.
+-->
+
+# Kyuubi Spark SQL Engine Plugin
+
+The `Kyuubi Spark SQL Engine Plugin` is a plugin that includes SPI definitions for extending some features of the Spark SQL Engine.
+
+## Plugins
+
+| Plugin                                                     | Description                                            |
+|:-----------------------------------------------------------|:-------------------------------------------------------|
+| `org.apache.kyuubi.engine.spark.plugin.SQLStringifyPlugin` | The plugin to extend custom stringified sql statement. |
+
+## How to implement a custom plugin
+
+Usually to implement a custom plugin we need to do the following steps:
+
++ Add `kyuubi-spark-sql-engine-plugin` dependency
++ Implement plugin interface
++ Create a service file
++ Package and integration with Kyuubi Spark SQL Engine
+
+### Example of implementing a custom SQLStringifyPlugin
+
+Add `kyuubi-spark-sql-engine-plugin` dependency:
+
+```xml
+<dependency>
+    <groupId>org.apache.kyuubi</groupId>
+    <artifactId>kyuubi-spark-sql-engine-plugin</artifactId>
+    <version>${kyuubi.version}</version>
+    <scope>provided</scope>
+</dependency>
+```
+
+Extend `SQLStringifyPlugin` interface and implement methods:
+
+```java
+package org.apache.kyuubi.custom;
+
+import java.util.Locale;
+import org.apache.spark.sql.SparkSession;
+import org.apache.kyuubi.engine.spark.plugin.SQLStringifyPlugin;
+
+
+public class CustomSQLStringifyPlugin implements SQLStringifyPlugin {
+    @Override
+    public String mode() {
+        return "custom_to_upper";
+    }
+
+    @Override
+    public String toString(SparkSession spark, String statement) {
+        return statement.toUpperCase(Locale.ROOT);
+    }
+}
+```
+
+Create a `org.apache.kyuubi.engine.spark.plugin.SQLStringifyPlugin` file in `src/main/resources/META-INF/services/` with content:
+
+```
+org.apache.kyuubi.custom.CustomSQLStringifyPlugin
+```
+
+Package the custom plugin jar, and set `spark.jars=/path/to/custom-plugin-jar`.
+
+Enjoy your custom plugin with Kyuubi Spark SQL Engine:
+
+```sql
+set kyuubi.operation.plan.only.mode=custom_to_upper;
+select * from table;
+```
+
+output:
+
+```text
+SELECT * FROM TABLE;
+```

extensions/spark/kyuubi-spark-sql-engine-plugin/pom.xml

@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.apache.kyuubi</groupId>
+        <artifactId>kyuubi-parent</artifactId>
+        <version>1.10.0-SNAPSHOT</version>
+        <relativePath>../../../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>kyuubi-spark-sql-engine-plugin</artifactId>
+    <packaging>jar</packaging>
+    <name>Kyuubi Project Spark SQL Engine Plugin</name>
+    <url>https://kyuubi.apache.org/</url>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.spark</groupId>
+            <artifactId>spark-sql_${scala.binary.version}</artifactId>
+            <scope>provided</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <configuration>
+                    <skip>true</skip>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

extensions/spark/kyuubi-spark-sql-engine-plugin/src/main/java/org/apache/kyuubi/engine/spark/plugin/SQLStringifyPlugin.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.spark.plugin;
+
+import org.apache.spark.sql.SparkSession;
+
+public interface SQLStringifyPlugin {
+
+  String mode();
+
+  String toString(SparkSession spark, String statement);
+}

externals/kyuubi-spark-sql-engine/pom.xml

@@ -49,6 +49,12 @@
             <version>${project.version}</version>
         </dependency>
 
+        <dependency>
+            <groupId>org.apache.kyuubi</groupId>
+            <artifactId>kyuubi-spark-sql-engine-plugin</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>io.grpc</groupId>
             <artifactId>grpc-core</artifactId>

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/PlanOnlyStatement.scala

@@ -28,6 +28,7 @@ import org.apache.spark.sql.types.StructType
 import org.apache.kyuubi.KyuubiSQLException
 import org.apache.kyuubi.config.KyuubiConf.{LINEAGE_PARSER_PLUGIN_PROVIDER, OPERATION_PLAN_ONLY_EXCLUDES, OPERATION_PLAN_ONLY_OUT_STYLE}
 import org.apache.kyuubi.engine.spark.KyuubiSparkUtil.getSessionConf
+import org.apache.kyuubi.engine.spark.operation.planonly.SQLStringifyPlugins
 import org.apache.kyuubi.operation.{AnalyzeMode, ArrayFetchIterator, ExecutionMode, IterableFetchIterator, JsonStyle, LineageMode, OperationHandle, OptimizeMode, OptimizeWithStatsMode, ParseMode, PhysicalMode, PlainStyle, PlanOnlyMode, PlanOnlyStyle, UnknownMode, UnknownStyle}
 import org.apache.kyuubi.operation.PlanOnlyMode.{notSupportedModeError, unknownModeError}
 import org.apache.kyuubi.operation.PlanOnlyStyle.{notSupportedStyleError, unknownStyleError}
@@ -78,11 +79,16 @@ class PlanOnlyStatement(
               result = spark.sql(statement)
               iter = new ArrayFetchIterator(result.collect())
 
-            case plan => style match {
-                case PlainStyle => explainWithPlainStyle(plan)
-                case JsonStyle => explainWithJsonStyle(plan)
-                case UnknownStyle => unknownStyleError(style)
-                case other => throw notSupportedStyleError(other, "Spark SQL")
+            case plan =>
+              // TODO: remove style configuration, keep only mode configuration
+              (mode.name, style) match {
+                case (SQLStringifyPlugins(stringifyPlugin), _) =>
+                  val result = stringifyPlugin.toString(spark, statement)
+                  iter = new IterableFetchIterator(Seq(Row(result)))
+                case (_, PlainStyle) => explainWithPlainStyle(plan)
+                case (_, JsonStyle) => explainWithJsonStyle(plan)
+                case (_, UnknownStyle) => unknownStyleError(style)
+                case (_, other) => throw notSupportedStyleError(other, "Spark SQL")
               }
           }
         }

externals/kyuubi-spark-sql-engine/src/main/scala/org/apache/kyuubi/engine/spark/operation/planonly/SQLStringifyPlugins.scala

@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.kyuubi.engine.spark.operation.planonly
+
+import java.util.Locale
+
+import org.apache.kyuubi.engine.spark.plugin.SQLStringifyPlugin
+import org.apache.kyuubi.util.reflect.ReflectUtils
+
+object SQLStringifyPlugins {
+
+  private lazy val plugins: Map[String, SQLStringifyPlugin] = {
+    ReflectUtils.loadFromServiceLoader[SQLStringifyPlugin]()
+      .map(e => e.mode().toLowerCase(Locale.ROOT) -> e).toMap
+  }
+
+  def unapply(mode: String): Option[SQLStringifyPlugin] = {
+    plugins.get(mode.toLowerCase(Locale.ROOT))
+  }
+
+}

pom.xml

@@ -60,6 +60,7 @@
         <module>extensions/spark/kyuubi-spark-connector-common</module>
         <module>extensions/spark/kyuubi-spark-connector-tpcds</module>
         <module>extensions/spark/kyuubi-spark-connector-tpch</module>
+        <module>extensions/spark/kyuubi-spark-sql-engine-plugin</module>
         <module>extensions/spark/kyuubi-spark-lineage</module>
         <module>externals/kyuubi-chat-engine</module>
         <module>externals/kyuubi-download</module>