From 968d1c543e6c2f2720f8e62f4ca674d75eeaacc7 Mon Sep 17 00:00:00 2001 From: shuvamk Date: Tue, 4 Aug 2026 22:18:51 +0530 Subject: [PATCH 1/2] Databricks: support CREATE TABLE USING, MAP columns and LONG as BIGINT Databricks SQL is built on Spark SQL, but DatabricksDialect never got three of the flags SparkSqlDialect sets, so these fail on Databricks and parse on Spark: CREATE TABLE t (id BIGINT) USING DELTA CREATE TABLE t (m MAP) with "Expected: end of statement, found: USING at Line: 1, Column: 28" and "Expected: ',' or ')' after column definition, found: < at Line: 1, Column: 22". CREATE TABLE t (id LONG) parses, but builds DataType::Custom("LONG") instead of DataType::BigInt(None). Databricks documents the type as { BIGINT | LONG }. Set supports_create_table_using, supports_long_type_as_bigint and supports_map_literal_with_angle_brackets on DatabricksDialect, mirroring src/dialect/spark.rs. No parser change and no other dialect is affected. Regression tests in tests/sqlparser_databricks.rs; all three fail with src/dialect/databricks.rs reverted. Co-Authored-By: Claude Opus 5 --- src/dialect/databricks.rs | 17 ++++++++++++ tests/sqlparser_databricks.rs | 52 +++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index 3bc187a7f..036d7808a 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -113,4 +113,21 @@ impl Dialect for DatabricksDialect { fn supports_select_item_multi_column_alias(&self) -> bool { true } + + /// See + fn supports_create_table_using(&self) -> bool { + true + } + + /// `LONG` is an alias for `BIGINT` in Databricks SQL. + /// + /// See + fn supports_long_type_as_bigint(&self) -> bool { + true + } + + /// See + fn supports_map_literal_with_angle_brackets(&self) -> bool { + true + } } diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 7c582546f..23c263263 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -737,3 +737,55 @@ fn parse_cte_without_as() { .parse_sql_statements("WITH cte (SELECT 1) SELECT * FROM cte") .is_err()); } + +#[test] +fn parse_create_table_using() { + match databricks().verified_stmt("CREATE TABLE t (id BIGINT) USING DELTA") { + Statement::CreateTable(CreateTable { hive_formats, .. }) => { + assert_eq!( + hive_formats.unwrap().storage, + Some(HiveIOFormat::Using { + format: Ident::new("DELTA") + }) + ); + } + s => panic!("Unexpected statement: {s:?}"), + } + + databricks().verified_stmt("CREATE TABLE IF NOT EXISTS t (id BIGINT) USING PARQUET"); + + assert!(all_dialects_where(|d| !d.supports_create_table_using()) + .parse_sql_statements("CREATE TABLE t (id BIGINT) USING DELTA") + .is_err()); +} + +#[test] +fn parse_create_table_map_type() { + match databricks().verified_stmt("CREATE TABLE t (m MAP)") { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!( + columns[0].data_type, + DataType::Map( + Box::new(DataType::String(None)), + Box::new(DataType::Int(None)), + MapBracketKind::AngleBrackets + ) + ); + } + s => panic!("Unexpected statement: {s:?}"), + } + + databricks().verified_stmt("CREATE TABLE t (m MAP>)"); +} + +#[test] +fn parse_long_type_as_bigint() { + match databricks() + .one_statement_parses_to("CREATE TABLE t (id LONG)", "CREATE TABLE t (id BIGINT)") + { + Statement::CreateTable(CreateTable { columns, .. }) => { + assert_eq!(columns[0].data_type, DataType::BigInt(None)); + } + s => panic!("Unexpected statement: {s:?}"), + } +} From 5a401afb743692903b2e116b4ddbd86574ba8e59 Mon Sep 17 00:00:00 2001 From: shuvamk Date: Sat, 8 Aug 2026 16:01:33 +0530 Subject: [PATCH 2/2] Databricks: delegate to Spark and add pipe operator and DIV Address review feedback on #2425. The flags Databricks shares with Spark now call the Spark methods instead of repeating the literal, so the two dialects cannot drift apart silently. `RedshiftSqlDialect` already delegates to `PostgreSqlDialect` this way. Two more Spark capabilities are documented for Databricks SQL and were missing: SELECT 10 div 3 SELECT * FROM t |> WHERE x > 1 |> SELECT x AS y Both failed with `No infix parser for token Word(... keyword: DIV })` and `Expected: end of statement, found: |`. `div` is documented at https://docs.databricks.com/aws/en/sql/language-manual/functions/div and pipeline syntax at https://docs.databricks.com/aws/en/sql/language-manual/sql-ref-syntax-qry-pipeline (Databricks SQL / Databricks Runtime 16.2 and above). The cross-dialect assertion moves from the Databricks tests to `tests/sqlparser_common.rs`, joined by the analogous assertions for the `MAP` and `LONG` flags. Dialects that do not set `supports_map_literal_with_angle_brackets` reject `MAP` with two different messages, so that one is checked per dialect as in `parse_create_table_exclude_constraint`. Dialects that do not set `supports_long_type_as_bigint` parse `LONG` as a custom type rather than erroring, so the assertion there is that the statement round-trips unchanged. `parse_pipeline_operator_negative_tests` asserted that `|> CALL 123invalid` fails with one message across every pipe-enabled dialect. Databricks sets `supports_numeric_prefix`, so it reads `123invalid` as an identifier and fails later, on the missing parentheses. The input is now `|> CALL 123`, which fails identically on all of them. Co-Authored-By: Claude Opus 5 --- src/dialect/databricks.rs | 27 ++++++++++++++++++++++++--- tests/sqlparser_common.rs | 34 +++++++++++++++++++++++++++++++++- tests/sqlparser_databricks.rs | 15 +++++++++++---- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/src/dialect/databricks.rs b/src/dialect/databricks.rs index 036d7808a..955761c0c 100644 --- a/src/dialect/databricks.rs +++ b/src/dialect/databricks.rs @@ -15,7 +15,11 @@ // specific language governing permissions and limitations // under the License. +use crate::ast::Expr; use crate::dialect::Dialect; +use crate::parser::{Parser, ParserError}; + +use super::SparkSqlDialect; /// A [`Dialect`] for [Databricks SQL](https://www.databricks.com/) /// @@ -116,18 +120,35 @@ impl Dialect for DatabricksDialect { /// See fn supports_create_table_using(&self) -> bool { - true + SparkSqlDialect {}.supports_create_table_using() } /// `LONG` is an alias for `BIGINT` in Databricks SQL. /// /// See fn supports_long_type_as_bigint(&self) -> bool { - true + SparkSqlDialect {}.supports_long_type_as_bigint() } /// See fn supports_map_literal_with_angle_brackets(&self) -> bool { - true + SparkSqlDialect {}.supports_map_literal_with_angle_brackets() + } + + /// See + fn supports_pipe_operator(&self) -> bool { + SparkSqlDialect {}.supports_pipe_operator() + } + + /// Parse the `DIV` keyword as integer division. + /// + /// See + fn parse_infix( + &self, + parser: &mut Parser, + expr: &Expr, + precedence: u8, + ) -> Option> { + SparkSqlDialect {}.parse_infix(parser, expr, precedence) } } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0800bc41f..b131a1014 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -17318,7 +17318,7 @@ fn parse_pipeline_operator_negative_tests() { // Test that CALL with invalid function syntax fails assert!(dialects - .parse_sql_statements("SELECT * FROM users |> CALL 123invalid") + .parse_sql_statements("SELECT * FROM users |> CALL 123") .is_err()); // Test that CALL with malformed arguments fails @@ -19679,3 +19679,35 @@ fn parse_function_arg_call_chain_no_exponential_blowup() { rx.recv_timeout(Duration::from_secs(5)) .expect("parser should reject this quickly, not loop exponentially"); } + +#[test] +fn parse_create_table_using() { + let sql = "CREATE TABLE t (id BIGINT) USING DELTA"; + all_dialects_where(|d| d.supports_create_table_using()).verified_stmt(sql); + + let unsupported_dialects = all_dialects_where(|d| !d.supports_create_table_using()); + assert!(unsupported_dialects.parse_sql_statements(sql).is_err()); +} + +#[test] +fn parse_map_type_with_angle_brackets() { + let sql = "CREATE TABLE t (m MAP)"; + all_dialects_where(|d| d.supports_map_literal_with_angle_brackets()).verified_stmt(sql); + + let unsupported_dialects = + all_dialects_where(|d| !d.supports_map_literal_with_angle_brackets()); + for dialect in unsupported_dialects.dialects { + assert!(TestedDialects::new(vec![dialect]) + .parse_sql_statements(sql) + .is_err()); + } +} + +#[test] +fn parse_long_type_as_bigint() { + all_dialects_where(|d| d.supports_long_type_as_bigint()) + .one_statement_parses_to("CREATE TABLE t (id LONG)", "CREATE TABLE t (id BIGINT)"); + + let unsupported_dialects = all_dialects_where(|d| !d.supports_long_type_as_bigint()); + unsupported_dialects.verified_stmt("CREATE TABLE t (id LONG)"); +} diff --git a/tests/sqlparser_databricks.rs b/tests/sqlparser_databricks.rs index 23c263263..bdddadd52 100644 --- a/tests/sqlparser_databricks.rs +++ b/tests/sqlparser_databricks.rs @@ -753,10 +753,6 @@ fn parse_create_table_using() { } databricks().verified_stmt("CREATE TABLE IF NOT EXISTS t (id BIGINT) USING PARQUET"); - - assert!(all_dialects_where(|d| !d.supports_create_table_using()) - .parse_sql_statements("CREATE TABLE t (id BIGINT) USING DELTA") - .is_err()); } #[test] @@ -789,3 +785,14 @@ fn parse_long_type_as_bigint() { s => panic!("Unexpected statement: {s:?}"), } } + +#[test] +fn parse_div_operator() { + databricks().one_statement_parses_to("SELECT 10 div 3", "SELECT 10 DIV 3"); + databricks().one_statement_parses_to("SELECT c1 div c2 FROM t", "SELECT c1 DIV c2 FROM t"); +} + +#[test] +fn parse_pipe_operator() { + databricks().verified_stmt("SELECT * FROM t |> WHERE x > 1 |> SELECT x AS y |> ORDER BY y"); +}