diff --git a/Cargo.lock b/Cargo.lock index 987e8842da07d..55527d807b1c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5714,9 +5714,8 @@ dependencies = [ [[package]] name = "sqlparser" -version = "0.60.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "505aa16b045c4c1375bf5f125cce3813d0176325bfe9ffc4a903f423de7774ff" +version = "0.61.0" +source = "git+https://github.com/apache/datafusion-sqlparser-rs.git?rev=ce678990397f81c621d2df079ad552baae0cdfda#ce678990397f81c621d2df079ad552baae0cdfda" dependencies = [ "log", "recursive", @@ -5725,9 +5724,8 @@ dependencies = [ [[package]] name = "sqlparser_derive" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028e551d5e270b31b9f3ea271778d9d827148d4287a5d96167b6bb9787f5cc38" +version = "0.5.0" +source = "git+https://github.com/apache/datafusion-sqlparser-rs.git?rev=ce678990397f81c621d2df079ad552baae0cdfda#ce678990397f81c621d2df079ad552baae0cdfda" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 157dd68b9cdb0..b9406af977be8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -182,7 +182,7 @@ regex = "1.12" rstest = "0.26.1" serde_json = "1" sha2 = "^0.10.9" -sqlparser = { version = "0.60.0", default-features = false, features = ["std", "visitor"] } +sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor"] } strum = "0.27.2" strum_macros = "0.27.2" tempfile = "3" @@ -275,3 +275,6 @@ incremental = false inherits = "release" debug = true strip = false + +[patch.crates-io] +sqlparser = { git = "https://github.com/apache/datafusion-sqlparser-rs.git", rev = "ce678990397f81c621d2df079ad552baae0cdfda" } diff --git a/datafusion/sql/src/expr/mod.rs b/datafusion/sql/src/expr/mod.rs index dbf2ce67732ec..9aa5be8131dcb 100644 --- a/datafusion/sql/src/expr/mod.rs +++ b/datafusion/sql/src/expr/mod.rs @@ -267,11 +267,16 @@ impl SqlToRel<'_, S> { planner_context, ), + SQLExpr::Cast { array: true, .. } => { + not_impl_err!("`CAST(... AS type ARRAY`) not supported") + } + SQLExpr::Cast { kind: CastKind::Cast | CastKind::DoubleColon, expr, data_type, format, + array: false, } => { self.sql_cast_to_expr(*expr, &data_type, format, schema, planner_context) } @@ -281,6 +286,7 @@ impl SqlToRel<'_, S> { expr, data_type, format, + array: false, } => { if let Some(format) = format { return not_impl_err!("CAST with format is not supported: {format}"); diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 1d6ccde6be13a..2975c5eae81d6 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -361,6 +361,7 @@ impl SqlToRel<'_, S> { // Process distinct clause let plan = match select.distinct { None => Ok(plan), + Some(Distinct::All) => Ok(plan), Some(Distinct::Distinct) => { LogicalPlanBuilder::from(plan).distinct()?.build() } diff --git a/datafusion/sql/src/statement.rs b/datafusion/sql/src/statement.rs index 4981db5537a74..a8e8b1ecd9495 100644 --- a/datafusion/sql/src/statement.rs +++ b/datafusion/sql/src/statement.rs @@ -342,26 +342,28 @@ impl SqlToRel<'_, S> { refresh_mode, initialize, require_user, + partition_of, + for_values, }) => { if temporary { - return not_impl_err!("Temporary tables not supported")?; + return not_impl_err!("Temporary tables not supported"); } if external { - return not_impl_err!("External tables not supported")?; + return not_impl_err!("External tables not supported"); } if global.is_some() { - return not_impl_err!("Global tables not supported")?; + return not_impl_err!("Global tables not supported"); } if transient { - return not_impl_err!("Transient tables not supported")?; + return not_impl_err!("Transient tables not supported"); } if volatile { - return not_impl_err!("Volatile tables not supported")?; + return not_impl_err!("Volatile tables not supported"); } if hive_distribution != ast::HiveDistributionStyle::NONE { return not_impl_err!( "Hive distribution not supported: {hive_distribution:?}" - )?; + ); } if hive_formats.is_some() && !matches!( @@ -374,122 +376,126 @@ impl SqlToRel<'_, S> { }) ) { - return not_impl_err!( - "Hive formats not supported: {hive_formats:?}" - )?; + return not_impl_err!("Hive formats not supported: {hive_formats:?}"); } if file_format.is_some() { - return not_impl_err!("File format not supported")?; + return not_impl_err!("File format not supported"); } if location.is_some() { - return not_impl_err!("Location not supported")?; + return not_impl_err!("Location not supported"); } if without_rowid { - return not_impl_err!("Without rowid not supported")?; + return not_impl_err!("Without rowid not supported"); } if like.is_some() { - return not_impl_err!("Like not supported")?; + return not_impl_err!("Like not supported"); } if clone.is_some() { - return not_impl_err!("Clone not supported")?; + return not_impl_err!("Clone not supported"); } if comment.is_some() { - return not_impl_err!("Comment not supported")?; + return not_impl_err!("Comment not supported"); } if on_commit.is_some() { - return not_impl_err!("On commit not supported")?; + return not_impl_err!("On commit not supported"); } if on_cluster.is_some() { - return not_impl_err!("On cluster not supported")?; + return not_impl_err!("On cluster not supported"); } if primary_key.is_some() { - return not_impl_err!("Primary key not supported")?; + return not_impl_err!("Primary key not supported"); } if order_by.is_some() { - return not_impl_err!("Order by not supported")?; + return not_impl_err!("Order by not supported"); } if partition_by.is_some() { - return not_impl_err!("Partition by not supported")?; + return not_impl_err!("Partition by not supported"); } if cluster_by.is_some() { - return not_impl_err!("Cluster by not supported")?; + return not_impl_err!("Cluster by not supported"); } if clustered_by.is_some() { - return not_impl_err!("Clustered by not supported")?; + return not_impl_err!("Clustered by not supported"); } if strict { - return not_impl_err!("Strict not supported")?; + return not_impl_err!("Strict not supported"); } if copy_grants { - return not_impl_err!("Copy grants not supported")?; + return not_impl_err!("Copy grants not supported"); } if enable_schema_evolution.is_some() { - return not_impl_err!("Enable schema evolution not supported")?; + return not_impl_err!("Enable schema evolution not supported"); } if change_tracking.is_some() { - return not_impl_err!("Change tracking not supported")?; + return not_impl_err!("Change tracking not supported"); } if data_retention_time_in_days.is_some() { - return not_impl_err!("Data retention time in days not supported")?; + return not_impl_err!("Data retention time in days not supported"); } if max_data_extension_time_in_days.is_some() { return not_impl_err!( "Max data extension time in days not supported" - )?; + ); } if default_ddl_collation.is_some() { - return not_impl_err!("Default DDL collation not supported")?; + return not_impl_err!("Default DDL collation not supported"); } if with_aggregation_policy.is_some() { - return not_impl_err!("With aggregation policy not supported")?; + return not_impl_err!("With aggregation policy not supported"); } if with_row_access_policy.is_some() { - return not_impl_err!("With row access policy not supported")?; + return not_impl_err!("With row access policy not supported"); } if with_tags.is_some() { - return not_impl_err!("With tags not supported")?; + return not_impl_err!("With tags not supported"); } if iceberg { - return not_impl_err!("Iceberg not supported")?; + return not_impl_err!("Iceberg not supported"); } if external_volume.is_some() { - return not_impl_err!("External volume not supported")?; + return not_impl_err!("External volume not supported"); } if base_location.is_some() { - return not_impl_err!("Base location not supported")?; + return not_impl_err!("Base location not supported"); } if catalog.is_some() { - return not_impl_err!("Catalog not supported")?; + return not_impl_err!("Catalog not supported"); } if catalog_sync.is_some() { - return not_impl_err!("Catalog sync not supported")?; + return not_impl_err!("Catalog sync not supported"); } if storage_serialization_policy.is_some() { - return not_impl_err!("Storage serialization policy not supported")?; + return not_impl_err!("Storage serialization policy not supported"); } if inherits.is_some() { - return not_impl_err!("Table inheritance not supported")?; + return not_impl_err!("Table inheritance not supported"); } if dynamic { - return not_impl_err!("Dynamic tables not supported")?; + return not_impl_err!("Dynamic tables not supported"); } if version.is_some() { - return not_impl_err!("Version not supported")?; + return not_impl_err!("Version not supported"); } if target_lag.is_some() { - return not_impl_err!("Target lag not supported")?; + return not_impl_err!("Target lag not supported"); } if warehouse.is_some() { - return not_impl_err!("Warehouse not supported")?; + return not_impl_err!("Warehouse not supported"); } if refresh_mode.is_some() { - return not_impl_err!("Refresh mode not supported")?; + return not_impl_err!("Refresh mode not supported"); } if initialize.is_some() { - return not_impl_err!("Initialize not supported")?; + return not_impl_err!("Initialize not supported"); } if require_user { - return not_impl_err!("Require user not supported")?; + return not_impl_err!("Require user not supported"); + } + if partition_of.is_some() { + return not_impl_err!("PARTITION OF not supported"); + } + if for_values.is_some() { + return not_impl_err!("PARTITION OF .. FOR VALUES .. not supported"); } // Merge inline constraints and existing constraints let mut all_constraints = constraints; @@ -989,7 +995,8 @@ impl SqlToRel<'_, S> { has_table_keyword, settings, format_clause, - insert_token: _insert_token, // record the location the `INSERT` token + insert_token: _, // record the location the `INSERT` token + optimizer_hint, }) => { let table_name = match table { TableObject::TableName(table_name) => table_name, @@ -1045,6 +1052,9 @@ impl SqlToRel<'_, S> { if format_clause.is_some() { plan_err!("Inserts with format clause not supported")?; } + if optimizer_hint.is_some() { + plan_err!("Optimizer hints not supported")?; + } // optional keywords don't change behavior let _ = into; let _ = has_table_keyword; @@ -1059,6 +1069,7 @@ impl SqlToRel<'_, S> { or, limit, update_token: _, + optimizer_hint, }) => { let from_clauses = from.map(|update_table_from_kind| match update_table_from_kind { @@ -1079,6 +1090,9 @@ impl SqlToRel<'_, S> { if limit.is_some() { return not_impl_err!("Update-limit clause not supported")?; } + if optimizer_hint.is_some() { + plan_err!("Optimizer hints not supported")?; + } self.update_to_plan(table, &assignments, update_from, selection) } @@ -1091,6 +1105,7 @@ impl SqlToRel<'_, S> { order_by, limit, delete_token: _, + optimizer_hint, }) => { if !tables.is_empty() { plan_err!("DELETE not supported")?; @@ -1111,6 +1126,9 @@ impl SqlToRel<'_, S> { if limit.is_some() { plan_err!("Delete-limit clause not yet supported")?; } + if optimizer_hint.is_some() { + plan_err!("Optimizer hints not supported")?; + } let table_name = self.get_delete_target(from)?; self.delete_to_plan(&table_name, selection) @@ -1397,6 +1415,7 @@ impl SqlToRel<'_, S> { cascade, on_cluster, table, + if_exists, }) => { let _ = table; // Support TRUNCATE TABLE and TRUNCATE syntax if table_names.len() != 1 { @@ -1425,6 +1444,9 @@ impl SqlToRel<'_, S> { if on_cluster.is_some() { return not_impl_err!("TRUNCATE with ON CLUSTER is not supported"); } + if if_exists { + return not_impl_err!("TRUNCATE .. WITH EXISTS is not supported"); + } let table = self.object_name_to_table_reference(target.name.clone())?; let source = self.context_provider.get_table_source(table.clone())?; diff --git a/datafusion/sql/src/unparser/ast.rs b/datafusion/sql/src/unparser/ast.rs index ec78a42d6534b..8446a44b07e35 100644 --- a/datafusion/sql/src/unparser/ast.rs +++ b/datafusion/sql/src/unparser/ast.rs @@ -315,7 +315,9 @@ impl SelectBuilder { } pub fn build(&self) -> Result { Ok(ast::Select { + optimizer_hint: None, distinct: self.distinct.clone(), + select_modifiers: None, top_before_distinct: false, top: self.top.clone(), projection: self.projection.clone().unwrap_or_default(), @@ -340,12 +342,12 @@ impl SelectBuilder { named_window: self.named_window.clone(), qualify: self.qualify.clone(), value_table_mode: self.value_table_mode, - connect_by: None, + connect_by: Vec::new(), window_before_qualify: false, prewhere: None, select_token: AttachedToken::empty(), flavor: match self.flavor { - Some(ref value) => value.clone(), + Some(ref value) => *value, None => return Err(Into::into(UninitializedFieldError::from("flavor"))), }, exclude: None, @@ -608,6 +610,7 @@ impl DerivedRelationBuilder { } }, alias: self.alias.clone(), + sample: None, }) } fn create_empty() -> Self { diff --git a/datafusion/sql/src/unparser/dialect.rs b/datafusion/sql/src/unparser/dialect.rs index 1a3e1a06db5f1..31d2662cc4cc0 100644 --- a/datafusion/sql/src/unparser/dialect.rs +++ b/datafusion/sql/src/unparser/dialect.rs @@ -372,6 +372,7 @@ impl PostgreSqlDialect { kind: ast::CastKind::Cast, expr: Box::new(expr.clone()), data_type: ast::DataType::Numeric(ast::ExactNumberInfo::None), + array: false, format: None, }; } diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index 5f6612830ac1f..59a9207b51ef0 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -494,6 +494,7 @@ impl Unparser<'_> { kind: ast::CastKind::TryCast, expr: Box::new(inner_expr), data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + array: false, format: None, }) } @@ -1145,6 +1146,7 @@ impl Unparser<'_> { kind: ast::CastKind::Cast, expr: Box::new(ast::Expr::value(SingleQuotedString(ts))), data_type: self.dialect.timestamp_cast_dtype(&time_unit, &None), + array: false, format: None, }) } @@ -1167,6 +1169,7 @@ impl Unparser<'_> { kind: ast::CastKind::Cast, expr: Box::new(ast::Expr::value(SingleQuotedString(time))), data_type: ast::DataType::Time(None, TimezoneInfo::None), + array: false, format: None, }) } @@ -1184,6 +1187,7 @@ impl Unparser<'_> { kind: ast::CastKind::Cast, expr: Box::new(inner_expr), data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + array: false, format: None, }), }, @@ -1191,6 +1195,7 @@ impl Unparser<'_> { kind: ast::CastKind::Cast, expr: Box::new(inner_expr), data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + array: false, format: None, }), } @@ -1332,6 +1337,7 @@ impl Unparser<'_> { date.to_string(), ))), data_type: ast::DataType::Date, + array: false, format: None, }) } @@ -1355,6 +1361,7 @@ impl Unparser<'_> { datetime.to_string(), ))), data_type: self.ast_type_for_date64_in_cast(), + array: false, format: None, }) } diff --git a/datafusion/sql/src/utils.rs b/datafusion/sql/src/utils.rs index 9205336a52e4e..16ac353d4ba9b 100644 --- a/datafusion/sql/src/utils.rs +++ b/datafusion/sql/src/utils.rs @@ -331,6 +331,8 @@ pub(crate) fn value_to_string(value: &Value) -> Option { Value::Number(_, _) | Value::Boolean(_) => Some(value.to_string()), Value::UnicodeStringLiteral(s) => Some(s.to_string()), Value::EscapedStringLiteral(s) => Some(s.to_string()), + Value::QuoteDelimitedStringLiteral(s) + | Value::NationalQuoteDelimitedStringLiteral(s) => Some(s.value.to_string()), Value::DoubleQuotedString(_) | Value::NationalStringLiteral(_) | Value::SingleQuotedByteStringLiteral(_) diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index 490df4b72d17b..5bc44e6af128c 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -830,6 +830,10 @@ SELECT DISTINCT * FROM aggregate_simple 0.00004 0.000000000004 false 0.00005 0.000000000005 true +# select distinct all ( +query error DataFusion error: SQL error: ParserError\("Cannot specify DISTINCT then ALL at Line: 1, Column: 8"\) +SELECT DISTINCT ALL * FROM aggregate_simple + # select distinct with projection and order by query R SELECT DISTINCT c1 FROM aggregate_simple order by c1