From 05f43b3c7a23801413560f8fd9a2b0f7b12d140f Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Tue, 22 Oct 2013 11:24:23 +0200 Subject: [PATCH 01/41] Fix redshift schema dump. Redshift does not support arrays, and dump was using pg_index.indkey which is unsupported INT2VECTOR type. Haven't found other workaround for now than removing the primary key check and setting it always to false. In mean time you can set it at the model level with `set_primary_key`. --- lib/sequel/adapters/redshift.rb | 3 ++ lib/sequel/adapters/shared/redshift.rb | 46 ++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/sequel/adapters/shared/redshift.rb diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index b693fe3..41e416b 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -1,10 +1,13 @@ require 'sequel/adapters/postgres' +require 'sequel/adapters/shared/redshift' module Sequel module Redshift include Postgres class Database < Postgres::Database + include Sequel::Redshift::DatabaseMethods + set_adapter_scheme :redshift def column_definition_primary_key_sql(sql, column) diff --git a/lib/sequel/adapters/shared/redshift.rb b/lib/sequel/adapters/shared/redshift.rb new file mode 100644 index 0000000..86c2af0 --- /dev/null +++ b/lib/sequel/adapters/shared/redshift.rb @@ -0,0 +1,46 @@ +module Sequel + module Redshift + module DatabaseMethods + # Redshift does not support arrays (type of pg_index.indkey is INT2VECTOR), + # and because of that we can't determine the primary key - so we set it to false. + # + # The workaround for now is to use `set_primary_key` inside the sequel model. + def schema_parse_table(table_name, opts) + m = output_identifier_meth(opts[:dataset]) + + ds = metadata_dataset.select(:pg_attribute__attname___name, + SQL::Cast.new(:pg_attribute__atttypid, :integer).as(:oid), + SQL::Cast.new(:basetype__oid, :integer).as(:base_oid), + SQL::Function.new(:format_type, :basetype__oid, :pg_type__typtypmod).as(:db_base_type), + SQL::Function.new(:format_type, :pg_type__oid, :pg_attribute__atttypmod).as(:db_type), + SQL::Function.new(:pg_get_expr, :pg_attrdef__adbin, :pg_class__oid).as(:default), + SQL::BooleanExpression.new(:NOT, :pg_attribute__attnotnull).as(:allow_null)). + from(:pg_class). + join(:pg_attribute, :attrelid=>:oid). + join(:pg_type, :oid=>:atttypid). + left_outer_join(:pg_type___basetype, :oid=>:typbasetype). + left_outer_join(:pg_attrdef, :adrelid=>:pg_class__oid, :adnum=>:pg_attribute__attnum). + filter(:pg_attribute__attisdropped=>false). + filter{|o| o.pg_attribute__attnum > 0}. + filter(:pg_class__oid=>regclass_oid(table_name, opts)). + order(:pg_attribute__attnum) + + ds.map do |row| + row[:default] = nil if blank_object?(row[:default]) + if row[:base_oid] + row[:domain_oid] = row[:oid] + row[:oid] = row.delete(:base_oid) + row[:db_domain_type] = row[:db_type] + row[:db_type] = row.delete(:db_base_type) + else + row.delete(:base_oid) + row.delete(:db_base_type) + end + row[:type] = schema_column_type(row[:db_type]) + row[:primary_key] = false + [m.call(row.delete(:name)), row] + end + end + end + end +end From 6561a7beb2560af6f1cd8aea507094442df5cae5 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Wed, 13 Nov 2013 15:58:36 +0100 Subject: [PATCH 02/41] Some other fixes for redshift. - Redshift doesn't support TEXT type, use char or varchar only - Redshift doesn't support RETURNING at all, so we shouldn't respond with true for that --- lib/sequel/adapters/redshift.rb | 10 ++++++++-- lib/sequel/adapters/shared/redshift.rb | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 41e416b..77641e3 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -25,11 +25,17 @@ def serial_primary_key_options class Dataset < Postgres::Dataset Database::DatasetClass = self - # Redshift doesn't support RETURNING statement def insert_returning_sql(sql) - # do nothing here sql end + + def supports_returning?(type) + false + end + + def supports_insert_select? + false + end end end end diff --git a/lib/sequel/adapters/shared/redshift.rb b/lib/sequel/adapters/shared/redshift.rb index 86c2af0..879f9f6 100644 --- a/lib/sequel/adapters/shared/redshift.rb +++ b/lib/sequel/adapters/shared/redshift.rb @@ -1,6 +1,9 @@ module Sequel module Redshift module DatabaseMethods + + REDSHIFT_STRING_MAXIMUM_SIZE = 255 + # Redshift does not support arrays (type of pg_index.indkey is INT2VECTOR), # and because of that we can't determine the primary key - so we set it to false. # @@ -41,6 +44,11 @@ def schema_parse_table(table_name, opts) [m.call(row.delete(:name)), row] end end + + # Redshift changes text to varchar with maximum size of 256, and it complains if you will give text column + def type_literal_generic_string(column) + "#{ column[:fixed] ? 'char' : 'varchar' }(#{ column[:size] || REDSHIFT_STRING_MAXIMUM_SIZE })" + end end end end From 21d85564b06128231245e881297323df454819df Mon Sep 17 00:00:00 2001 From: Maximilian Seifert Date: Tue, 26 Nov 2013 16:36:05 +0100 Subject: [PATCH 03/41] Add default adapter options --- lib/sequel/adapters/redshift.rb | 10 ++++++++++ spec/spec_helper.rb | 2 -- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 77641e3..d3decfe 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -10,6 +10,16 @@ class Database < Postgres::Database set_adapter_scheme :redshift + # We need to change these default settings because they correspond to + # Postgres configuration variables which do not exist in Redshift + def adapter_initialize + @opts.merge!( + force_standard_strings: false, + client_min_messages: false + ) + super + end + def column_definition_primary_key_sql(sql, column) result = super result << ' IDENTITY' if result diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index b10f42b..29b6f09 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -7,8 +7,6 @@ require 'logger' options = { - client_min_messages: false, - force_standard_strings: false, logger: Logger.new(STDOUT) } DB = Sequel.connect('redshift://remind101:wvpfSzw2TT8COE@r101-dw-production.ck1sr65ao7pg.us-east-1.redshift.amazonaws.com:5439/test1', options) From 8f070ebe3ca48d0cb143962a4e7c622025fabae1 Mon Sep 17 00:00:00 2001 From: Maximilian Seifert Date: Tue, 26 Nov 2013 22:22:23 +0100 Subject: [PATCH 04/41] Added DISTKEY and SORTKEY SQL fragments to column definition methods --- lib/sequel/adapters/redshift.rb | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index d3decfe..3dfbdb7 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -10,6 +10,12 @@ class Database < Postgres::Database set_adapter_scheme :redshift + DIST_KEY = ' DISTKEY'.freeze + SORT_KEY = ' SORTKEY'.freeze + + # The order of column modifiers to use when defining a column. + COLUMN_DEFINITION_ORDER = [:collate, :default, :primary_key, :dist_key, :sort_key, :null, :unique, :auto_increment, :references] + # We need to change these default settings because they correspond to # Postgres configuration variables which do not exist in Redshift def adapter_initialize @@ -26,6 +32,20 @@ def column_definition_primary_key_sql(sql, column) result end + # Add DISTKEY SQL fragment to column creation SQL. + def column_definition_dist_key_sql(sql, column) + if column[:dist_key] + sql << DIST_KEY + end + end + + # Add SORTKEY SQL fragment to column creation SQL. + def column_definition_sort_key_sql(sql, column) + if column[:sort_key] + sql << SORT_KEY + end + end + def serial_primary_key_options # redshift doesn't support serial type super.merge(serial: false) From 1da6c74f59f9bbf95e0495771e41aa8997308b74 Mon Sep 17 00:00:00 2001 From: Maximilian Seifert Date: Tue, 7 Jan 2014 11:56:34 +0100 Subject: [PATCH 05/41] Redshift does not support DROP TABLE IF EXISTS --- lib/sequel/adapters/redshift.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 3dfbdb7..0da3b05 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -50,6 +50,11 @@ def serial_primary_key_options # redshift doesn't support serial type super.merge(serial: false) end + + # DROP TABLE IF EXISTS is not supported by Redshift + def supports_drop_table_if_exists? + false + end end class Dataset < Postgres::Dataset From a6fc7171f85192c71217d7d3cabf979da56fcf7e Mon Sep 17 00:00:00 2001 From: Roland Swingler Date: Thu, 28 Aug 2014 16:46:55 +0100 Subject: [PATCH 06/41] Remove hardcoded database connection from spec_helper. This is instead loaded from spec/database.yml - an example file is provided. This allows different people to run tests on their own redshift clusters. In addition, tests have been moved under a test-specific schema - this needs to be created before runnning tests on your cluster. --- .gitignore | 1 + spec/database.yml.example | 6 ++++++ spec/spec_helper.rb | 23 +++++++++++++++++++---- 3 files changed, 26 insertions(+), 4 deletions(-) create mode 100644 spec/database.yml.example diff --git a/.gitignore b/.gitignore index d87d4be..feca1db 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +spec/database.yml *.gem *.rbc .bundle diff --git a/spec/database.yml.example b/spec/database.yml.example new file mode 100644 index 0000000..db31a2f --- /dev/null +++ b/spec/database.yml.example @@ -0,0 +1,6 @@ +adapter: redshift +database: test1 +username: +password: +host: r101-dw-production.ck1sr65ao7pg.us-east-1.redshift.amazonaws.com +port: 5439 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 29b6f09..af6f987 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,8 +5,23 @@ require 'sequel' require 'logger' +require 'yaml' + +$: << "." +require 'sequel/redshift' + +config_path = File.join(File.dirname(__FILE__), "database.yml") + +unless File.exists?(config_path) + warn "spec/database.yml does not exist." + warn "Create it based on spec/database.yml.example\nto conenct to a redshift cluster." + exit 1 +end + +options = YAML.load(File.read(config_path)) +options.merge(logger: Logger.new(STDOUT)) + +DB = Sequel.connect(options) +# Run all the tests in a specific test schema +DB.run "set search_path to 'sequel_redshift_adapter_test'" -options = { - logger: Logger.new(STDOUT) -} -DB = Sequel.connect('redshift://remind101:wvpfSzw2TT8COE@r101-dw-production.ck1sr65ao7pg.us-east-1.redshift.amazonaws.com:5439/test1', options) From 649b9b8b3229ed924dc1ceff495e224d21cb3193 Mon Sep 17 00:00:00 2001 From: Roland Swingler Date: Thu, 28 Aug 2014 16:48:49 +0100 Subject: [PATCH 07/41] Ensure INSERT RETURNING is disabled in latest Sequel version. The latest (4.13 as of writing) version of Sequel doesn't seem to pay any attention to supports_returning? anymore. There is a method/opts value called disable_insert_returning that has been added back in - just ensure that this is set all the time on a dataset. --- lib/sequel/adapters/redshift.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 0da3b05..5107624 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -60,6 +60,11 @@ def supports_drop_table_if_exists? class Dataset < Postgres::Dataset Database::DatasetClass = self + def initialize(*args) + super(*args) + @opts = @opts.merge(:disable_insert_returning => true).freeze + end + def insert_returning_sql(sql) sql end From a9c0752c63f1faac202c0a0f64f46052f565e114 Mon Sep 17 00:00:00 2001 From: Troy Thompson Date: Fri, 24 Apr 2015 20:54:25 -0400 Subject: [PATCH 08/41] adding common table expressions support --- lib/sequel/adapters/redshift.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 0da3b05..e989daf 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -60,10 +60,18 @@ def supports_drop_table_if_exists? class Dataset < Postgres::Dataset Database::DatasetClass = self + Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'values order limit'], ['elsif server_version >= 80400', %w'with select distinct columns from join where group having window compounds order limit lock'], ['else', %w'with select distinct columns from join where group having compounds order limit lock']]) + def insert_returning_sql(sql) sql end + + + def supports_cte?(type = :select) + true + end + def supports_returning?(type) false end From 4885aa0561783344ad90950a606c821cd103f379 Mon Sep 17 00:00:00 2001 From: Sam Reh Date: Sat, 9 May 2015 07:15:17 -0600 Subject: [PATCH 09/41] Redshift Supports Window Functions --- lib/sequel/adapters/redshift.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index b693fe3..c5e2202 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -27,6 +27,11 @@ def insert_returning_sql(sql) # do nothing here sql end + + # Redshift supports window functions + def supports_window_functions? + true + end end end end From ad8ee563488f252a25192e3be971f61b0f0604de Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Mon, 6 Mar 2017 22:49:21 +0200 Subject: [PATCH 10/41] code cleanup --- lib/sequel/adapters/redshift.rb | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 089077e..4fce51c 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -64,28 +64,23 @@ class Dataset < Postgres::Dataset def initialize(*args) super(*args) - @opts = @opts.merge(:disable_insert_returning => true).freeze + @opts = @opts.merge(disable_insert_returning: true).freeze end - def insert_returning_sql(sql) - sql - end - - # Redshift supports window functions - def supports_window_functions? - true + def supports_cte?(type = :select) + true if type == :select end - def supports_cte?(type = :select) - true + def supports_insert_select? + false end def supports_returning?(type) false end - def supports_insert_select? - false + def supports_window_functions? + true end end end From 0a0461871da98cd2c1f5494bcabcc663a019210c Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Tue, 7 Mar 2017 00:06:55 +0200 Subject: [PATCH 11/41] bump version --- lib/sequel/redshift/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequel/redshift/version.rb b/lib/sequel/redshift/version.rb index 74b4aab..fa31c6b 100644 --- a/lib/sequel/redshift/version.rb +++ b/lib/sequel/redshift/version.rb @@ -1,5 +1,5 @@ module Sequel module Redshift - VERSION = "0.0.1" + VERSION = "0.0.2" end end From 8193788aee75ab9e68bb17f2f38850edb5608dfc Mon Sep 17 00:00:00 2001 From: Paul Zaczkiewicz Date: Fri, 12 Dec 2014 17:01:06 -0500 Subject: [PATCH 12/41] Each ALTER TABLE command has to be on its own line --- lib/sequel/adapters/redshift.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 4fce51c..23eb629 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -55,6 +55,12 @@ def serial_primary_key_options def supports_drop_table_if_exists? false end + + # None of the alter table operation are combinable. + def combinable_alter_table_op?(op) + false + end + end class Dataset < Postgres::Dataset From a3c43cf525fb865ddbe17d9f1e97b64f5abfb8fd Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Sat, 15 Sep 2018 01:00:49 +0300 Subject: [PATCH 13/41] bump version --- lib/sequel/redshift/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequel/redshift/version.rb b/lib/sequel/redshift/version.rb index fa31c6b..01768b6 100644 --- a/lib/sequel/redshift/version.rb +++ b/lib/sequel/redshift/version.rb @@ -1,5 +1,5 @@ module Sequel module Redshift - VERSION = "0.0.2" + VERSION = "0.0.3" end end From ef673aedaaae5c59dbe3a84fea310b62c7f5bbc3 Mon Sep 17 00:00:00 2001 From: Rustam G Date: Mon, 24 Jun 2019 19:06:40 +0300 Subject: [PATCH 14/41] adds redshift string agg extension --- lib/sequel/extensions/redshift_string_agg.rb | 95 ++++++++++++++++++++ 1 file changed, 95 insertions(+) create mode 100644 lib/sequel/extensions/redshift_string_agg.rb diff --git a/lib/sequel/extensions/redshift_string_agg.rb b/lib/sequel/extensions/redshift_string_agg.rb new file mode 100644 index 0000000..f590c08 --- /dev/null +++ b/lib/sequel/extensions/redshift_string_agg.rb @@ -0,0 +1,95 @@ +# frozen-string-literal: true +# +# The redshift_string_agg extension adds the ability to perform database-independent +# aggregate string concatentation on Amazon Redshift. + +# Related module: Sequel::SQL::RedshiftStringAgg +module Sequel + module SQL + module Builders + # Return a RedshiftStringAgg expression for an aggregate string concatentation. + def redshift_string_agg(*a) + RedshiftStringAgg.new(*a) + end + end + + # The RedshiftStringAgg class represents an aggregate string concatentation. + class RedshiftStringAgg < GenericExpression + include StringMethods + include StringConcatenationMethods + include InequalityMethods + include AliasMethods + include CastMethods + include OrderMethods + include PatternMatchMethods + include SubscriptMethods + + # These methods are added to datasets using the redshift_string_agg + # extension, for the purposes of correctly literalizing RedshiftStringAgg + # expressions for the appropriate database type. + module DatasetMethods + # Append the SQL fragment for the RedshiftStringAgg expression to the SQL query. + def redshift_string_agg_sql_append(sql, sa) + unless db.adapter_scheme == :redshift + raise Error, "redshift_string_agg is not implemented on #{db.adapter_scheme}" + end + + expr = sa.expr + separator = sa.separator || "," + order = sa.order_expr + distinct = sa.is_distinct? + + if distinct + raise Error, "redshift_string_agg with distinct is not implemented on #{db.database_type}" + end + literal_append(sql, Function.new(:listagg, expr, separator)) + if order + sql << " WITHIN GROUP (ORDER BY " + expression_list_append(sql, order) + sql << ")" + else + sql << " WITHIN GROUP (ORDER BY 1)" + end + end + end + + # The string expression for each row that will concatenated to the output. + attr_reader :expr + + # The separator between each string expression. + attr_reader :separator + + # The expression that the aggregation is ordered by. + attr_reader :order_expr + + # Set the expression and separator + def initialize(expr, separator=nil) + @expr = expr + @separator = separator + end + + # Whether the current expression uses distinct expressions + def is_distinct? + @distinct == true + end + + # Return a modified RedshiftStringAgg that uses distinct expressions + def distinct + sa = dup + sa.instance_variable_set(:@distinct, true) + sa + end + + # Return a modified RedshiftStringAgg with the given order + def order(*o) + sa = dup + sa.instance_variable_set(:@order_expr, o.empty? ? nil : o) + sa + end + + to_s_method :redshift_string_agg_sql + end + end + + Dataset.register_extension(:redshift_string_agg, SQL::RedshiftStringAgg::DatasetMethods) +end From 4850b1a0956d5a03cc1b0e42fa4336e00d83087a Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Fri, 6 Sep 2019 03:20:38 +0300 Subject: [PATCH 15/41] support drop tabke if exists this prevents savepoint from firing inside transaction loop --- lib/sequel/adapters/redshift.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 23eb629..48fb14f 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -51,9 +51,9 @@ def serial_primary_key_options super.merge(serial: false) end - # DROP TABLE IF EXISTS is not supported by Redshift + # DROP TABLE IF EXISTS is now supported by Redshift def supports_drop_table_if_exists? - false + true end # None of the alter table operation are combinable. @@ -88,6 +88,8 @@ def supports_returning?(type) def supports_window_functions? true end + + supports_drop_table_if_exists? end end end From 7052fb51e0f44be872974594f244d5f86d322c73 Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Fri, 6 Sep 2019 03:22:57 +0300 Subject: [PATCH 16/41] bump version --- lib/sequel/adapters/redshift.rb | 4 +--- lib/sequel/redshift/version.rb | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 48fb14f..7dff55e 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -87,9 +87,7 @@ def supports_returning?(type) def supports_window_functions? true - end - - supports_drop_table_if_exists? + end end end end diff --git a/lib/sequel/redshift/version.rb b/lib/sequel/redshift/version.rb index 01768b6..b7ef509 100644 --- a/lib/sequel/redshift/version.rb +++ b/lib/sequel/redshift/version.rb @@ -1,5 +1,5 @@ module Sequel module Redshift - VERSION = "0.0.3" + VERSION = "0.0.4" end end From 8d2828706970c6fd7661909315b5eecde9759bb7 Mon Sep 17 00:00:00 2001 From: Rustam G Date: Sun, 27 Oct 2019 23:10:25 +0300 Subject: [PATCH 17/41] monkey patch sequel's string_agg to use redshift out of the box --- lib/sequel/extensions/redshift_string_agg.rb | 3 + lib/sequel/extensions/string_agg.rb | 182 +++++++++++++++++++ spec/features/extensions_spec.rb | 35 ++++ 3 files changed, 220 insertions(+) create mode 100644 lib/sequel/extensions/string_agg.rb create mode 100644 spec/features/extensions_spec.rb diff --git a/lib/sequel/extensions/redshift_string_agg.rb b/lib/sequel/extensions/redshift_string_agg.rb index f590c08..c6ee4bc 100644 --- a/lib/sequel/extensions/redshift_string_agg.rb +++ b/lib/sequel/extensions/redshift_string_agg.rb @@ -1,5 +1,8 @@ # frozen-string-literal: true # +# Please consider using patched `string_agg` extension, it properly handles all supported DB adapters +# including Redshift out of the box. +# # The redshift_string_agg extension adds the ability to perform database-independent # aggregate string concatentation on Amazon Redshift. diff --git a/lib/sequel/extensions/string_agg.rb b/lib/sequel/extensions/string_agg.rb new file mode 100644 index 0000000..60d671e --- /dev/null +++ b/lib/sequel/extensions/string_agg.rb @@ -0,0 +1,182 @@ +# frozen-string-literal: true +# +# This is patched version of original plugin +# @see https://github.com/jeremyevans/sequel/blob/4.49.0/lib/sequel/extensions/string_agg.rb +# The reasons for this patch are 4.49 is not updated anymore and Redshift is not officially +# supported by sequel. +# +# The string_agg extension adds the ability to perform database-independent +# aggregate string concatentation. For example, with a table like: +# +# c1 | c2 +# ---+--- +# a | 1 +# a | 2 +# a | 3 +# b | 4 +# +# You can return a result set like: +# +# c1 | c2s +# ---+--- +# a | 1,2,3 +# b | 4 +# +# First, you need to load the extension into the database: +# +# DB.extension :string_agg +# +# Then you can use the Sequel.string_agg method to return a Sequel +# expression: +# +# sa = Sequel.string_agg(:column_name) +# # or: +# sa = Sequel.string_agg(:column_name, '-') # custom separator +# +# You can specify the order in which the concatention happens by +# calling +order+ on the expression: +# +# sa = Sequel.string_agg(:column_name).order(:other_column) +# +# Additionally, if you want to have the concatenation only operate +# on distinct values, you can call distinct: +# +# sa = Sequel.string_agg(:column_name).order(:other_column).distinct +# +# These expressions can be used in your datasets, or anywhere else that +# Sequel expressions are allowed: +# +# DB[:table]. +# select_group(:c1). +# select_append(Sequel.string_agg(:c2)) +# +# This extension currenly supports the following databases: +# +# * PostgreSQL 9+ +# * SQLAnywhere 12+ +# * Oracle 11g+ (except distinct) +# * DB2 9.7+ (except distinct) +# * MySQL +# * HSQLDB +# * H2 +# +# Related module: Sequel::SQL::StringAgg + +# +module Sequel + module SQL + module Builders + # Return a StringAgg expression for an aggregate string concatentation. + def string_agg(*a) + StringAgg.new(*a) + end + end + + # The StringAgg class represents an aggregate string concatentation. + class StringAgg < GenericExpression + include StringMethods + include StringConcatenationMethods + include InequalityMethods + include AliasMethods + include CastMethods + include OrderMethods + include PatternMatchMethods + include SubscriptMethods + + # These methods are added to datasets using the string_agg + # extension, for the purposes of correctly literalizing StringAgg + # expressions for the appropriate database type. + module DatasetMethods + # Append the SQL fragment for the StringAgg expression to the SQL query. + def string_agg_sql_append(sql, sa) + if defined?(super) + return super + end + + expr = sa.expr + separator = sa.separator || "," + order = sa.order_expr + distinct = sa.is_distinct? + + case db_type = db.adapter_scheme + when :postgres, :sqlanywhere + f = Function.new(db_type == :postgres ? :string_agg : :list, expr, separator) + if order + f = f.order(*order) + end + if distinct + f = f.distinct + end + literal_append(sql, f) + # SEQUEL5: Remove cubrid + when :mysql, :hsqldb, :cubrid, :h2 + sql << "GROUP_CONCAT(" + if distinct + sql << "DISTINCT " + end + literal_append(sql, expr) + if order + sql << " ORDER BY " + expression_list_append(sql, order) + end + sql << " SEPARATOR " + literal_append(sql, separator) + sql << ")" + when :oracle, :db2, :redshift + if distinct + raise Error, "string_agg with distinct is not implemented on #{db.database_type}" + end + literal_append(sql, Function.new(:listagg, expr, separator)) + if order + sql << " WITHIN GROUP (ORDER BY " + expression_list_append(sql, order) + sql << ")" + else + sql << " WITHIN GROUP (ORDER BY 1)" + end + else + raise Error, "string_agg is not implemented on #{db.database_type}" + end + end + end + + # The string expression for each row that will concatenated to the output. + attr_reader :expr + + # The separator between each string expression. + attr_reader :separator + + # The expression that the aggregation is ordered by. + attr_reader :order_expr + + # Set the expression and separator + def initialize(expr, separator=nil) + @expr = expr + @separator = separator + end + + # Whether the current expression uses distinct expressions + def is_distinct? + @distinct == true + end + + # Return a modified StringAgg that uses distinct expressions + def distinct + sa = dup + sa.instance_variable_set(:@distinct, true) + sa + end + + # Return a modified StringAgg with the given order + def order(*o) + sa = dup + sa.instance_variable_set(:@order_expr, o.empty? ? nil : o) + sa + end + + to_s_method :string_agg_sql + end + end + + Dataset.register_extension(:string_agg, SQL::StringAgg::DatasetMethods) +end diff --git a/spec/features/extensions_spec.rb b/spec/features/extensions_spec.rb new file mode 100644 index 0000000..b08bac5 --- /dev/null +++ b/spec/features/extensions_spec.rb @@ -0,0 +1,35 @@ +require 'spec_helper' + +describe 'Extensions' do + describe 'string_agg' do + before { DB.extension :string_agg } + + it 'generates correct sql for Redshift' do + allow(DB).to receive(:adapter_scheme).and_return(:redshift) + expect( + DB[:test]. + select( + Sequel. + string_agg(Sequel[:revenue].cast_string). + order(Sequel.asc(:period_start), Sequel.asc(:sub_period_start)). + as(:relative) + ).sql).to eq( + "SELECT listagg(CAST(\"revenue\" AS varchar(255)), ',') WITHIN GROUP (ORDER BY \"period_start\" ASC, \"sub_period_start\" ASC) AS \"relative\" FROM \"test\"" + ) + end + + it 'generates correct sql for Postgresql' do + allow(DB).to receive(:adapter_scheme).and_return(:postgres) + expect( + DB[:test]. + select( + Sequel. + string_agg(Sequel[:revenue].cast_string). + order(Sequel.asc(:period_start), Sequel.asc(:sub_period_start)). + as(:relative) + ).sql).to eq( + "SELECT string_agg(CAST(\"revenue\" AS varchar(255)), ',' ORDER BY \"period_start\" ASC, \"sub_period_start\" ASC) AS \"relative\" FROM \"test\"" + ) + end + end +end From 91ff8bba62aa1bfdccd164a269748909f5c9b943 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 19 Nov 2019 13:33:03 +0200 Subject: [PATCH 18/41] Bump Sequel dependency --- sequel-redshift.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequel-redshift.gemspec b/sequel-redshift.gemspec index 1cc251c..3b9a490 100644 --- a/sequel-redshift.gemspec +++ b/sequel-redshift.gemspec @@ -19,7 +19,7 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.add_dependency "pg" - spec.add_dependency "sequel" + spec.add_dependency "sequel", '~> 5' spec.add_development_dependency "bundler", "~> 1.3" spec.add_development_dependency "rake" From 515e8aac0d87841b2f1e545ae1d643c85455a0bb Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 21 Nov 2019 13:39:37 +0200 Subject: [PATCH 19/41] Bump bundler to v2 --- sequel-redshift.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sequel-redshift.gemspec b/sequel-redshift.gemspec index 3b9a490..49b5d2e 100644 --- a/sequel-redshift.gemspec +++ b/sequel-redshift.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |spec| spec.add_dependency "pg" spec.add_dependency "sequel", '~> 5' - spec.add_development_dependency "bundler", "~> 1.3" + spec.add_development_dependency "bundler", "~> 2" spec.add_development_dependency "rake" spec.add_development_dependency "rspec" end From 15dd7ca9a8ea2c093185b113da94deb668830e87 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 21 Nov 2019 13:39:21 +0200 Subject: [PATCH 20/41] Drop test table if exists --- spec/features/redshift_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/features/redshift_spec.rb b/spec/features/redshift_spec.rb index f0d4376..69d7c64 100644 --- a/spec/features/redshift_spec.rb +++ b/spec/features/redshift_spec.rb @@ -9,6 +9,7 @@ end it "inserts new record" do + DB.drop_table? :items DB.create_table :items do primary_key :id column :name, 'varchar(255)' From 6d783b216cda898fbc3cece091e9a50d723a125c Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 21 Nov 2019 13:40:57 +0200 Subject: [PATCH 21/41] Allow logging with DEBUG_SQL env var --- spec/spec_helper.rb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index af6f987..56c8cb2 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -22,6 +22,9 @@ options.merge(logger: Logger.new(STDOUT)) DB = Sequel.connect(options) +if ENV['DEBUG_SQL'] == 'true' + DB.loggers << Logger.new($stdout) +end # Run all the tests in a specific test schema DB.run "set search_path to 'sequel_redshift_adapter_test'" From aaadcdd9005ddf67b19b35626347c278c51705ed Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 21 Nov 2019 13:45:20 +0200 Subject: [PATCH 22/41] Add fixes for Sequel 5 --- lib/sequel/adapters/redshift.rb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 7dff55e..f85ab1c 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -16,6 +16,10 @@ class Database < Postgres::Database # The order of column modifiers to use when defining a column. COLUMN_DEFINITION_ORDER = [:collate, :default, :primary_key, :dist_key, :sort_key, :null, :unique, :auto_increment, :references] + def dataset_class_default + Sequel::Redshift::Dataset + end + # We need to change these default settings because they correspond to # Postgres configuration variables which do not exist in Redshift def adapter_initialize @@ -27,9 +31,9 @@ def adapter_initialize end def column_definition_primary_key_sql(sql, column) - result = super - result << ' IDENTITY' if result - result + if column[:primary_key] + sql << ' IDENTITY' + end end # Add DISTKEY SQL fragment to column creation SQL. @@ -68,9 +72,11 @@ class Dataset < Postgres::Dataset Dataset.def_sql_method(self, :select, [['if opts[:values]', %w'values order limit'], ['elsif server_version >= 80400', %w'with select distinct columns from join where group having window compounds order limit lock'], ['else', %w'with select distinct columns from join where group having compounds order limit lock']]) - def initialize(*args) - super(*args) - @opts = @opts.merge(disable_insert_returning: true).freeze + def initialize(db) + @db = db + @opts = {disable_insert_returning: true}.freeze + @cache = {} + freeze end def supports_cte?(type = :select) From ece8f7e0d38e9a6f3f55eefbce9d492efb78ecd9 Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 22 Nov 2019 10:55:22 +0200 Subject: [PATCH 23/41] Only Integer columns can be IDENTITY --- lib/sequel/adapters/redshift.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index f85ab1c..4b9d575 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -31,7 +31,7 @@ def adapter_initialize end def column_definition_primary_key_sql(sql, column) - if column[:primary_key] + if column[:primary_key] && column[:type] == Integer sql << ' IDENTITY' end end From 45afbc394924cb7d567702e53587d6d02e8aea7c Mon Sep 17 00:00:00 2001 From: Max Date: Fri, 22 Nov 2019 10:56:05 +0200 Subject: [PATCH 24/41] Do not support transactional DDL --- lib/sequel/adapters/redshift.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 4b9d575..3ce87ac 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -65,6 +65,10 @@ def combinable_alter_table_op?(op) false end + def supports_transactional_ddl? + false + end + end class Dataset < Postgres::Dataset From c1d9cb92ee2c579c21a18258ce91e97dfd36a30e Mon Sep 17 00:00:00 2001 From: Rustam G Date: Tue, 14 Jan 2020 13:42:25 +0300 Subject: [PATCH 25/41] add median function to extensions --- lib/sequel/extensions/median.rb | 55 +++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 lib/sequel/extensions/median.rb diff --git a/lib/sequel/extensions/median.rb b/lib/sequel/extensions/median.rb new file mode 100644 index 0000000..700586e --- /dev/null +++ b/lib/sequel/extensions/median.rb @@ -0,0 +1,55 @@ +# frozen-string-literal: true +# +module Sequel + module SQL + module Builders + # Return a median expression. + def median(*a) + Median.new(*a) + end + end + + # The Median class represents a median function for Redshift and Postgresql databases. + class Median < GenericExpression + include OrderMethods + + module DatasetMethods + # Append the SQL fragment for the Median expression to the SQL query. + def median_sql_append(sql, sa) + if defined?(super) + return super + end + + expr = sa.expr + + case db_type = db.adapter_scheme + when :postgres + + literal_append(sql, Function.new(:percentile_disc, 0.5)) + sql << " WITHIN GROUP (ORDER BY " + identifier_append(sql, expr) + sql << ")" + + when :redshift + + literal_append(sql, Function.new(:median, expr)) + + else + raise Error, "median is not implemented on #{db.database_type}" + end + end + end + + attr_reader :expr + + # Set the expression and separator + def initialize(expr) + @expr = expr + end + + to_s_method :median_sql + end + end + + Dataset.register_extension(:median, SQL::Median::DatasetMethods) +end From c379775a9b957e6a0522396412e5842ec1d53e52 Mon Sep 17 00:00:00 2001 From: Rustam G Date: Tue, 14 Jan 2020 13:52:20 +0300 Subject: [PATCH 26/41] add spec for median --- spec/features/extensions_spec.rb | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/spec/features/extensions_spec.rb b/spec/features/extensions_spec.rb index b08bac5..49f74a6 100644 --- a/spec/features/extensions_spec.rb +++ b/spec/features/extensions_spec.rb @@ -4,7 +4,7 @@ describe 'string_agg' do before { DB.extension :string_agg } - it 'generates correct sql for Redshift' do + it 'generates correct sql for Redshift' do allow(DB).to receive(:adapter_scheme).and_return(:redshift) expect( DB[:test]. @@ -32,4 +32,32 @@ ) end end + + describe 'median' do + before { DB.extension :median } + + it 'generates correct sql for Redshift' do + allow(DB).to receive(:adapter_scheme).and_return(:redshift) + expect( + DB[:test]. + select( + Sequel. + median(:revenue) + ).sql).to eq( + "SELECT median(\"revenue\") FROM \"test\"" + ) + end + + it 'generates correct sql for Postgresql' do + allow(DB).to receive(:adapter_scheme).and_return(:postgres) + expect( + DB[:test]. + select( + Sequel. + median(:revenue) + ).sql).to eq( + "SELECT percentile_disc(0.5) WITHIN GROUP (ORDER BY \"revenue\") FROM \"test\"" + ) + end + end end From 53a89a698e38421767b072ecd7acb3b6281c7af7 Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Sat, 31 Oct 2020 04:29:52 +0200 Subject: [PATCH 27/41] redshift doesn't support savepoints --- lib/sequel/adapters/redshift.rb | 5 ++++- lib/sequel/redshift/version.rb | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 7dff55e..c07a895 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -61,6 +61,9 @@ def combinable_alter_table_op?(op) false end + def supports_savepoints? + false + end end class Dataset < Postgres::Dataset @@ -87,7 +90,7 @@ def supports_returning?(type) def supports_window_functions? true - end + end end end end diff --git a/lib/sequel/redshift/version.rb b/lib/sequel/redshift/version.rb index b7ef509..a231740 100644 --- a/lib/sequel/redshift/version.rb +++ b/lib/sequel/redshift/version.rb @@ -1,5 +1,5 @@ module Sequel module Redshift - VERSION = "0.0.4" + VERSION = "0.0.5" end end From 259e6ce17d4b2c822557d41e086bd30484f3eca2 Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Wed, 4 Nov 2020 00:53:14 +0200 Subject: [PATCH 28/41] transaction safe table_exits? --- lib/sequel/adapters/redshift.rb | 13 +++++++++++++ lib/sequel/redshift/version.rb | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 4d1e4b8..838d564 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -30,6 +30,19 @@ def adapter_initialize super end + def table_exists?(name) + sql = <<~SQL + SELECT EXISTS ( + SELECT * FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' + AND c.relname = '#{name}' + AND c.relkind = 'r' + ); + SQL + DB.fetch(sql).first.fetch(:"?column?") + end + def column_definition_primary_key_sql(sql, column) if column[:primary_key] && column[:type] == Integer sql << ' IDENTITY' diff --git a/lib/sequel/redshift/version.rb b/lib/sequel/redshift/version.rb index a231740..7b5c033 100644 --- a/lib/sequel/redshift/version.rb +++ b/lib/sequel/redshift/version.rb @@ -1,5 +1,5 @@ module Sequel module Redshift - VERSION = "0.0.5" + VERSION = "0.0.6" end end From cdf88ab969108f2e88bbaa8dce3503991ad9fce9 Mon Sep 17 00:00:00 2001 From: Rustam Galeev Date: Thu, 12 Nov 2020 22:42:32 +0100 Subject: [PATCH 29/41] fix table_exists? method --- lib/sequel/adapters/redshift.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/sequel/adapters/redshift.rb b/lib/sequel/adapters/redshift.rb index 838d564..92694f1 100644 --- a/lib/sequel/adapters/redshift.rb +++ b/lib/sequel/adapters/redshift.rb @@ -40,7 +40,7 @@ def table_exists?(name) AND c.relkind = 'r' ); SQL - DB.fetch(sql).first.fetch(:"?column?") + fetch(sql).first.fetch(:"?column?") end def column_definition_primary_key_sql(sql, column) From 20255072675637133e76e348c7ba1ce3d7cfc753 Mon Sep 17 00:00:00 2001 From: Rustam Galeev Date: Tue, 1 Dec 2020 20:04:02 +0100 Subject: [PATCH 30/41] add redshift? helper --- lib/sequel/helpers.rb | 7 +++++++ lib/sequel/redshift.rb | 1 + 2 files changed, 8 insertions(+) create mode 100644 lib/sequel/helpers.rb diff --git a/lib/sequel/helpers.rb b/lib/sequel/helpers.rb new file mode 100644 index 0000000..c4d68ba --- /dev/null +++ b/lib/sequel/helpers.rb @@ -0,0 +1,7 @@ +module Sequel + class Database + def redshift? + adapter_scheme == :redshift + end + end +end diff --git a/lib/sequel/redshift.rb b/lib/sequel/redshift.rb index a7aa1d8..9869bcb 100644 --- a/lib/sequel/redshift.rb +++ b/lib/sequel/redshift.rb @@ -1 +1,2 @@ require "sequel/redshift/version" +require "sequel/helpers" From bd710d670e2475657a1a0790d0bda9a384efa89d Mon Sep 17 00:00:00 2001 From: Rustam Galeev Date: Fri, 21 Jan 2022 18:16:53 +0100 Subject: [PATCH 31/41] add server version for redshift --- lib/sequel/adapters/shared/redshift.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/sequel/adapters/shared/redshift.rb b/lib/sequel/adapters/shared/redshift.rb index 879f9f6..3089f72 100644 --- a/lib/sequel/adapters/shared/redshift.rb +++ b/lib/sequel/adapters/shared/redshift.rb @@ -49,6 +49,11 @@ def schema_parse_table(table_name, opts) def type_literal_generic_string(column) "#{ column[:fixed] ? 'char' : 'varchar' }(#{ column[:size] || REDSHIFT_STRING_MAXIMUM_SIZE })" end + + # The version of the PostgreSQL server, used for determining capability. + def server_version(server=nil) + @server_version ||= 80002 + end end end end From c7aa904aa330bf915852d6dc9748683f8c7c4fc0 Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Tue, 16 Aug 2022 19:15:22 +0200 Subject: [PATCH 32/41] add basic support for table diststyle/distkey/sortkey attributes --- lib/sequel/adapters/shared/redshift.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/sequel/adapters/shared/redshift.rb b/lib/sequel/adapters/shared/redshift.rb index 3089f72..ceed93d 100644 --- a/lib/sequel/adapters/shared/redshift.rb +++ b/lib/sequel/adapters/shared/redshift.rb @@ -47,13 +47,22 @@ def schema_parse_table(table_name, opts) # Redshift changes text to varchar with maximum size of 256, and it complains if you will give text column def type_literal_generic_string(column) - "#{ column[:fixed] ? 'char' : 'varchar' }(#{ column[:size] || REDSHIFT_STRING_MAXIMUM_SIZE })" + "#{column[:fixed] ? 'CHAR' : 'VARCHAR'}(#{column[:size] || REDSHIFT_STRING_MAXIMUM_SIZE})" end # The version of the PostgreSQL server, used for determining capability. def server_version(server=nil) @server_version ||= 80002 end + + def create_table_suffix_sql(name, options) + sql = String.new + sql << " DISTSTYLE #{options[:diststyle].upcase}" if options[:diststyle] + sql << " DISTKEY(#{options[:distkey]})" if options[:distkey] + sql << " SORTKEY(#{Array(options[:sortkey]).join(', ')})" if options[:sortkey] + "#{super}#{sql}" + end + end end end From dfb761aa02a19bff1f4421fb38a435d127434e10 Mon Sep 17 00:00:00 2001 From: Troex Nevelin Date: Tue, 16 Aug 2022 19:47:50 +0200 Subject: [PATCH 33/41] support attributes on create table as --- lib/sequel/adapters/shared/redshift.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/sequel/adapters/shared/redshift.rb b/lib/sequel/adapters/shared/redshift.rb index ceed93d..4bc40ff 100644 --- a/lib/sequel/adapters/shared/redshift.rb +++ b/lib/sequel/adapters/shared/redshift.rb @@ -56,13 +56,23 @@ def server_version(server=nil) end def create_table_suffix_sql(name, options) + sql = create_table_attributes(name, options) + "#{super}#{sql}" + end + + def create_table_attributes(name, options) sql = String.new sql << " DISTSTYLE #{options[:diststyle].upcase}" if options[:diststyle] sql << " DISTKEY(#{options[:distkey]})" if options[:distkey] sql << " SORTKEY(#{Array(options[:sortkey]).join(', ')})" if options[:sortkey] - "#{super}#{sql}" + sql end + def create_table_as_sql(name, sql, options) + result = create_table_prefix_sql(name, options) + result += create_table_attributes(name, options) + result += " AS #{sql}" + end end end end From cd8968e6a76f74c9675fdf74b9e3ff851e3189be Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 19:31:00 +0300 Subject: [PATCH 34/41] add pg_array support --- lib/sequel/extensions/pg_array.rb | 522 ++++++++++++++++++++++++++++++ 1 file changed, 522 insertions(+) create mode 100644 lib/sequel/extensions/pg_array.rb diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb new file mode 100644 index 0000000..205179c --- /dev/null +++ b/lib/sequel/extensions/pg_array.rb @@ -0,0 +1,522 @@ +# frozen-string-literal: true +# +# This is patched version of original plugin +# @see https://github.com/jeremyevans/sequel/blob/5.71.0/lib/sequel/extensions/pg_array.rb +# + +require 'delegate' + +module Sequel + module Postgres + # Represents a PostgreSQL array column value. + class PGArray < DelegateClass(Array) + include Sequel::SQL::AliasMethods + + module DatabaseMethods + BLOB_RANGE = 1...-1 + + # Create the local hash of database type strings to schema type symbols, + # used for array types local to this database. + def self.extended(db) + db.instance_exec do + @pg_array_schema_types ||= {} + register_array_type('timestamp without time zone', :oid=>1115, :scalar_oid=>1114, :type_symbol=>:datetime) + register_array_type('timestamp with time zone', :oid=>1185, :scalar_oid=>1184, :type_symbol=>:datetime_timezone, :scalar_typecast=>:datetime) + + register_array_type('text', :oid=>1009, :scalar_oid=>25, :type_symbol=>:string) + register_array_type('integer', :oid=>1007, :scalar_oid=>23) + register_array_type('bigint', :oid=>1016, :scalar_oid=>20, :scalar_typecast=>:integer) + register_array_type('numeric', :oid=>1231, :scalar_oid=>1700, :type_symbol=>:decimal) + register_array_type('double precision', :oid=>1022, :scalar_oid=>701, :type_symbol=>:float) + + register_array_type('boolean', :oid=>1000, :scalar_oid=>16) + register_array_type('bytea', :oid=>1001, :scalar_oid=>17, :type_symbol=>:blob) + register_array_type('date', :oid=>1182, :scalar_oid=>1082) + register_array_type('time without time zone', :oid=>1183, :scalar_oid=>1083, :type_symbol=>:time) + register_array_type('time with time zone', :oid=>1270, :scalar_oid=>1266, :type_symbol=>:time_timezone, :scalar_typecast=>:time) + + register_array_type('smallint', :oid=>1005, :scalar_oid=>21, :scalar_typecast=>:integer) + register_array_type('oid', :oid=>1028, :scalar_oid=>26, :scalar_typecast=>:integer) + register_array_type('real', :oid=>1021, :scalar_oid=>700, :scalar_typecast=>:float) + register_array_type('character', :oid=>1014, :converter=>nil, :array_type=>:text, :scalar_typecast=>:string) + register_array_type('character varying', :oid=>1015, :converter=>nil, :scalar_typecast=>:string, :type_symbol=>:varchar) + + register_array_type('xml', :oid=>143, :scalar_oid=>142) + register_array_type('money', :oid=>791, :scalar_oid=>790) + register_array_type('bit', :oid=>1561, :scalar_oid=>1560) + register_array_type('bit varying', :oid=>1563, :scalar_oid=>1562, :type_symbol=>:varbit) + register_array_type('uuid', :oid=>2951, :scalar_oid=>2950) + + register_array_type('xid', :oid=>1011, :scalar_oid=>28) + register_array_type('cid', :oid=>1012, :scalar_oid=>29) + + register_array_type('name', :oid=>1003, :scalar_oid=>19) + register_array_type('tid', :oid=>1010, :scalar_oid=>27) + register_array_type('int2vector', :oid=>1006, :scalar_oid=>22) + register_array_type('oidvector', :oid=>1013, :scalar_oid=>30) + + [:string_array, :integer_array, :decimal_array, :float_array, :boolean_array, :blob_array, :date_array, :time_array, :datetime_array].each do |v| + @schema_type_classes[v] = PGArray + end + end + end + + def add_named_conversion_proc(name, &block) + ret = super + name = name.to_s if name.is_a?(Symbol) + from(:pg_type).where(:typname=>name).select_map([:oid, :typarray]).each do |scalar_oid, array_oid| + register_array_type(name, :oid=>array_oid.to_i, :scalar_oid=>scalar_oid.to_i) + end + ret + end + + # Handle arrays in bound variables + def bound_variable_arg(arg, conn) + case arg + when PGArray + bound_variable_array(arg.to_a) + when Array + bound_variable_array(arg) + else + super + end + end + + # Freeze the pg array schema types to prevent adding new ones. + def freeze + @pg_array_schema_types.freeze + super + end + + # Register a database specific array type. Options: + # + # :array_type :: The type to automatically cast the array to when literalizing the array. + # Usually the same as db_type. + # :converter :: A callable object (e.g. Proc), that is called with each element of the array + # (usually a string), and should return the appropriate typecasted object. + # :oid :: The PostgreSQL OID for the array type. This is used by the Sequel postgres adapter + # to set up automatic type conversion on retrieval from the database. + # :scalar_oid :: Should be the PostgreSQL OID for the scalar version of this array type. If given, + # automatically sets the :converter option by looking for scalar conversion + # proc. + # :scalar_typecast :: Should be a symbol indicating the typecast method that should be called on + # each element of the array, when a plain array is passed into a database + # typecast method. For example, for an array of integers, this could be set to + # :integer, so that the typecast_value_integer method is called on all of the + # array elements. Defaults to :type_symbol option. + # :type_symbol :: The base of the schema type symbol for this type. For example, if you provide + # :integer, Sequel will recognize this type as :integer_array during schema parsing. + # Defaults to the db_type argument. + # + # If a block is given, it is treated as the :converter option. + def register_array_type(db_type, opts=OPTS, &block) + oid = opts[:oid] + soid = opts[:scalar_oid] + + if has_converter = opts.has_key?(:converter) + raise Error, "can't provide both a block and :converter option to register_array_type" if block + converter = opts[:converter] + else + has_converter = true if block + converter = block + end + + unless (soid || has_converter) && oid + array_oid, scalar_oid = from(:pg_type).where(:typname=>db_type.to_s).get([:typarray, :oid]) + soid ||= scalar_oid unless has_converter + oid ||= array_oid + end + + db_type = db_type.to_s + type = (opts[:type_symbol] || db_type).to_sym + typecast_method_map = @pg_array_schema_types + + if soid + raise Error, "can't provide both a converter and :scalar_oid option to register" if has_converter + converter = conversion_procs[soid] + end + + array_type = (opts[:array_type] || db_type).to_s.dup.freeze + creator = Creator.new(array_type, converter) + add_conversion_proc(oid, creator) + + typecast_method_map[db_type] = :"#{type}_array" + + singleton_class.class_eval do + meth = :"typecast_value_#{type}_array" + scalar_typecast_method = :"typecast_value_#{opts.fetch(:scalar_typecast, type)}" + define_method(meth){|v| typecast_value_pg_array(v, creator, scalar_typecast_method)} + private meth + alias_method(meth, meth) + end + + @schema_type_classes[:"#{type}_array"] = PGArray + nil + end + + private + + # Format arrays used in bound variables. + def bound_variable_array(a) + case a + when Array + "{#{a.map{|i| bound_variable_array(i)}.join(',')}}" + when Sequel::SQL::Blob + bound_variable_array_string(literal(a)[BLOB_RANGE].gsub("''", "'")) + when Sequel::LiteralString + a + when String + bound_variable_array_string(a) + when Float + if a.infinite? + a > 0 ? '"Infinity"' : '"-Infinity"' + elsif a.nan? + '"NaN"' + else + literal(a) + end + else + if (s = bound_variable_arg(a, nil)).is_a?(String) + bound_variable_array_string(s) + else + literal(a) + end + end + end + + # Escape strings used as array members in bound variables. Most complex + # will create a regular string with bound_variable_arg, and then use this + # escaping to format it as an array member. + def bound_variable_array_string(s) + "\"#{s.gsub(/("|\\)/, '\\\\\1')}\"" + end + + # Look into both the current database's array schema types and the global + # array schema types to get the type symbol for the given database type + # string. + def pg_array_schema_type(type) + @pg_array_schema_types[type] + end + + # Make the column type detection handle registered array types. + def schema_array_type(db_type) + if (db_type =~ /\A([^(]+)(?:\([^(]+\))?\[\]\z/io) && (type = pg_array_schema_type($1)) + type + else + super + end + end + + # Set the :callable_default value if the default value is recognized as an empty array. + def schema_post_process(_) + super.each do |a| + h = a[1] + if h[:default] =~ /\A(?:'\{\}'|ARRAY\[\])::([\w ]+)\[\]\z/ + type = $1.freeze + h[:callable_default] = lambda{Sequel.pg_array([], type)} + end + end + end + + # Convert ruby arrays to PostgreSQL arrays when used as default values. + def column_definition_default_sql(sql, column) + if (d = column[:default]) && d.is_a?(Array) && !Sequel.condition_specifier?(d) + if db_type == :redshift + sql << " DEFAULT (#{literal(Sequel.pg_array(d))}" + else + sql << " DEFAULT (#{literal(Sequel.pg_array(d))}::#{type_literal(column)})" + end + else + super + end + end + + # Given a value to typecast and the type of PGArray subclass: + # * If given a PGArray with a matching array_type, use it directly. + # * If given a PGArray with a different array_type, return a PGArray + # with the creator's type. + # * If given an Array, create a new PGArray instance for it. This does not + # typecast all members of the array in ruby for performance reasons, but + # it will cast the array the appropriate database type when the array is + # literalized. + def typecast_value_pg_array(value, creator, scalar_typecast_method=nil) + case value + when PGArray + if value.array_type != creator.type + PGArray.new(value.to_a, creator.type) + else + value + end + when Array + if scalar_typecast_method && respond_to?(scalar_typecast_method, true) + value = Sequel.recursive_map(value, method(scalar_typecast_method)) + end + PGArray.new(value, creator.type) + else + raise Sequel::InvalidValue, "invalid value for array type: #{value.inspect}" + end + end + end + + unless Sequel::Postgres.respond_to?(:parse_pg_array) + require 'strscan' + + # PostgreSQL array parser that handles PostgreSQL array output format. + # Note that does not handle all forms out input that PostgreSQL will + # accept, and it will not raise an error for all forms of invalid input. + class Parser < StringScanner + # Set the source for the input, and any converter callable + # to call with objects to be created. For nested parsers + # the source may contain text after the end current parse, + # which will be ignored. + def initialize(source, converter=nil) + super(source) + @converter = converter + @stack = [[]] + @encoding = string.encoding + @recorded = String.new.force_encoding(@encoding) + end + + # Take the buffer of recorded characters and add it to the array + # of entries, and use a new buffer for recorded characters. + def new_entry(include_empty=false) + if !@recorded.empty? || include_empty + entry = @recorded + if entry == 'NULL' && !include_empty + entry = nil + elsif @converter + entry = @converter.call(entry) + end + @stack.last.push(entry) + @recorded = String.new.force_encoding(@encoding) + end + end + + # Parse the input character by character, returning an array + # of parsed (and potentially converted) objects. + def parse + raise Sequel::Error, "invalid array, empty string" if eos? + raise Sequel::Error, "invalid array, doesn't start with {" unless scan(/((\[\d+:\d+\])+=)?\{/) + + # :nocov: + while !eos? + # :nocov: + char = scan(/[{}",]|[^{}",]+/) + if char == ',' + # Comma outside quoted string indicates end of current entry + new_entry + elsif char == '"' + raise Sequel::Error, "invalid array, opening quote with existing recorded data" unless @recorded.empty? + # :nocov: + while true + # :nocov: + char = scan(/["\\]|[^"\\]+/) + if char == '\\' + @recorded << getch + elsif char == '"' + n = peek(1) + raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == '}' + break + else + @recorded << char + end + end + new_entry(true) + elsif char == '{' + raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty? + + # Start of new array, add it to the stack + new = [] + @stack.last << new + @stack << new + elsif char == '}' + # End of current array, add current entry to the current array + new_entry + + if @stack.length == 1 + raise Sequel::Error, "array parsing finished without parsing entire string" unless eos? + + # Top level of array, parsing should be over. + # Pop current array off stack and return it as result + return @stack.pop + else + # Nested array, pop current array off stack + @stack.pop + end + else + # Add the character to the recorded character buffer. + @recorded << char + end + end + + raise Sequel::Error, "array parsing finished with array unclosed" + end + end + end + + # Callable object that takes the input string and parses it using Parser. + class Creator + # The converter callable that is called on each member of the array + # to convert it to the correct type. + attr_reader :converter + + # The database type to set on the PGArray instances returned. + attr_reader :type + + # Set the type and optional converter callable that will be used. + def initialize(type, converter=nil) + @type = type + @converter = converter + end + + if Sequel::Postgres.respond_to?(:parse_pg_array) + # :nocov: + # Use sequel_pg's C-based parser if it has already been defined. + def call(string) + PGArray.new(Sequel::Postgres.parse_pg_array(string, @converter), @type) + end + # :nocov: + else + # Parse the string using Parser with the appropriate + # converter, and return a PGArray with the appropriate database + # type. + def call(string) + PGArray.new(Parser.new(string, @converter).parse, @type) + end + end + end + + # The type of this array. May be nil if no type was given. If a type + # is provided, the array is automatically casted to this type when + # literalizing. This type is the underlying type, not the array type + # itself, so for an int4[] database type, it should be :int4 or 'int4' + attr_accessor :array_type + + # Set the array to delegate to, and a database type. + def initialize(array, type=nil) + super(array) + @array_type = type + end + + # Append the array SQL to the given sql string. + # If the receiver has a type, add a cast to the + # database array type. + def sql_literal_append(ds, sql) + return rs_sql_literal_append(ds, sql) if ds.db.redshift? + + at = array_type + if empty? && at + sql << "'{}'" + else + sql << "ARRAY" + _literal_append(sql, ds, to_a) + end + if at + sql << '::' << at.to_s << '[]' + end + end + + # Redshift version + def rs_sql_literal_append(ds, sql) + at = array_type + if empty? && at + sql << "'{}'" + else + sql << "ARRAY" + _rs_literal_append(sql, ds, to_a) + end + end + + # Allow automatic parameterization of the receiver if all elements can be + # can be automatically parameterized. + def sequel_auto_param_type(ds) + if array_type && all?{|x| nil == x || ds.send(:auto_param_type, x)} && db_type != :redshift + "::#{array_type}[]" + end + end + + private + + # Recursive method that handles multi-dimensional + # arrays, surrounding each with [] and interspersing + # entries with ,. + def _literal_append(sql, ds, array) + sql << '[' + comma = false + commas = ',' + array.each do |i| + sql << commas if comma + if i.is_a?(Array) + _literal_append(sql, ds, i) + else + ds.literal_append(sql, i) + end + comma = true + end + sql << ']' + end + + # Redshift version + def _rs_literal_append(sql, ds, array) + sql << '(' + comma = false + commas = ',' + array.each do |i| + sql << commas if comma + if i.is_a?(Array) + _rs_literal_append(sql, ds, i) + else + ds.literal_append(sql, i) + end + comma = true + end + sql << ')' + end + end + end + + module SQL::Builders + # Return a Postgres::PGArray proxy for the given array and database array type. + def pg_array(v, array_type=nil) + case v + when Postgres::PGArray + if array_type.nil? || v.array_type == array_type + v + else + Postgres::PGArray.new(v.to_a, array_type) + end + when Array + Postgres::PGArray.new(v, array_type) + else + # May not be defined unless the pg_array_ops extension is used + pg_array_op(v) + end + end + end + + Database.register_extension(:pg_array, Postgres::PGArray::DatabaseMethods) +end + +# :nocov: +if Sequel.core_extensions? + class Array + # Return a PGArray proxy to the receiver, using a + # specific database type if given. This is mostly useful + # as a short cut for creating PGArray objects that didn't + # come from the database. + def pg_array(type=nil) + Sequel::Postgres::PGArray.new(self, type) + end + end +end + +if defined?(Sequel::CoreRefinements) + module Sequel::CoreRefinements + refine Array do + def pg_array(type=nil) + Sequel::Postgres::PGArray.new(self, type) + end + end + end +end +# :nocov: \ No newline at end of file From fbdf387b31e8816cd53bb5465a63609093da1bc7 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 20:18:50 +0300 Subject: [PATCH 35/41] upd --- lib/sequel/extensions/pg_array.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index 205179c..ecdecbb 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -55,6 +55,8 @@ def self.extended(db) register_array_type('int2vector', :oid=>1006, :scalar_oid=>22) register_array_type('oidvector', :oid=>1013, :scalar_oid=>30) + register_array_type('super', :oid=>4000, :scalar_oid=>25, :type_symbol=>:string) + [:string_array, :integer_array, :decimal_array, :float_array, :boolean_array, :blob_array, :date_array, :time_array, :datetime_array].each do |v| @schema_type_classes[v] = PGArray end From 1d64ad49ba9e2d422d53f7927638c23ccfe542d9 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 23:35:40 +0300 Subject: [PATCH 36/41] update --- lib/sequel/extensions/pg_array.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index ecdecbb..5801562 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -298,12 +298,12 @@ def new_entry(include_empty=false) # of parsed (and potentially converted) objects. def parse raise Sequel::Error, "invalid array, empty string" if eos? - raise Sequel::Error, "invalid array, doesn't start with {" unless scan(/((\[\d+:\d+\])+=)?\{/) + raise Sequel::Error, "invalid array, doesn't start with [" unless scan(/((\[\d+:\d+\])+=)?\[/) # :nocov: while !eos? # :nocov: - char = scan(/[{}",]|[^{}",]+/) + char = scan(/[\[\]",]|[^\[\]",]+/) if char == ',' # Comma outside quoted string indicates end of current entry new_entry @@ -317,21 +317,21 @@ def parse @recorded << getch elsif char == '"' n = peek(1) - raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == '}' + raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == ']' break else @recorded << char end end new_entry(true) - elsif char == '{' + elsif char == '[' raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty? # Start of new array, add it to the stack new = [] @stack.last << new @stack << new - elsif char == '}' + elsif char == ']' # End of current array, add current entry to the current array new_entry From 5a3485ed87fd8adb8b055dfee1cb3017788a0076 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 23:46:50 +0300 Subject: [PATCH 37/41] upd --- lib/sequel/extensions/pg_array.rb | 77 ++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 6 deletions(-) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index 5801562..08507e8 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -298,12 +298,12 @@ def new_entry(include_empty=false) # of parsed (and potentially converted) objects. def parse raise Sequel::Error, "invalid array, empty string" if eos? - raise Sequel::Error, "invalid array, doesn't start with [" unless scan(/((\[\d+:\d+\])+=)?\[/) + raise Sequel::Error, "invalid array, doesn't start with {" unless scan(/((\[\d+:\d+\])+=)?\{/) # :nocov: while !eos? # :nocov: - char = scan(/[\[\]",]|[^\[\]",]+/) + char = scan(/[{}",]|[^{}",]+/) if char == ',' # Comma outside quoted string indicates end of current entry new_entry @@ -317,21 +317,21 @@ def parse @recorded << getch elsif char == '"' n = peek(1) - raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == ']' + raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == '}' break else @recorded << char end end new_entry(true) - elsif char == '[' + elsif char == '{' raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty? # Start of new array, add it to the stack new = [] @stack.last << new @stack << new - elsif char == ']' + elsif char == '}' # End of current array, add current entry to the current array new_entry @@ -356,6 +356,67 @@ def parse end end + class RsParser < Parser + + def parse + raise Sequel::Error, "invalid array, empty string" if eos? + raise Sequel::Error, "invalid array, doesn't start with [" unless scan(/((\[\d+:\d+\])+=)?\[/) + + # :nocov: + while !eos? + # :nocov: + char = scan(/[\[\]",]|[^\[\]",]+/) + if char == ',' + # Comma outside quoted string indicates end of current entry + new_entry + elsif char == '"' + raise Sequel::Error, "invalid array, opening quote with existing recorded data" unless @recorded.empty? + # :nocov: + while true + # :nocov: + char = scan(/["\\]|[^"\\]+/) + if char == '\\' + @recorded << getch + elsif char == '"' + n = peek(1) + raise Sequel::Error, "invalid array, closing quote not followed by comma or closing brace" unless n == ',' || n == ']' + break + else + @recorded << char + end + end + new_entry(true) + elsif char == '[' + raise Sequel::Error, "invalid array, opening brace with existing recorded data" unless @recorded.empty? + + # Start of new array, add it to the stack + new = [] + @stack.last << new + @stack << new + elsif char == ']' + # End of current array, add current entry to the current array + new_entry + + if @stack.length == 1 + raise Sequel::Error, "array parsing finished without parsing entire string" unless eos? + + # Top level of array, parsing should be over. + # Pop current array off stack and return it as result + return @stack.pop + else + # Nested array, pop current array off stack + @stack.pop + end + else + # Add the character to the recorded character buffer. + @recorded << char + end + end + + raise Sequel::Error, "array parsing finished with array unclosed" + end + end + # Callable object that takes the input string and parses it using Parser. class Creator # The converter callable that is called on each member of the array @@ -383,7 +444,11 @@ def call(string) # converter, and return a PGArray with the appropriate database # type. def call(string) - PGArray.new(Parser.new(string, @converter).parse, @type) + if @type == :redshift? + PGArray.new(RsParser.new(string, @converter).parse, @type) + else + PGArray.new(Parser.new(string, @converter).parse, @type) + end end end end From 7062389369c7af70b7c2f35f6e71b1185cde0451 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 23:48:20 +0300 Subject: [PATCH 38/41] upd --- lib/sequel/extensions/pg_array.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index 08507e8..ebc2a31 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -444,6 +444,7 @@ def call(string) # converter, and return a PGArray with the appropriate database # type. def call(string) + puts @type if @type == :redshift? PGArray.new(RsParser.new(string, @converter).parse, @type) else From d6c164319fd07de25ab4ee419842122be2bb2c20 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 23:55:36 +0300 Subject: [PATCH 39/41] upd --- lib/sequel/extensions/pg_array.rb | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index ebc2a31..bef9da1 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -55,7 +55,7 @@ def self.extended(db) register_array_type('int2vector', :oid=>1006, :scalar_oid=>22) register_array_type('oidvector', :oid=>1013, :scalar_oid=>30) - register_array_type('super', :oid=>4000, :scalar_oid=>25, :type_symbol=>:string) + register_array_type('super', :oid=>4000, :scalar_oid=>25, :type_symbol=>:string, :redshift_only=>true) [:string_array, :integer_array, :decimal_array, :float_array, :boolean_array, :blob_array, :date_array, :time_array, :datetime_array].each do |v| @schema_type_classes[v] = PGArray @@ -139,7 +139,7 @@ def register_array_type(db_type, opts=OPTS, &block) end array_type = (opts[:array_type] || db_type).to_s.dup.freeze - creator = Creator.new(array_type, converter) + creator = Creator.new(array_type, converter, opts[:redshift_only] == true) add_conversion_proc(oid, creator) typecast_method_map[db_type] = :"#{type}_array" @@ -427,9 +427,10 @@ class Creator attr_reader :type # Set the type and optional converter callable that will be used. - def initialize(type, converter=nil) + def initialize(type, converter=nil, redshift_only=false) @type = type @converter = converter + @redshift_only = redshift_only end if Sequel::Postgres.respond_to?(:parse_pg_array) @@ -445,7 +446,7 @@ def call(string) # type. def call(string) puts @type - if @type == :redshift? + if @redshift_only PGArray.new(RsParser.new(string, @converter).parse, @type) else PGArray.new(Parser.new(string, @converter).parse, @type) From 632724436b3890fde632cbb049f7da05ba965cc0 Mon Sep 17 00:00:00 2001 From: Rustam Date: Wed, 23 Aug 2023 23:55:56 +0300 Subject: [PATCH 40/41] upd --- lib/sequel/extensions/pg_array.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/sequel/extensions/pg_array.rb b/lib/sequel/extensions/pg_array.rb index bef9da1..a25da1a 100644 --- a/lib/sequel/extensions/pg_array.rb +++ b/lib/sequel/extensions/pg_array.rb @@ -445,7 +445,6 @@ def call(string) # converter, and return a PGArray with the appropriate database # type. def call(string) - puts @type if @redshift_only PGArray.new(RsParser.new(string, @converter).parse, @type) else From 68f639fb8c0609f8803b9ce1402d5796a87842b3 Mon Sep 17 00:00:00 2001 From: Rustam Date: Thu, 24 Aug 2023 12:09:07 +0300 Subject: [PATCH 41/41] add array ops --- lib/sequel/extensions/pg_array_ops.rb | 284 ++++++++++++++++++++++++++ 1 file changed, 284 insertions(+) create mode 100644 lib/sequel/extensions/pg_array_ops.rb diff --git a/lib/sequel/extensions/pg_array_ops.rb b/lib/sequel/extensions/pg_array_ops.rb new file mode 100644 index 0000000..571c9da --- /dev/null +++ b/lib/sequel/extensions/pg_array_ops.rb @@ -0,0 +1,284 @@ +# frozen-string-literal: true +# +# This is patched version of original plugin +# @see https://github.com/jeremyevans/sequel/blob/5.71.0/lib/sequel/extensions/pg_array_ops.rb +# + +module Sequel + module Postgres + # The ArrayOp class is a simple container for a single object that + # defines methods that yield Sequel expression objects representing + # PostgreSQL array operators and functions. + # + # In the method documentation examples, assume that: + # + # array_op = :array.pg_array + class ArrayOp < Sequel::SQL::Wrapper + CONCAT = ["(".freeze, " || ".freeze, ")".freeze].freeze + CONTAINS = ["(".freeze, " @> ".freeze, ")".freeze].freeze + CONTAINED_BY = ["(".freeze, " <@ ".freeze, ")".freeze].freeze + OVERLAPS = ["(".freeze, " && ".freeze, ")".freeze].freeze + + # Access a member of the array, returns an SQL::Subscript instance: + # + # array_op[1] # array[1] + def [](key) + s = Sequel::SQL::Subscript.new(self, [key]) + s = ArrayOp.new(s) if key.is_a?(Range) + s + end + + # Call the ALL function: + # + # array_op.all # ALL(array) + # + # Usually used like: + # + # dataset.where(1=>array_op.all) + # # WHERE (1 = ALL(array)) + def all + function(:ALL) + end + + # Call the ANY function: + # + # array_op.any # ANY(array) + # + # Usually used like: + # + # dataset.where(1=>array_op.any) + # # WHERE (1 = ANY(array)) + def any + function(:ANY) + end + + # Call the cardinality method: + # + # array_op.cardinality # cardinality(array) + def cardinality + function(:cardinality) + end + + # Use the contains (@>) operator: + # + # array_op.contains(:a) # (array @> a) + def contains(other) + bool_op(CONTAINS, wrap_array(other)) + end + + # Use the contained by (<@) operator: + # + # array_op.contained_by(:a) # (array <@ a) + def contained_by(other) + bool_op(CONTAINED_BY, wrap_array(other)) + end + + # Call the array_dims method: + # + # array_op.dims # array_dims(array) + def dims + function(:array_dims) + end + + # Convert the array into an hstore using the hstore function. + # If given an argument, use the two array form: + # + # array_op.hstore # hstore(array) + # array_op.hstore(:array2) # hstore(array, array2) + def hstore(arg=(no_arg_given=true; nil)) + v = if no_arg_given + Sequel.function(:hstore, self) + else + Sequel.function(:hstore, self, wrap_array(arg)) + end + # :nocov: + if Sequel.respond_to?(:hstore_op) + # :nocov: + v = Sequel.hstore_op(v) + end + v + end + + # Call the array_length method: + # + # array_op.length # array_length(array, 1) + # array_op.length(2) # array_length(array, 2) + def length(dimension = 1) + function(:array_length, dimension) + end + + # Call the array_lower method: + # + # array_op.lower # array_lower(array, 1) + # array_op.lower(2) # array_lower(array, 2) + def lower(dimension = 1) + function(:array_lower, dimension) + end + + # Use the overlaps (&&) operator: + # + # array_op.overlaps(:a) # (array && a) + def overlaps(other) + bool_op(OVERLAPS, wrap_array(other)) + end + + # Use the concatentation (||) operator: + # + # array_op.push(:a) # (array || a) + # array_op.concat(:a) # (array || a) + def push(other) + array_op(CONCAT, [self, wrap_array(other)]) + end + alias concat push + + # Return the receiver. + def pg_array + self + end + + # Remove the given element from the array: + # + # array_op.remove(1) # array_remove(array, 1) + def remove(element) + ArrayOp.new(function(:array_remove, element)) + end + + # Replace the given element in the array with another + # element: + # + # array_op.replace(1, 2) # array_replace(array, 1, 2) + def replace(element, replacement) + ArrayOp.new(function(:array_replace, element, replacement)) + end + + # Call the array_to_string method: + # + # array_op.join # array_to_string(array, '') + # array_op.to_string # array_to_string(array, '') + # array_op.join(":") # array_to_string(array, ':') + # array_op.join(":", "*") # array_to_string(array, ':', '*') + def to_string(joiner="", null=nil) + if null.nil? + function(:array_to_string, joiner) + else + function(:array_to_string, joiner, null) + end + end + alias join to_string + + def rs_to_string(joiner="", null=nil) + query = <<~SQL.squish + ( + SELECT + listagg(json_serialize(el), ?) + FROM + ? AS el + ) + SQL + + Sequel.lit(query, joiner, pg_array) + end + alias rs_join rs_to_string + + # Call the unnest method: + # + # array_op.unnest # unnest(array) + def unnest(*args) + function(:unnest, *args.map{|a| wrap_array(a)}) + end + + # Use the concatentation (||) operator, reversing the order: + # + # array_op.unshift(:a) # (a || array) + def unshift(other) + array_op(CONCAT, [wrap_array(other), self]) + end + + private + + # Return a placeholder literal with the given str and args, wrapped + # in an ArrayOp, used by operators that return arrays. + def array_op(str, args) + ArrayOp.new(Sequel::SQL::PlaceholderLiteralString.new(str, args)) + end + + # Return a placeholder literal with the given str and args, wrapped + # in a boolean expression, used by operators that return booleans. + def bool_op(str, other) + Sequel::SQL::BooleanExpression.new(:NOOP, Sequel::SQL::PlaceholderLiteralString.new(str, [value, other])) + end + + # Return a function with the given name, and the receiver as the first + # argument, with any additional arguments given. + def function(name, *args) + SQL::Function.new(name, self, *args) + end + + # Automatically wrap argument in a PGArray if it is a plain Array. + # Requires that the pg_array extension has been loaded to work. + def wrap_array(arg) + if arg.instance_of?(Array) + Sequel.pg_array(arg) + else + arg + end + end + end + + module ArrayOpMethods + # Wrap the receiver in an ArrayOp so you can easily use the PostgreSQL + # array functions and operators with it. + def pg_array + ArrayOp.new(self) + end + end + + # :nocov: + if defined?(PGArray) + # :nocov: + class PGArray + # Wrap the PGArray instance in an ArrayOp, allowing you to easily use + # the PostgreSQL array functions and operators with literal arrays. + def op + ArrayOp.new(self) + end + end + end + end + + module SQL::Builders + # Return the object wrapped in an Postgres::ArrayOp. + def pg_array_op(v) + case v + when Postgres::ArrayOp + v + else + Postgres::ArrayOp.new(v) + end + end + end + + class SQL::GenericExpression + include Sequel::Postgres::ArrayOpMethods + end + + class LiteralString + include Sequel::Postgres::ArrayOpMethods + end +end + +# :nocov: +if Sequel.core_extensions? + class Symbol + include Sequel::Postgres::ArrayOpMethods + end +end + +if defined?(Sequel::CoreRefinements) + module Sequel::CoreRefinements + refine Symbol do + send INCLUDE_METH, Sequel::Postgres::ArrayOpMethods + end + end +end +# :nocov: \ No newline at end of file