From 8410ca1654861c5e3ce16291f41d9d1b0c76ed14 Mon Sep 17 00:00:00 2001 From: Michael Harrison Date: Thu, 5 Jun 2025 11:45:51 +0100 Subject: [PATCH 1/2] CCM-7939: enable high throughput fifo queues --- infrastructure/modules/sqs/README.md | 2 ++ infrastructure/modules/sqs/sqs_queue.tf | 2 ++ infrastructure/modules/sqs/variables.tf | 21 +++++++++++++++++++++ 3 files changed, 25 insertions(+) diff --git a/infrastructure/modules/sqs/README.md b/infrastructure/modules/sqs/README.md index 23de60e..8398829 100644 --- a/infrastructure/modules/sqs/README.md +++ b/infrastructure/modules/sqs/README.md @@ -16,10 +16,12 @@ | [component](#input\_component) | The name of the tfscaffold component | `string` | n/a | yes | | [content\_based\_deduplication](#input\_content\_based\_deduplication) | Enables content-based deduplication for FIFO queues | `bool` | `false` | no | | [create\_dlq](#input\_create\_dlq) | Create a DLQ | `bool` | `false` | no | +| [deduplication\_scope](#input\_deduplication\_scope) | For FIFO queues, specifies whether message deduplication occurs at the message group or queue level. Valid values are messageGroup and queue (default). | `string` | `"queue"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [delay\_seconds](#input\_delay\_seconds) | Time in seconds that the delivery of all messages in the queue will be delayed. An integer from 0 to 900 (15 minutes). | `number` | `0` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | | [fifo\_queue](#input\_fifo\_queue) | Boolean designating a FIFO queue | `bool` | `false` | no | +| [fifo\_throughput\_limit](#input\_fifo\_throughput\_limit) | For FIFO queues, specifies whether the FIFO queue throughput quota applies to the entire queue or per message group. Valid values are perQueue (default) and perMessageGroupId. | `string` | `"perQueue"` | no | | [kms\_data\_key\_reuse\_period\_seconds](#input\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again. An integer representing seconds, between 60 seconds (1 minute) and 86,400 seconds (24 hours) | `number` | `300` | no | | [max\_message\_size](#input\_max\_message\_size) | The limit of how many bytes a message can contain before Amazon SQS rejects it. An integer from 1024 bytes (1 KiB) up to 262144 bytes (256 KiB) | `number` | `262144` | no | | [message\_retention\_seconds](#input\_message\_retention\_seconds) | The number of seconds Amazon SQS retains a message. Integer representing seconds, from 60 (1 minute) to 1209600 (14 days) | `number` | `null` | no | diff --git a/infrastructure/modules/sqs/sqs_queue.tf b/infrastructure/modules/sqs/sqs_queue.tf index cd3446e..147d751 100644 --- a/infrastructure/modules/sqs/sqs_queue.tf +++ b/infrastructure/modules/sqs/sqs_queue.tf @@ -7,6 +7,8 @@ resource "aws_sqs_queue" "sqs_queue" { fifo_queue = var.fifo_queue content_based_deduplication = var.content_based_deduplication max_message_size = var.max_message_size + deduplication_scope = var.fifo_queue ? var.deduplication_scope : null + fifo_throughput_limit = var.fifo_queue ? var.fifo_throughput_limit : null kms_master_key_id = var.sqs_kms_key_arn kms_data_key_reuse_period_seconds = var.kms_data_key_reuse_period_seconds diff --git a/infrastructure/modules/sqs/variables.tf b/infrastructure/modules/sqs/variables.tf index b26ff39..d969c33 100644 --- a/infrastructure/modules/sqs/variables.tf +++ b/infrastructure/modules/sqs/variables.tf @@ -110,3 +110,24 @@ variable "create_dlq" { type = bool default = false } + +variable "deduplication_scope" { + description = "For FIFO queues, specifies whether message deduplication occurs at the message group or queue level. Valid values are messageGroup and queue (default)." + type = string + default = "queue" + + validation { + condition = contains([], var.deduplication_scope) + error_message = "deduplication_scope must be either \"queue\" or \"messageGroup\"" + } +} +variable "fifo_throughput_limit" { + description = "For FIFO queues, specifies whether the FIFO queue throughput quota applies to the entire queue or per message group. Valid values are perQueue (default) and perMessageGroupId." + type = string + default = "perQueue" + + validation { + condition = contains([], var.fifo_throughput_limit) + error_message = "fifo_throughput_limit must be either \"perQueue\" or \"perMessageGroup\"" + } +} From a16fad28ddb0177ceb57ca693fffc1c1753f1142 Mon Sep 17 00:00:00 2001 From: Michael Harrison Date: Thu, 5 Jun 2025 17:41:18 +0100 Subject: [PATCH 2/2] CCM-7939: consolidate to single high throughput variable --- infrastructure/modules/sqs/README.md | 3 +-- infrastructure/modules/sqs/sqs_queue.tf | 4 ++-- infrastructure/modules/sqs/variables.tf | 21 ++++++--------------- 3 files changed, 9 insertions(+), 19 deletions(-) diff --git a/infrastructure/modules/sqs/README.md b/infrastructure/modules/sqs/README.md index 8398829..ecce6b5 100644 --- a/infrastructure/modules/sqs/README.md +++ b/infrastructure/modules/sqs/README.md @@ -16,12 +16,11 @@ | [component](#input\_component) | The name of the tfscaffold component | `string` | n/a | yes | | [content\_based\_deduplication](#input\_content\_based\_deduplication) | Enables content-based deduplication for FIFO queues | `bool` | `false` | no | | [create\_dlq](#input\_create\_dlq) | Create a DLQ | `bool` | `false` | no | -| [deduplication\_scope](#input\_deduplication\_scope) | For FIFO queues, specifies whether message deduplication occurs at the message group or queue level. Valid values are messageGroup and queue (default). | `string` | `"queue"` | no | | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [delay\_seconds](#input\_delay\_seconds) | Time in seconds that the delivery of all messages in the queue will be delayed. An integer from 0 to 900 (15 minutes). | `number` | `0` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | | [fifo\_queue](#input\_fifo\_queue) | Boolean designating a FIFO queue | `bool` | `false` | no | -| [fifo\_throughput\_limit](#input\_fifo\_throughput\_limit) | For FIFO queues, specifies whether the FIFO queue throughput quota applies to the entire queue or per message group. Valid values are perQueue (default) and perMessageGroupId. | `string` | `"perQueue"` | no | +| [high\_throughput\_fifo](#input\_high\_throughput\_fifo) | For FIFO queues, specifies whether high throughput mode is enabled. | `bool` | `false` | no | | [kms\_data\_key\_reuse\_period\_seconds](#input\_kms\_data\_key\_reuse\_period\_seconds) | The length of time, in seconds, for which Amazon SQS can reuse a data key to encrypt or decrypt messages before calling AWS KMS again. An integer representing seconds, between 60 seconds (1 minute) and 86,400 seconds (24 hours) | `number` | `300` | no | | [max\_message\_size](#input\_max\_message\_size) | The limit of how many bytes a message can contain before Amazon SQS rejects it. An integer from 1024 bytes (1 KiB) up to 262144 bytes (256 KiB) | `number` | `262144` | no | | [message\_retention\_seconds](#input\_message\_retention\_seconds) | The number of seconds Amazon SQS retains a message. Integer representing seconds, from 60 (1 minute) to 1209600 (14 days) | `number` | `null` | no | diff --git a/infrastructure/modules/sqs/sqs_queue.tf b/infrastructure/modules/sqs/sqs_queue.tf index 147d751..5635bb1 100644 --- a/infrastructure/modules/sqs/sqs_queue.tf +++ b/infrastructure/modules/sqs/sqs_queue.tf @@ -7,8 +7,8 @@ resource "aws_sqs_queue" "sqs_queue" { fifo_queue = var.fifo_queue content_based_deduplication = var.content_based_deduplication max_message_size = var.max_message_size - deduplication_scope = var.fifo_queue ? var.deduplication_scope : null - fifo_throughput_limit = var.fifo_queue ? var.fifo_throughput_limit : null + deduplication_scope = (var.fifo_queue && var.high_throughput_fifo) ? "messageGroup" : null + fifo_throughput_limit = (var.fifo_queue && var.high_throughput_fifo) ? "perMessageGroupId" : null kms_master_key_id = var.sqs_kms_key_arn kms_data_key_reuse_period_seconds = var.kms_data_key_reuse_period_seconds diff --git a/infrastructure/modules/sqs/variables.tf b/infrastructure/modules/sqs/variables.tf index d969c33..99968b0 100644 --- a/infrastructure/modules/sqs/variables.tf +++ b/infrastructure/modules/sqs/variables.tf @@ -111,23 +111,14 @@ variable "create_dlq" { default = false } -variable "deduplication_scope" { - description = "For FIFO queues, specifies whether message deduplication occurs at the message group or queue level. Valid values are messageGroup and queue (default)." - type = string - default = "queue" +variable "high_throughput_fifo" { + description = "For FIFO queues, specifies whether high throughput mode is enabled." + type = bool + default = false validation { - condition = contains([], var.deduplication_scope) - error_message = "deduplication_scope must be either \"queue\" or \"messageGroup\"" + condition = var.high_throughput_fifo == false || var.fifo_queue == true + error_message = "high_throughput_fifo can only be true if fifo_queue is also true" } } -variable "fifo_throughput_limit" { - description = "For FIFO queues, specifies whether the FIFO queue throughput quota applies to the entire queue or per message group. Valid values are perQueue (default) and perMessageGroupId." - type = string - default = "perQueue" - validation { - condition = contains([], var.fifo_throughput_limit) - error_message = "fifo_throughput_limit must be either \"perQueue\" or \"perMessageGroup\"" - } -}