From d28e31d752d5df7dbe649bf8b5e7b6564fb8ef7d Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 24 Feb 2026 09:12:59 +1100 Subject: [PATCH] exports and logging updates --- .../resources/{ => OLD}/aws/iam/iam_role.iql | 0 .../policy_statements/ec2_permissions.json | 72 +++++ .../iam_service_linked_role.json | 15 + .../resources/OLD/aws/iam/roles.iql | 72 +++++ .../aws/iam/update_metastore_access_role.iql | 0 .../resources/{ => OLD}/aws/s3/s3_bucket.iql | 0 .../{ => OLD}/aws/s3/s3_bucket_policy.iql | 0 .../OLD/databricks_account/credentials.iql | 31 +++ .../databricks_account/get_users.iql | 0 .../{ => OLD}/databricks_account/network.iql | 0 .../storage_configuration.iql | 0 .../update_group_membership.iql | 0 .../databricks_account/workspace.iql | 0 .../databricks_account/workspace_group.iql | 0 .../workspace_permission_assignments.iql | 0 .../external_location.iql | 0 .../storage_credential.iql | 0 .../serverless/resources/aws/iam/roles.iql | 33 +-- .../databricks_account/credentials.iql | 76 ++--- .../serverless/stackql_manifest.yml | 31 ++- src/commands/base.rs | 18 +- src/commands/common_args.rs | 7 +- src/core/utils.rs | 128 ++++++++- src/resource/manifest.rs | 9 + src/resource/mod.rs | 1 + src/resource/validation.rs | 263 ++++++++++++++++++ src/utils/logging.rs | 4 +- .../manifest_fields/resources/exports.mdx | 48 +++- .../docs/manifest_fields/resources/name.mdx | 10 + 29 files changed, 743 insertions(+), 75 deletions(-) rename examples/databricks/serverless/resources/{ => OLD}/aws/iam/iam_role.iql (100%) create mode 100644 examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/ec2_permissions.json create mode 100644 examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/iam_service_linked_role.json create mode 100644 examples/databricks/serverless/resources/OLD/aws/iam/roles.iql rename examples/databricks/serverless/resources/{ => OLD}/aws/iam/update_metastore_access_role.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/aws/s3/s3_bucket.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/aws/s3/s3_bucket_policy.iql (100%) create mode 100644 examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/get_users.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/network.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/storage_configuration.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/update_group_membership.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/workspace.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/workspace_group.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_account/workspace_permission_assignments.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_workspace/external_location.iql (100%) rename examples/databricks/serverless/resources/{ => OLD}/databricks_workspace/storage_credential.iql (100%) create mode 100644 src/resource/validation.rs diff --git a/examples/databricks/serverless/resources/aws/iam/iam_role.iql b/examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql similarity index 100% rename from examples/databricks/serverless/resources/aws/iam/iam_role.iql rename to examples/databricks/serverless/resources/OLD/aws/iam/iam_role.iql diff --git a/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/ec2_permissions.json b/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/ec2_permissions.json new file mode 100644 index 0000000..d626ee1 --- /dev/null +++ b/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/ec2_permissions.json @@ -0,0 +1,72 @@ +{ + "Sid": "Stmt1403287045000", + "Effect": "Allow", + "Action": [ + "ec2:AllocateAddress", + "ec2:AssociateDhcpOptions", + "ec2:AssociateIamInstanceProfile", + "ec2:AssociateRouteTable", + "ec2:AttachInternetGateway", + "ec2:AttachVolume", + "ec2:AuthorizeSecurityGroupEgress", + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CancelSpotInstanceRequests", + "ec2:CreateDhcpOptions", + "ec2:CreateInternetGateway", + "ec2:CreateKeyPair", + "ec2:CreateNatGateway", + "ec2:CreatePlacementGroup", + "ec2:CreateRoute", + "ec2:CreateRouteTable", + "ec2:CreateSecurityGroup", + "ec2:CreateSubnet", + "ec2:CreateTags", + "ec2:CreateVolume", + "ec2:CreateVpc", + "ec2:CreateVpcEndpoint", + "ec2:DeleteDhcpOptions", + "ec2:DeleteInternetGateway", + "ec2:DeleteKeyPair", + "ec2:DeleteNatGateway", + "ec2:DeletePlacementGroup", + "ec2:DeleteRoute", + "ec2:DeleteRouteTable", + "ec2:DeleteSecurityGroup", + "ec2:DeleteSubnet", + "ec2:DeleteTags", + "ec2:DeleteVolume", + "ec2:DeleteVpc", + "ec2:DeleteVpcEndpoints", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeIamInstanceProfileAssociations", + "ec2:DescribeInstanceStatus", + "ec2:DescribeInstances", + "ec2:DescribeInternetGateways", + "ec2:DescribeNatGateways", + "ec2:DescribePlacementGroups", + "ec2:DescribePrefixLists", + "ec2:DescribeReservedInstancesOfferings", + "ec2:DescribeRouteTables", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotInstanceRequests", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets", + "ec2:DescribeVolumes", + "ec2:DescribeVpcs", + "ec2:DescribeVpcAttribute", + "ec2:DescribeNetworkAcls", + "ec2:DetachInternetGateway", + "ec2:DisassociateIamInstanceProfile", + "ec2:DisassociateRouteTable", + "ec2:ModifyVpcAttribute", + "ec2:ReleaseAddress", + "ec2:ReplaceIamInstanceProfileAssociation", + "ec2:ReplaceRoute", + "ec2:RequestSpotInstances", + "ec2:RevokeSecurityGroupEgress", + "ec2:RevokeSecurityGroupIngress", + "ec2:RunInstances", + "ec2:TerminateInstances" + ], + "Resource": ["*"] +} \ No newline at end of file diff --git a/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/iam_service_linked_role.json b/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/iam_service_linked_role.json new file mode 100644 index 0000000..3c099aa --- /dev/null +++ b/examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/iam_service_linked_role.json @@ -0,0 +1,15 @@ +{ + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole", + "iam:PutRolePolicy" + ], + "Resource": [ + "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot" + ], + "Condition": { + "StringLike": { + "iam:AWSServiceName": "spot.amazonaws.com" + } + } +} \ No newline at end of file diff --git a/examples/databricks/serverless/resources/OLD/aws/iam/roles.iql b/examples/databricks/serverless/resources/OLD/aws/iam/roles.iql new file mode 100644 index 0000000..4e78a9d --- /dev/null +++ b/examples/databricks/serverless/resources/OLD/aws/iam/roles.iql @@ -0,0 +1,72 @@ +/*+ exists */ +SELECT count(*) as count +FROM awscc.iam.roles +WHERE region = 'us-east-1' AND +Identifier = '{{ role_name }}' +; + +/*+ create */ +INSERT INTO awscc.iam.roles ( + AssumeRolePolicyDocument, + Description, + ManagedPolicyArns, + MaxSessionDuration, + Path, + PermissionsBoundary, + Policies, + RoleName, + Tags, + region +) +SELECT + '{{ assume_role_policy_document }}', + '{{ description }}', + '{{ managed_policy_arns }}', + '{{ max_session_duration }}', + '{{ path }}', + '{{ permissions_boundary }}', + '{{ policies }}', + '{{ role_name }}', + '{{ tags }}', + 'us-east-1'; + +/*+ update */ +UPDATE awscc.iam.roles +SET PatchDocument = string('{{ { + "AssumeRolePolicyDocument": assume_role_policy_document, + "Description": description, + "ManagedPolicyArns": managed_policy_arns, + "MaxSessionDuration": max_session_duration, + "PermissionsBoundary": permissions_boundary, + "Path": path, + "Policies": policies, + "Tags": tags +} | generate_patch_document }}') +WHERE region = 'us-east-1' +AND Identifier = '{{ role_name }}'; + +/*+ statecheck, retries=5, retry_delay=10 */ +SELECT COUNT(*) as count FROM ( + SELECT + max_session_duration, + path, + AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc, + AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies + FROM awscc.iam.roles + WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1')t +WHERE test_assume_role_policy_doc = 1 +AND test_policies = 1 +AND path = '{{ path }}'; + +/*+ exports */ +SELECT +arn, +role_name +FROM awscc.iam.roles +WHERE region = 'us-east-1' AND +Identifier = '{{ role_name }}'; + +/*+ delete */ +DELETE FROM awscc.iam.roles +WHERE Identifier = '{{ role_name }}' +AND region = 'us-east-1'; \ No newline at end of file diff --git a/examples/databricks/serverless/resources/aws/iam/update_metastore_access_role.iql b/examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql similarity index 100% rename from examples/databricks/serverless/resources/aws/iam/update_metastore_access_role.iql rename to examples/databricks/serverless/resources/OLD/aws/iam/update_metastore_access_role.iql diff --git a/examples/databricks/serverless/resources/aws/s3/s3_bucket.iql b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql similarity index 100% rename from examples/databricks/serverless/resources/aws/s3/s3_bucket.iql rename to examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket.iql diff --git a/examples/databricks/serverless/resources/aws/s3/s3_bucket_policy.iql b/examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql similarity index 100% rename from examples/databricks/serverless/resources/aws/s3/s3_bucket_policy.iql rename to examples/databricks/serverless/resources/OLD/aws/s3/s3_bucket_policy.iql diff --git a/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql b/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql new file mode 100644 index 0000000..687b3f1 --- /dev/null +++ b/examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql @@ -0,0 +1,31 @@ +/*+ exists */ +SELECT COUNT(*) as count +FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ databricks_account_id }}' +AND credentials_name = '{{ credentials_name }}' + +/*+ create */ +INSERT INTO databricks_account.provisioning.credentials ( +account_id, +data__credentials_name, +data__aws_credentials +) +SELECT +'{{ databricks_account_id }}', +'{{ credentials_name }}', +'{{ aws_credentials }}' + +/*+ exports, retries=3, retry_delay=5 */ +SELECT +'{{ credentials_name }}' as databricks_credentials_name, +credentials_id as databricks_credentials_id, +JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id +FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ databricks_account_id }}' +AND credentials_name = '{{ credentials_name }}' +AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}' + +/*+ delete */ +DELETE FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ databricks_account_id }}' AND +credentials_id = '{{ databricks_credentials_id }}'; \ No newline at end of file diff --git a/examples/databricks/serverless/resources/databricks_account/get_users.iql b/examples/databricks/serverless/resources/OLD/databricks_account/get_users.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/get_users.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/get_users.iql diff --git a/examples/databricks/serverless/resources/databricks_account/network.iql b/examples/databricks/serverless/resources/OLD/databricks_account/network.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/network.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/network.iql diff --git a/examples/databricks/serverless/resources/databricks_account/storage_configuration.iql b/examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/storage_configuration.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/storage_configuration.iql diff --git a/examples/databricks/serverless/resources/databricks_account/update_group_membership.iql b/examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/update_group_membership.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/update_group_membership.iql diff --git a/examples/databricks/serverless/resources/databricks_account/workspace.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/workspace.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/workspace.iql diff --git a/examples/databricks/serverless/resources/databricks_account/workspace_group.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/workspace_group.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/workspace_group.iql diff --git a/examples/databricks/serverless/resources/databricks_account/workspace_permission_assignments.iql b/examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_account/workspace_permission_assignments.iql rename to examples/databricks/serverless/resources/OLD/databricks_account/workspace_permission_assignments.iql diff --git a/examples/databricks/serverless/resources/databricks_workspace/external_location.iql b/examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_workspace/external_location.iql rename to examples/databricks/serverless/resources/OLD/databricks_workspace/external_location.iql diff --git a/examples/databricks/serverless/resources/databricks_workspace/storage_credential.iql b/examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql similarity index 100% rename from examples/databricks/serverless/resources/databricks_workspace/storage_credential.iql rename to examples/databricks/serverless/resources/OLD/databricks_workspace/storage_credential.iql diff --git a/examples/databricks/serverless/resources/aws/iam/roles.iql b/examples/databricks/serverless/resources/aws/iam/roles.iql index b4d32c4..4e78a9d 100644 --- a/examples/databricks/serverless/resources/aws/iam/roles.iql +++ b/examples/databricks/serverless/resources/aws/iam/roles.iql @@ -28,7 +28,7 @@ SELECT '{{ policies }}', '{{ role_name }}', '{{ tags }}', - '{{ region }}'; + 'us-east-1'; /*+ update */ UPDATE awscc.iam.roles @@ -38,33 +38,30 @@ SET PatchDocument = string('{{ { "ManagedPolicyArns": managed_policy_arns, "MaxSessionDuration": max_session_duration, "PermissionsBoundary": permissions_boundary, + "Path": path, "Policies": policies, "Tags": tags } | generate_patch_document }}') -WHERE region = '{{ region }}' +WHERE region = 'us-east-1' AND Identifier = '{{ role_name }}'; /*+ statecheck, retries=5, retry_delay=10 */ -SELECT count(*) as count -FROM awscc.iam.roles -WHERE -region = 'us-east-1' AND -Identifier = '{{ role_name }}' -; +SELECT COUNT(*) as count FROM ( + SELECT + max_session_duration, + path, + AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc, + AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies + FROM awscc.iam.roles + WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1')t +WHERE test_assume_role_policy_doc = 1 +AND test_policies = 1 +AND path = '{{ path }}'; /*+ exports */ SELECT arn, -assume_role_policy_document, -description, -managed_policy_arns, -max_session_duration, -path, -permissions_boundary, -policies, -role_id, -role_name, -tags +role_name FROM awscc.iam.roles WHERE region = 'us-east-1' AND Identifier = '{{ role_name }}'; diff --git a/examples/databricks/serverless/resources/databricks_account/credentials.iql b/examples/databricks/serverless/resources/databricks_account/credentials.iql index 687b3f1..569d133 100644 --- a/examples/databricks/serverless/resources/databricks_account/credentials.iql +++ b/examples/databricks/serverless/resources/databricks_account/credentials.iql @@ -1,31 +1,45 @@ -/*+ exists */ -SELECT COUNT(*) as count -FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' -AND credentials_name = '{{ credentials_name }}' - -/*+ create */ -INSERT INTO databricks_account.provisioning.credentials ( -account_id, -data__credentials_name, -data__aws_credentials -) -SELECT -'{{ databricks_account_id }}', -'{{ credentials_name }}', -'{{ aws_credentials }}' - -/*+ exports, retries=3, retry_delay=5 */ -SELECT -'{{ credentials_name }}' as databricks_credentials_name, -credentials_id as databricks_credentials_id, -JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id -FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' -AND credentials_name = '{{ credentials_name }}' -AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}' - -/*+ delete */ -DELETE FROM databricks_account.provisioning.credentials -WHERE account_id = '{{ databricks_account_id }}' AND -credentials_id = '{{ databricks_credentials_id }}'; \ No newline at end of file +/*+ exists */ +SELECT count(*) as count --should use list +FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ account_id }}' +AND credentials_name = '{{ credentials_name }}' +; + +/*+ create */ +INSERT INTO databricks_account.provisioning.credentials ( +credentials_name, +aws_credentials, +account_id +) +SELECT +'{{ credentials_name }}', +'{{ aws_credentials }}', +'{{ account_id }}' +; + +/*+ statecheck, retries=5, retry_delay=10 */ +SELECT count(*) as count --should use list +FROM databricks_account.provisioning.credentials +WHERE +credentials_name = '{{ credentials_name }}' AND --where are the '' +aws_credentials = '{{ aws_credentials }}' AND --where are the '' +account_id = '{{ account_id }}'; + + +-- {"sts_role":{"external_id":"ebfcc5a9-9d49-4c93-b651-b3ee6cf1c9ce","role_arn":"arn:aws:iam::824532806693:role/stackql-serverless-dev-role"}} + +/*+ exports */ +SELECT -- account_id, -- dont get account_id back from exports, but do get credentials_id back, which is needed for updates and deletes +credentials_name, +credentials_id, +JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id +FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ account_id }}' -- required +AND credentials_name = '{{ credentials_name }}' -- required +; --should use list + +/*+ delete */ +DELETE FROM databricks_account.provisioning.credentials +WHERE account_id = '{{ account_id }}' --required +AND credentials_id = '{{ credentials_id }}' --required +; \ No newline at end of file diff --git a/examples/databricks/serverless/stackql_manifest.yml b/examples/databricks/serverless/stackql_manifest.yml index 108c646..b8c8a17 100644 --- a/examples/databricks/serverless/stackql_manifest.yml +++ b/examples/databricks/serverless/stackql_manifest.yml @@ -72,21 +72,24 @@ resources: merge: - global_tags exports: - - aws_iam_role_name: aws_iam_cross_account_role_name - - aws_iam_role_arn: aws_iam_cross_account_role_arn + - role_name: aws_iam_cross_account_role_name + - arn: aws_iam_cross_account_role_arn - # - name: databricks_account/credentials - # props: - # - name: credentials_name - # value: "{{ stack_name }}-{{ stack_env }}-credentials" - # - name: aws_credentials - # value: - # sts_role: - # role_arn: "{{ aws_iam_cross_account_role_arn }}" - # exports: - # - databricks_credentials_name - # - databricks_credentials_id - # - databricks_role_external_id + - name: databricks_account_credentials + file: databricks_account/credentials.iql + props: + - name: account_id + value: "{{ databricks_account_id }}" + - name: credentials_name + value: "{{ stack_name }}-{{ stack_env }}-credentials" + - name: aws_credentials + value: + sts_role: + role_arn: "{{ aws_iam_cross_account_role_arn }}" + exports: + - credentials_name + - credentials_id + - databricks_role_external_id # ==================================================================================== # Storage diff --git a/src/commands/base.rs b/src/commands/base.rs index 03a7056..89f8813 100644 --- a/src/commands/base.rs +++ b/src/commands/base.rs @@ -180,14 +180,28 @@ impl CommandRunner { info!("running {} check for [{}]...", check_type, resource.name); show_query(show_queries, exists_query); - perform_retries( + let exists = perform_retries( &resource.name, exists_query, retries, retry_delay, &mut self.client, delete_test, - ) + ); + + if delete_test { + if exists { + info!("[{}] still exists", resource.name); + } else { + info!("[{}] confirmed deleted", resource.name); + } + } else if exists { + info!("[{}] exists", resource.name); + } else { + info!("[{}] does not exist", resource.name); + } + + exists } /// Check if a resource is in the correct state. diff --git a/src/commands/common_args.rs b/src/commands/common_args.rs index 8063d69..3cb36eb 100644 --- a/src/commands/common_args.rs +++ b/src/commands/common_args.rs @@ -50,8 +50,11 @@ pub fn log_level() -> Arg { Arg::new("log-level") .long("log-level") .help("Set the logging level") - .default_value("INFO") - .value_parser(["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]) + .default_value("info") + .value_parser(clap::builder::PossibleValuesParser::new([ + "trace", "debug", "info", "warn", "error", + ])) + .ignore_case(true) } /// Common argument for specifying an environment file diff --git a/src/core/utils.rs b/src/core/utils.rs index 74a467c..dfa098f 100644 --- a/src/core/utils.rs +++ b/src/core/utils.rs @@ -458,20 +458,51 @@ fn is_version_higher(installed: &str, requested: &str) -> bool { } /// Update global context with exported values. +/// +/// Each export is stored under two keys: +/// +/// - **`{key}`** — the global (unscoped) key. This can be overridden by a +/// subsequent resource that exports a variable with the same name, so it +/// always reflects the *most recent* export value. +/// +/// - **`{resource_name}.{key}`** — the resource-scoped (fully qualified) key. +/// This is written **once** and never overwritten, so it is immutable once +/// set. Consumers that need an unambiguous reference should use this form. +/// /// Matches Python's `export_vars`. pub fn export_vars( global_context: &mut HashMap, - _resource_name: &str, + resource_name: &str, export_data: &HashMap, protected_exports: &[String], ) { for (key, value) in export_data { - if protected_exports.contains(key) { - let mask = "*".repeat(value.len()); - info!("set protected variable [{}] to [{}] in exports", key, mask); + let is_protected = protected_exports.contains(key); + let display_value = if is_protected { + "*".repeat(value.len()) } else { - info!("set [{}] to [{}] in exports", key, value); + value.clone() + }; + + // --- resource-scoped key (immutable: only written if not already set) --- + let scoped_key = format!("{}.{}", resource_name, key); + if !global_context.contains_key(&scoped_key) { + info!( + "set {} [{}] to [{}] in exports", + if is_protected { "protected variable" } else { "variable" }, + scoped_key, + display_value, + ); + global_context.insert(scoped_key, value.clone()); } + + // --- global (unscoped) key (can be overridden by later resources) --- + info!( + "set {} [{}] to [{}] in exports", + if is_protected { "protected variable" } else { "variable" }, + key, + display_value, + ); global_context.insert(key.clone(), value.clone()); } } @@ -566,3 +597,90 @@ pub fn run_ext_script( _ => None, } } + +#[cfg(test)] +mod tests { + use super::*; + + // ------------------------------------------------------------------ + // export_vars + // ------------------------------------------------------------------ + + #[test] + fn test_export_vars_sets_global_and_scoped_key() { + let mut ctx: HashMap = HashMap::new(); + let mut data: HashMap = HashMap::new(); + data.insert("role_name".to_string(), "my-role".to_string()); + + export_vars(&mut ctx, "aws_cross_account_role", &data, &[]); + + // Global key + assert_eq!(ctx.get("role_name").map(|s| s.as_str()), Some("my-role")); + // Resource-scoped key + assert_eq!( + ctx.get("aws_cross_account_role.role_name").map(|s| s.as_str()), + Some("my-role"), + ); + } + + #[test] + fn test_export_vars_global_key_is_overridable() { + let mut ctx: HashMap = HashMap::new(); + + // First resource exports role_name + let mut data1 = HashMap::new(); + data1.insert("role_name".to_string(), "first-role".to_string()); + export_vars(&mut ctx, "resource_a", &data1, &[]); + + // Second resource exports role_name with a different value + let mut data2 = HashMap::new(); + data2.insert("role_name".to_string(), "second-role".to_string()); + export_vars(&mut ctx, "resource_b", &data2, &[]); + + // Global key reflects the most recent export + assert_eq!(ctx.get("role_name").map(|s| s.as_str()), Some("second-role")); + } + + #[test] + fn test_export_vars_scoped_key_is_immutable() { + let mut ctx: HashMap = HashMap::new(); + + // First resource exports role_name + let mut data1 = HashMap::new(); + data1.insert("role_name".to_string(), "original-role".to_string()); + export_vars(&mut ctx, "resource_a", &data1, &[]); + + // Simulate an accidental re-export of the same resource (e.g. called + // twice): the scoped key must not be overwritten. + let mut data2 = HashMap::new(); + data2.insert("role_name".to_string(), "should-not-overwrite".to_string()); + export_vars(&mut ctx, "resource_a", &data2, &[]); + + // Scoped key is unchanged + assert_eq!( + ctx.get("resource_a.role_name").map(|s| s.as_str()), + Some("original-role"), + ); + // Global key reflects the latest call (expected) + assert_eq!( + ctx.get("role_name").map(|s| s.as_str()), + Some("should-not-overwrite"), + ); + } + + #[test] + fn test_export_vars_protected_values_are_stored_normally() { + // Protection only affects log-masking, not what is stored + let mut ctx: HashMap = HashMap::new(); + let mut data = HashMap::new(); + data.insert("secret_key".to_string(), "super-secret".to_string()); + + export_vars(&mut ctx, "vault", &data, &["secret_key".to_string()]); + + assert_eq!(ctx.get("secret_key").map(|s| s.as_str()), Some("super-secret")); + assert_eq!( + ctx.get("vault.secret_key").map(|s| s.as_str()), + Some("super-secret"), + ); + } +} diff --git a/src/resource/manifest.rs b/src/resource/manifest.rs index a9b821d..438cd8b 100644 --- a/src/resource/manifest.rs +++ b/src/resource/manifest.rs @@ -33,6 +33,9 @@ pub enum ManifestError { #[error("Failed to resolve file() directive: {0}")] FileIncludeError(String), + + #[error("Manifest validation failed: {0}")] + ValidationFailed(String), } /// Type alias for ManifestResult @@ -387,6 +390,12 @@ impl Manifest { } } + // Run the extensible validation rule-set + if let Err(errors) = crate::resource::validation::validate_manifest(self) { + let messages: Vec = errors.iter().map(|e| e.to_string()).collect(); + return Err(ManifestError::ValidationFailed(messages.join("; "))); + } + Ok(()) } diff --git a/src/resource/mod.rs b/src/resource/mod.rs index 9707ca8..3ba596e 100644 --- a/src/resource/mod.rs +++ b/src/resource/mod.rs @@ -10,6 +10,7 @@ // pub mod exports; pub mod manifest; +pub mod validation; // pub mod operations; // pub mod queries; diff --git a/src/resource/validation.rs b/src/resource/validation.rs new file mode 100644 index 0000000..8cd5519 --- /dev/null +++ b/src/resource/validation.rs @@ -0,0 +1,263 @@ +// resource/validation.rs + +//! # Manifest Validation Module +//! +//! Contains validation rules applied to a [`Manifest`] before any command +//! (`build`, `plan`, `teardown`, `test`) is executed. +//! +//! Each rule is a standalone function that returns either `Ok(())` or a list of +//! [`ValidationError`] values describing what failed. New rules should be added +//! as additional functions and wired into [`validate_manifest`]. +//! +//! ## Current rules +//! +//! | Rule ID | Description | +//! |---------------------------|-------------------------------------------------| +//! | `unique-resource-names` | Every resource name in the stack must be unique | + +use std::collections::HashSet; +use std::fmt; + +use crate::resource::manifest::Manifest; + +// --------------------------------------------------------------------------- +// Error type +// --------------------------------------------------------------------------- + +/// A single manifest validation failure. +#[derive(Debug, Clone)] +pub struct ValidationError { + /// Short, machine-readable identifier for the rule that was violated. + pub rule: &'static str, + + /// Human-readable explanation of what failed. + pub detail: String, +} + +impl fmt::Display for ValidationError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "[{}] {}", self.rule, self.detail) + } +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +/// Run all manifest validation rules against `manifest`. +/// +/// All rules are evaluated (fail-all, not fail-fast) so that callers receive a +/// complete picture of every problem at once. +/// +/// Returns `Ok(())` when the manifest passes every rule, or +/// `Err(Vec)` containing one entry per failing check. +pub fn validate_manifest(manifest: &Manifest) -> Result<(), Vec> { + let mut errors: Vec = Vec::new(); + + collect(&mut errors, validate_unique_resource_names(manifest)); + // Wire in additional rules here as the list grows: + // collect(&mut errors, validate_some_other_rule(manifest)); + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +/// Append errors from a rule result into the accumulator. +fn collect(acc: &mut Vec, result: Result<(), Vec>) { + if let Err(mut errs) = result { + acc.append(&mut errs); + } +} + +// --------------------------------------------------------------------------- +// Rule: unique-resource-names +// --------------------------------------------------------------------------- + +/// Validates that every resource `name` in the stack is unique. +/// +/// Resource names must be unique because: +/// +/// * Resources are processed in declaration order — a duplicate name leads to +/// ambiguous processing behaviour. +/// * Resource-scoped export keys (`{resource_name}.{export}`) are immutable +/// once written. A second resource with the same name would attempt to write +/// the same scoped keys, making them permanently incorrect. +/// +/// **Rule ID**: `unique-resource-names` +fn validate_unique_resource_names(manifest: &Manifest) -> Result<(), Vec> { + let mut seen: HashSet<&str> = HashSet::new(); + let mut errors: Vec = Vec::new(); + + for resource in &manifest.resources { + if !seen.insert(resource.name.as_str()) { + errors.push(ValidationError { + rule: "unique-resource-names", + detail: format!( + "resource name '{}' appears more than once in stack '{}'; \ + every resource name must be unique within a stack", + resource.name, manifest.name, + ), + }); + } + } + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + /// Parse a manifest from an inline YAML string. + /// + /// Panics with a clear message if the YAML is malformed, so test failures + /// are easy to diagnose. + fn parse(yaml: &str) -> Manifest { + serde_yaml::from_str(yaml).unwrap_or_else(|e| { + panic!("test manifest YAML is invalid: {}\n\nYAML:\n{}", e, yaml) + }) + } + + // ----------------------------------------------------------------------- + // Rule: unique-resource-names — positive (valid) fixture + // ----------------------------------------------------------------------- + + /// Manifest where every resource name is distinct. Should pass. + const VALID_UNIQUE_NAMES: &str = r#" +version: 1 +name: test-stack +providers: + - aws +resources: + - name: vpc + props: + - name: vpc_name + value: my-vpc + - name: subnet + props: + - name: subnet_name + value: my-subnet + - name: role + props: + - name: role_name + value: my-role +"#; + + #[test] + fn test_unique_resource_names_valid() { + let manifest = parse(VALID_UNIQUE_NAMES); + let result = validate_manifest(&manifest); + assert!( + result.is_ok(), + "A manifest with distinct resource names should pass validation, got: {:?}", + result, + ); + } + + // ----------------------------------------------------------------------- + // Rule: unique-resource-names — negative (invalid) fixture + // ----------------------------------------------------------------------- + + /// Manifest where `vpc` appears twice. Should fail with exactly one error. + const INVALID_DUPLICATE_NAMES: &str = r#" +version: 1 +name: test-stack +providers: + - aws +resources: + - name: vpc + props: + - name: vpc_name + value: my-vpc + - name: role + props: + - name: role_name + value: my-role + - name: vpc + props: + - name: vpc_name + value: another-vpc +"#; + + #[test] + fn test_unique_resource_names_duplicate_fails() { + let manifest = parse(INVALID_DUPLICATE_NAMES); + let result = validate_manifest(&manifest); + + assert!( + result.is_err(), + "A manifest with duplicate resource names must fail validation", + ); + + let errors = result.unwrap_err(); + assert_eq!( + errors.len(), + 1, + "Expected exactly one validation error for one duplicate, got: {:?}", + errors, + ); + assert_eq!( + errors[0].rule, "unique-resource-names", + "Error must reference the correct rule ID", + ); + assert!( + errors[0].detail.contains("vpc"), + "Error detail must mention the duplicate resource name 'vpc', got: {}", + errors[0].detail, + ); + } + + #[test] + fn test_unique_resource_names_multiple_duplicates() { + // Two independent duplicate pairs: 'vpc' and 'role' each appear twice. + let yaml = r#" +version: 1 +name: test-stack +providers: + - aws +resources: + - name: vpc + props: + - name: vpc_name + value: my-vpc + - name: vpc + props: + - name: vpc_name + value: another-vpc + - name: role + props: + - name: role_name + value: my-role + - name: role + props: + - name: role_name + value: another-role +"#; + let manifest = parse(yaml); + let result = validate_manifest(&manifest); + + assert!(result.is_err()); + let errors = result.unwrap_err(); + assert_eq!( + errors.len(), + 2, + "Expected two errors (one per duplicate pair), got: {:?}", + errors, + ); + } +} diff --git a/src/utils/logging.rs b/src/utils/logging.rs index b2a5674..e609411 100644 --- a/src/utils/logging.rs +++ b/src/utils/logging.rs @@ -1,7 +1,7 @@ // utils/logging.rs use chrono::Local; -use env_logger::{Builder, Env}; +use env_logger::Builder; use log::LevelFilter; use std::io::Write; use std::path::Path; @@ -47,7 +47,7 @@ pub fn initialize_logger(log_level: &str) { _ => LevelFilter::Info, }; - let mut builder = Builder::from_env(Env::default()); + let mut builder = Builder::new(); builder.format(|buf, record| { let timestamp = Local::now().format("%Y-%m-%dT%H:%M:%SZ"); diff --git a/website/docs/manifest_fields/resources/exports.mdx b/website/docs/manifest_fields/resources/exports.mdx index e1ad6f5..2774435 100644 --- a/website/docs/manifest_fields/resources/exports.mdx +++ b/website/docs/manifest_fields/resources/exports.mdx @@ -3,7 +3,7 @@ import LeftAlignedTable from '@site/src/components/LeftAlignedTable'; -Variables exported from the `resource` +Variables exported from the `resource`. @@ -32,4 +32,50 @@ WHERE name = '{{ vpc_name }}' AND project = '{{ project }}' ``` +::: + +## Export scoping + +Every exported variable is stored under **two keys** simultaneously: + +| Key form | Behaviour | +|---|---| +| `{var}` | **Global (unscoped)** — available by the short name throughout the rest of the stack. If a later resource exports a variable with the same name, the global key is overwritten with the new value. | +| `{resource_name}.{var}` | **Resource-scoped (fully qualified)** — written once when the resource is processed and **never overwritten**, even if another resource later exports a variable with the same name. | + +### Example + +Given a resource named `aws_cross_account_role` that exports `role_name`: + + + +```yaml +resources: +- name: aws_cross_account_role + ... + exports: + - role_name +``` + + + +After `aws_cross_account_role` is processed, two variables are available in subsequent resources and query templates: + +| Variable | Value | Mutability | +|---|---|---| +| `{{ role_name }}` | The exported value | Mutable — can be superseded by a later export | +| `{{ aws_cross_account_role.role_name }}` | The exported value | **Immutable** — fixed for the lifetime of the deployment | + +:::tip + +Use the fully-qualified form `{{ resource_name.var }}` whenever you need an unambiguous reference that is guaranteed not to be overridden by a subsequent resource. + +Use the short form `{{ var }}` for convenience when you know the variable name is unique across the stack, or when you intentionally want the most recently exported value. + +::: + +:::info Resource name uniqueness + +Because resource-scoped export keys are immutable and tied to the resource name, **every resource name in a stack must be unique**. A duplicate resource name is a validation error that prevents the stack from being deployed. See [`resource.name`](./name) for more details. + ::: \ No newline at end of file diff --git a/website/docs/manifest_fields/resources/name.mdx b/website/docs/manifest_fields/resources/name.mdx index 4b1bcad..18a10ac 100644 --- a/website/docs/manifest_fields/resources/name.mdx +++ b/website/docs/manifest_fields/resources/name.mdx @@ -19,4 +19,14 @@ resources: You can reference the current resource's name using `{{ resource_name }}` in property values or query templates. See [Built-in variables](../../../manifest-file#name) for more details. +::: + +:::warning Resource names must be unique + +Every resource name in a stack must be **unique**. Duplicate names are rejected at load time before any command (`build`, `plan`, `teardown`, `test`) is executed. + +This requirement exists because resource names are used as a namespace prefix for exports. When a resource named `my_role` exports `role_name`, the fully-qualified key `my_role.role_name` is written once and never overwritten. A second resource with the name `my_role` would conflict with those immutable scoped export keys. + +See [resource.exports](./exports#export-scoping) for more details on export scoping. + ::: \ No newline at end of file