exports and logging updates

examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/ec2_permissions.json

@@ -0,0 +1,72 @@
+{
+  "Sid": "Stmt1403287045000",
+  "Effect": "Allow",
+  "Action": [
+    "ec2:AllocateAddress",
+    "ec2:AssociateDhcpOptions",
+    "ec2:AssociateIamInstanceProfile",
+    "ec2:AssociateRouteTable",
+    "ec2:AttachInternetGateway",
+    "ec2:AttachVolume",
+    "ec2:AuthorizeSecurityGroupEgress",
+    "ec2:AuthorizeSecurityGroupIngress",
+    "ec2:CancelSpotInstanceRequests",
+    "ec2:CreateDhcpOptions",
+    "ec2:CreateInternetGateway",
+    "ec2:CreateKeyPair",
+    "ec2:CreateNatGateway",
+    "ec2:CreatePlacementGroup",
+    "ec2:CreateRoute",
+    "ec2:CreateRouteTable",
+    "ec2:CreateSecurityGroup",
+    "ec2:CreateSubnet",
+    "ec2:CreateTags",
+    "ec2:CreateVolume",
+    "ec2:CreateVpc",
+    "ec2:CreateVpcEndpoint",
+    "ec2:DeleteDhcpOptions",
+    "ec2:DeleteInternetGateway",
+    "ec2:DeleteKeyPair",
+    "ec2:DeleteNatGateway",
+    "ec2:DeletePlacementGroup",
+    "ec2:DeleteRoute",
+    "ec2:DeleteRouteTable",
+    "ec2:DeleteSecurityGroup",
+    "ec2:DeleteSubnet",
+    "ec2:DeleteTags",
+    "ec2:DeleteVolume",
+    "ec2:DeleteVpc",
+    "ec2:DeleteVpcEndpoints",
+    "ec2:DescribeAvailabilityZones",
+    "ec2:DescribeIamInstanceProfileAssociations",
+    "ec2:DescribeInstanceStatus",
+    "ec2:DescribeInstances",
+    "ec2:DescribeInternetGateways",
+    "ec2:DescribeNatGateways",
+    "ec2:DescribePlacementGroups",
+    "ec2:DescribePrefixLists",
+    "ec2:DescribeReservedInstancesOfferings",
+    "ec2:DescribeRouteTables",
+    "ec2:DescribeSecurityGroups",
+    "ec2:DescribeSpotInstanceRequests",
+    "ec2:DescribeSpotPriceHistory",
+    "ec2:DescribeSubnets",
+    "ec2:DescribeVolumes",
+    "ec2:DescribeVpcs",
+    "ec2:DescribeVpcAttribute",
+    "ec2:DescribeNetworkAcls",
+    "ec2:DetachInternetGateway",
+    "ec2:DisassociateIamInstanceProfile",
+    "ec2:DisassociateRouteTable",
+    "ec2:ModifyVpcAttribute",
+    "ec2:ReleaseAddress",
+    "ec2:ReplaceIamInstanceProfileAssociation",
+    "ec2:ReplaceRoute",
+    "ec2:RequestSpotInstances",
+    "ec2:RevokeSecurityGroupEgress",
+    "ec2:RevokeSecurityGroupIngress",
+    "ec2:RunInstances",
+    "ec2:TerminateInstances"
+  ],
+  "Resource": ["*"]
+}

examples/databricks/serverless/resources/OLD/aws/iam/policy_statements/iam_service_linked_role.json

@@ -0,0 +1,15 @@
+{
+  "Effect": "Allow",
+  "Action": [
+    "iam:CreateServiceLinkedRole",
+    "iam:PutRolePolicy"
+  ],
+  "Resource": [
+    "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot"
+  ],
+  "Condition": {
+    "StringLike": {
+      "iam:AWSServiceName": "spot.amazonaws.com"
+    }
+  }
+}

examples/databricks/serverless/resources/OLD/aws/iam/roles.iql

@@ -0,0 +1,72 @@
+/*+ exists */
+SELECT count(*) as count
+FROM awscc.iam.roles
+WHERE region = 'us-east-1' AND
+Identifier = '{{ role_name }}'
+;
+
+/*+ create */
+INSERT INTO awscc.iam.roles (
+ AssumeRolePolicyDocument,
+ Description,
+ ManagedPolicyArns,
+ MaxSessionDuration,
+ Path,
+ PermissionsBoundary,
+ Policies,
+ RoleName,
+ Tags,
+ region
+)
+SELECT
+ '{{ assume_role_policy_document }}',
+ '{{ description }}',
+ '{{ managed_policy_arns }}',
+ '{{ max_session_duration }}',
+ '{{ path }}',
+ '{{ permissions_boundary }}',
+ '{{ policies }}',
+ '{{ role_name }}',
+ '{{ tags }}',
+ 'us-east-1';
+
+/*+ update */
+UPDATE awscc.iam.roles
+SET PatchDocument = string('{{ {
+    "AssumeRolePolicyDocument": assume_role_policy_document,
+    "Description": description,
+    "ManagedPolicyArns": managed_policy_arns,
+    "MaxSessionDuration": max_session_duration,
+    "PermissionsBoundary": permissions_boundary,
+    "Path": path,
+    "Policies": policies,
+    "Tags": tags
+} | generate_patch_document }}')
+WHERE region = 'us-east-1'
+AND Identifier = '{{ role_name }}';
+
+/*+ statecheck, retries=5, retry_delay=10 */
+SELECT COUNT(*) as count FROM (
+    SELECT 
+        max_session_duration,
+        path,
+        AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc,
+        AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies
+    FROM awscc.iam.roles
+    WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1')t
+WHERE test_assume_role_policy_doc = 1
+AND test_policies = 1
+AND path = '{{ path }}';
+
+/*+ exports */
+SELECT
+arn,
+role_name
+FROM awscc.iam.roles
+WHERE region = 'us-east-1' AND
+Identifier = '{{ role_name }}';
+
+/*+ delete */
+DELETE FROM awscc.iam.roles
+WHERE Identifier = '{{ role_name }}'
+AND region = 'us-east-1';

examples/databricks/serverless/resources/OLD/databricks_account/credentials.iql

@@ -0,0 +1,31 @@
+/*+ exists */
+SELECT COUNT(*) as count
+FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ databricks_account_id }}'
+AND credentials_name = '{{ credentials_name }}'
+
+/*+ create */
+INSERT INTO databricks_account.provisioning.credentials (
+account_id,
+data__credentials_name,
+data__aws_credentials
+)
+SELECT 
+'{{ databricks_account_id }}',
+'{{ credentials_name }}',
+'{{ aws_credentials }}'
+
+/*+ exports, retries=3, retry_delay=5 */
+SELECT 
+'{{ credentials_name }}' as databricks_credentials_name,
+credentials_id as databricks_credentials_id,
+JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id
+FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ databricks_account_id }}'
+AND credentials_name = '{{ credentials_name }}'
+AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}'
+
+/*+ delete */
+DELETE FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ databricks_account_id }}' AND
+credentials_id = '{{ databricks_credentials_id }}';

examples/databricks/serverless/resources/aws/iam/roles.iql

@@ -28,7 +28,7 @@ SELECT
  '{{ policies }}',
  '{{ role_name }}',
  '{{ tags }}',
- '{{ region }}';
+ 'us-east-1';
 
 /*+ update */
 UPDATE awscc.iam.roles
@@ -38,33 +38,30 @@ SET PatchDocument = string('{{ {
     "ManagedPolicyArns": managed_policy_arns,
     "MaxSessionDuration": max_session_duration,
     "PermissionsBoundary": permissions_boundary,
+    "Path": path,
     "Policies": policies,
     "Tags": tags
 } | generate_patch_document }}')
-WHERE region = '{{ region }}'
+WHERE region = 'us-east-1'
 AND Identifier = '{{ role_name }}';
 
 /*+ statecheck, retries=5, retry_delay=10 */
-SELECT count(*) as count
-FROM awscc.iam.roles
-WHERE 
-region = 'us-east-1' AND
-Identifier = '{{ role_name }}'
-;
+SELECT COUNT(*) as count FROM (
+    SELECT 
+        max_session_duration,
+        path,
+        AWS_POLICY_EQUAL(assume_role_policy_document, '{{ assume_role_policy_document }}') as test_assume_role_policy_doc,
+        AWS_POLICY_EQUAL(policies, '{{ policies }}') as test_policies
+    FROM awscc.iam.roles
+    WHERE Identifier = '{{ role_name }}' AND region = 'us-east-1')t
+WHERE test_assume_role_policy_doc = 1
+AND test_policies = 1
+AND path = '{{ path }}';
 
 /*+ exports */
 SELECT
 arn,
-assume_role_policy_document,
-description,
-managed_policy_arns,
-max_session_duration,
-path,
-permissions_boundary,
-policies,
-role_id,
-role_name,
-tags
+role_name
 FROM awscc.iam.roles
 WHERE region = 'us-east-1' AND
 Identifier = '{{ role_name }}';

examples/databricks/serverless/resources/databricks_account/credentials.iql

@@ -1,31 +1,45 @@
-/*+ exists */
-SELECT COUNT(*) as count
-FROM databricks_account.provisioning.credentials
-WHERE account_id = '{{ databricks_account_id }}'
-AND credentials_name = '{{ credentials_name }}'
-
-/*+ create */
-INSERT INTO databricks_account.provisioning.credentials (
-account_id,
-data__credentials_name,
-data__aws_credentials
-)
-SELECT 
-'{{ databricks_account_id }}',
-'{{ credentials_name }}',
-'{{ aws_credentials }}'
-
-/*+ exports, retries=3, retry_delay=5 */
-SELECT 
-'{{ credentials_name }}' as databricks_credentials_name,
-credentials_id as databricks_credentials_id,
-JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id
-FROM databricks_account.provisioning.credentials
-WHERE account_id = '{{ databricks_account_id }}'
-AND credentials_name = '{{ credentials_name }}'
-AND JSON_EXTRACT(aws_credentials, '$.sts_role.role_arn') = '{{ aws_iam_cross_account_role_arn }}'
-
-/*+ delete */
-DELETE FROM databricks_account.provisioning.credentials
-WHERE account_id = '{{ databricks_account_id }}' AND
-credentials_id = '{{ databricks_credentials_id }}';
+/*+ exists */
+SELECT count(*) as count --should use list
+FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ account_id }}'
+AND credentials_name = '{{ credentials_name }}' 
+;
+
+/*+ create */
+INSERT INTO databricks_account.provisioning.credentials (
+credentials_name,
+aws_credentials,
+account_id
+)
+SELECT 
+'{{ credentials_name }}',
+'{{ aws_credentials }}',
+'{{ account_id }}'
+;
+
+/*+ statecheck, retries=5, retry_delay=10 */
+SELECT count(*) as count --should use list
+FROM databricks_account.provisioning.credentials
+WHERE 
+credentials_name = '{{ credentials_name }}' AND --where are the ''
+aws_credentials = '{{ aws_credentials }}' AND --where are the ''
+account_id = '{{ account_id }}';
+
+
+-- {"sts_role":{"external_id":"ebfcc5a9-9d49-4c93-b651-b3ee6cf1c9ce","role_arn":"arn:aws:iam::824532806693:role/stackql-serverless-dev-role"}}
+
+/*+ exports */
+SELECT -- account_id, -- dont get account_id back from exports, but do get credentials_id back, which is needed for updates and deletes
+credentials_name,
+credentials_id,
+JSON_EXTRACT(aws_credentials, '$.sts_role.external_id') as databricks_role_external_id
+FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ account_id }}' -- required
+AND credentials_name = '{{ credentials_name }}' -- required
+; --should use list
+
+/*+ delete */
+DELETE FROM databricks_account.provisioning.credentials
+WHERE account_id = '{{ account_id }}' --required
+AND credentials_id = '{{ credentials_id }}' --required
+;

examples/databricks/serverless/stackql_manifest.yml

@@ -72,21 +72,24 @@ resources:
         merge:
           - global_tags
     exports:
-      - aws_iam_role_name: aws_iam_cross_account_role_name
-      - aws_iam_role_arn: aws_iam_cross_account_role_arn
+      - role_name: aws_iam_cross_account_role_name
+      - arn: aws_iam_cross_account_role_arn
 
-  # - name: databricks_account/credentials
-  #   props:
-  #     - name: credentials_name
-  #       value: "{{ stack_name }}-{{ stack_env }}-credentials"
-  #     - name: aws_credentials
-  #       value:
-  #         sts_role:
-  #           role_arn: "{{ aws_iam_cross_account_role_arn }}"
-  #   exports:
-  #     - databricks_credentials_name
-  #     - databricks_credentials_id
-  #     - databricks_role_external_id
+  - name: databricks_account_credentials
+    file: databricks_account/credentials.iql
+    props:
+      - name: account_id
+        value: "{{ databricks_account_id }}"      
+      - name: credentials_name
+        value: "{{ stack_name }}-{{ stack_env }}-credentials"
+      - name: aws_credentials
+        value:
+          sts_role:
+            role_arn: "{{ aws_iam_cross_account_role_arn }}"
+    exports:
+      - credentials_name
+      - credentials_id
+      - databricks_role_external_id
 
 # ====================================================================================
 # Storage