View creation failure scenarios during asynchronous operations
The following examples are representative of the types of errors that may result
from CreateTable or UpdateTable view API calls. They are
not exhaustive as the error surface of SQL query failures is quite large.
Scenario 1: Amazon Redshift query failure
The query provided for Amazon Redshift includes a misspelled table name can't be found in
the Data Catalog during the validation. The resulting error is shown in the
Status field in the GetTable response for the
view.
GetTable request:
{ "CatalogId": "123456789012", "DatabaseName": "async-view-test-db", "TableInput": { "Name": "view-athena-redshift-72", "Description": "This is an atomic operation", "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ] }, "ViewDefinition": { "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "ValidationConnection": "athena-connection" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table__1\";", "ValidationConnection": "redshift-connection" } ] } } }
GetTable response:
IncludeStatusDetails = FALSE { "Table": { "Name": "view-athena-redshift-72", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:39:19-07:00", "UpdateTime": "2024-07-11T11:39:19-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:39:19-07:00", "UpdateTime": "2024-07-11T11:40:06-07:00", "Action": "CREATE", "State": "FAILED" } } } IncludeStatusDetails = TRUE { "Table": { "Name": "view-athena-redshift-72", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:39:19-07:00", "UpdateTime": "2024-07-11T11:39:19-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:39:19-07:00", "UpdateTime": "2024-07-11T11:40:06-07:00", "Action": "CREATE", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: redshift-connection | Query Execution Id: ddb711d3-2415-4aa9-b251-6a76ab4f41b1 | Timestamp: Thu Jul 11 18:39:37 UTC 2024]: Redshift returned error for the statement: ERROR: AwsClientException: EntityNotFoundException from glue - Entity Not Found" }, "Details": { "RequestedChange": { "Name": "view-athena-redshift-72", "DatabaseName": "async-view-test-db", "Description": "This is an atomic operation", "Retention": 0, "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ], "Compressed": false, "NumberOfBuckets": 0, "SortColumns": [], "StoredAsSubDirectories": false }, "TableType": "VIRTUAL_VIEW", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "ViewDefinition": { "IsProtected": true, "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "IsStale": false }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table__1\";", "IsStale": false } ] }, "IsMultiDialectView": true }, "ViewValidations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewValidationText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "UpdateTime": "2024-07-11T11:40:06-07:00", "State": "SUCCESS" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewValidationText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table__1\";", "UpdateTime": "2024-07-11T11:39:37-07:00", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: redshift-connection | Query Execution Id: ddb711d3-2415-4aa9-b251-6a76ab4f41b1 | Timestamp: Thu Jul 11 18:39:37 UTC 2024]: Redshift returned error for the statement: ERROR: AwsClientException: EntityNotFoundException from glue - Entity Not Found" } } ] } } } }
Scenario 2: Invalid Amazon Redshift connection
The Amazon Redshift connection in the following example is malformed because it refers to
a Amazon Redshift database that doesn't exist in the provided cluster/serverless endpoint.
Amazon Redshift is not able to validate the view and the Status field in the
GetTable response shows the error ("State":
"FAILED" from Amazon Redshift.
GetTable request:
{ "CatalogId": "123456789012", "DatabaseName": "async-view-test-db", "TableInput": { "Name": "view-athena-redshift-73", "Description": "This is an atomic operation", "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ] }, "ViewDefinition": { "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "ValidationConnection": "athena-connection" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "ValidationConnection": "redshift-connection-malformed" } ] } } }
GetTable response:
IncludeStatusDetails = FALSE { "Table": { "Name": "view-athena-redshift-73", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:43:27-07:00", "UpdateTime": "2024-07-11T11:43:27-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:43:27-07:00", "UpdateTime": "2024-07-11T11:43:40-07:00", "Action": "CREATE", "State": "FAILED" } } } IncludeStatusDetails = TRUE { "Table": { "Name": "view-athena-redshift-73", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:43:27-07:00", "UpdateTime": "2024-07-11T11:43:27-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:43:27-07:00", "UpdateTime": "2024-07-11T11:43:40-07:00", "Action": "CREATE", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: redshift-connection-malformed | Query Execution Id: 69bfafd4-3d51-4cb0-9320-7ce5404b1809 | Timestamp: Thu Jul 11 18:43:38 UTC 2024]: Redshift returned error for the statement: FATAL: database \"devooo\" does not exist" }, "Details": { "RequestedChange": { "Name": "view-athena-redshift-73", "DatabaseName": "async-view-test-db", "Description": "This is an atomic operation", "Retention": 0, "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ], "Compressed": false, "NumberOfBuckets": 0, "SortColumns": [], "StoredAsSubDirectories": false }, "TableType": "VIRTUAL_VIEW", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "ViewDefinition": { "IsProtected": true, "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "IsStale": false }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "IsStale": false } ] }, "IsMultiDialectView": true }, "ViewValidations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewValidationText": "SELECT * FROM \"gdc-view-playground-db\".\"table_1\"", "UpdateTime": "2024-07-11T11:43:40-07:00", "State": "SUCCESS" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewValidationText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "UpdateTime": "2024-07-11T11:43:38-07:00", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: redshift-connection-malformed | Query Execution Id: 69bfafd4-3d51-4cb0-9320-7ce5404b1809 | Time stamp: Thu Jul 11 18:43:38 UTC 2024]: Redshift returned error for the statement: FATAL: database \"devooo\" does not exist" } } ] } } } }
Scenario 3: Athena query failure
The SQL for Athena here is invalid because the query misspells the database
name. The Athena query validation catches this and the resulting error is
surfaced through the Status object in a GetTable
call.
GetTable request:
{ "CatalogId": "123456789012", "DatabaseName": "async-view-test-db", "TableInput": { "Name": "view-athena-redshift-70", "Description": "This is an atomic operation", "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ] }, "ViewDefinition": { "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc--view-playground-db\".\"table_1\"", "ValidationConnection": "athena-connection" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "ValidationConnection": "redshift-connection" } ] } } }
GetTable response:
IncludeStatusDetails = FALSE { "Table": { "Name": "view-athena-redshift-70", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:09:53-07:00", "UpdateTime": "2024-07-11T11:09:53-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:09:54-07:00", "UpdateTime": "2024-07-11T11:10:41-07:00", "Action": "CREATE", "State": "FAILED", } } } IncludeStatusDetails = TRUE { "Table": { "Name": "view-athena-redshift-70", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:09:53-07:00", "UpdateTime": "2024-07-11T11:09:53-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:09:54-07:00", "UpdateTime": "2024-07-11T11:10:41-07:00", "Action": "CREATE", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: athena-connection | Query Execution Id: d9bb1e6d-ce26-4b35-8276-8a199af966aa | Timestamp: Thu Jul 11 18:10: 41 UTC 2024]: Athena validation FAILED: {ErrorCategory: 2,ErrorType: 1301,Retryable: false,ErrorMessage: line 1:118: Schema 'gdc--view-playground-db' does not exist}" }, "Details": { "RequestedChange": { "Name": "view-athena-redshift-70", "DatabaseName": "async-view-test-db", "Description": "This is an atomic operation", "Retention": 0, "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ], "Compressed": false, "NumberOfBuckets": 0, "SortColumns": [], "StoredAsSubDirectories": false }, "TableType": "VIRTUAL_VIEW", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "ViewDefinition": { "IsProtected": true, "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT * FROM \"gdc--view-playground-db\".\"table_1\"", "IsStale": false }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "IsStale": false } ] }, "IsMultiDialectView": true }, "ViewValidations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewValidationText": "SELECT * FROM \"gdc--view-playground-db\".\"table_1\"", "UpdateTime": "2024-07-11T11:10:41-07:00", "State": "FAILED", "Error": { "ErrorCode": "QueryExecutionException", "ErrorMessage": "Error received during view SQL validation using a connection: [Connection Name: athena-connection | Query Execution Id: d9bb1e6d-ce26-4b35-8276-8a199af966aa | Timestamp: Thu J ul 11 18:10:41 UTC 2024]: Athena validation FAILED: {ErrorCategory: 2,ErrorType: 1301,Retryable: false,ErrorMessage: line 1:118: Schema 'gdc--view-playground-db' does not exist}" } }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewValidationText": "SELECT * FROM \"gdc-view-playground-external-schema\".\"table_1\";", "UpdateTime": "2024-07-11T11:10:41-07:00", "State": "SUCCESS" } ] } } } }
Scenario 4: Mismatch storage descriptors
The SQL provided for the Athena dialect selects col1 and
col2 while the SQL for Redshift selects only col1.
This leads to a storage descriptor mismatch error.
GetTable request:
{ "CatalogId": "123456789012", "DatabaseName": "async-view-test-db", "TableInput": { "Name": "view-athena-redshift-71", "Description": "This is an atomic operation", "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ] }, "ViewDefinition": { "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT col1, col2 FROM \"gdc-view-playground-db\".\"table_1\"", "ValidationConnection": "athena-connection" }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT col1 FROM \"gdc-view-playground-external-schema\".\"table_1\";", "ValidationConnection": "redshift-connection" } ] } } }
GetTable response:
IncludeStatusDetails = FALSE { "Table": { "Name": "view-athena-redshift-71", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:22:02-07:00", "UpdateTime": "2024-07-11T11:22:02-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:22:02-07:00", "UpdateTime": "2024-07-11T11:23:19-07:00", "Action": "CREATE", "State": "FAILED" } } } IncludeStatusDetails = TRUE { "Table": { "Name": "view-athena-redshift-71", "DatabaseName": "async-view-test-db", "Description": "", "CreateTime": "2024-07-11T11:22:02-07:00", "UpdateTime": "2024-07-11T11:22:02-07:00", "Retention": 0, "ViewOriginalText": "", "ViewExpandedText": "", "TableType": "", "CreatedBy": "arn:aws:iam::123456789012:user/zcaisse", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "IsMultiDialectView": false, "Status": { "RequestedBy": "arn:aws:iam::123456789012:user/zcaisse", "UpdatedBy": "arn:aws:iam::123456789012:user/zcaisse", "RequestTime": "2024-07-11T11:22:02-07:00", "UpdateTime": "2024-07-11T11:23:19-07:00", "Action": "CREATE", "State": "FAILED", "Error": { "ErrorCode": "InvalidInputException", "ErrorMessage": "Engine and existing storage descriptor mismatch" }, "Details": { "RequestedChange": { "Name": "view-athena-redshift-71", "DatabaseName": "async-view-test-db", "Description": "This is an atomic operation", "Retention": 0, "StorageDescriptor": { "Columns": [ { "Name": "col1", "Type": "int" }, { "Name": "col2", "Type": "string" }, { "Name": "col3", "Type": "double" } ], "Compressed": false, "NumberOfBuckets": 0, "SortColumns": [], "StoredAsSubDirectories": false }, "TableType": "VIRTUAL_VIEW", "IsRegisteredWithLakeFormation": false, "CatalogId": "123456789012", "IsRowFilteringEnabled": false, "VersionId": "-1", "DatabaseId": "<databaseID>", "ViewDefinition": { "IsProtected": true, "Definer": "arn:aws:iam::123456789012:role/GDCViewDefiner", "SubObjects": [ "arn:aws:glue:us-east-1:123456789012:table/gdc-view-playground-db/table_1" ], "Representations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewOriginalText": "SELECT col1, col2 FROM \"gdc-view-playground-db\".\"table_1\"", "IsStale": false }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewOriginalText": "SELECT col1 FROM \"gdc-view-playground-external-schema\".\"table_1\";", "IsStale": false } ] }, "IsMultiDialectView": true }, "ViewValidations": [ { "Dialect": "ATHENA", "DialectVersion": "3", "ViewValidationText": "SELECT col1, col2 FROM \"gdc-view-playground-db\".\"table_1\"", "UpdateTime": "2024-07-11T11:23:19-07:00", "State": "FAILED", "Error": { "ErrorCode": "InvalidInputException", "ErrorMessage": "Engine and existing storage descriptor mismatch" } }, { "Dialect": "REDSHIFT", "DialectVersion": "1.0", "ViewValidationText": "SELECT col1 FROM \"gdc-view-playground-external-schema\".\"table_1\";", "UpdateTime": "2024-07-11T11:22:49-07:00", "State": "FAILED", "Error": { "ErrorCode": "InvalidInputException", "ErrorMessage": "Engine and existing storage descriptor mismatch" } } ] } } } }
