PySparkEtlJobProps Struct
Properties for creating a Python Spark ETL job.
Example
import * as cdk from 'aws-cdk-lib';
import * as iam from 'aws-cdk-lib/aws-iam';
declare const stack: cdk.Stack;
declare const role: iam.IRole;
declare const script: glue.Code;
// Create a job to run from the workflow
const job = new glue.PySparkEtlJob(stack, 'Job', { role, script });
// Create a workflow and add a trigger that runs the job
const workflow = new glue.Workflow(stack, 'Workflow');
workflow.addOnDemandTrigger('OnDemandTrigger', {
actions: [glue.Action.job(job)],
});
import aws_cdk as cdk
import aws_cdk.aws_iam as iam
# stack: cdk.Stack
# role: iam.IRole
# script: glue.Code
# Create a job to run from the workflow
job = glue.PySparkEtlJob(stack, "Job", role=role, script=script)
# Create a workflow and add a trigger that runs the job
workflow = glue.Workflow(stack, "Workflow")
workflow.add_on_demand_trigger("OnDemandTrigger",
actions=[glue.Action.job(job)]
)
import software.amazon.awscdk.*;
import software.amazon.awscdk.services.iam.*;
Stack stack;
IRole role;
Code script;
// Create a job to run from the workflow
PySparkEtlJob job = PySparkEtlJob.Builder.create(stack, "Job").role(role).script(script).build();
// Create a workflow and add a trigger that runs the job
Workflow workflow = new Workflow(stack, "Workflow");
workflow.addOnDemandTrigger("OnDemandTrigger", OnDemandTriggerOptions.builder()
.actions(List.of(Action.job(job)))
.build());
// Install: dotnet add package Amazon.CDK.Lib
using Amazon.CDK;
using Amazon.CDK.AWS.IAM;
Stack stack;
IRole role;
Code script;
// Create a job to run from the workflow
var job = new PySparkEtlJob(stack, "Job", new PySparkEtlJobProps { Role = role, Script = script });
// Create a workflow and add a trigger that runs the job
var workflow = new Workflow(stack, "Workflow");
workflow.AddOnDemandTrigger("OnDemandTrigger", new OnDemandTriggerOptions {
Actions = new [] { Action.Job(job) }
});
// Install: go get github.com/aws/aws-cdk-go/awscdk/v2
import cdk "github.com/aws/aws-cdk-go/awscdk"
import iam "github.com/aws/aws-cdk-go/awscdk"
var stack Stack
var role IRole
var script Code
// Create a job to run from the workflow
job := glue.NewPySparkEtlJob(stack, jsii.String("Job"), &PySparkEtlJobProps{
Role: Role,
Script: Script,
})
// Create a workflow and add a trigger that runs the job
workflow := glue.NewWorkflow(stack, jsii.String("Workflow"))
workflow.AddOnDemandTrigger(jsii.String("OnDemandTrigger"), &OnDemandTriggerOptions{
Actions: []Action{
glue.Action_Job(job),
},
})
Properties
connectionsoptionalIConnection[]connectionsoptionaltyping.List[IConnection]connectionsoptionalList<IConnection>ConnectionsoptionalIConnection[]Connectionsoptional*[]IConnection#Default: [] - no connections are added to the job
continuousLoggingoptionalContinuousLoggingPropscontinuous_loggingoptionalContinuousLoggingPropscontinuousLoggingoptionalContinuousLoggingPropsContinuousLoggingoptionalContinuousLoggingPropsContinuousLoggingoptionalContinuousLoggingProps#Default: continuous logging is enabled.
defaultArgumentsoptional{ [key: string]: string }default_argumentsoptionaltyping.Mapping[str, str]defaultArgumentsoptionalMap<String, String>DefaultArgumentsoptionalIDictionary<string, string>DefaultArgumentsoptional*map[string]*string#Default: no arguments
notes
This map is the escape hatch for Glue job arguments that this construct does not model. It MUST NOT be used to set arguments that already have a dedicated prop — configure those through the corresponding prop instead (continuousLogging, enableMetrics, enableObservabilityMetrics, sparkUI, className, extraJars, extraJarsFirst, extraPythonFiles, extraFiles). Passing a construct-managed argument (e.g. --enable-continuous-cloudwatch-log, --enable-metrics, --enable-spark-ui, --job-language) or a Glue-reserved argument (--debug, --mode, --JOB_NAME, --endpoint) here throws at synthesis time, so there is exactly one way to express each intent.
Also note that these are emitted verbatim into the CloudFormation template, so avoid placing secrets here in plaintext. Pass secrets to the job at runtime through AWS Secrets Manager instead. A synthesis-time warning is emitted when an argument key looks like a credential and holds a plaintext literal.
descriptionoptionalstringdescriptionoptionalstrdescriptionoptionalStringDescriptionoptionalstringDescriptionoptional*string#Default: no value
enableMetricsoptionalbooleanenable_metricsoptionalboolenableMetricsoptionalBooleanEnableMetricsoptionalboolEnableMetricsoptional*bool= true#When enabled, adds '--enable-metrics' to job arguments.
enableObservabilityMetricsoptionalbooleanenable_observability_metricsoptionalboolenableObservabilityMetricsoptionalBooleanEnableObservabilityMetricsoptionalboolEnableObservabilityMetricsoptional*bool= true#When enabled, adds '--enable-observability-metrics': 'true' to job arguments.
extraFilesoptionalCode[]extra_filesoptionaltyping.List[Code]extraFilesoptionalList<Code>ExtraFilesoptionalCode[]ExtraFilesoptional*[]Code#Default: no extra files specified.
extraJarsoptionalCode[]extra_jarsoptionaltyping.List[Code]extraJarsoptionalList<Code>ExtraJarsoptionalCode[]ExtraJarsoptional*[]Code#Default: no extra jar files
extraJarsFirstoptionalbooleanextra_jars_firstoptionalboolextraJarsFirstoptionalBooleanExtraJarsFirstoptionalboolExtraJarsFirstoptional*bool#Default: false - priority is not given to user-provided jars
extraPythonFilesoptionalCode[]extra_python_filesoptionaltyping.List[Code]extraPythonFilesoptionalList<Code>ExtraPythonFilesoptionalCode[]ExtraPythonFilesoptional*[]Code#Default: no extra files
glueVersionoptionalGlueVersionglue_versionoptionalGlueVersionglueVersionoptionalGlueVersionGlueVersionoptionalGlueVersionGlueVersionoptionalGlueVersion#Default: determined by the job type: 4.0 for ETL and Streaming, 5.0 for Flex, 3.0 for Python Shell
jobNameoptionalstringjob_nameoptionalstrjobNameoptionalStringJobNameoptionalstringJobNameoptional*string#Default: a name is automatically generated
jobRunQueuingEnabledoptionalbooleanjob_run_queuing_enabledoptionalbooljobRunQueuingEnabledoptionalBooleanJobRunQueuingEnabledoptionalboolJobRunQueuingEnabledoptional*bool= false#notes
A value of true means job run queuing is enabled for the job runs. If false or not populated, the job runs will not be considered for queueing. If this field does not match the value set in the job run, then the value from the job run field will be used. This property must be set to false for flex jobs. If this property is enabled, maxRetries must be set to zero.
maxConcurrentRunsoptionalnumbermax_concurrent_runsoptionalint | floatmaxConcurrentRunsoptionalNumberMaxConcurrentRunsoptionaldoubleMaxConcurrentRunsoptional*float64= 1#An error is returned when this threshold is reached. The maximum value you can specify is controlled by a service limit.
maxRetriesoptionalnumbermax_retriesoptionalint | floatmaxRetriesoptionalNumberMaxRetriesoptionaldoubleMaxRetriesoptional*float64= 0#notifyDelayAfteroptionalDurationnotify_delay_afteroptionalDurationnotifyDelayAfteroptionalDurationNotifyDelayAfteroptionalDurationNotifyDelayAfteroptionalDuration= undefined#After a job run starts, the number of minutes to wait before sending a job run delay notification.
Can be uploaded from the local directory structure using fromAsset or referenced via S3 location using fromBucket
securityConfigurationoptionalISecurityConfigurationsecurity_configurationoptionalISecurityConfigurationsecurityConfigurationoptionalISecurityConfigurationSecurityConfigurationoptionalISecurityConfigurationSecurityConfigurationoptionalISecurityConfiguration#Default: no security configuration.
sparkUIoptionalSparkUIPropsspark_uioptionalSparkUIPropssparkUIoptionalSparkUIPropsSparkUIoptionalSparkUIPropsSparkUIoptionalSparkUIProps#Default: Spark UI debugging and monitoring is disabled.
tagsoptional{ [key: string]: string }tagsoptionaltyping.Mapping[str, str]tagsoptionalMap<String, String>TagsoptionalIDictionary<string, string>Tagsoptional*map[string]*string#Default: {} - no tags
timeoutoptionalDurationtimeoutoptionalDurationtimeoutoptionalDurationTimeoutoptionalDurationTimeoutoptionalDuration#Default: no value set; Glue applies its service default (2880 minutes for non-streaming jobs)
Specified in minutes.
workerConfigurationoptionalWorkerConfigurationworker_configurationoptionalWorkerConfigurationworkerConfigurationoptionalWorkerConfigurationWorkerConfigurationoptionalWorkerConfigurationWorkerConfigurationoptionalWorkerConfiguration#Default: the job runs with the G_1X worker type and 10 workers.