SelfManagedKafkaEventSourceProps Struct
Properties for a self managed Kafka cluster event source.
If your Kafka cluster is only reachable via VPC make sure to configure it.
Example
ts
import { SelfManagedKafkaEventSource, AuthenticationMethod } from 'aws-cdk-lib/aws-lambda-event-sources';
import { StartingPosition, Function } from 'aws-cdk-lib/aws-lambda';
import { ISecret } from 'aws-cdk-lib/aws-secretsmanager';
// With provisioned pollers and poller group for cost optimization
declare const myFunction: Function;
declare const kafkaCredentials: ISecret;
myFunction.addEventSource(new SelfManagedKafkaEventSource({
bootstrapServers: ['kafka-broker1.example.com:9092', 'kafka-broker2.example.com:9092'],
topic: 'events-topic',
secret: kafkaCredentials,
startingPosition: StartingPosition.LATEST,
authenticationMethod: AuthenticationMethod.SASL_SCRAM_512_AUTH,
provisionedPollerConfig: {
minimumPollers: 1,
maximumPollers: 8,
pollerGroupName: 'self-managed-kafka-group', // Group pollers to reduce costs
},
}));
python
from aws_cdk.aws_lambda_event_sources import ProvisionedPollerConfig
from aws_cdk.aws_lambda_event_sources import SelfManagedKafkaEventSource, AuthenticationMethod
from aws_cdk.aws_lambda import StartingPosition, Function
from aws_cdk.aws_secretsmanager import ISecret
# With provisioned pollers and poller group for cost optimization
# my_function: Function
# kafka_credentials: ISecret
my_function.add_event_source(SelfManagedKafkaEventSource(
bootstrap_servers=["kafka-broker1.example.com:9092", "kafka-broker2.example.com:9092"],
topic="events-topic",
secret=kafka_credentials,
starting_position=StartingPosition.LATEST,
authentication_method=AuthenticationMethod.SASL_SCRAM_512_AUTH,
provisioned_poller_config=ProvisionedPollerConfig(
minimum_pollers=1,
maximum_pollers=8,
poller_group_name="self-managed-kafka-group"
)
))
java
import software.amazon.awscdk.services.lambda.eventsources.SelfManagedKafkaEventSource;
import software.amazon.awscdk.services.lambda.eventsources.AuthenticationMethod;
import software.amazon.awscdk.services.lambda.StartingPosition;
import software.amazon.awscdk.services.lambda.Function;
import software.amazon.awscdk.services.secretsmanager.ISecret;
// With provisioned pollers and poller group for cost optimization
Function myFunction;
ISecret kafkaCredentials;
myFunction.addEventSource(SelfManagedKafkaEventSource.Builder.create()
.bootstrapServers(List.of("kafka-broker1.example.com:9092", "kafka-broker2.example.com:9092"))
.topic("events-topic")
.secret(kafkaCredentials)
.startingPosition(StartingPosition.LATEST)
.authenticationMethod(AuthenticationMethod.SASL_SCRAM_512_AUTH)
.provisionedPollerConfig(ProvisionedPollerConfig.builder()
.minimumPollers(1)
.maximumPollers(8)
.pollerGroupName("self-managed-kafka-group")
.build())
.build());
csharp
// Install: dotnet add package Amazon.CDK.Lib
using Amazon.CDK.AWS.Lambda.EventSources;
using Amazon.CDK.AWS.Lambda;
using Amazon.CDK.AWS.SecretsManager;
// With provisioned pollers and poller group for cost optimization
Function myFunction;
ISecret kafkaCredentials;
myFunction.AddEventSource(new SelfManagedKafkaEventSource(new SelfManagedKafkaEventSourceProps {
BootstrapServers = new [] { "kafka-broker1.example.com:9092", "kafka-broker2.example.com:9092" },
Topic = "events-topic",
Secret = kafkaCredentials,
StartingPosition = StartingPosition.LATEST,
AuthenticationMethod = AuthenticationMethod.SASL_SCRAM_512_AUTH,
ProvisionedPollerConfig = new ProvisionedPollerConfig {
MinimumPollers = 1,
MaximumPollers = 8,
PollerGroupName = "self-managed-kafka-group"
}
}));
go
// Install: go get github.com/aws/aws-cdk-go/awscdk/v2
import "github.com/aws/aws-cdk-go/awscdk"
import "github.com/aws/aws-cdk-go/awscdk"
import "github.com/aws/aws-cdk-go/awscdk"
// With provisioned pollers and poller group for cost optimization
var myFunction Function
var kafkaCredentials ISecret
myFunction.AddEventSource(awscdk.NewSelfManagedKafkaEventSource(&SelfManagedKafkaEventSourceProps{
BootstrapServers: []*string{
jsii.String("kafka-broker1.example.com:9092"),
jsii.String("kafka-broker2.example.com:9092"),
},
Topic: jsii.String("events-topic"),
Secret: kafkaCredentials,
StartingPosition: awscdk.StartingPosition_LATEST,
AuthenticationMethod: awscdk.AuthenticationMethod_SASL_SCRAM_512_AUTH,
ProvisionedPollerConfig: &ProvisionedPollerConfig{
MinimumPollers: jsii.Number(1),
MaximumPollers: jsii.Number(8),
PollerGroupName: jsii.String("self-managed-kafka-group"),
},
}))
Properties
authenticationMethodoptionalAuthenticationMethodauthentication_methodoptionalAuthenticationMethodauthenticationMethodoptionalAuthenticationMethodAuthenticationMethodoptionalAuthenticationMethodAuthenticationMethodoptionalAuthenticationMethod#The authentication method for your Kafka cluster.
Default: AuthenticationMethod.SASL_SCRAM_512_AUTH
batchSizeoptionalnumberbatch_sizeoptionalint | floatbatchSizeoptionalNumberBatchSizeoptionaldoubleBatchSizeoptional*float64= 100#The largest number of records that AWS Lambda will retrieve from your event source at the time of invoking your function.
notes
Your function receives an event with all the retrieved records.
Valid Range:
- Minimum value of 1
- Maximum value of:
- 1000 for
DynamoEventSource - 10000 for
KinesisEventSource,ManagedKafkaEventSourceandSelfManagedKafkaEventSource
- 1000 for
bisectBatchOnErroroptionalbooleanbisect_batch_on_erroroptionalboolbisectBatchOnErroroptionalBooleanBisectBatchOnErroroptionalboolBisectBatchOnErroroptional*bool= false#* If the function returns an error, split the batch in two and retry.
bootstrapServersstring[]bootstrap_serverstyping.List[str]bootstrapServersList<String>BootstrapServersstring[]BootstrapServers*[]*stringrequired#The list of host and port pairs that are the addresses of the Kafka brokers in a "bootstrap" Kafka cluster that a Kafka client connects to initially to bootstrap itself.
They are in the format abc.xyz.com:xxxx.
consumerGroupIdoptionalstringconsumer_group_idoptionalstrconsumerGroupIdoptionalStringConsumerGroupIdoptionalstringConsumerGroupIdoptional*string= none#The identifier for the Kafka consumer group to join.
notes
The consumer group ID must be unique among all your Kafka event sources. After creating a Kafka event source mapping with the consumer group ID specified, you cannot update this value. The value must have a length between 1 and 200 and full the pattern '[a-zA-Z0-9-/:_+=.@-]'.
enabledoptionalbooleanenabledoptionalboolenabledoptionalBooleanEnabledoptionalboolEnabledoptional*bool= true#If the stream event source mapping should be enabled.
filterEncryptionoptionalIKeyfilter_encryptionoptionalIKeyfilterEncryptionoptionalIKeyFilterEncryptionoptionalIKeyFilterEncryptionoptionalIKey= none#Add Customer managed KMS key to encrypt Filter Criteria.
filtersoptional{ [key: string]: any }[]filtersoptionaltyping.List[typing.Mapping[str, Any]]filtersoptionalList<Map<String, Object>>FiltersoptionalIDictionary<string, object>[]Filtersoptional*[]*map[string]interface{}= none#Add filter criteria to Event Source.
logLeveloptionalEventSourceMappingLogLevellog_leveloptionalEventSourceMappingLogLevellogLeveloptionalEventSourceMappingLogLevelLogLeveloptionalEventSourceMappingLogLevelLogLeveloptionalEventSourceMappingLogLevel#Configuration for logging verbosity from the event source mapping poller.
Default: No logging
maxBatchingWindowoptionalDurationmax_batching_windowoptionalDurationmaxBatchingWindowoptionalDurationMaxBatchingWindowoptionalDurationMaxBatchingWindowoptionalDuration#The maximum amount of time to gather records before invoking the function.
Default: Duration.seconds(0) for Kinesis, DynamoDB, and SQS event sources, Duration.millis(500) for MSK, self-managed Kafka, and Amazon MQ.
Maximum of Duration.minutes(5).
maxRecordAgeoptionalDurationmax_record_ageoptionalDurationmaxRecordAgeoptionalDurationMaxRecordAgeoptionalDurationMaxRecordAgeoptionalDuration= -1#The maximum age of a record that Lambda sends to a function for processing.
The default value is -1, which sets the maximum age to infinite. When the value is set to infinite, Lambda never discards old records. Record are valid until it expires in the event source.
metricsConfigoptionalMetricsConfigmetrics_configoptionalMetricsConfigmetricsConfigoptionalMetricsConfigMetricsConfigoptionalMetricsConfigMetricsConfigoptionalMetricsConfig#Configuration for enhanced monitoring metrics collection.
Default: Enhanced monitoring is disabled
onFailureoptionalIEventSourceDlqon_failureoptionalIEventSourceDlqonFailureoptionalIEventSourceDlqOnFailureoptionalIEventSourceDlqOnFailureoptionalIEventSourceDlq#Add an on Failure Destination for this Kafka event.
Default: discarded records are ignored
Supported destinations:
- {@link KafkaDlq } - Send failed records to a Kafka topic
- SNS topics - Send failed records to an SNS topic
- SQS queues - Send failed records to an SQS queue
- S3 buckets - Send failed records to an S3 bucket
provisionedPollerConfigoptionalProvisionedPollerConfigprovisioned_poller_configoptionalProvisionedPollerConfigprovisionedPollerConfigoptionalProvisionedPollerConfigProvisionedPollerConfigoptionalProvisionedPollerConfigProvisionedPollerConfigoptionalProvisionedPollerConfig#Configuration for provisioned pollers that read from the event source.
Default: no provisioned pollers
When specified, allows control over the minimum and maximum number of pollers that can be provisioned to process events from the source.
reportBatchItemFailuresoptionalbooleanreport_batch_item_failuresoptionalboolreportBatchItemFailuresoptionalBooleanReportBatchItemFailuresoptionalboolReportBatchItemFailuresoptional*bool= false#* Allow functions to return partially successful responses for a batch of records.
retryAttemptsoptionalnumberretry_attemptsoptionalint | floatretryAttemptsoptionalNumberRetryAttemptsoptionaldoubleRetryAttemptsoptional*float64#* Maximum number of retry attempts.
Default: -1 (infinite retries)
Set to -1 for infinite retries (until the record expires in the event source).
rootCACertificateoptionalISecretroot_ca_certificateoptionalISecretrootCACertificateoptionalISecretRootCACertificateoptionalISecretRootCACertificateoptionalISecret= none#The secret with the root CA certificate used by your Kafka brokers for TLS encryption This field is required if your Kafka brokers use certificates signed by a private CA.
schemaRegistryConfigoptionalISchemaRegistryschema_registry_configoptionalISchemaRegistryschemaRegistryConfigoptionalISchemaRegistrySchemaRegistryConfigoptionalISchemaRegistrySchemaRegistryConfigoptionalISchemaRegistry= none#Specific configuration settings for a Kafka schema registry.
secretoptionalISecretsecretoptionalISecretsecretoptionalISecretSecretoptionalISecretSecretoptionalISecret= none#The secret with the Kafka credentials, see https://docs.aws.amazon.com/msk/latest/developerguide/msk-password.html for details This field is required if your Kafka brokers are accessed over the Internet.
securityGroupoptionalISecurityGroupsecurity_groupoptionalISecurityGroupsecurityGroupoptionalISecurityGroupSecurityGroupoptionalISecurityGroupSecurityGroupoptionalISecurityGroup#If your Kafka brokers are only reachable via VPC, provide the security group here.
Default: none, required if setting vpc
startingPositionStartingPositionstarting_positionStartingPositionstartingPositionStartingPositionStartingPositionStartingPositionStartingPositionStartingPositionrequired#Where to begin consuming the stream.
startingPositionTimestampoptionalnumberstarting_position_timestampoptionalint | floatstartingPositionTimestampoptionalNumberStartingPositionTimestampoptionaldoubleStartingPositionTimestampoptional*float64#The time from which to start reading, in Unix time seconds.
Default: no timestamp
The Kafka topic to subscribe to.
If your Kafka brokers are only reachable via VPC provide the VPC here.
vpcSubnetsoptionalSubnetSelectionvpc_subnetsoptionalSubnetSelectionvpcSubnetsoptionalSubnetSelectionVpcSubnetsoptionalSubnetSelectionVpcSubnetsoptionalSubnetSelection#If your Kafka brokers are only reachable via VPC, provide the subnets selection here.
Default: none, required if setting vpc