WebCrawlerConfigurationProperty Struct
The configuration details for the web data source.
| Source | |
|---|---|
| Type last changed |
Example
// The code below shows an example of how to instantiate this type.
// The values are placeholders you should change.
import { aws_wisdom as wisdom } from 'aws-cdk-lib';
const webCrawlerConfigurationProperty: wisdom.CfnKnowledgeBase.WebCrawlerConfigurationProperty = {
urlConfiguration: {
seedUrls: [{
url: 'url',
}],
},
// the properties below are optional
crawlerLimits: {
rateLimit: 123,
},
exclusionFilters: ['exclusionFilters'],
inclusionFilters: ['inclusionFilters'],
scope: 'scope',
};
# The code below shows an example of how to instantiate this type.
# The values are placeholders you should change.
from aws_cdk import aws_wisdom as wisdom
web_crawler_configuration_property = wisdom.CfnKnowledgeBase.WebCrawlerConfigurationProperty(
url_configuration=wisdom.CfnKnowledgeBase.UrlConfigurationProperty(
seed_urls=[wisdom.CfnKnowledgeBase.SeedUrlProperty(
url="url"
)]
),
# the properties below are optional
crawler_limits=wisdom.CfnKnowledgeBase.CrawlerLimitsProperty(
rate_limit=123
),
exclusion_filters=["exclusionFilters"],
inclusion_filters=["inclusionFilters"],
scope="scope"
)
// The code below shows an example of how to instantiate this type.
// The values are placeholders you should change.
import software.amazon.awscdk.services.wisdom.*;
WebCrawlerConfigurationProperty webCrawlerConfigurationProperty = WebCrawlerConfigurationProperty.builder()
.urlConfiguration(UrlConfigurationProperty.builder()
.seedUrls(List.of(SeedUrlProperty.builder()
.url("url")
.build()))
.build())
// the properties below are optional
.crawlerLimits(CrawlerLimitsProperty.builder()
.rateLimit(123)
.build())
.exclusionFilters(List.of("exclusionFilters"))
.inclusionFilters(List.of("inclusionFilters"))
.scope("scope")
.build();
// Install: dotnet add package Amazon.CDK.Lib
// The code below shows an example of how to instantiate this type.
// The values are placeholders you should change.
using Amazon.CDK.AWS.Wisdom;
var webCrawlerConfigurationProperty = new WebCrawlerConfigurationProperty {
UrlConfiguration = new UrlConfigurationProperty {
SeedUrls = new [] { new SeedUrlProperty {
Url = "url"
} }
},
// the properties below are optional
CrawlerLimits = new CrawlerLimitsProperty {
RateLimit = 123
},
ExclusionFilters = new [] { "exclusionFilters" },
InclusionFilters = new [] { "inclusionFilters" },
Scope = "scope"
};
// Install: go get github.com/aws/aws-cdk-go/awscdk/v2
// The code below shows an example of how to instantiate this type.
// The values are placeholders you should change.
import "github.com/aws/aws-cdk-go/awscdk"
webCrawlerConfigurationProperty := &WebCrawlerConfigurationProperty{
UrlConfiguration: &UrlConfigurationProperty{
SeedUrls: []interface{}{
&SeedUrlProperty{
Url: jsii.String("url"),
},
},
},
// the properties below are optional
CrawlerLimits: &CrawlerLimitsProperty{
RateLimit: jsii.Number(123),
},
ExclusionFilters: []*string{
jsii.String("exclusionFilters"),
},
InclusionFilters: []*string{
jsii.String("inclusionFilters"),
},
Scope: jsii.String("scope"),
}
Properties
crawlerLimitsoptionalIResolvable | CfnKnowledgeBase.crawler_limitsoptionaltyping.Union[IResolvable, CrawlerLimitsProperty]crawlerLimitsoptionalIResolvableCrawlerLimitsoptionalIResolvableCrawlerLimitsoptionalinterface{} #exclusionFiltersoptionalstring[]exclusion_filtersoptionaltyping.List[str]exclusionFiltersoptionalList<String>ExclusionFiltersoptionalstring[]ExclusionFiltersoptional*[]*string #If you specify an inclusion and exclusion filter/pattern and both match a URL, the exclusion filter takes precedence and the web content of the URL isn’t crawled.
inclusionFiltersoptionalstring[]inclusion_filtersoptionaltyping.List[str]inclusionFiltersoptionalList<String>InclusionFiltersoptionalstring[]InclusionFiltersoptional*[]*string #If you specify an inclusion and exclusion filter/pattern and both match a URL, the exclusion filter takes precedence and the web content of the URL isn’t crawled.
notes
You can choose to crawl only web pages that belong to the same host or primary domain. For example, only web pages that contain the seed URL https://docs.aws.amazon.com/bedrock/latest/userguide/ and no other domains. You can choose to include sub domains in addition to the host or primary domain. For example, web pages that contain aws.amazon.com can also include sub domain docs.aws.amazon.com .
urlConfigurationIResolvable | CfnKnowledgeBase.url_configurationtyping.Union[IResolvable, UrlConfigurationProperty]urlConfigurationIResolvableUrlConfigurationIResolvableUrlConfigurationinterface{} required #You should be authorized to crawl the URLs.