AWS services or capabilities described in AWS Documentation may vary by region/location. Click Getting Started with Amazon AWS to see specific differences applicable to the China (Beijing) Region.
See also: Hadoop Streaming
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
IAmazonElasticMapReduce emr = new AmazonElasticMapReduceClient(credentials);
HadoopJarStepConfig config = new StreamingStep {
Inputs = new List<string> { "s3://elasticmapreduce/samples/wordcount/input" },
Output = "s3://my-bucket/output/",
Mapper = "s3://elasticmapreduce/samples/wordcount/wordSplitter.py",
Reducer = "aggregate"
}.ToHadoopJarStepConfig();
StepConfig wordCount = new StepConfig {
Name = "Word Count",
ActionOnFailure = "TERMINATE_JOB_FLOW",
HadoopJarStep = config
};
RunJobFlowRequest request = new RunJobFlowRequest {
Name = "Word Count",
Steps = new List<StepConfig> { wordCount },
LogUri = "s3://log-bucket/",
Instances = new JobFlowInstancesConfig {
Ec2KeyName = "keypair",
HadoopVersion = "0.20",
InstanceCount = 5,
KeepJobFlowAliveWhenNoSteps = true,
MasterInstanceType = "m1.small",
SlaveInstanceType = "m1.small"
}
};
RunJobFlowResponse response = emr.RunJobFlow(request);
Namespace: Amazon.ElasticMapReduce.Model
Assembly: AWSSDK.dll
Version: (assembly version)
public class StreamingStep : Object
The StreamingStep type exposes the following members
| Name | Description | |
|---|---|---|
|
StreamingStep() | Creates a new default StreamingStep. |
| Name | Type | Description | |
|---|---|---|---|
|
HadoopConfig | System.Collections.Generic.Dictionary<System.String, System.String> | Gets and sets the hadoopConfig. |
|
Inputs | System.Collections.Generic.List<System.String> | Gets and sets list of step input paths. |
|
Mapper | System.String | Gets and sets the mapper. |
|
Output | System.String | Gets and sets the output path. |
|
Reducer | System.String | Gets and sets the reducer. |
| Name | Description | |
|---|---|---|
|
AddHadoopConfig(string, string) | Add a Hadoop config override (-D value) |
|
AddInputs(String[]) | Add more input paths to this step. |
|
ToHadoopJarStepConfig() | Creates the final HadoopJarStepConfig once you are done configuring the step. You can use this as you would any other HadoopJarStepConfig. |
.NET Framework:
Supported in: 4.5, 4.0, 3.5
.NET for Windows Store apps:
Supported in: Windows 8