Repository navigation
Expand file tree
/
Copy pathstorm.thrift
More file actions
209 lines (173 loc) · 5.75 KB
/
Copy pathstorm.thrift
File metadata and controls
209 lines (173 loc) · 5.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
#!/usr/local/bin/thrift --gen java:beans,nocamel,hashcode
namespace java backtype.storm.generated
union JavaObjectArg {
1: i32 int_arg;
2: i64 long_arg;
3: string string_arg;
4: bool bool_arg;
5: binary binary_arg;
6: double double_arg;
}
struct JavaObject {
1: required string full_class_name;
2: required list<JavaObjectArg> args_list;
}
struct NullStruct {
}
union Grouping {
1: list<string> fields; //empty list means global grouping
2: NullStruct shuffle; // tuple is sent to random task
3: NullStruct all; // tuple is sent to every task
4: NullStruct none; // tuple is sent to a single task (storm's choice) -> allows storm to optimize the topology by bundling tasks into a single process
5: NullStruct direct; // this bolt expects the source bolt to send tuples directly to it
6: JavaObject custom_object;
7: binary custom_serialized;
}
struct StreamInfo {
1: required list<string> output_fields;
2: required bool direct;
}
struct ShellComponent {
1: string execution_command;
2: string script;
}
union ComponentObject {
1: binary serialized_java;
2: ShellComponent shell;
3: JavaObject java_object;
}
struct ComponentCommon {
1: required map<string, StreamInfo> streams; //key is stream id
2: optional i32 parallelism_hint; //how many threads across the cluster should be dedicated to this component
}
struct SpoutSpec {
1: required ComponentObject spout_object;
2: required ComponentCommon common;
3: required bool distributed;
}
struct GlobalStreamId {
1: required string componentId;
2: required string streamId;
#Going to need to add an enum for the stream type (NORMAL or FAILURE)
}
struct Bolt {
1: required map<GlobalStreamId, Grouping> inputs; //a join would have multiple inputs
2: required ComponentObject bolt_object;
3: required ComponentCommon common;
}
// not implemented yet
// this will eventually be the basis for subscription implementation in storm
struct StateSpoutSpec {
1: required ComponentObject state_spout_object;
2: required ComponentCommon common;
}
struct StormTopology {
//ids must be unique across maps
1: required map<string, SpoutSpec> spouts;
2: required map<string, Bolt> bolts;
3: required map<string, StateSpoutSpec> state_spouts;
// #workers to use is in conf
}
exception AlreadyAliveException {
1: required string msg;
}
exception NotAliveException {
1: required string msg;
}
exception InvalidTopologyException {
1: required string msg;
}
struct TopologySummary {
1: required string id;
2: required string name;
3: required i32 num_tasks;
4: required i32 num_workers;
5: required i32 uptime_secs;
6: required string status;
}
struct SupervisorSummary {
1: required string host;
2: required i32 uptime_secs;
3: required i32 num_workers;
4: required i32 num_used_workers;
}
struct ClusterSummary {
1: required list<SupervisorSummary> supervisors;
2: required i32 nimbus_uptime_secs;
3: required list<TopologySummary> topologies;
}
struct ErrorInfo {
1: required string error;
2: required i32 error_time_secs;
}
struct BoltStats {
1: required map<string, map<GlobalStreamId, i64>> acked;
2: required map<string, map<GlobalStreamId, i64>> failed;
3: required map<string, map<GlobalStreamId, double>> process_ms_avg;
}
struct SpoutStats {
1: required map<string, map<string, i64>> acked;
2: required map<string, map<string, i64>> failed;
3: required map<string, map<string, double>> complete_ms_avg;
}
union TaskSpecificStats {
1: BoltStats bolt;
2: SpoutStats spout;
}
// Stats are a map from the time window (all time or a number indicating number of seconds in the window)
// to the stats. Usually stats are a stream id to a count or average.
struct TaskStats {
1: required map<string, map<string, i64>> emitted;
2: required map<string, map<string, i64>> transferred;
3: required TaskSpecificStats specific;
}
struct TaskSummary {
1: required i32 task_id;
2: required string component_id;
3: required string host;
4: required i32 port;
5: required i32 uptime_secs;
6: required list<ErrorInfo> errors;
7: optional TaskStats stats;
}
struct TopologyInfo {
1: required string id;
2: required string name;
3: required i32 uptime_secs;
4: required list<TaskSummary> tasks;
5: required string status;
}
struct KillOptions {
1: optional i32 wait_secs;
}
service Nimbus {
void submitTopology(1: string name, 2: string uploadedJarLocation, 3: string jsonConf, 4: StormTopology topology) throws (1: AlreadyAliveException e, 2: InvalidTopologyException ite);
void killTopology(1: string name) throws (1: NotAliveException e);
void killTopologyWithOpts(1: string name, 2: KillOptions options) throws (1: NotAliveException e);
// need to add functions for asking about status of storms, what nodes they're running on, looking at task logs
string beginFileUpload();
void uploadChunk(1: string location, 2: binary chunk);
void finishFileUpload(1: string location);
string beginFileDownload(1: string file);
//can stop downloading chunks when receive 0-length byte array back
binary downloadChunk(1: string id);
// stats functions
ClusterSummary getClusterInfo();
TopologyInfo getTopologyInfo(1: string id) throws (1: NotAliveException e);
//returns json
string getTopologyConf(1: string id) throws (1: NotAliveException e);
StormTopology getTopology(1: string id) throws (1: NotAliveException e);
}
struct DRPCRequest {
1: required string func_args;
2: required string request_id;
}
exception DRPCExecutionException {
1: required string msg;
}
service DistributedRPC {
string execute(1: string functionName, 2: string funcArgs) throws (1: DRPCExecutionException e);
void result(1: string id, 2: string result);
DRPCRequest fetchRequest(1: string functionName);
void failRequest(1: string id);
}