Repository navigation
Expand file tree
/
Copy pathutil.py
More file actions
354 lines (309 loc) · 11.1 KB
/
Copy pathutil.py
File metadata and controls
354 lines (309 loc) · 11.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
"Utility functions for Mininet."
from time import sleep
from resource import setrlimit, RLIMIT_NPROC, RLIMIT_NOFILE
from select import poll, POLLIN
from subprocess import call, check_call, Popen, PIPE, STDOUT
from mininet.log import output, info, error
import re
# Command execution support
def run( cmd ):
"""Simple interface to subprocess.call()
cmd: list of command params"""
return call( cmd.split( ' ' ) )
def checkRun( cmd ):
"""Simple interface to subprocess.check_call()
cmd: list of command params"""
return check_call( cmd.split( ' ' ) )
# pylint doesn't understand explicit type checking
# pylint: disable-msg=E1103
def oldQuietRun( *cmd ):
"""Run a command, routing stderr to stdout, and return the output.
cmd: list of command params"""
if len( cmd ) == 1:
cmd = cmd[ 0 ]
if isinstance( cmd, str ):
cmd = cmd.split( ' ' )
popen = Popen( cmd, stdout=PIPE, stderr=STDOUT )
# We can't use Popen.communicate() because it uses
# select(), which can't handle
# high file descriptor numbers! poll() can, however.
out = ''
readable = poll()
readable.register( popen.stdout )
while True:
while readable.poll():
data = popen.stdout.read( 1024 )
if len( data ) == 0:
break
out += data
popen.poll()
if popen.returncode != None:
break
return out
# This is a bit complicated, but it enables us to
# monitor commount output as it is happening
def errRun( *cmd, **kwargs ):
"""Run a command and return stdout, stderr and return code
cmd: string or list of command and args
stderr: STDOUT to merge stderr with stdout
shell: run command using shell
echo: monitor output to console"""
# Allow passing in a list or a string
if len( cmd ) == 1:
cmd = cmd[ 0 ]
if isinstance( cmd, str ):
cmd = cmd.split( ' ' )
cmd = [ str( arg ) for arg in cmd ]
# By default we separate stderr, don't run in a shell, and don't echo
stderr = kwargs.get( 'stderr', PIPE )
shell = kwargs.get( 'shell', False )
echo = kwargs.get( 'echo', False )
if echo:
# cmd goes to stderr, output goes to stdout
info( cmd, '\n' )
popen = Popen( cmd, stdout=PIPE, stderr=stderr, shell=shell )
# We use poll() because select() doesn't work with large fd numbers
out, err = '', ''
poller = poll()
poller.register( popen.stdout, POLLIN )
fdtofile = { popen.stdout.fileno(): popen.stdout }
if popen.stderr:
fdtofile[ popen.stderr.fileno() ] = popen.stderr
poller.register( popen.stderr, POLLIN )
while True:
readable = poller.poll()
# Tell pylint to ignore unused variable event
# pylint: disable-msg=W0612
for fd, event in readable:
# pylint: enable-msg=W0612
f = fdtofile[ fd ]
data = f.read( 1024 )
if echo:
output( data )
if f == popen.stdout:
out += data
elif f == popen.stderr:
err += data
returncode = popen.poll()
if returncode is not None:
break
return out, err, returncode
def errFail( *cmd, **kwargs ):
"Run a command using errRun and raise exception on nonzero exit"
out, err, ret = errRun( *cmd, **kwargs )
if ret:
raise Exception( "errFail: %s failed with return code %s: %s"
% ( cmd, ret, err ) )
return out, err, ret
def quietRun( cmd, **kwargs ):
"Run a command and return merged stdout and stderr"
return errRun( cmd, stderr=STDOUT, **kwargs )[ 0 ]
# pylint: enable-msg=E1103
# pylint: disable-msg=E1101,W0612
def isShellBuiltin( cmd ):
"Return True if cmd is a bash builtin."
if isShellBuiltin.builtIns is None:
isShellBuiltin.builtIns = quietRun( 'bash -c enable' )
space = cmd.find( ' ' )
if space > 0:
cmd = cmd[ :space]
return cmd in isShellBuiltin.builtIns
isShellBuiltin.builtIns = None
# pylint: enable-msg=E1101,W0612
# Interface management
#
# Interfaces are managed as strings which are simply the
# interface names, of the form 'nodeN-ethM'.
#
# To connect nodes, we create a pair of veth interfaces, and then place them
# in the pair of nodes that we want to communicate. We then update the node's
# list of interfaces and connectivity map.
#
# For the kernel datapath, switch interfaces
# live in the root namespace and thus do not have to be
# explicitly moved.
def makeIntfPair( intf1, intf2 ):
"""Make a veth pair connecting intf1 and intf2.
intf1: string, interface
intf2: string, interface
returns: success boolean"""
# Delete any old interfaces with the same names
quietRun( 'ip link del ' + intf1 )
quietRun( 'ip link del ' + intf2 )
# Create new pair
cmd = 'ip link add name ' + intf1 + ' type veth peer name ' + intf2
return quietRun( cmd )
def retry( retries, delaySecs, fn, *args, **keywords ):
"""Try something several times before giving up.
n: number of times to retry
delaySecs: wait this long between tries
fn: function to call
args: args to apply to function call"""
tries = 0
while not fn( *args, **keywords ) and tries < retries:
sleep( delaySecs )
tries += 1
if tries >= retries:
error( "*** gave up after %i retries\n" % tries )
exit( 1 )
def moveIntfNoRetry( intf, node, printError=False ):
"""Move interface to node, without retrying.
intf: string, interface
node: Node object
printError: if true, print error"""
cmd = 'ip link set ' + intf + ' netns ' + repr( node.pid )
quietRun( cmd )
links = node.cmd( 'ip link show' )
if not ( ' %s:' % intf ) in links:
if printError:
error( '*** Error: moveIntf: ' + intf +
' not successfully moved to ' + node.name + '\n' )
return False
return True
def moveIntf( intf, node, printError=False, retries=3, delaySecs=0.001 ):
"""Move interface to node, retrying on failure.
intf: string, interface
node: Node object
printError: if true, print error"""
retry( retries, delaySecs, moveIntfNoRetry, intf, node, printError )
# Support for dumping network
def dumpNodeConnections( nodes ):
"Dump connections to/from nodes."
def dumpConnections( node ):
"Helper function: dump connections to node"
for intf in node.intfList():
output( ' %s:' % intf )
if intf.link:
intfs = [ intf.link.intf1, intf.link.intf2 ]
intfs.remove( intf )
output( intfs[ 0 ] )
else:
output( ' ' )
for node in nodes:
output( node.name )
dumpConnections( node )
output( '\n' )
def dumpNetConnections( net ):
"Dump connections in network"
nodes = net.controllers + net.switches + net.hosts
dumpNodeConnections( nodes )
# IP and Mac address formatting and parsing
def _colonHex( val, bytecount ):
"""Generate colon-hex string.
val: input as unsigned int
bytescount: number of bytes to convert
returns: chStr colon-hex string"""
pieces = []
for i in range( bytecount - 1, -1, -1 ):
piece = ( ( 0xff << ( i * 8 ) ) & val ) >> ( i * 8 )
pieces.append( '%02x' % piece )
chStr = ':'.join( pieces )
return chStr
def macColonHex( mac ):
"""Generate MAC colon-hex string from unsigned int.
mac: MAC address as unsigned int
returns: macStr MAC colon-hex string"""
return _colonHex( mac, 6 )
def ipStr( ip ):
"""Generate IP address string from an unsigned int.
ip: unsigned int of form w << 24 | x << 16 | y << 8 | z
returns: ip address string w.x.y.z, or 10.x.y.z if w==0"""
w = ( ip >> 24 ) & 0xff
w = 10 if w == 0 else w
x = ( ip >> 16 ) & 0xff
y = ( ip >> 8 ) & 0xff
z = ip & 0xff
return "%i.%i.%i.%i" % ( w, x, y, z )
def ipNum( w, x, y, z ):
"""Generate unsigned int from components of IP address
returns: w << 24 | x << 16 | y << 8 | z"""
return ( w << 24 ) | ( x << 16 ) | ( y << 8 ) | z
def ipAdd( i, prefixLen=8, ipBaseNum=0x0a000000 ):
"""Return IP address string from ints
i: int to be added to ipbase
prefixLen: optional IP prefix length
ipBaseNum: option base IP address as int
returns IP address as string"""
# Ugly but functional
assert i < ( 1 << ( 32 - prefixLen ) )
mask = 0xffffffff ^ ( ( 1 << prefixLen ) - 1 )
ipnum = i + ( ipBaseNum & mask )
return ipStr( ipnum )
def ipParse( ip ):
"Parse an IP address and return an unsigned int."
args = [ int( arg ) for arg in ip.split( '.' ) ]
return ipNum( *args )
def netParse( ipstr ):
"""Parse an IP network specification, returning
address and prefix len as unsigned ints"""
prefixLen = 0
if '/' in ipstr:
ip, pf = ipstr.split( '/' )
prefixLen = int( pf )
return ipParse( ip ), prefixLen
def checkInt( s ):
"Check if input string is an int"
try:
int( s )
return True
except ValueError:
return False
def checkFloat( s ):
"Check if input string is a float"
try:
float( s )
return True
except ValueError:
return False
def makeNumeric( s ):
"Convert string to int or float if numeric."
if checkInt( s ):
return int( s )
elif checkFloat( s ):
return float( s )
else:
return s
# Other stuff we use
def fixLimits():
"Fix ridiculously small resource limits."
setrlimit( RLIMIT_NPROC, ( 8192, 8192 ) )
setrlimit( RLIMIT_NOFILE, ( 16384, 16384 ) )
def mountCgroups():
"Make sure cgroups file system is mounted"
mounts = quietRun( 'mount' )
cgdir = '/sys/fs/cgroup'
csdir = cgdir + '/cpuset'
if 'cgroups on %s' % cgdir not in mounts:
raise Exception( "cgroups not mounted on " + cgdir )
if 'cpuset on %s' % csdir not in mounts:
errRun( 'mkdir -p ' + csdir )
errRun( 'mount -t cgroup -ocpuset cpuset ' + csdir )
def natural( text ):
"To sort sanely/alphabetically: sorted( l, key=natural )"
def num( s ):
"Convert text segment to int if necessary"
return int( s ) if s.isdigit() else s
return [ num( s ) for s in re.split( r'(\d+)', text ) ]
def naturalSeq( t ):
"Natural sort key function for sequences"
return [ natural( x ) for x in t ]
def numCores():
"Returns number of CPU cores based on /proc/cpuinfo"
if hasattr( numCores, 'ncores' ):
return numCores.ncores
try:
numCores.ncores = int( quietRun('grep -c processor /proc/cpuinfo') )
except ValueError:
return 0
return numCores.ncores
def irange(start, end):
"""Inclusive range from start to end (vs. Python insanity.)
irange(1,5) -> 1, 2, 3, 4, 5"""
return range( start, end + 1 )
def custom( cls, **params ):
"Returns customized constructor for class cls."
def customized( *args, **kwargs):
"Customized constructor"
kwargs.update( params )
return cls( *args, **kwargs )
return customized