Repository navigation
Expand file tree
/
Copy pathcleanup.sh
More file actions
341 lines (312 loc) · 9.45 KB
/
Copy pathcleanup.sh
File metadata and controls
341 lines (312 loc) · 9.45 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
#!/bin/bash
# This library holds functions that are used to clean up local
# system state after other scripts have run.
# os::cleanup::all will clean up all of the processes and data that
# a script leaves around after running. All of the sub-tasks called
# from this function should gracefully handle when they do not need
# to do anything.
#
# Globals:
# - ARTIFACT_DIR
# - SKIP_CLEANUP
# - SKIP_TEARDOWN
# - SKIP_IMAGE_CLEANUP
# Arguments:
# None
# Returns:
# None
function os::cleanup::all() {
if [[ -n "${SKIP_CLEANUP:-}" ]]; then
os::log::warning "[CLEANUP] Skipping cleanup routines..."
return 0
fi
# All of our cleanup is best-effort, so we do not care
# if any specific step fails.
set +o errexit
os::log::info "[CLEANUP] Beginning cleanup routines..."
os::cleanup::dump_events
os::cleanup::dump_etcd
os::cleanup::dump_container_logs
os::cleanup::dump_pprof_output
os::cleanup::find_cache_alterations
os::cleanup::truncate_large_logs
if [[ -z "${SKIP_TEARDOWN:-}" ]]; then
os::cleanup::containers
os::cleanup::processes
os::cleanup::prune_etcd
fi
}
readonly -f os::cleanup::all
# os::cleanup::dump_etcd dumps the full contents of etcd to a file.
#
# Globals:
# - ARTIFACT_DIR
# - API_SCHEME
# - API_HOST
# - ETCD_PORT
# Arguments:
# None
# Returns:
# None
function os::cleanup::dump_etcd() {
if [[ -n "${API_SCHEME:-}" && -n "${API_HOST:-}" && -n "${ETCD_PORT:-}" ]]; then
local dump_dir="${ARTIFACT_DIR}/etcd"
mkdir -p "${dump_dir}"
os::log::info "[CLEANUP] Dumping etcd contents to $( os::util::repository_relative_path "${dump_dir}" )"
os::util::curl_etcd "/v2/keys/?recursive=true" > "${dump_dir}/v2_dump.json"
os::cleanup::internal::dump_etcd_v3 > "${dump_dir}/v3_dump.json"
fi
}
readonly -f os::cleanup::dump_etcd
# os::cleanup::internal::dump_etcd_v3 dumps the full contents of etcd v3 to a file.
#
# Globals:
# - ARTIFACT_DIR
# - API_SCHEME
# - API_HOST
# - ETCD_PORT
# Arguments:
8000
# None
# Returns:
# None
function os::cleanup::internal::dump_etcd_v3() {
local full_url="${API_SCHEME}://${API_HOST}:${ETCD_PORT}"
local etcd_client_cert="${MASTER_CONFIG_DIR}/master.etcd-client.crt"
local etcd_client_key="${MASTER_CONFIG_DIR}/master.etcd-client.key"
local ca_bundle="${MASTER_CONFIG_DIR}/ca-bundle.crt"
os::util::ensure::built_binary_exists 'etcdhelper' >&2
etcdhelper --cert "${etcd_client_cert}" --key "${etcd_client_key}" \
--cacert "${ca_bundle}" --endpoint "${full_url}" dump
}
readonly -f os::cleanup::internal::dump_etcd_v3
# os::cleanup::prune_etcd removes the etcd data store from disk.
#
# Globals:
# ARTIFACT_DIR
# Arguments:
# None
# Returns:
# None
function os::cleanup::prune_etcd() {
if [[ -n "${ETCD_DATA_DIR:-}" ]]; then
os::log::info "[CLEANUP] Pruning etcd data directory"
${USE_SUDO:+sudo} rm -rf "${ETCD_DATA_DIR}"
fi
}
readonly -f os::cleanup::prune_etcd
# os::cleanup::containers operates on our containers to stop the containers
# and optionally remove the containers and any volumes they had attached.
#
# Globals:
# - SKIP_IMAGE_CLEANUP
# Arguments:
# None
# Returns:
# None
function os::cleanup::containers() {
if ! os::util::find::system_binary docker >/dev/null 2>&1; then
os::log::warning "[CLEANUP] No \`docker\` binary found, skipping container cleanup."
return
fi
os::log::info "[CLEANUP] Stopping docker containers"
for id in $( os::cleanup::internal::list_our_containers ); do
os::log::debug "Stopping ${id}"
docker stop "${id}" >/dev/null
done
if [[ -n "${SKIP_IMAGE_CLEANUP:-}" ]]; then
return
fi
os::log::info "[CLEANUP] Removing docker containers"
for id in $( os::cleanup::internal::list_our_containers ); do
os::log::debug "Removing ${id}"
docker rm --volumes "${id}" >/dev/null
done
}
readonly -f os::cleanup::containers
# os::cleanup::dump_container_logs operates on k8s containers to dump any logs
# from the containers.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function os::cleanup::dump_container_logs() {
if ! os::util::find::system_binary docker >/dev/null 2>&1; then
os::log::warning "[CLEANUP] No \`docker\` binary found, skipping container log harvest."
return
fi
local container_log_dir="${LOG_DIR}/containers"
mkdir -p "${container_log_dir}"
os::log::info "[CLEANUP] Dumping container logs to $( os::util::repository_relative_path "${container_log_dir}" )"
for id in $( os::cleanup::internal::list_our_containers ); do
local name; name="$( docker inspect --format '{{ .Name }}' "${id}" )"
os::log::debug "Dumping logs for ${id} to ${name}.log"
docker logs "${id}" >"${container_log_dir}/${name}.log" 2>&1
done
}
readonly -f os::cleanup::dump_container_logs
# os::cleanup::internal::list_our_containers returns a space-delimited list of
# docker containers that belonged to some part of the Origin deployment.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function os::cleanup::internal::list_our_containers() {
os::cleanup::internal::list_containers '^/origin$'
os::cleanup::internal::list_k8s_containers
}
readonly -f os::cleanup::internal::list_our_containers
# os::cleanup::internal::list_k8s_containers returns a space-delimited list of
# docker containers that belonged to k8s.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function os::cleanup::internal::list_k8s_containers() {
os::cleanup::internal::list_containers '^/k8s_.*'
}
readonly -f os::cleanup::internal::list_k8s_containers
# os::cleanup::internal::list_containers returns a space-delimited list of
# docker containers that match a name regex.
#
# Globals:
# None
# Arguments:
# 1 - regex to match on the name
# Returns:
# None
function os::cleanup::internal::list_containers() {
local regex="$1"
local ids;
for short_id in $( docker ps -aq ); do
local id; id="$( docker inspect --format '{{ .Id }}' "${short_id}" )"
local name; name="$( docker inspect --format '{{ .Name }}' "${id}" )"
if [[ "${name}" =~ ${regex} ]]; then
ids+=( "${id}" )
fi
done
echo "${ids[*]:+"${ids[*]}"}"
}
readonly -f os::cleanup::internal::list_containers
# os::cleanup::tmpdir performs cleanup of temp directories as a precondition for running a test. It tries to
# clean up mounts in the temp directories.
#
# Globals:
# - BASETMPDIR
# - USE_SUDO
# Returns:
# None
function os::cleanup::tmpdir() {
os::log::info "[CLEANUP] Cleaning up temporary directories"
# ensure that the directories are clean
if os::util::find::system_binary "findmnt" &>/dev/null; then
for target in $( ${USE_SUDO:+sudo} findmnt --output TARGET --list ); do
if [[ "${target}" == "${BASETMPDIR}"* ]]; then
${USE_SUDO:+sudo} umount "${target}"
fi
done
fi
# delete any sub directory underneath BASETMPDIR
for directory in $( find "${BASETMPDIR}" -mindepth 2 -maxdepth 2 ); do
${USE_SUDO:+sudo} rm -rf "${directory}"
done
}
readonly -f os::cleanup::tmpdir
# os::cleanup::dump_events dumps all the events from a cluster to a file.
#
# Globals:
# ARTIFACT_DIR
# Arguments:
# None
# Returns:
# None
function os::cleanup::dump_events() {
os::log::info "[CLEANUP] Dumping cluster events to $( os::util::repository_relative_path "${ARTIFACT_DIR}/events.txt" )"
local kubeconfig
if [[ -n "${ADMIN_KUBECONFIG:-}" ]]; then
kubeconfig="--config=${ADMIN_KUBECONFIG}"
fi
oc login -u system:admin ${kubeconfig:-}
oc get events --all-namespaces ${kubeconfig:-} > "${ARTIFACT_DIR}/events.txt" 2>&1
}
readonly -f os::cleanup::dump_events
# os::cleanup::find_cache_alterations ulls information out of the server
# log so that we can get failure management in jenkins to highlight it
# and really have it smack people in their logs. This is a severe
# correctness problem.
#
# Globals:
# - LOG_DIR
# Arguments:
# None
# Returns:
# None
function os::cleanup::find_cache_alterations() {
grep -ra5 "CACHE.*ALTERED" "${LOG_DIR}" || true
}
readonly -f os::cleanup::find_cache_alterations
# os::cleanup::dump_pprof_output dumps profiling output for the
# `openshift` binary
#
# Globals:
# - LOG_DIR
# Arguments:
# None
# Returns:
# None
function os::cleanup::dump_pprof_output() {
if go tool -n pprof >/dev/null 2>&1 && [[ -s cpu.pprof ]]; then
os::log::info "[CLEANUP] \`pprof\` output logged to $( os::util::repository_relative_path "${LOG_DIR}/pprof.out" )"
go tool pprof -text "./_output/local/bin/$(os::util::host_platform)/openshift" cpu.pprof >"${LOG_DIR}/pprof.out" 2>&1
fi
}
readonly -f os::cleanup::dump_pprof_output
# os::cleanup::truncate_large_logs truncates very large files under
# $LOG_DIR and $ARTIFACT_DIR so we do not upload them to cloud storage
# after CI runs.
#
# Globals:
# - LOG_DIR
# - ARTIFACT_DIR
# Arguments:
# None
# Returns:
# None
function os::cleanup::truncate_large_logs() {
local max_file_size="200M"
os::log::info "[CLEANUP] Truncating log files over ${max_file_size}"
for file in $(find "${ARTIFACT_DIR}" "${LOG_DIR}" -type f -name '*.log' \( -size +${max_file_size} \)); do
mv "${file}" "${file}.tmp"
echo "LOGFILE TOO LONG ($(du -h "${file}.tmp")), PREVIOUS BYTES TRUNCATED. LAST ${max_file_size} OF LOGFILE:" > "${file}"
tail -c ${max_file_size} "${file}.tmp" >> "${file}"
rm "${file}.tmp"
done
}
readonly -f os::cleanup::truncate_large_logs
# os::cleanup::processes kills all processes created by the test
# script.
#
# Globals:
# None
# Arguments:
# None
# Returns:
# None
function os::cleanup::processes() {
os::log::info "[CLEANUP] Killing child processes"
for job in $( jobs -pr ); do
for child in $( pgrep -P "${job}" ); do
${USE_SUDO:+sudo} kill "${child}" &> /dev/null
done
${USE_SUDO:+sudo} kill "${job}" &> /dev/null
done
}
readonly -f os::cleanup::processes