-
Notifications
You must be signed in to change notification settings - Fork 50
/
update_distfiles
executable file
·145 lines (119 loc) · 4.3 KB
/
update_distfiles
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
#!/bin/bash
# Copyright (c) 2013 The CoreOS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
SCRIPT_ROOT=$(dirname $(readlink -f "$0"))
. "${SCRIPT_ROOT}/common.sh" || exit 1
DEFINE_boolean dry_run ${FLAGS_FALSE} "Trial run, makes no changes."
DEFINE_boolean parallel ${FLAGS_TRUE} "Enable parallelism in gsutil."
DEFINE_boolean upload ${FLAGS_FALSE} "Upload distfile mirror via gsutil."
# FIXME(marineam): We need to add gs support to emirrordist so it
# doesn't have to operate on a local copy of the complete mirror.
DEFINE_boolean download ${FLAGS_FALSE} \
"Download the current mirror before making updates to it."
MIRROR_ROOT="${DEFAULT_BUILD_ROOT}/mirror"
UPLOAD_ROOT="gs://storage.core-os.net/mirror"
SECOND_ROOT="gs://coreos-net-storage/mirror"
# Parse flags
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"
switch_to_strict_mode
declare -A repos=(
[portage-stable]="portage-stable"
[coreos-overlay]="coreos"
)
if [[ $# -eq 0 ]]; then
set -- "${!repos[@]}"
fi
GSUTIL_OPTS=
if [[ ${FLAGS_parallel} -eq ${FLAGS_TRUE} ]]; then
GSUTIL_OPTS="-m"
fi
EXIT_CODE=0
update_local_mirror() {
local repo_name="$1"
local repo_mirror="${MIRROR_ROOT}/${repos[$repo_name]}"
local extra_flags=""
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
info "Pretend distfiles update for $repo_name"
extra_flags+=" --dry-run "
else
info "Starting distfiles update for $repo_name"
fi
rm -rf "${repo_mirror}/log" # clear old logs
mkdir -p "${repo_mirror}/"{distfiles,info,log,tmp}
emirrordist --mirror --verbose $extra_flags \
--jobs=${NUM_JOBS} --repo="${repo_name}" \
--distfiles="${repo_mirror}/distfiles" \
--distfiles-local="$(portageq envvar DISTDIR)" \
--fetch-log-dir="${repo_mirror}/log" \
--failure-log="${repo_mirror}/log/failure.log" \
--success-log="${repo_mirror}/log/success.log" \
--distfiles-db="${repo_mirror}/info/distfiles.db" \
--restrict-mirror-exemptions="gentoo" \
--temp-dir="${repo_mirror}/tmp" \
--verify-existing-digest
if [[ ! -s "${repo_mirror}/log/failure.log" ]]; then
info "Completed distfiles update for $repo_name without error"
return
fi
# report what went wrong :(
local lastpkg pkg file error
while read pkg file error; do
local log="${repo_mirror}/log/${file}.log"
if [[ "${pkg}" != "${lastpkg}" ]]; then
error "${pkg} failed:"
lastpkg="${pkg}"
fi
error " ${file} ${error}"
[[ -s "${log}" ]] && cat "${log}"
done <"${repo_mirror}/log/failure.log"
EXIT_CODE=1
}
upload_mirror() {
local repo_name="$1"
local local_mirror="${MIRROR_ROOT}/${repos[$repo_name]}"
local remote_mirror="$2/${repos[$repo_name]}"
info "Uploading public distfiles for $repo_name"
gsutil ${GSUTIL_OPTS} rsync -c \
"${local_mirror}/distfiles/" "${remote_mirror}/distfiles"
info "Uploading private metadata for $repo_name"
# uses cp instead of rsync in order to provide acl
gsutil ${GSUTIL_OPTS} cp -a project-private \
"${local_mirror}/info/*" "${remote_mirror}/info"
}
download_mirror() {
local repo_name="$1"
local local_mirror="${MIRROR_ROOT}/${repos[$repo_name]}"
local remote_mirror="${UPLOAD_ROOT}/${repos[$repo_name]}"
info "Downloading public distfiles for $repo_name"
mkdir -p "${local_mirror}/"{distfiles,info}
gsutil ${GSUTIL_OPTS} rsync -c -d \
"${remote_mirror}/distfiles/" "${local_mirror}/distfiles"
info "Downloading private metadata for $repo_name"
gsutil ${GSUTIL_OPTS} rsync -c -d \
"${remote_mirror}/info/" "${local_mirror}/info"
}
if [[ ${FLAGS_download} -eq ${FLAGS_TRUE} ]]; then
for repo in "$@"; do
download_mirror "$repo"
done
fi
for repo in "$@"; do
if ! portageq get_repo_path / "$repo" >/dev/null; then
die_notrace "Unknown repo name '$repo'"
fi
update_local_mirror "$repo"
done
if [[ ${FLAGS_dry_run} == ${FLAGS_TRUE} ]]; then
info "Dry-run complete."
exit
fi
if [[ ${FLAGS_upload} -eq ${FLAGS_TRUE} ]]; then
for repo in "$@"; do
upload_mirror "$repo" "$UPLOAD_ROOT"
upload_mirror "$repo" "$SECOND_ROOT"
done
fi
command_completed
exit $EXIT_CODE