This repository was archived by the owner on Sep 6, 2022. It is now read-only.
Repository navigation
Expand file tree
/
Copy pathdebian.go
More file actions
359 lines (319 loc) · 11.7 KB
/
Copy pathdebian.go
File metadata and controls
359 lines (319 loc) · 11.7 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
/*
Onix Config Manager - Artisan
Copyright (c) 2018-Present by www.gatblau.org
Licensed under the Apache License, Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0
Contributors to this project, hereby assign copyright in this code to the project,
to be licensed under the same terms as the rest of the code.
*/
package release
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/gatblau/onix/artisan/build"
"github.com/gatblau/onix/artisan/core"
"github.com/gatblau/onix/artisan/data"
"github.com/gatblau/onix/artisan/merge"
"github.com/gatblau/onix/artisan/registry"
"github.com/gatblau/onix/oxlib/resx"
"gopkg.in/yaml.v2"
)
// BuildDebianPackage build a package containing debian packages
// If export options are specified, it also exports the package
func BuildDebianPackage(pkgNames []string, opts *ExportOptions) error {
targetUri := opts.TargetUri
creds := opts.TargetCreds
artHome := opts.ArtHome
if len(pkgNames) == 0 {
return fmt.Errorf("debian package name is missing : %s", pkgNames)
}
pName, err := core.ParseName(aptPkgName())
if err != nil {
return fmt.Errorf("invalid artisan package name %s : %s, ", aptPkgName(), err)
}
// if a target has been specified
if len(targetUri) > 0 {
// if a final slash does not exist add it
if targetUri[len(targetUri)-1] != '/' {
targetUri = fmt.Sprintf("%s/", targetUri)
}
// automatically adds a tar filename to the URI based on the package name:tag
targetUri = fmt.Sprintf("%s%s", targetUri, aptPkgTarFileName())
} else {
return fmt.Errorf("a destination URI must be specified to export the image")
}
// execution path
tmp, err := core.NewTempDir(artHome)
if err != nil {
return fmt.Errorf("cannot create temp folder for processing image archive: %s", err)
}
core.DebugLogger.Printf("location of temporary folder %s", tmp)
// create a target folder for the artisan package
targetFolder := filepath.Join(tmp, "build")
err = os.MkdirAll(targetFolder, 0755)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("failed to create build folder : %s", err)
}
// create a patch folder where packages to be used for patching will be downloaded
patchFolder := filepath.Join(targetFolder, "patch")
err = os.MkdirAll(patchFolder, 0755)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("failed to create patch folder : %s", err)
}
// get the debian packages either locally or from remote
err = getPackages(pkgNames, patchFolder)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("failed to get package and its dependencies for packages %s \n error :- %s", pkgNames, err)
}
// get all the package name as string from folder "patch" which will be used to build
// backup command
bckupCmds, err := buildBackupCmds(patchFolder)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("failed to read .deb file from path %s, \n error is %s", patchFolder, err)
}
// generate build function build.yaml for this package
bfBytes, err := generateBuildFunctions(bckupCmds)
if err != nil {
return fmt.Errorf("failed to marshall debian package build file: %s", err)
}
// create a build file to build the package containing the debian packages tar
pbfBytes, err := generateArtBuild(aptPkgName())
if err != nil {
return fmt.Errorf("failed to marshall debian packaging build file: %s", err)
}
// save package build and function build file
core.InfoLogger.Println("packaging debian packages tarball file")
err = os.WriteFile(filepath.Join(tmp, "build.yaml"), pbfBytes, 0755)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("cannot save debian packaging build file: %s", err)
}
err = os.WriteFile(filepath.Join(targetFolder, "build.yaml"), bfBytes, 0755)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("cannot save debian build function file: %s", err)
}
b := build.NewBuilder(artHome)
b.Build(tmp, "", "", pName, "", false, false, "")
r := registry.NewLocalRegistry(artHome)
if opts != nil {
// export package
core.InfoLogger.Printf("exporting debian package to tarball file")
err = r.ExportPackage([]core.PackageName{*pName}, "", targetUri, creds)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("cannot save debian package to destination: %s", err)
}
// append package name to the spec file
spec := new(Spec)
err = yaml.Unmarshal(opts.Specification.content, spec)
if err != nil {
os.RemoveAll(tmp)
return fmt.Errorf("failed unmarshal spec file's content part: %s", err)
}
m := spec.Packages
if m == nil {
m = make(map[string]string)
spec.Packages = m
}
m["PACKAGE_APT"] = pName.String()
contents, err := core.ToYamlBytes(spec)
if err != nil {
return fmt.Errorf("failed to marshal the spec file's content part: %s", err)
}
opts.Specification.content = contents
}
return nil
}
func getPackages(pkgNames []string, target string) error {
if len(pkgNames) == 0 {
return fmt.Errorf("package names to be exported is empty")
}
// make sure pkgNames slice contains either all with debian package file name with .deb extension
// or all will be debian package name (with no .deb extension)
ext := filepath.Ext(pkgNames[0])
prePkg := pkgNames[0]
for _, p := range pkgNames {
ex := filepath.Ext(p)
if strings.Compare(strings.TrimSpace(ext), strings.TrimSpace(ex)) != 0 {
return fmt.Errorf("all package names are not of same format, some has extension some don't [ %s ] [ %s ]", prePkg, p)
}
prePkg = p
}
// if the pkgNames slice contains deb package file names then copy them all from source folder
// to target folder patch
var err error
if len(ext) > 0 {
err = copyPackagesFromLocal(pkgNames, target)
} else {
err = downloadPackagesFromRemote(pkgNames, target)
}
return err
}
func downloadPackagesFromRemote(pkgNames []string, target string) error {
// convert pkgNames slice to a single string, each element of slice separated by space
allPkgs := strings.Join(pkgNames, " ")
// get name of dependent packages
compList, err := getDependencies(allPkgs, target)
if err != nil {
return fmt.Errorf("failed to query dependencies for packages %s \n error :- %s", allPkgs, err)
}
// download all the packages incudling dependencies
err = downloadPackages(compList, target)
if err != nil {
return fmt.Errorf("failed to download package [ %s ] and its dependencies \n error :- %s", compList, err)
}
return nil
}
func copyPackagesFromLocal(pkgNames []string, target string) error {
for _, p := range pkgNames {
ext := filepath.Ext(p)
if ext != ".deb" {
return fmt.Errorf("package names extension is not .deb %s", p)
}
b, e := resx.ReadFile(p, "")
if e != nil {
return nil
}
core.DebugLogger.Printf("< byte size [ %d ] \n file path [ %s ]\n >", len(b), filepath.Join(target, filepath.Base(p)))
e = resx.WriteFile(b, filepath.Join(target, filepath.Base(p)), "")
if e != nil {
return nil
}
}
return nil
}
func aptPkgTarFileName() string {
r := strings.NewReplacer(
"/", "_",
".", "_",
)
fileWithExt := fmt.Sprintf("%s.%s", r.Replace(aptPkgName()), "tar")
return fileWithExt
}
func aptPkgName() string {
return fmt.Sprintf("127.0.0.1/os/packages/apt:%d", time.Now().Unix())
}
func generateBuildFunctions(bckupCmds []string) ([]byte, error) {
export_yes := true
export_no := false
patchCmd := "(cd patch && echo \"\" && echo \"************ patching started ***************\"" +
" && sudo DEBIAN_FRONTEND=noninteractive dpkg --force-all -i * && echo \"************ patch completed ***************\" && echo \"\")"
rollbackCmd := "(cd backup && echo \"\" && echo \"************ patching failed, rollback started ***************\"" +
" && sudo DEBIAN_FRONTEND=noninteractive dpkg --force-all -i * && echo \"************ rollback completed ***************\" && echo \"\")"
combinedCmd := "bash -c '" + patchCmd + " || " + rollbackCmd + "'"
bf := data.BuildFile{
Runtime: "ubi-min",
Functions: []*data.Function{
{
Name: "apply",
Description: "apply debian packages to the operating system. In case of failure, will automatically rollback",
Export: &export_yes,
Run: []string{
"bash -c 'mkdir -p backup'",
"$(backup)",
combinedCmd,
},
},
{
Name: "backup",
Description: "take backup of current version of packages",
Export: &export_no,
Run: bckupCmds,
},
},
}
return yaml.Marshal(bf)
}
func buildBackupCmds(patchFolder string) ([]string, error) {
files, err := ioutil.ReadDir(patchFolder)
if err != nil {
return nil, err
}
core.DebugLogger.Printf("count of .deb files found at the path [ %s] is [ %d ] ", patchFolder, len(files))
var bckupCmds []string
core.InfoLogger.Printf("generating run command for .deb files")
exp := regexp.MustCompile(`\r?\n`)
for _, file := range files {
if filepath.Ext(file.Name()) == ".deb" {
// go to patch folder and get the debian package name
cmd := fmt.Sprintf("bash -c 'cd %s && dpkg -f %s Package'", patchFolder, file.Name())
pname, er := build.Exe(cmd, patchFolder, merge.NewEnVarFromSlice([]string{}), false)
pname = exp.ReplaceAllString(pname, " ")
if er != nil {
return nil, er
}
bckupCmd := fmt.Sprintf("bash -c \"cd backup && sudo dpkg-query -W %s | awk '{print $1}' | sudo xargs dpkg-repack\"", pname)
bckupCmds = append(bckupCmds, bckupCmd)
}
}
return bckupCmds, nil
}
func downloadPackages(compList, patchFolder string) error {
downloadPkg := "bash -c 'apt-get download " + strings.TrimSpace(compList) + "'"
core.InfoLogger.Printf("downloading package %s and its dependencies", compList)
core.DebugLogger.Printf("downloading debian package [%s] dependencies using command :\n %s", compList, downloadPkg)
_, err := build.Exe(downloadPkg, patchFolder, merge.NewEnVarFromSlice([]string{}), false)
return err
}
func generateArtBuild(pkgName string) ([]byte, error) {
// create a build file to build the package containing the debian packages tar
pbf := data.BuildFile{
Labels: map[string]string{
"package": pkgName,
},
Profiles: []*data.Profile{
{
Name: "debian-package",
Target: "./build",
Type: "content/apt",
},
},
}
return yaml.Marshal(pbf)
}
func getDependencies(pkgNames, executionPath string) (string, error) {
// get all dependencies for current package
pkgNames = strings.TrimSpace(pkgNames)
qryDependencies := fmt.Sprintf("bash -c 'apt-rdepends %s | grep -v \"^ \" | sed 's/debconf-2.0/debconf/g'' | sed 's/time-daemon/systemd-timesyncd/g'", pkgNames)
core.DebugLogger.Printf("querying debian package [%s] dependencies using command :\n %s\n ", pkgNames, qryDependencies)
// execute the command synchronously
core.InfoLogger.Printf("querying dependencies of package %s", pkgNames)
// using async because when using sync occassionally it been observed that return list contains
// status messages key word also like "processing" along with the dependent package names
dep, err := build.ExeAsync(qryDependencies, executionPath, merge.NewEnVarFromSlice([]string{}), false)
// note: the dep list will contain the parent package name also for which we looked dependencies
if err != nil {
return "", err
}
replacer := strings.NewReplacer("\n", " ", " ", " ", "\t", "")
dep = replacer.Replace(dep)
return dedup(dep)
}
func dedup(dep string) (string, error) {
core.DebugLogger.Printf("dedup input data is \n %s \n", dep)
var b strings.Builder
words := strings.Split(dep, " ")
for _, word := range words {
// find exact match for the word in the string builder
w := strings.Replace(fmt.Sprintf("\b%s\b", word), "+", "\\+", -1)
x, err := regexp.MatchString(w, b.String())
if err != nil {
return "", fmt.Errorf("failed dedup process while trying for word %s in new package list %s: %s", word, b.String(), err)
}
if x {
continue
}
b.WriteString(word)
b.WriteString(" ")
}
return strings.TrimSpace(b.String()), nil
}