From 374976d445b605f2ea1618cc6d2c5006d046fa28 Mon Sep 17 00:00:00 2001 From: Dan Duvall Date: Mon, 6 Aug 2018 10:40:03 -0700 Subject: Refactor builder to support file requirements and run pre-install The builder configuration has proven useful for supporting generic pre-entrypoint commands such as dependency managers not otherwise supported by specific Blubber configuration. Adding additional `builder.requirements` config expands support for such commands by allowing the user to specify files that should be copied into the image before the builder command runs. To support this extra configuration, `builder` had to be changed from a simple string to a mapping. The builder command must now by given as `builder.command`. The pattern of creating parent directories, copying files, and executing one or more commands prior to the entrypoint has become a common one. Some of the implementation of this pattern was moved from `PythonConfig` into shared build macros `build.SortFilesByDir` and `build.SyncFiles`. All config types that must have requirements files copied over independently of the entire source tree (`PythonConfig`, `BuilderConfig`, `NodeConfig`) now delegate to these functions. Change-Id: I67f33034f22cee2851ec866cfb07ab20c23eba8c --- build/macros.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++- build/macros_test.go | 66 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 1 deletion(-) (limited to 'build') diff --git a/build/macros.go b/build/macros.go index 08556d1..b99c03d 100644 --- a/build/macros.go +++ b/build/macros.go @@ -2,6 +2,8 @@ package build import ( "fmt" + "path" + "sort" ) // ApplyUser wraps any build.Copy instructions as build.CopyAs using the given @@ -29,11 +31,18 @@ func Chown(uid uint, gid uint, path string) Run { return Run{"chown %s:%s", []string{fmt.Sprint(uid), fmt.Sprint(gid), path}} } +// CreateDirectories returns a build.Run instruction for creating all the +// given directories. +// +func CreateDirectories(paths []string) Run { + return Run{"mkdir -p", paths} +} + // CreateDirectory returns a build.Run instruction for creating the given // directory. // func CreateDirectory(path string) Run { - return Run{"mkdir -p", []string{path}} + return CreateDirectories([]string{path}) } // CreateUser returns build.Run instructions for creating the given user @@ -59,3 +68,70 @@ func homeDir(name string) string { return "/home/" + name } + +// SortFilesByDir returns both the given files indexed by parent directory and +// a sorted slice of those parent directories. The latter is useful in +// ensuring deterministic iteration since the ordering of map keys is not +// guaranteed. +// +func SortFilesByDir(files []string) ([]string, map[string][]string) { + bydir := make(map[string][]string) + + for _, file := range files { + dir := path.Dir(file) + "/" + file = path.Clean(file) + + if dirfiles, found := bydir[dir]; found { + bydir[dir] = append(dirfiles, file) + } else { + bydir[dir] = []string{file} + } + } + + dirs := make([]string, len(bydir)) + i := 0 + + for dir := range bydir { + dirs[i] = dir + i++ + } + + sort.Strings(dirs) + + return dirs, bydir +} + +// SyncFiles returns build instructions to copy over the given files after +// creating their parent directories. Parent directories are created in a +// sorted order. +// +func SyncFiles(files []string, dest string) []Instruction { + if len(files) < 1 { + return []Instruction{} + } + + dirs, bydir := SortFilesByDir(files) + mkdirs := []string{} + copies := make([]Instruction, len(dirs)) + + // make project subdirectories for requirements files if necessary, and + // copy in requirements files + for i, dir := range dirs { + fulldir := dest + "/" + dir + fulldir = path.Clean(fulldir) + "/" + + if dir != "./" { + mkdirs = append(mkdirs, fulldir) + } + + copies[i] = Copy{bydir[dir], fulldir} + } + + ins := []Instruction{} + + if len(mkdirs) > 0 { + ins = append(ins, CreateDirectories(mkdirs)) + } + + return append(ins, copies...) +} diff --git a/build/macros_test.go b/build/macros_test.go index 7b656b1..c237bf4 100644 --- a/build/macros_test.go +++ b/build/macros_test.go @@ -31,6 +31,12 @@ func TestChown(t *testing.T) { assert.Equal(t, []string{`chown "123":"124" "/foo"`}, i.Compile()) } +func TestCreateDirectories(t *testing.T) { + i := build.CreateDirectories([]string{"/foo", "/bar"}) + + assert.Equal(t, []string{`mkdir -p "/foo" "/bar"`}, i.Compile()) +} + func TestCreateDirectory(t *testing.T) { i := build.CreateDirectory("/foo") @@ -61,3 +67,63 @@ func TestHome(t *testing.T) { ) }) } + +func TestSortFilesByDir(t *testing.T) { + files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"} + + sortedDirs, filesByDir := build.SortFilesByDir(files) + + assert.Equal(t, + []string{ + "./", + "a/", + "b/", + "c/c/", + "d/", + }, + sortedDirs, + ) + + assert.Equal(t, + map[string][]string{ + "./": []string{"foo", "bar"}, + "d/": []string{"d/d-foo"}, + "c/c/": []string{"c/c/c-foo"}, + "b/": []string{"b/b-foo", "b/b-bar"}, + "a/": []string{"a/a-foo"}, + }, + filesByDir, + ) +} + +func TestSyncFiles(t *testing.T) { + files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"} + + assert.Equal(t, + []build.Instruction{ + build.Run{"mkdir -p", []string{"a/", "b/", "c/c/", "d/"}}, + build.Copy{[]string{"foo", "bar"}, "./"}, + build.Copy{[]string{"a/a-foo"}, "a/"}, + build.Copy{[]string{"b/b-foo", "b/b-bar"}, "b/"}, + build.Copy{[]string{"c/c/c-foo"}, "c/c/"}, + build.Copy{[]string{"d/d-foo"}, "d/"}, + }, + build.SyncFiles(files, "."), + ) +} + +func TestSyncFilesWithDestination(t *testing.T) { + files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"} + + assert.Equal(t, + []build.Instruction{ + build.Run{"mkdir -p", []string{"/dir/a/", "/dir/b/", "/dir/c/c/", "/dir/d/"}}, + build.Copy{[]string{"foo", "bar"}, "/dir/"}, + build.Copy{[]string{"a/a-foo"}, "/dir/a/"}, + build.Copy{[]string{"b/b-foo", "b/b-bar"}, "/dir/b/"}, + build.Copy{[]string{"c/c/c-foo"}, "/dir/c/c/"}, + build.Copy{[]string{"d/d-foo"}, "/dir/d/"}, + }, + build.SyncFiles(files, "/dir"), + ) +} -- cgit v1.2.1