Refactor builder to support file requirements and run pre-install

The builder configuration has proven useful for supporting generic
pre-entrypoint commands such as dependency managers not otherwise
supported by specific Blubber configuration. Adding additional
`builder.requirements` config expands support for such commands by
allowing the user to specify files that should be copied into the image
before the builder command runs.

To support this extra configuration, `builder` had to be changed from a
simple string to a mapping. The builder command must now by given as
`builder.command`.

The pattern of creating parent directories, copying files, and executing
one or more commands prior to the entrypoint has become a common
one. Some of the implementation of this pattern was moved from
`PythonConfig` into shared build macros `build.SortFilesByDir` and
`build.SyncFiles`. All config types that must have requirements files
copied over independently of the entire source tree (`PythonConfig`,
`BuilderConfig`, `NodeConfig`) now delegate to these functions.

Change-Id: I67f33034f22cee2851ec866cfb07ab20c23eba8c

2 files changed, 143 insertions, 1 deletions

diff --git a/build/macros.go b/build/macros.go
index 08556d1..b99c03d 100644
--- a/build/macros.go
+++ b/build/macros.go
@@ -2,6 +2,8 @@ package build
 
 import (
 	"fmt"
+	"path"
+	"sort"
 )
 
 // ApplyUser wraps any build.Copy instructions as build.CopyAs using the given
@@ -29,11 +31,18 @@ func Chown(uid uint, gid uint, path string) Run {
 	return Run{"chown %s:%s", []string{fmt.Sprint(uid), fmt.Sprint(gid), path}}
 }
 
+// CreateDirectories returns a build.Run instruction for creating all the
+// given directories.
+//
+func CreateDirectories(paths []string) Run {
+	return Run{"mkdir -p", paths}
+}
+
 // CreateDirectory returns a build.Run instruction for creating the given
 // directory.
 //
 func CreateDirectory(path string) Run {
-	return Run{"mkdir -p", []string{path}}
+	return CreateDirectories([]string{path})
 }
 
 // CreateUser returns build.Run instructions for creating the given user
@@ -59,3 +68,70 @@ func homeDir(name string) string {
 
 	return "/home/" + name
 }
+
+// SortFilesByDir returns both the given files indexed by parent directory and
+// a sorted slice of those parent directories. The latter is useful in
+// ensuring deterministic iteration since the ordering of map keys is not
+// guaranteed.
+//
+func SortFilesByDir(files []string) ([]string, map[string][]string) {
+	bydir := make(map[string][]string)
+
+	for _, file := range files {
+		dir := path.Dir(file) + "/"
+		file = path.Clean(file)
+
+		if dirfiles, found := bydir[dir]; found {
+			bydir[dir] = append(dirfiles, file)
+		} else {
+			bydir[dir] = []string{file}
+		}
+	}
+
+	dirs := make([]string, len(bydir))
+	i := 0
+
+	for dir := range bydir {
+		dirs[i] = dir
+		i++
+	}
+
+	sort.Strings(dirs)
+
+	return dirs, bydir
+}
+
+// SyncFiles returns build instructions to copy over the given files after
+// creating their parent directories. Parent directories are created in a
+// sorted order.
+//
+func SyncFiles(files []string, dest string) []Instruction {
+	if len(files) < 1 {
+		return []Instruction{}
+	}
+
+	dirs, bydir := SortFilesByDir(files)
+	mkdirs := []string{}
+	copies := make([]Instruction, len(dirs))
+
+	// make project subdirectories for requirements files if necessary, and
+	// copy in requirements files
+	for i, dir := range dirs {
+		fulldir := dest + "/" + dir
+		fulldir = path.Clean(fulldir) + "/"
+
+		if dir != "./" {
+			mkdirs = append(mkdirs, fulldir)
+		}
+
+		copies[i] = Copy{bydir[dir], fulldir}
+	}
+
+	ins := []Instruction{}
+
+	if len(mkdirs) > 0 {
+		ins = append(ins, CreateDirectories(mkdirs))
+	}
+
+	return append(ins, copies...)
+}
diff --git a/build/macros_test.go b/build/macros_test.go
index 7b656b1..c237bf4 100644
--- a/build/macros_test.go
+++ b/build/macros_test.go
@@ -31,6 +31,12 @@ func TestChown(t *testing.T) {
 	assert.Equal(t, []string{`chown "123":"124" "/foo"`}, i.Compile())
 }
 
+func TestCreateDirectories(t *testing.T) {
+	i := build.CreateDirectories([]string{"/foo", "/bar"})
+
+	assert.Equal(t, []string{`mkdir -p "/foo" "/bar"`}, i.Compile())
+}
+
 func TestCreateDirectory(t *testing.T) {
 	i := build.CreateDirectory("/foo")
 
@@ -61,3 +67,63 @@ func TestHome(t *testing.T) {
 		)
 	})
 }
+
+func TestSortFilesByDir(t *testing.T) {
+	files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"}
+
+	sortedDirs, filesByDir := build.SortFilesByDir(files)
+
+	assert.Equal(t,
+		[]string{
+			"./",
+			"a/",
+			"b/",
+			"c/c/",
+			"d/",
+		},
+		sortedDirs,
+	)
+
+	assert.Equal(t,
+		map[string][]string{
+			"./":   []string{"foo", "bar"},
+			"d/":   []string{"d/d-foo"},
+			"c/c/": []string{"c/c/c-foo"},
+			"b/":   []string{"b/b-foo", "b/b-bar"},
+			"a/":   []string{"a/a-foo"},
+		},
+		filesByDir,
+	)
+}
+
+func TestSyncFiles(t *testing.T) {
+	files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"}
+
+	assert.Equal(t,
+		[]build.Instruction{
+			build.Run{"mkdir -p", []string{"a/", "b/", "c/c/", "d/"}},
+			build.Copy{[]string{"foo", "bar"}, "./"},
+			build.Copy{[]string{"a/a-foo"}, "a/"},
+			build.Copy{[]string{"b/b-foo", "b/b-bar"}, "b/"},
+			build.Copy{[]string{"c/c/c-foo"}, "c/c/"},
+			build.Copy{[]string{"d/d-foo"}, "d/"},
+		},
+		build.SyncFiles(files, "."),
+	)
+}
+
+func TestSyncFilesWithDestination(t *testing.T) {
+	files := []string{"foo", "./bar", "./d/d-foo", "./c/c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"}
+
+	assert.Equal(t,
+		[]build.Instruction{
+			build.Run{"mkdir -p", []string{"/dir/a/", "/dir/b/", "/dir/c/c/", "/dir/d/"}},
+			build.Copy{[]string{"foo", "bar"}, "/dir/"},
+			build.Copy{[]string{"a/a-foo"}, "/dir/a/"},
+			build.Copy{[]string{"b/b-foo", "b/b-bar"}, "/dir/b/"},
+			build.Copy{[]string{"c/c/c-foo"}, "/dir/c/c/"},
+			build.Copy{[]string{"d/d-foo"}, "/dir/d/"},
+		},
+		build.SyncFiles(files, "/dir"),
+	)
+}