Support Python projects

Summary:
A new root and variant `python` config field is provided with two new
fields below, `version` and `requirements`.

The former, `version`, should specify the Python executable to use when
executing related package installation commands and ostensibly the same
executable that will be used to run the application.

The latter, `requirements`, should specify all pip requirements files
such that a compiler that supports layered filesystems (e.g. Docker) can
output separate instructions that will invalidate cache layers for
changes to those files independently of changes to the rest of the
codebase.

Python related instructions will be generated only if either `version`
or `requirements` are given.

Fixes T186545

Test Plan: Run `go test ./...`.

Reviewers: thcipriani, hashar, demon, #release-engineering-team

Reviewed By: thcipriani, #release-engineering-team

Tags: #release-engineering-team

Maniphest Tasks: T186545

Differential Revision: https://phabricator.wikimedia.org/D976

4 files changed, 434 insertions, 8 deletions

diff --git a/blubber.example.yaml b/blubber.example.yaml
index 07e1a86..5dc7233 100644
--- a/blubber.example.yaml
+++ b/blubber.example.yaml
@@ -2,6 +2,8 @@
 base: debian:jessie
 apt:
   packages: [libjpeg, libyaml]
+python:
+  version: python2.7
 runs:
   environment:
     FOO: bar
@@ -13,6 +15,8 @@ variants:
       packages: [libjpeg-dev, libyaml-dev]
     node:
       dependencies: true
+    python:
+      requirements: [requirements.txt]
 
   development:
     includes: [build]
@@ -22,6 +26,8 @@ variants:
     includes: [build]
     apt:
       packages: [chromium]
+    python:
+      requirements: [requirements.txt, test-requirements.txt, docs/requirements.txt]
     entrypoint: [npm, test]
 
   prep:
diff --git a/config/common.go b/config/common.go
index fa100b8..a597a82 100644
--- a/config/common.go
+++ b/config/common.go
@@ -8,13 +8,14 @@ import (
 // and each configured variant.
 //
 type CommonConfig struct {
-	Base         string      `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image
-	Apt          AptConfig   `yaml:"apt"`                                 // APT related configuration
-	Node         NodeConfig  `yaml:"node"`                                // Node related configuration
-	Lives        LivesConfig `yaml:"lives"`                               // application owner/dir configuration
-	Runs         RunsConfig  `yaml:"runs"`                                // runtime environment configuration
-	SharedVolume Flag        `yaml:"sharedvolume"`                        // define a volume for application
-	EntryPoint   []string    `yaml:"entrypoint"`                          // entry-point executable
+	Base         string       `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image
+	Apt          AptConfig    `yaml:"apt"`                                 // APT related configuration
+	Node         NodeConfig   `yaml:"node"`                                // Node related configuration
+	Python       PythonConfig `yaml:"python"`                              // Python related configuration
+	Lives        LivesConfig  `yaml:"lives"`                               // application owner/dir configuration
+	Runs         RunsConfig   `yaml:"runs"`                                // runtime environment configuration
+	SharedVolume Flag         `yaml:"sharedvolume"`                        // define a volume for application
+	EntryPoint   []string     `yaml:"entrypoint"`                          // entry-point executable
 }
 
 // Merge takes another CommonConfig and merges its fields this one's.
@@ -26,6 +27,7 @@ func (cc *CommonConfig) Merge(cc2 CommonConfig) {
 
 	cc.Apt.Merge(cc2.Apt)
 	cc.Node.Merge(cc2.Node)
+	cc.Python.Merge(cc2.Python)
 	cc.Lives.Merge(cc2.Lives)
 	cc.Runs.Merge(cc2.Runs)
 	cc.SharedVolume.Merge(cc2.SharedVolume)
@@ -40,7 +42,7 @@ func (cc *CommonConfig) Merge(cc2 CommonConfig) {
 // injected.
 //
 func (cc *CommonConfig) PhaseCompileableConfig() []build.PhaseCompileable {
-	return []build.PhaseCompileable{cc.Apt, cc.Node, cc.Lives, cc.Runs}
+	return []build.PhaseCompileable{cc.Apt, cc.Node, cc.Python, cc.Lives, cc.Runs}
 }
 
 // InstructionsForPhase injects instructions into the given build phase for
diff --git a/config/python.go b/config/python.go
new file mode 100644
index 0000000..4e2f2ce
--- /dev/null
+++ b/config/python.go
@@ -0,0 +1,173 @@
+package config
+
+import (
+	"path"
+	"sort"
+
+	"phabricator.wikimedia.org/source/blubber/build"
+)
+
+// PythonLocalLibPrefix is the path to installed dependency wheels.
+//
+const PythonLocalLibPrefix = LocalLibPrefix + "/python"
+
+// PythonConfig holds configuration fields related to pre-installation of project
+// dependencies via PIP.
+//
+type PythonConfig struct {
+	Version      string   `yaml:"version"`      // Python binary to use when installing dependencies
+	Requirements []string `yaml:"requirements"` // install requirements from given files
+}
+
+// Merge takes another PythonConfig and merges its fields into this one's,
+// overwriting both the dependencies flag and requirements.
+//
+func (pc *PythonConfig) Merge(pc2 PythonConfig) {
+	if pc2.Version != "" {
+		pc.Version = pc2.Version
+	}
+
+	if pc2.Requirements != nil {
+		pc.Requirements = pc2.Requirements
+	}
+}
+
+// InstructionsForPhase injects instructions into the build related to Python
+// dependency installation.
+//
+// PhasePrivileged
+//
+// Ensures that the newest versions of setuptools, wheel, tox, and pip are
+// installed.
+//
+// PhasePreInstall
+//
+// Sets up Python wheels under the shared library directory (/opt/lib/python)
+// for dependencies found in the declared requirements files. Installing
+// dependencies during the build.PhasePreInstall phase allows a compiler
+// implementation (e.g. Docker) to produce cache-efficient output so only
+// changes to the given requirements files will invalidate these steps of the
+// image build.
+//
+// Injects build.Env instructions for PIP_WHEEL_DIR and PIP_FIND_LINKS that
+// will cause future executions of `pip install` (and by extension, `tox`) to
+// consider packages from the shared library directory first.
+//
+// PhasePostInstall
+//
+// Injects a build.Env instruction for PIP_NO_INDEX that will cause future
+// executions of `pip install` and `tox` to consider _only_ packages from the
+// shared library directory, helping to speed up image builds by reducing
+// network requests from said commands.
+//
+func (pc PythonConfig) InstructionsForPhase(phase build.Phase) []build.Instruction {
+	if pc.Requirements != nil || pc.Version != "" {
+		switch phase {
+		case build.PhasePrivileged:
+			return []build.Instruction{
+				build.RunAll{[]build.Run{
+					{pc.version(), []string{"-m", "easy_install", "pip"}},
+					{pc.version(), []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}},
+				}},
+			}
+
+		case build.PhasePreInstall:
+			envs := build.Env{map[string]string{
+				"PIP_WHEEL_DIR":  PythonLocalLibPrefix,
+				"PIP_FIND_LINKS": "file://" + PythonLocalLibPrefix,
+			}}
+
+			mkdirs := build.RunAll{
+				Runs: []build.Run{
+					build.CreateDirectory(PythonLocalLibPrefix),
+				},
+			}
+
+			dirs, bydir := pc.RequirementsByDir()
+			copies := make([]build.Instruction, len(dirs))
+
+			// make project subdirectories for requirements files if necessary, and
+			// copy in requirements files
+			for i, dir := range dirs {
+				if dir != "./" {
+					mkdirs.Runs = append(mkdirs.Runs, build.CreateDirectory(dir))
+				}
+
+				copies[i] = build.Copy{bydir[dir], dir}
+			}
+
+			ins := []build.Instruction{envs, mkdirs}
+			ins = append(ins, copies...)
+
+			if args := pc.RequirementsArgs(); len(args) > 0 {
+				ins = append(ins, build.Run{
+					pc.version(), append([]string{"-m", "pip", "wheel"}, args...),
+				})
+			}
+
+			return ins
+
+		case build.PhasePostInstall:
+			return []build.Instruction{
+				build.Env{map[string]string{
+					"PIP_NO_INDEX": "1",
+				}},
+			}
+		}
+	}
+
+	return []build.Instruction{}
+}
+
+// RequirementsArgs returns the configured requirements as pip `-r` arguments.
+//
+func (pc PythonConfig) RequirementsArgs() []string {
+	args := make([]string, len(pc.Requirements)*2)
+
+	for i, req := range pc.Requirements {
+		args[i*2] = "-r"
+		args[(i*2)+1] = req
+	}
+
+	return args
+}
+
+// RequirementsByDir returns both the configured requirements files indexed by
+// parent directory and a sorted slice of those parent directories. The latter
+// is useful in ensuring deterministic iteration since the ordering of map
+// keys is not guaranteed.
+//
+func (pc PythonConfig) RequirementsByDir() ([]string, map[string][]string) {
+	bydir := make(map[string][]string)
+
+	for _, reqpath := range pc.Requirements {
+		dir := path.Dir(reqpath) + "/"
+		reqpath = path.Clean(reqpath)
+
+		if reqs, found := bydir[dir]; found {
+			bydir[dir] = append(reqs, reqpath)
+		} else {
+			bydir[dir] = []string{reqpath}
+		}
+	}
+
+	dirs := make([]string, len(bydir))
+	i := 0
+
+	for dir := range bydir {
+		dirs[i] = dir
+		i++
+	}
+
+	sort.Strings(dirs)
+
+	return dirs, bydir
+}
+
+func (pc PythonConfig) version() string {
+	if pc.Version == "" {
+		return "python"
+	}
+
+	return pc.Version
+}
diff --git a/config/python_test.go b/config/python_test.go
new file mode 100644
index 0000000..5912ae2
--- /dev/null
+++ b/config/python_test.go
@@ -0,0 +1,245 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"phabricator.wikimedia.org/source/blubber/build"
+	"phabricator.wikimedia.org/source/blubber/config"
+)
+
+func TestPythonConfigUnmarshalMerge(t *testing.T) {
+	cfg, err := config.ReadConfig([]byte(`---
+    base: foo
+    python:
+      version: python2.7
+      requirements: [requirements.txt]
+    variants:
+      test:
+        python:
+          version: python3
+          requirements: [other-requirements.txt, requirements-test.txt]`))
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements)
+		assert.Equal(t, "python2.7", cfg.Python.Version)
+
+		variant, err := config.ExpandVariant(cfg, "test")
+
+		if assert.NoError(t, err) {
+			assert.Equal(t, []string{"other-requirements.txt", "requirements-test.txt"}, variant.Python.Requirements)
+			assert.Equal(t, "python3", variant.Python.Version)
+		}
+	}
+}
+
+func TestPythonConfigMergeEmpty(t *testing.T) {
+	cfg, err := config.ReadConfig([]byte(`---
+    base: foo
+    python:
+      requirements: [requirements.txt]
+    variants:
+      test:
+        python:
+          requirements: []`))
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements)
+
+		variant, err := config.ExpandVariant(cfg, "test")
+
+		if assert.NoError(t, err) {
+			assert.Equal(t, []string{}, variant.Python.Requirements)
+		}
+	}
+}
+
+func TestPythonConfigDoNotMergeNil(t *testing.T) {
+	cfg, err := config.ReadConfig([]byte(`---
+    base: foo
+    python:
+      requirements: [requirements.txt]
+    variants:
+      test:
+        python:
+          requirements: ~`))
+
+	if assert.NoError(t, err) {
+		assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements)
+
+		variant, err := config.ExpandVariant(cfg, "test")
+
+		if assert.NoError(t, err) {
+			assert.Equal(t, []string{"requirements.txt"}, variant.Python.Requirements)
+		}
+	}
+}
+
+func TestPythonConfigInstructionsNoRequirementsWithVersion(t *testing.T) {
+	cfg := config.PythonConfig{
+		Version: "python2.7",
+	}
+
+	t.Run("PhasePrivileged", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.RunAll{[]build.Run{
+					{"python2.7", []string{"-m", "easy_install", "pip"}},
+					{"python2.7", []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}},
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePrivileged),
+		)
+	})
+
+	t.Run("PhasePrivilegeDropped", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped))
+	})
+
+	t.Run("PhasePreInstall", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.Env{map[string]string{
+					"PIP_WHEEL_DIR":  "/opt/lib/python",
+					"PIP_FIND_LINKS": "file:///opt/lib/python",
+				}},
+				build.RunAll{[]build.Run{
+					{"mkdir -p", []string{"/opt/lib/python"}},
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePreInstall),
+		)
+	})
+
+	t.Run("PhasePostInstall", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.Env{map[string]string{
+					"PIP_NO_INDEX": "1",
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePostInstall),
+		)
+	})
+}
+
+func TestPythonConfigInstructionsNoRequirementsNoVersion(t *testing.T) {
+	cfg := config.PythonConfig{}
+
+	t.Run("PhasePrivileged", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivileged))
+	})
+
+	t.Run("PhasePrivilegeDropped", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped))
+	})
+
+	t.Run("PhasePreInstall", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePreInstall))
+	})
+
+	t.Run("PhasePostInstall", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePostInstall))
+	})
+}
+
+func TestPythonConfigInstructionsWithRequirements(t *testing.T) {
+	cfg := config.PythonConfig{
+		Version:      "python2.7",
+		Requirements: []string{"requirements.txt", "requirements-test.txt", "docs/requirements.txt"},
+	}
+
+	t.Run("PhasePrivileged", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.RunAll{[]build.Run{
+					{"python2.7", []string{"-m", "easy_install", "pip"}},
+					{"python2.7", []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}},
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePrivileged),
+		)
+	})
+
+	t.Run("PhasePrivilegeDropped", func(t *testing.T) {
+		assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped))
+	})
+
+	t.Run("PhasePreInstall", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.Env{map[string]string{
+					"PIP_WHEEL_DIR":  "/opt/lib/python",
+					"PIP_FIND_LINKS": "file:///opt/lib/python",
+				}},
+				build.RunAll{[]build.Run{
+					{"mkdir -p", []string{"/opt/lib/python"}},
+					{"mkdir -p", []string{"docs/"}},
+				}},
+				build.Copy{[]string{"requirements.txt", "requirements-test.txt"}, "./"},
+				build.Copy{[]string{"docs/requirements.txt"}, "docs/"},
+				build.Run{"python2.7", []string{"-m", "pip", "wheel",
+					"-r", "requirements.txt",
+					"-r", "requirements-test.txt",
+					"-r", "docs/requirements.txt",
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePreInstall),
+		)
+	})
+
+	t.Run("PhasePostInstall", func(t *testing.T) {
+		assert.Equal(t,
+			[]build.Instruction{
+				build.Env{map[string]string{
+					"PIP_NO_INDEX": "1",
+				}},
+			},
+			cfg.InstructionsForPhase(build.PhasePostInstall),
+		)
+	})
+}
+
+func TestPythonConfigRequirementsByDir(t *testing.T) {
+	cfg := config.PythonConfig{
+		Requirements: []string{"foo", "./bar", "./c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"},
+	}
+
+	sortedDirs, reqsByDir := cfg.RequirementsByDir()
+
+	assert.Equal(t,
+		[]string{
+			"./",
+			"a/",
+			"b/",
+			"c/",
+		},
+		sortedDirs,
+	)
+
+	assert.Equal(t,
+		map[string][]string{
+			"./": []string{"foo", "bar"},
+			"c/": []string{"c/c-foo"},
+			"b/": []string{"b/b-foo", "b/b-bar"},
+			"a/": []string{"a/a-foo"},
+		},
+		reqsByDir,
+	)
+}
+
+func TestPythonConfigRequirementsArgs(t *testing.T) {
+	cfg := config.PythonConfig{
+		Requirements: []string{"foo", "bar", "baz/qux"},
+	}
+
+	assert.Equal(t,
+		[]string{
+			"-r", "foo",
+			"-r", "bar",
+			"-r", "baz/qux",
+		},
+		cfg.RequirementsArgs(),
+	)
+}