diff options
| -rw-r--r-- | blubber.example.yaml | 6 | ||||
| -rw-r--r-- | config/common.go | 18 | ||||
| -rw-r--r-- | config/python.go | 173 | ||||
| -rw-r--r-- | config/python_test.go | 245 |
4 files changed, 434 insertions, 8 deletions
diff --git a/blubber.example.yaml b/blubber.example.yaml index 07e1a86..5dc7233 100644 --- a/blubber.example.yaml +++ b/blubber.example.yaml @@ -2,6 +2,8 @@ base: debian:jessie apt: packages: [libjpeg, libyaml] +python: + version: python2.7 runs: environment: FOO: bar @@ -13,6 +15,8 @@ variants: packages: [libjpeg-dev, libyaml-dev] node: dependencies: true + python: + requirements: [requirements.txt] development: includes: [build] @@ -22,6 +26,8 @@ variants: includes: [build] apt: packages: [chromium] + python: + requirements: [requirements.txt, test-requirements.txt, docs/requirements.txt] entrypoint: [npm, test] prep: diff --git a/config/common.go b/config/common.go index fa100b8..a597a82 100644 --- a/config/common.go +++ b/config/common.go @@ -8,13 +8,14 @@ import ( // and each configured variant. // type CommonConfig struct { - Base string `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image - Apt AptConfig `yaml:"apt"` // APT related configuration - Node NodeConfig `yaml:"node"` // Node related configuration - Lives LivesConfig `yaml:"lives"` // application owner/dir configuration - Runs RunsConfig `yaml:"runs"` // runtime environment configuration - SharedVolume Flag `yaml:"sharedvolume"` // define a volume for application - EntryPoint []string `yaml:"entrypoint"` // entry-point executable + Base string `yaml:"base" validate:"omitempty,baseimage"` // name/path to base image + Apt AptConfig `yaml:"apt"` // APT related configuration + Node NodeConfig `yaml:"node"` // Node related configuration + Python PythonConfig `yaml:"python"` // Python related configuration + Lives LivesConfig `yaml:"lives"` // application owner/dir configuration + Runs RunsConfig `yaml:"runs"` // runtime environment configuration + SharedVolume Flag `yaml:"sharedvolume"` // define a volume for application + EntryPoint []string `yaml:"entrypoint"` // entry-point executable } // Merge takes another CommonConfig and merges its fields this one's. @@ -26,6 +27,7 @@ func (cc *CommonConfig) Merge(cc2 CommonConfig) { cc.Apt.Merge(cc2.Apt) cc.Node.Merge(cc2.Node) + cc.Python.Merge(cc2.Python) cc.Lives.Merge(cc2.Lives) cc.Runs.Merge(cc2.Runs) cc.SharedVolume.Merge(cc2.SharedVolume) @@ -40,7 +42,7 @@ func (cc *CommonConfig) Merge(cc2 CommonConfig) { // injected. // func (cc *CommonConfig) PhaseCompileableConfig() []build.PhaseCompileable { - return []build.PhaseCompileable{cc.Apt, cc.Node, cc.Lives, cc.Runs} + return []build.PhaseCompileable{cc.Apt, cc.Node, cc.Python, cc.Lives, cc.Runs} } // InstructionsForPhase injects instructions into the given build phase for diff --git a/config/python.go b/config/python.go new file mode 100644 index 0000000..4e2f2ce --- /dev/null +++ b/config/python.go @@ -0,0 +1,173 @@ +package config + +import ( + "path" + "sort" + + "phabricator.wikimedia.org/source/blubber/build" +) + +// PythonLocalLibPrefix is the path to installed dependency wheels. +// +const PythonLocalLibPrefix = LocalLibPrefix + "/python" + +// PythonConfig holds configuration fields related to pre-installation of project +// dependencies via PIP. +// +type PythonConfig struct { + Version string `yaml:"version"` // Python binary to use when installing dependencies + Requirements []string `yaml:"requirements"` // install requirements from given files +} + +// Merge takes another PythonConfig and merges its fields into this one's, +// overwriting both the dependencies flag and requirements. +// +func (pc *PythonConfig) Merge(pc2 PythonConfig) { + if pc2.Version != "" { + pc.Version = pc2.Version + } + + if pc2.Requirements != nil { + pc.Requirements = pc2.Requirements + } +} + +// InstructionsForPhase injects instructions into the build related to Python +// dependency installation. +// +// PhasePrivileged +// +// Ensures that the newest versions of setuptools, wheel, tox, and pip are +// installed. +// +// PhasePreInstall +// +// Sets up Python wheels under the shared library directory (/opt/lib/python) +// for dependencies found in the declared requirements files. Installing +// dependencies during the build.PhasePreInstall phase allows a compiler +// implementation (e.g. Docker) to produce cache-efficient output so only +// changes to the given requirements files will invalidate these steps of the +// image build. +// +// Injects build.Env instructions for PIP_WHEEL_DIR and PIP_FIND_LINKS that +// will cause future executions of `pip install` (and by extension, `tox`) to +// consider packages from the shared library directory first. +// +// PhasePostInstall +// +// Injects a build.Env instruction for PIP_NO_INDEX that will cause future +// executions of `pip install` and `tox` to consider _only_ packages from the +// shared library directory, helping to speed up image builds by reducing +// network requests from said commands. +// +func (pc PythonConfig) InstructionsForPhase(phase build.Phase) []build.Instruction { + if pc.Requirements != nil || pc.Version != "" { + switch phase { + case build.PhasePrivileged: + return []build.Instruction{ + build.RunAll{[]build.Run{ + {pc.version(), []string{"-m", "easy_install", "pip"}}, + {pc.version(), []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}}, + }}, + } + + case build.PhasePreInstall: + envs := build.Env{map[string]string{ + "PIP_WHEEL_DIR": PythonLocalLibPrefix, + "PIP_FIND_LINKS": "file://" + PythonLocalLibPrefix, + }} + + mkdirs := build.RunAll{ + Runs: []build.Run{ + build.CreateDirectory(PythonLocalLibPrefix), + }, + } + + dirs, bydir := pc.RequirementsByDir() + copies := make([]build.Instruction, len(dirs)) + + // make project subdirectories for requirements files if necessary, and + // copy in requirements files + for i, dir := range dirs { + if dir != "./" { + mkdirs.Runs = append(mkdirs.Runs, build.CreateDirectory(dir)) + } + + copies[i] = build.Copy{bydir[dir], dir} + } + + ins := []build.Instruction{envs, mkdirs} + ins = append(ins, copies...) + + if args := pc.RequirementsArgs(); len(args) > 0 { + ins = append(ins, build.Run{ + pc.version(), append([]string{"-m", "pip", "wheel"}, args...), + }) + } + + return ins + + case build.PhasePostInstall: + return []build.Instruction{ + build.Env{map[string]string{ + "PIP_NO_INDEX": "1", + }}, + } + } + } + + return []build.Instruction{} +} + +// RequirementsArgs returns the configured requirements as pip `-r` arguments. +// +func (pc PythonConfig) RequirementsArgs() []string { + args := make([]string, len(pc.Requirements)*2) + + for i, req := range pc.Requirements { + args[i*2] = "-r" + args[(i*2)+1] = req + } + + return args +} + +// RequirementsByDir returns both the configured requirements files indexed by +// parent directory and a sorted slice of those parent directories. The latter +// is useful in ensuring deterministic iteration since the ordering of map +// keys is not guaranteed. +// +func (pc PythonConfig) RequirementsByDir() ([]string, map[string][]string) { + bydir := make(map[string][]string) + + for _, reqpath := range pc.Requirements { + dir := path.Dir(reqpath) + "/" + reqpath = path.Clean(reqpath) + + if reqs, found := bydir[dir]; found { + bydir[dir] = append(reqs, reqpath) + } else { + bydir[dir] = []string{reqpath} + } + } + + dirs := make([]string, len(bydir)) + i := 0 + + for dir := range bydir { + dirs[i] = dir + i++ + } + + sort.Strings(dirs) + + return dirs, bydir +} + +func (pc PythonConfig) version() string { + if pc.Version == "" { + return "python" + } + + return pc.Version +} diff --git a/config/python_test.go b/config/python_test.go new file mode 100644 index 0000000..5912ae2 --- /dev/null +++ b/config/python_test.go @@ -0,0 +1,245 @@ +package config_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "phabricator.wikimedia.org/source/blubber/build" + "phabricator.wikimedia.org/source/blubber/config" +) + +func TestPythonConfigUnmarshalMerge(t *testing.T) { + cfg, err := config.ReadConfig([]byte(`--- + base: foo + python: + version: python2.7 + requirements: [requirements.txt] + variants: + test: + python: + version: python3 + requirements: [other-requirements.txt, requirements-test.txt]`)) + + if assert.NoError(t, err) { + assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements) + assert.Equal(t, "python2.7", cfg.Python.Version) + + variant, err := config.ExpandVariant(cfg, "test") + + if assert.NoError(t, err) { + assert.Equal(t, []string{"other-requirements.txt", "requirements-test.txt"}, variant.Python.Requirements) + assert.Equal(t, "python3", variant.Python.Version) + } + } +} + +func TestPythonConfigMergeEmpty(t *testing.T) { + cfg, err := config.ReadConfig([]byte(`--- + base: foo + python: + requirements: [requirements.txt] + variants: + test: + python: + requirements: []`)) + + if assert.NoError(t, err) { + assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements) + + variant, err := config.ExpandVariant(cfg, "test") + + if assert.NoError(t, err) { + assert.Equal(t, []string{}, variant.Python.Requirements) + } + } +} + +func TestPythonConfigDoNotMergeNil(t *testing.T) { + cfg, err := config.ReadConfig([]byte(`--- + base: foo + python: + requirements: [requirements.txt] + variants: + test: + python: + requirements: ~`)) + + if assert.NoError(t, err) { + assert.Equal(t, []string{"requirements.txt"}, cfg.Python.Requirements) + + variant, err := config.ExpandVariant(cfg, "test") + + if assert.NoError(t, err) { + assert.Equal(t, []string{"requirements.txt"}, variant.Python.Requirements) + } + } +} + +func TestPythonConfigInstructionsNoRequirementsWithVersion(t *testing.T) { + cfg := config.PythonConfig{ + Version: "python2.7", + } + + t.Run("PhasePrivileged", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.RunAll{[]build.Run{ + {"python2.7", []string{"-m", "easy_install", "pip"}}, + {"python2.7", []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}}, + }}, + }, + cfg.InstructionsForPhase(build.PhasePrivileged), + ) + }) + + t.Run("PhasePrivilegeDropped", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped)) + }) + + t.Run("PhasePreInstall", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.Env{map[string]string{ + "PIP_WHEEL_DIR": "/opt/lib/python", + "PIP_FIND_LINKS": "file:///opt/lib/python", + }}, + build.RunAll{[]build.Run{ + {"mkdir -p", []string{"/opt/lib/python"}}, + }}, + }, + cfg.InstructionsForPhase(build.PhasePreInstall), + ) + }) + + t.Run("PhasePostInstall", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.Env{map[string]string{ + "PIP_NO_INDEX": "1", + }}, + }, + cfg.InstructionsForPhase(build.PhasePostInstall), + ) + }) +} + +func TestPythonConfigInstructionsNoRequirementsNoVersion(t *testing.T) { + cfg := config.PythonConfig{} + + t.Run("PhasePrivileged", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivileged)) + }) + + t.Run("PhasePrivilegeDropped", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped)) + }) + + t.Run("PhasePreInstall", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePreInstall)) + }) + + t.Run("PhasePostInstall", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePostInstall)) + }) +} + +func TestPythonConfigInstructionsWithRequirements(t *testing.T) { + cfg := config.PythonConfig{ + Version: "python2.7", + Requirements: []string{"requirements.txt", "requirements-test.txt", "docs/requirements.txt"}, + } + + t.Run("PhasePrivileged", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.RunAll{[]build.Run{ + {"python2.7", []string{"-m", "easy_install", "pip"}}, + {"python2.7", []string{"-m", "pip", "install", "-U", "setuptools", "wheel", "tox"}}, + }}, + }, + cfg.InstructionsForPhase(build.PhasePrivileged), + ) + }) + + t.Run("PhasePrivilegeDropped", func(t *testing.T) { + assert.Empty(t, cfg.InstructionsForPhase(build.PhasePrivilegeDropped)) + }) + + t.Run("PhasePreInstall", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.Env{map[string]string{ + "PIP_WHEEL_DIR": "/opt/lib/python", + "PIP_FIND_LINKS": "file:///opt/lib/python", + }}, + build.RunAll{[]build.Run{ + {"mkdir -p", []string{"/opt/lib/python"}}, + {"mkdir -p", []string{"docs/"}}, + }}, + build.Copy{[]string{"requirements.txt", "requirements-test.txt"}, "./"}, + build.Copy{[]string{"docs/requirements.txt"}, "docs/"}, + build.Run{"python2.7", []string{"-m", "pip", "wheel", + "-r", "requirements.txt", + "-r", "requirements-test.txt", + "-r", "docs/requirements.txt", + }}, + }, + cfg.InstructionsForPhase(build.PhasePreInstall), + ) + }) + + t.Run("PhasePostInstall", func(t *testing.T) { + assert.Equal(t, + []build.Instruction{ + build.Env{map[string]string{ + "PIP_NO_INDEX": "1", + }}, + }, + cfg.InstructionsForPhase(build.PhasePostInstall), + ) + }) +} + +func TestPythonConfigRequirementsByDir(t *testing.T) { + cfg := config.PythonConfig{ + Requirements: []string{"foo", "./bar", "./c/c-foo", "b/b-foo", "b/b-bar", "a/a-foo"}, + } + + sortedDirs, reqsByDir := cfg.RequirementsByDir() + + assert.Equal(t, + []string{ + "./", + "a/", + "b/", + "c/", + }, + sortedDirs, + ) + + assert.Equal(t, + map[string][]string{ + "./": []string{"foo", "bar"}, + "c/": []string{"c/c-foo"}, + "b/": []string{"b/b-foo", "b/b-bar"}, + "a/": []string{"a/a-foo"}, + }, + reqsByDir, + ) +} + +func TestPythonConfigRequirementsArgs(t *testing.T) { + cfg := config.PythonConfig{ + Requirements: []string{"foo", "bar", "baz/qux"}, + } + + assert.Equal(t, + []string{ + "-r", "foo", + "-r", "bar", + "-r", "baz/qux", + }, + cfg.RequirementsArgs(), + ) +} |