Smarter copies/sharedvolume/default behavior

Summary:
Defined new abstract `build.Volume` and corresponding
`docker.DockerVolume` instructions.

Refactored compilation of main `COPY` or `VOLUME` instruction for
application files to use the new instructions and moved injection of
these instructions out of the compiler and into `VariantConfig`. The
latter can be smarter about the following cases:

 1. When `copies` is set, simply depend on artifacts for the application
    files and do not copy anything from the build host.
 2. When `sharedvolume` is `true`, inject a `build.Volume` instruction
    for the application working directory.
 3. When neither of the above are set, copy application files from the
    host.

Fixes T174623
Depends on D768

Test Plan:
Run `go test ./...`. Run `blubber blubber.example.yaml production` and ensure:

 1. The `prep` stage has a `COPY . .` instruction.
 2. The final stage has no `COPY . .` instruction, only `COPY --from=prep`
    instructions.

Reviewers: thcipriani, mobrovac, hashar, mmodell, #release-engineering-team

Reviewed By: thcipriani, mobrovac, #release-engineering-team

Tags: #release-engineering-team

Maniphest Tasks: T174623

Differential Revision: https://phabricator.wikimedia.org/D769

10 files changed, 133 insertions, 25 deletions

diff --git a/README.md b/README.md
index b8018a7..c7e3e1f 100644
--- a/README.md
+++ b/README.md
@@ -40,11 +40,16 @@ variants:
       packages: [chromium]
     entrypoint: [npm, test]
 
+  prep:
+    includes: [build]
+    node:
+      env: production
+
   production:
     base: debian:jessie-slim
     node:
       env: production
-    copies: test
+    copies: prep
     entrypoint: [node, server.js]
 ```
 
diff --git a/blubber.example.yaml b/blubber.example.yaml
index 87a4cf1..894472f 100644
--- a/blubber.example.yaml
+++ b/blubber.example.yaml
@@ -28,9 +28,14 @@ variants:
       packages: [chromium]
     entrypoint: [npm, test]
 
+  prep:
+    includes: [build]
+    node:
+      env: production
+
   production:
     base: debian:jessie-slim
     node:
       env: production
-    copies: test
+    copies: prep
     entrypoint: [node, server.js]
diff --git a/build/instructions.go b/build/instructions.go
index 1b954a2..38ea322 100644
--- a/build/instructions.go
+++ b/build/instructions.go
@@ -80,6 +80,14 @@ func (env Env) Compile() []string {
 	return defs
 }
 
+type Volume struct {
+	Path string
+}
+
+func (vol Volume) Compile() []string {
+	return []string{quote(vol.Path)}
+}
+
 func quote(arg string) string {
 	return strconv.Quote(arg)
 }
diff --git a/build/instructions_test.go b/build/instructions_test.go
index 8f9470c..04ebdfb 100644
--- a/build/instructions_test.go
+++ b/build/instructions_test.go
@@ -55,3 +55,9 @@ func TestEnv(t *testing.T) {
 		`quxname="quxvalue"`,
 	}, i.Compile())
 }
+
+func TestVolume(t *testing.T) {
+	i := build.Volume{"/foo/dir"}
+
+	assert.Equal(t, []string{`"/foo/dir"`}, i.Compile())
+}
diff --git a/build/phases.go b/build/phases.go
index 095263c..02e84e4 100644
--- a/build/phases.go
+++ b/build/phases.go
@@ -6,6 +6,7 @@ const (
 	PhasePrivileged Phase = iota
 	PhasePrivilegeDropped
 	PhasePreInstall
+	PhaseInstall
 	PhasePostInstall
 )
 
diff --git a/config/variant.go b/config/variant.go
index cc2cbdb..dea45af 100644
--- a/config/variant.go
+++ b/config/variant.go
@@ -25,7 +25,20 @@ func (vc *VariantConfig) InstructionsForPhase(phase build.Phase) []build.Instruc
 		ainstructions = append(ainstructions, artifact.InstructionsForPhase(phase)...)
 	}
 
-	return append(ainstructions, instructions...)
+	instructions = append(ainstructions, instructions...)
+
+	switch phase {
+	case build.PhaseInstall:
+		if vc.Copies == "" {
+			if vc.SharedVolume.True {
+				instructions = append(instructions, build.Volume{vc.Runs.In})
+			} else {
+				instructions = append(instructions, build.Copy{[]string{"."}, "."})
+			}
+		}
+	}
+
+	return instructions
 }
 
 func (vc *VariantConfig) VariantDependencies() []string {
@@ -52,8 +65,8 @@ func (vc *VariantConfig) defaultArtifacts() []ArtifactsConfig {
 		return []ArtifactsConfig{
 			{
 				From:        vc.Copies,
-				Source:      vc.CommonConfig.Runs.In,
-				Destination: vc.CommonConfig.Runs.In,
+				Source:      vc.Runs.In,
+				Destination: vc.Runs.In,
 			},
 			{
 				From:        vc.Copies,
diff --git a/config/variant_test.go b/config/variant_test.go
index 0a98828..666b4d7 100644
--- a/config/variant_test.go
+++ b/config/variant_test.go
@@ -45,22 +45,72 @@ func TestVariantDependencies(t *testing.T) {
 }
 
 func TestVariantConfigInstructions(t *testing.T) {
-	cfg := config.VariantConfig{
-		CommonConfig: config.CommonConfig{Runs: config.RunsConfig{In: "/srv/service"}},
-		Copies:       "foo",
-		Artifacts: []config.ArtifactsConfig{
-			{From: "build", Source: "/foo/src", Destination: "/foo/dst"},
-		},
-	}
+	t.Run("PhaseInstall", func(t *testing.T) {
+		t.Run("copies", func(t *testing.T) {
+			cfg := config.VariantConfig{Copies: "foo"}
+
+			assert.Empty(t, cfg.InstructionsForPhase(build.PhaseInstall))
+		})
+
+		t.Run("shared volume", func(t *testing.T) {
+			cfg := config.VariantConfig{}
+			cfg.Runs.In = "/srv/service"
+			cfg.SharedVolume.True = true
+
+			assert.Equal(t,
+				[]build.Instruction{
+					build.Volume{"/srv/service"},
+				},
+				cfg.InstructionsForPhase(build.PhaseInstall),
+			)
+		})
+
+		t.Run("standard source copy", func(t *testing.T) {
+			cfg := config.VariantConfig{}
+
+			assert.Equal(t,
+				[]build.Instruction{
+					build.Copy{[]string{"."}, "."},
+				},
+				cfg.InstructionsForPhase(build.PhaseInstall),
+			)
+		})
+	})
 
 	t.Run("PhasePostInstall", func(t *testing.T) {
-		assert.Equal(t,
-			[]build.Instruction{
-				build.CopyFrom{"foo", build.Copy{[]string{"/srv/service"}, "/srv/service"}},
-				build.CopyFrom{"foo", build.Copy{[]string{config.LocalLibPrefix}, config.LocalLibPrefix}},
-				build.CopyFrom{"build", build.Copy{[]string{"/foo/src"}, "/foo/dst"}},
-			},
-			cfg.InstructionsForPhase(build.PhasePostInstall),
-		)
+		t.Run("for copies and artifacts", func(t *testing.T) {
+			cfg := config.VariantConfig{
+				Copies: "foo",
+				Artifacts: []config.ArtifactsConfig{
+					{From: "build", Source: "/foo/src", Destination: "/foo/dst"},
+				},
+				CommonConfig: config.CommonConfig{Runs: config.RunsConfig{In: "/srv/service"}},
+			}
+
+			assert.Equal(t,
+				[]build.Instruction{
+					build.CopyFrom{"foo", build.Copy{[]string{"/srv/service"}, "/srv/service"}},
+					build.CopyFrom{"foo", build.Copy{[]string{config.LocalLibPrefix}, config.LocalLibPrefix}},
+					build.CopyFrom{"build", build.Copy{[]string{"/foo/src"}, "/foo/dst"}},
+				},
+				cfg.InstructionsForPhase(build.PhasePostInstall),
+			)
+		})
+
+		t.Run("for just artifacts", func(t *testing.T) {
+			cfg := config.VariantConfig{
+				Artifacts: []config.ArtifactsConfig{
+					{From: "build", Source: "/foo/src", Destination: "/foo/dst"},
+				},
+				CommonConfig: config.CommonConfig{Runs: config.RunsConfig{In: "/srv/service"}},
+			}
+
+			assert.Equal(t,
+				[]build.Instruction{
+					build.CopyFrom{"build", build.Copy{[]string{"/foo/src"}, "/foo/dst"}},
+				},
+				cfg.InstructionsForPhase(build.PhasePostInstall),
+			)
+		})
 	})
 }
diff --git a/docker/compiler.go b/docker/compiler.go
index 2891140..c9ca0fb 100644
--- a/docker/compiler.go
+++ b/docker/compiler.go
@@ -58,11 +58,7 @@ func CompileStage(buffer *bytes.Buffer, stage string, vcfg *config.VariantConfig
 
 	CompilePhase(buffer, vcfg, build.PhasePreInstall)
 
-	if vcfg.SharedVolume.True {
-		Writeln(buffer, "VOLUME [\"", vcfg.Runs.In, "\"]")
-	} else {
-		Writeln(buffer, "COPY . .")
-	}
+	CompilePhase(buffer, vcfg, build.PhaseInstall)
 
 	CompilePhase(buffer, vcfg, build.PhasePostInstall)
 
diff --git a/docker/instructions.go b/docker/instructions.go
index 2b00c1b..4edcb23 100644
--- a/docker/instructions.go
+++ b/docker/instructions.go
@@ -26,6 +26,10 @@ func NewDockerInstruction(instruction build.Instruction) (DockerInstruction, err
 		var dockerInstruction DockerEnv
 		dockerInstruction.arguments = instruction.Compile()
 		return dockerInstruction, nil
+	case build.Volume:
+		var dockerInstruction DockerVolume
+		dockerInstruction.arguments = instruction.Compile()
+		return dockerInstruction, nil
 	}
 
 	return nil, errors.New("Unable to create DockerInstruction")
@@ -77,6 +81,14 @@ func (de DockerEnv) Compile() string {
 		join(de.arguments, " "))
 }
 
+type DockerVolume struct{ abstractDockerInstruction }
+
+func (dv DockerVolume) Compile() string {
+	return fmt.Sprintf(
+		"VOLUME [%s]\n",
+		join(dv.arguments, ", "))
+}
+
 func join(arguments []string, delimiter string) string {
 	return removeNewlines(strings.Join(arguments, delimiter))
 }
diff --git a/docker/instructions_test.go b/docker/instructions_test.go
index fb9116b..10cc1ab 100644
--- a/docker/instructions_test.go
+++ b/docker/instructions_test.go
@@ -70,6 +70,18 @@ func TestEnv(t *testing.T) {
 	assert.Equal(t, "ENV bar=\"foo\" foo=\"bar\"\n", di.Compile())
 }
 
+func TestVolume(t *testing.T) {
+	i := build.Volume{"/foo/dir"}
+
+	di, err := docker.NewDockerInstruction(i)
+
+	var dockerVolume docker.DockerVolume
+
+	assert.Nil(t, err)
+	assert.IsType(t, dockerVolume, di)
+	assert.Equal(t, "VOLUME [\"/foo/dir\"]\n", di.Compile())
+}
+
 func TestEscapeRun(t *testing.T) {
 	i := build.Run{"/bin/true\nRUN echo HACKED!", []string{}}
 	dr, _ := docker.NewDockerInstruction(i)