diff options
| -rw-r--r-- | Makefile | 8 | ||||
| -rw-r--r-- | api/openapi-spec/blubberoid.yaml | 217 | ||||
| -rw-r--r-- | cmd/blubberoid/main.go | 90 | ||||
| -rw-r--r-- | cmd/blubberoid/main_test.go | 10 | ||||
| -rw-r--r-- | cmd/blubberoid/openapi.go | 222 | ||||
| -rw-r--r-- | cmd/blubberoid/openapi_test.go | 16 | ||||
| -rwxr-xr-x | scripts/generate-const.sh | 11 |
7 files changed, 544 insertions, 30 deletions
@@ -16,10 +16,11 @@ GO_LDFLAGS := \ # # workaround bug in case CURDIR is a symlink # see https://github.com/golang/go/issues/24359 +GO_GENERATE := cd "$(REAL_CURDIR)" && go generate GO_BUILD := cd "$(REAL_CURDIR)" && go build -v -ldflags "$(GO_LDFLAGS)" GO_INSTALL := cd "$(REAL_CURDIR)" && go install -v -ldflags "$(GO_LDFLAGS)" -all: blubber blubberoid +all: code blubber blubberoid blubber: $(GO_BUILD) ./cmd/blubber @@ -27,11 +28,14 @@ blubber: blubberoid: $(GO_BUILD) ./cmd/blubberoid +code: + $(GO_GENERATE) $(GO_PACKAGES) + clean: go clean rm -f blubber blubberoid -install: +install: all $(GO_INSTALL) $(GO_PACKAGES) release: diff --git a/api/openapi-spec/blubberoid.yaml b/api/openapi-spec/blubberoid.yaml new file mode 100644 index 0000000..b0ee9ae --- /dev/null +++ b/api/openapi-spec/blubberoid.yaml @@ -0,0 +1,217 @@ +--- +openapi: '3.0.0' +info: + title: Blubberoid + description: > + Blubber is a highly opinionated abstraction for container build + configurations. + version: {{ .Version }} +paths: + /v1/{variant}: + post: + summary: > + Generates a valid Dockerfile based on Blubber YAML configuration + provided in the request body and the given variant name. + requestBody: + description: A valid Blubber configuration. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/v3.Config' + application/yaml: + schema: + type: string + application/x-yaml: + schema: + type: string + responses: + '200': + description: OK. Response body should be a valid Dockerfile. + content: + text/plain: + schema: + type: string + '400': + description: Bad request. The YAML request body failed to parse. + '404': + description: No variant name was provided in the request path. + '422': + description: Provided Blubber config parsed correctly but failed validation. + '5XX': + description: An unexpected service-side error. + + x-amples: + - title: Mathoid test variant + request: + params: + variant: test + headers: + Content-Type: application/json + body: { + "version": "v3", + "base": "docker-registry.wikimedia.org/nodejs-slim", + "apt": { "packages": ["librsvg2-2"] }, + "lives": { "in": "/srv/service" }, + "variants": { + "build": { + "base": "docker-registry.wikimedia.org/nodejs-devel", + "apt": { + "packages": ["librsvg2-dev", "git", "pkg-config", "build-essential"] + }, + "node": { "requirements": ["package.json"] }, + "runs": { "environment": { "LINK": "g++" } } + }, + "test": { "includes": ["build"], "entrypoint": ["npm", "test"] } + } + } + response: + status: 200 + headers: + content-type: text/plain + body: /^FROM docker-registry.wikimedia.org\/nodejs-devel/ + +components: + schemas: + v3.Config: + title: Top-level blubber configuration (version v3) + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + required: [version, variants] + version: + type: string + description: Blubber configuration version + variants: + type: object + description: Configuration variants (e.g. development, test, production) + additionalProperties: true + # OpenAPI 3.0 supports only v4 of the JSON Schema draft spec and + # cannot define schema for object properties with arbitrary + # names, but the following commented section is included to be + # useful to humans for the time being. It patiently awaits v6 + # json schema draft support before its uncommenting. + # + # patternProperties: + # "^[a-zA-Z][a-zA-Z0-9\-\.]+[a-zA-Z0-9]$": + # $ref: '#/components/schemas/v3.VariantConfig' + + v3.CommonConfig: + type: object + properties: + base: + type: string + description: Base image reference + apt: + type: object + properties: + packages: + type: array + description: Packages to install from APT sources of base image + items: + type: string + node: + type: object + properties: + env: + type: string + description: Node environment (e.g. production, etc.) + requirements: + type: array + description: Files needed for Node package installation (e.g. package.json, package-lock.json) + items: + type: string + python: + type: object + properties: + version: + type: string + description: Python binary present in the system (e.g. python3) + requirements: + type: array + description: Files needed for Python package installation (e.g. requirements.txt, etc.) + items: + type: string + builder: + type: object + properties: + command: + type: array + description: Command and arguments of an arbitrary build command + items: + type: string + requirements: + type: array + description: Files needed by the build command (e.g. Makefile, ./src/, etc.) + items: + type: string + lives: + type: object + properties: + as: + type: string + description: Owner (name) of application files within the container + uid: + type: integer + description: Owner (UID) of application files within the container + gid: + type: integer + description: Group owner (GID) of application files within the container + in: + type: string + description: Application working directory within the container + runs: + type: object + properties: + as: + type: string + description: Runtime process owner (name) of application entrypoint + uid: + type: integer + description: Runtime process owner (UID) of application entrypoint + gid: + type: integer + description: Runtime process group (GID) of application entrypoint + environment: + type: object + description: Environment variables and values to be set before entrypoint execution + additionalProperties: true + insecurely: + type: boolean + description: Skip dropping of priviledge to the runtime process owner before entrypoint execution + entrypoint: + type: array + description: Runtime entry point command and arguments + items: + type: string + v3.VariantConfig: + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + includes: + type: array + description: Names of other variants to inherit configuration from + items: + description: Variant name + type: string + copies: + type: string + description: Name of variant from which to copy application files, resulting in a multi-stage build + artifacts: + type: array + items: + type: object + description: Artifacts to copy from another variant, resulting in a multi-stage build + required: [from, source, destination] + properties: + from: + type: string + description: Variant name + source: + type: string + description: Path of files/directories to copy + destination: + type: string + description: Destination path diff --git a/cmd/blubberoid/main.go b/cmd/blubberoid/main.go index d436390..19604c1 100644 --- a/cmd/blubberoid/main.go +++ b/cmd/blubberoid/main.go @@ -3,27 +3,32 @@ package main import ( - "encoding/json" + "bytes" "fmt" "io/ioutil" "log" "mime" "net/http" + "net/url" "os" "path" + "strings" + "text/template" "github.com/pborman/getopt/v2" "gerrit.wikimedia.org/r/blubber/config" "gerrit.wikimedia.org/r/blubber/docker" + "gerrit.wikimedia.org/r/blubber/meta" ) var ( - showHelp = getopt.BoolLong("help", 'h', "show help/usage") - address = getopt.StringLong("address", 'a', ":8748", "socket address/port to listen on (default ':8748')", "address:port") - endpoint = getopt.StringLong("endpoint", 'e', "/", "server endpoint (default '/')", "path") - policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri") - policy *config.Policy + showHelp = getopt.BoolLong("help", 'h', "show help/usage") + address = getopt.StringLong("address", 'a', ":8748", "socket address/port to listen on (default ':8748')", "address:port") + endpoint = getopt.StringLong("endpoint", 'e', "/", "server endpoint (default '/')", "path") + policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri") + policy *config.Policy + openAPISpec []byte ) func main() { @@ -51,7 +56,10 @@ func main() { *endpoint += "/" } - log.Printf("listening on %s for requests to %s[variant]\n", *address, *endpoint) + // Evaluate OpenAPI spec template and store results for ?spec requests + openAPISpec = readOpenAPISpec() + + log.Printf("listening on %s for requests to %sv1/[variant]\n", *address, *endpoint) http.HandleFunc(*endpoint, blubberoid) log.Fatal(http.ListenAndServe(*address, nil)) @@ -59,12 +67,35 @@ func main() { func blubberoid(res http.ResponseWriter, req *http.Request) { if len(req.URL.Path) <= len(*endpoint) { + if req.URL.RawQuery == "spec" { + res.Header().Set("Content-Type", "text/plain") + res.Write(openAPISpec) + return + } + res.WriteHeader(http.StatusNotFound) - res.Write(responseBody("request a variant at %s[variant]", *endpoint)) + res.Write(responseBody("request a variant at %sv1/[variant]", *endpoint)) + return + } + + requestPath := req.URL.Path[len(*endpoint):] + pathSegments := strings.Split(requestPath, "/") + + // Request should have been to v1/[variant] + if len(pathSegments) != 2 || pathSegments[0] != "v1" { + res.WriteHeader(http.StatusNotFound) + res.Write(responseBody("request a variant at %sv1/[variant]", *endpoint)) + return + } + + variant, err := url.PathUnescape(pathSegments[1]) + + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + log.Printf("failed to unescape variant name '%s': %s\n", pathSegments[1], err) return } - variant := req.URL.Path[len(*endpoint):] body, err := ioutil.ReadAll(req.Body) if err != nil { @@ -73,25 +104,25 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { return } - switch mt, _, _ := mime.ParseMediaType(req.Header.Get("content-type")); mt { + var cfg *config.Config + mediaType, _, _ := mime.ParseMediaType(req.Header.Get("content-type")) + + // Default to application/json + if mediaType == "" { + mediaType = "application/json" + } + + switch mediaType { case "application/json": - // Enforce strict JSON syntax if specified, even though the config parser - // would technically handle anything that's at least valid YAML - if !json.Valid(body) { - res.WriteHeader(http.StatusBadRequest) - res.Write(responseBody("'%s' media type given but request contains invalid JSON", mt)) - return - } + cfg, err = config.ReadConfig(body) case "application/yaml", "application/x-yaml": - // Let the config parser validate YAML syntax + cfg, err = config.ReadYAMLConfig(body) default: res.WriteHeader(http.StatusUnsupportedMediaType) - res.Write(responseBody("'%s' media type is not supported", mt)) + res.Write(responseBody("'%s' media type is not supported", mediaType)) return } - cfg, err := config.ReadYAMLConfig(body) - if err != nil { if config.IsValidationError(err) { res.WriteHeader(http.StatusUnprocessableEntity) @@ -101,8 +132,8 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { res.WriteHeader(http.StatusBadRequest) res.Write(responseBody( - "Failed to read config YAML from request body. "+ - "Was it formatted correctly and encoded as binary data?\nerror: %s", + "Failed to read '%s' config from request body. Error: %s", + mediaType, err.Error(), )) return @@ -136,3 +167,16 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { func responseBody(msg string, a ...interface{}) []byte { return []byte(fmt.Sprintf(msg+"\n", a...)) } + +func readOpenAPISpec() []byte { + var buffer bytes.Buffer + tmpl, _ := template.New("spec").Parse(openAPISpecTemplate) + + tmpl.Execute(&buffer, struct { + Version string + }{ + Version: meta.FullVersion(), + }) + + return buffer.Bytes() +} diff --git a/cmd/blubberoid/main_test.go b/cmd/blubberoid/main_test.go index 7898e75..e01c0b1 100644 --- a/cmd/blubberoid/main_test.go +++ b/cmd/blubberoid/main_test.go @@ -12,7 +12,7 @@ import ( func TestBlubberoidYAMLRequest(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`--- + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`--- version: v3 base: foo variants: @@ -33,7 +33,7 @@ func TestBlubberoidYAMLRequest(t *testing.T) { func TestBlubberoidJSONRequest(t *testing.T) { t.Run("valid JSON syntax", func(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`{ + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`{ "version": "v3", "base": "foo", "variants": { @@ -55,7 +55,7 @@ func TestBlubberoidJSONRequest(t *testing.T) { t.Run("invalid JSON syntax", func(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`{ + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`{ version: "v3", base: "foo", variants: { @@ -70,13 +70,13 @@ func TestBlubberoidJSONRequest(t *testing.T) { body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - assert.Equal(t, string(body), "'application/json' media type given but request contains invalid JSON\n") + assert.Equal(t, string(body), "Failed to read 'application/json' config from request body. Error: invalid character 'v' looking for beginning of object key string\n") }) } func TestBlubberoidUnsupportedMediaType(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(``)) + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(``)) req.Header.Set("Content-Type", "application/foo") blubberoid(rec, req) diff --git a/cmd/blubberoid/openapi.go b/cmd/blubberoid/openapi.go new file mode 100644 index 0000000..632c447 --- /dev/null +++ b/cmd/blubberoid/openapi.go @@ -0,0 +1,222 @@ +// Code generated by ../../scripts/generate-const.sh DO NOT EDIT. +package main + +//go:generate ../../scripts/generate-const.sh openAPISpecTemplate ../../api/openapi-spec/blubberoid.yaml +const openAPISpecTemplate = `--- +openapi: '3.0.0' +info: + title: Blubberoid + description: > + Blubber is a highly opinionated abstraction for container build + configurations. + version: {{ .Version }} +paths: + /v1/{variant}: + post: + summary: > + Generates a valid Dockerfile based on Blubber YAML configuration + provided in the request body and the given variant name. + requestBody: + description: A valid Blubber configuration. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/v3.Config' + application/yaml: + schema: + type: string + application/x-yaml: + schema: + type: string + responses: + '200': + description: OK. Response body should be a valid Dockerfile. + content: + text/plain: + schema: + type: string + '400': + description: Bad request. The YAML request body failed to parse. + '404': + description: No variant name was provided in the request path. + '422': + description: Provided Blubber config parsed correctly but failed validation. + '5XX': + description: An unexpected service-side error. + + x-amples: + - title: Mathoid test variant + request: + params: + variant: test + headers: + Content-Type: application/json + body: { + "version": "v3", + "base": "docker-registry.wikimedia.org/nodejs-slim", + "apt": { "packages": ["librsvg2-2"] }, + "lives": { "in": "/srv/service" }, + "variants": { + "build": { + "base": "docker-registry.wikimedia.org/nodejs-devel", + "apt": { + "packages": ["librsvg2-dev", "git", "pkg-config", "build-essential"] + }, + "node": { "requirements": ["package.json"] }, + "runs": { "environment": { "LINK": "g++" } } + }, + "test": { "includes": ["build"], "entrypoint": ["npm", "test"] } + } + } + response: + status: 200 + headers: + content-type: text/plain + body: /^FROM docker-registry.wikimedia.org\/nodejs-devel/ + +components: + schemas: + v3.Config: + title: Top-level blubber configuration (version v3) + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + required: [version, variants] + version: + type: string + description: Blubber configuration version + variants: + type: object + description: Configuration variants (e.g. development, test, production) + additionalProperties: true + # OpenAPI 3.0 supports only v4 of the JSON Schema draft spec and + # cannot define schema for object properties with arbitrary + # names, but the following commented section is included to be + # useful to humans for the time being. It patiently awaits v6 + # json schema draft support before its uncommenting. + # + # patternProperties: + # "^[a-zA-Z][a-zA-Z0-9\-\.]+[a-zA-Z0-9]$": + # $ref: '#/components/schemas/v3.VariantConfig' + + v3.CommonConfig: + type: object + properties: + base: + type: string + description: Base image reference + apt: + type: object + properties: + packages: + type: array + description: Packages to install from APT sources of base image + items: + type: string + node: + type: object + properties: + env: + type: string + description: Node environment (e.g. production, etc.) + requirements: + type: array + description: Files needed for Node package installation (e.g. package.json, package-lock.json) + items: + type: string + python: + type: object + properties: + version: + type: string + description: Python binary present in the system (e.g. python3) + requirements: + type: array + description: Files needed for Python package installation (e.g. requirements.txt, etc.) + items: + type: string + builder: + type: object + properties: + command: + type: array + description: Command and arguments of an arbitrary build command + items: + type: string + requirements: + type: array + description: Files needed by the build command (e.g. Makefile, ./src/, etc.) + items: + type: string + lives: + type: object + properties: + as: + type: string + description: Owner (name) of application files within the container + uid: + type: integer + description: Owner (UID) of application files within the container + gid: + type: integer + description: Group owner (GID) of application files within the container + in: + type: string + description: Application working directory within the container + runs: + type: object + properties: + as: + type: string + description: Runtime process owner (name) of application entrypoint + uid: + type: integer + description: Runtime process owner (UID) of application entrypoint + gid: + type: integer + description: Runtime process group (GID) of application entrypoint + environment: + type: object + description: Environment variables and values to be set before entrypoint execution + additionalProperties: true + insecurely: + type: boolean + description: Skip dropping of priviledge to the runtime process owner before entrypoint execution + entrypoint: + type: array + description: Runtime entry point command and arguments + items: + type: string + v3.VariantConfig: + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + includes: + type: array + description: Names of other variants to inherit configuration from + items: + description: Variant name + type: string + copies: + type: string + description: Name of variant from which to copy application files, resulting in a multi-stage build + artifacts: + type: array + items: + type: object + description: Artifacts to copy from another variant, resulting in a multi-stage build + required: [from, source, destination] + properties: + from: + type: string + description: Variant name + source: + type: string + description: Path of files/directories to copy + destination: + type: string + description: Destination path +` diff --git a/cmd/blubberoid/openapi_test.go b/cmd/blubberoid/openapi_test.go new file mode 100644 index 0000000..559f681 --- /dev/null +++ b/cmd/blubberoid/openapi_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBlubberoidOpenAPISpecTemplateMatchesFile(t *testing.T) { + specFile, err := ioutil.ReadFile("../../api/openapi-spec/blubberoid.yaml") + + if assert.NoError(t, err) { + assert.Equal(t, string(specFile), openAPISpecTemplate) + } +} diff --git a/scripts/generate-const.sh b/scripts/generate-const.sh new file mode 100755 index 0000000..f049389 --- /dev/null +++ b/scripts/generate-const.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Generates a Go const from the contents of a static file, and appends it at +# the line following the go:generate directive that calls this script. +# +set -euo pipefail + +sed -i.sed.bak -e "$((GOLINE+1)),\$d" "$GOFILE" +rm "$GOFILE.sed.bak" + +(echo -n "const $1 = \`"; cat "$2"; echo "\`") >> "$GOFILE" |