From 4df63a83c93ba0f591eb78110e184bfef0f76535 Mon Sep 17 00:00:00 2001 From: Dan Duvall Date: Wed, 19 Sep 2018 11:06:46 -0700 Subject: Provide OpenAPI spec for Blubberoid Wrote an OpenAPI 3.0 spec for Blubberoid that provides `x-amples` entries compatible with service-checker. The written spec includes basic schema for Blubber config objects that may be later factored out for use in validation. Note that OpenAPI 3.0 supports only the v4 draft of the JSON Schema standard, so some parts of the configuration could not be fully described. Specifically, v4 does not include the `patternProperties` definition introduced in the JSON Schema v6 draft that would allow us to describe `variants` and `runs.environment` and everything beneath. Blubberoid was refactored slightly to incorporate the new spec as well as assume JSON as the canonical and default configuration format. It was also refactored to include a versioned namespace ("v1") after the server endpoint. Bug: T205920 Change-Id: I28a341aa503b8920d802715660d4c4d62be45475 --- Makefile | 8 +- api/openapi-spec/blubberoid.yaml | 217 ++++++++++++++++++++++++++++++++++++++ cmd/blubberoid/main.go | 90 ++++++++++++---- cmd/blubberoid/main_test.go | 10 +- cmd/blubberoid/openapi.go | 222 +++++++++++++++++++++++++++++++++++++++ cmd/blubberoid/openapi_test.go | 16 +++ scripts/generate-const.sh | 11 ++ 7 files changed, 544 insertions(+), 30 deletions(-) create mode 100644 api/openapi-spec/blubberoid.yaml create mode 100644 cmd/blubberoid/openapi.go create mode 100644 cmd/blubberoid/openapi_test.go create mode 100755 scripts/generate-const.sh diff --git a/Makefile b/Makefile index 3ad6d93..2b4dc45 100644 --- a/Makefile +++ b/Makefile @@ -16,10 +16,11 @@ GO_LDFLAGS := \ # # workaround bug in case CURDIR is a symlink # see https://github.com/golang/go/issues/24359 +GO_GENERATE := cd "$(REAL_CURDIR)" && go generate GO_BUILD := cd "$(REAL_CURDIR)" && go build -v -ldflags "$(GO_LDFLAGS)" GO_INSTALL := cd "$(REAL_CURDIR)" && go install -v -ldflags "$(GO_LDFLAGS)" -all: blubber blubberoid +all: code blubber blubberoid blubber: $(GO_BUILD) ./cmd/blubber @@ -27,11 +28,14 @@ blubber: blubberoid: $(GO_BUILD) ./cmd/blubberoid +code: + $(GO_GENERATE) $(GO_PACKAGES) + clean: go clean rm -f blubber blubberoid -install: +install: all $(GO_INSTALL) $(GO_PACKAGES) release: diff --git a/api/openapi-spec/blubberoid.yaml b/api/openapi-spec/blubberoid.yaml new file mode 100644 index 0000000..b0ee9ae --- /dev/null +++ b/api/openapi-spec/blubberoid.yaml @@ -0,0 +1,217 @@ +--- +openapi: '3.0.0' +info: + title: Blubberoid + description: > + Blubber is a highly opinionated abstraction for container build + configurations. + version: {{ .Version }} +paths: + /v1/{variant}: + post: + summary: > + Generates a valid Dockerfile based on Blubber YAML configuration + provided in the request body and the given variant name. + requestBody: + description: A valid Blubber configuration. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/v3.Config' + application/yaml: + schema: + type: string + application/x-yaml: + schema: + type: string + responses: + '200': + description: OK. Response body should be a valid Dockerfile. + content: + text/plain: + schema: + type: string + '400': + description: Bad request. The YAML request body failed to parse. + '404': + description: No variant name was provided in the request path. + '422': + description: Provided Blubber config parsed correctly but failed validation. + '5XX': + description: An unexpected service-side error. + + x-amples: + - title: Mathoid test variant + request: + params: + variant: test + headers: + Content-Type: application/json + body: { + "version": "v3", + "base": "docker-registry.wikimedia.org/nodejs-slim", + "apt": { "packages": ["librsvg2-2"] }, + "lives": { "in": "/srv/service" }, + "variants": { + "build": { + "base": "docker-registry.wikimedia.org/nodejs-devel", + "apt": { + "packages": ["librsvg2-dev", "git", "pkg-config", "build-essential"] + }, + "node": { "requirements": ["package.json"] }, + "runs": { "environment": { "LINK": "g++" } } + }, + "test": { "includes": ["build"], "entrypoint": ["npm", "test"] } + } + } + response: + status: 200 + headers: + content-type: text/plain + body: /^FROM docker-registry.wikimedia.org\/nodejs-devel/ + +components: + schemas: + v3.Config: + title: Top-level blubber configuration (version v3) + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + required: [version, variants] + version: + type: string + description: Blubber configuration version + variants: + type: object + description: Configuration variants (e.g. development, test, production) + additionalProperties: true + # OpenAPI 3.0 supports only v4 of the JSON Schema draft spec and + # cannot define schema for object properties with arbitrary + # names, but the following commented section is included to be + # useful to humans for the time being. It patiently awaits v6 + # json schema draft support before its uncommenting. + # + # patternProperties: + # "^[a-zA-Z][a-zA-Z0-9\-\.]+[a-zA-Z0-9]$": + # $ref: '#/components/schemas/v3.VariantConfig' + + v3.CommonConfig: + type: object + properties: + base: + type: string + description: Base image reference + apt: + type: object + properties: + packages: + type: array + description: Packages to install from APT sources of base image + items: + type: string + node: + type: object + properties: + env: + type: string + description: Node environment (e.g. production, etc.) + requirements: + type: array + description: Files needed for Node package installation (e.g. package.json, package-lock.json) + items: + type: string + python: + type: object + properties: + version: + type: string + description: Python binary present in the system (e.g. python3) + requirements: + type: array + description: Files needed for Python package installation (e.g. requirements.txt, etc.) + items: + type: string + builder: + type: object + properties: + command: + type: array + description: Command and arguments of an arbitrary build command + items: + type: string + requirements: + type: array + description: Files needed by the build command (e.g. Makefile, ./src/, etc.) + items: + type: string + lives: + type: object + properties: + as: + type: string + description: Owner (name) of application files within the container + uid: + type: integer + description: Owner (UID) of application files within the container + gid: + type: integer + description: Group owner (GID) of application files within the container + in: + type: string + description: Application working directory within the container + runs: + type: object + properties: + as: + type: string + description: Runtime process owner (name) of application entrypoint + uid: + type: integer + description: Runtime process owner (UID) of application entrypoint + gid: + type: integer + description: Runtime process group (GID) of application entrypoint + environment: + type: object + description: Environment variables and values to be set before entrypoint execution + additionalProperties: true + insecurely: + type: boolean + description: Skip dropping of priviledge to the runtime process owner before entrypoint execution + entrypoint: + type: array + description: Runtime entry point command and arguments + items: + type: string + v3.VariantConfig: + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + includes: + type: array + description: Names of other variants to inherit configuration from + items: + description: Variant name + type: string + copies: + type: string + description: Name of variant from which to copy application files, resulting in a multi-stage build + artifacts: + type: array + items: + type: object + description: Artifacts to copy from another variant, resulting in a multi-stage build + required: [from, source, destination] + properties: + from: + type: string + description: Variant name + source: + type: string + description: Path of files/directories to copy + destination: + type: string + description: Destination path diff --git a/cmd/blubberoid/main.go b/cmd/blubberoid/main.go index d436390..19604c1 100644 --- a/cmd/blubberoid/main.go +++ b/cmd/blubberoid/main.go @@ -3,27 +3,32 @@ package main import ( - "encoding/json" + "bytes" "fmt" "io/ioutil" "log" "mime" "net/http" + "net/url" "os" "path" + "strings" + "text/template" "github.com/pborman/getopt/v2" "gerrit.wikimedia.org/r/blubber/config" "gerrit.wikimedia.org/r/blubber/docker" + "gerrit.wikimedia.org/r/blubber/meta" ) var ( - showHelp = getopt.BoolLong("help", 'h', "show help/usage") - address = getopt.StringLong("address", 'a', ":8748", "socket address/port to listen on (default ':8748')", "address:port") - endpoint = getopt.StringLong("endpoint", 'e', "/", "server endpoint (default '/')", "path") - policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri") - policy *config.Policy + showHelp = getopt.BoolLong("help", 'h', "show help/usage") + address = getopt.StringLong("address", 'a', ":8748", "socket address/port to listen on (default ':8748')", "address:port") + endpoint = getopt.StringLong("endpoint", 'e', "/", "server endpoint (default '/')", "path") + policyURI = getopt.StringLong("policy", 'p', "", "policy file URI", "uri") + policy *config.Policy + openAPISpec []byte ) func main() { @@ -51,7 +56,10 @@ func main() { *endpoint += "/" } - log.Printf("listening on %s for requests to %s[variant]\n", *address, *endpoint) + // Evaluate OpenAPI spec template and store results for ?spec requests + openAPISpec = readOpenAPISpec() + + log.Printf("listening on %s for requests to %sv1/[variant]\n", *address, *endpoint) http.HandleFunc(*endpoint, blubberoid) log.Fatal(http.ListenAndServe(*address, nil)) @@ -59,12 +67,35 @@ func main() { func blubberoid(res http.ResponseWriter, req *http.Request) { if len(req.URL.Path) <= len(*endpoint) { + if req.URL.RawQuery == "spec" { + res.Header().Set("Content-Type", "text/plain") + res.Write(openAPISpec) + return + } + res.WriteHeader(http.StatusNotFound) - res.Write(responseBody("request a variant at %s[variant]", *endpoint)) + res.Write(responseBody("request a variant at %sv1/[variant]", *endpoint)) + return + } + + requestPath := req.URL.Path[len(*endpoint):] + pathSegments := strings.Split(requestPath, "/") + + // Request should have been to v1/[variant] + if len(pathSegments) != 2 || pathSegments[0] != "v1" { + res.WriteHeader(http.StatusNotFound) + res.Write(responseBody("request a variant at %sv1/[variant]", *endpoint)) + return + } + + variant, err := url.PathUnescape(pathSegments[1]) + + if err != nil { + res.WriteHeader(http.StatusInternalServerError) + log.Printf("failed to unescape variant name '%s': %s\n", pathSegments[1], err) return } - variant := req.URL.Path[len(*endpoint):] body, err := ioutil.ReadAll(req.Body) if err != nil { @@ -73,25 +104,25 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { return } - switch mt, _, _ := mime.ParseMediaType(req.Header.Get("content-type")); mt { + var cfg *config.Config + mediaType, _, _ := mime.ParseMediaType(req.Header.Get("content-type")) + + // Default to application/json + if mediaType == "" { + mediaType = "application/json" + } + + switch mediaType { case "application/json": - // Enforce strict JSON syntax if specified, even though the config parser - // would technically handle anything that's at least valid YAML - if !json.Valid(body) { - res.WriteHeader(http.StatusBadRequest) - res.Write(responseBody("'%s' media type given but request contains invalid JSON", mt)) - return - } + cfg, err = config.ReadConfig(body) case "application/yaml", "application/x-yaml": - // Let the config parser validate YAML syntax + cfg, err = config.ReadYAMLConfig(body) default: res.WriteHeader(http.StatusUnsupportedMediaType) - res.Write(responseBody("'%s' media type is not supported", mt)) + res.Write(responseBody("'%s' media type is not supported", mediaType)) return } - cfg, err := config.ReadYAMLConfig(body) - if err != nil { if config.IsValidationError(err) { res.WriteHeader(http.StatusUnprocessableEntity) @@ -101,8 +132,8 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { res.WriteHeader(http.StatusBadRequest) res.Write(responseBody( - "Failed to read config YAML from request body. "+ - "Was it formatted correctly and encoded as binary data?\nerror: %s", + "Failed to read '%s' config from request body. Error: %s", + mediaType, err.Error(), )) return @@ -136,3 +167,16 @@ func blubberoid(res http.ResponseWriter, req *http.Request) { func responseBody(msg string, a ...interface{}) []byte { return []byte(fmt.Sprintf(msg+"\n", a...)) } + +func readOpenAPISpec() []byte { + var buffer bytes.Buffer + tmpl, _ := template.New("spec").Parse(openAPISpecTemplate) + + tmpl.Execute(&buffer, struct { + Version string + }{ + Version: meta.FullVersion(), + }) + + return buffer.Bytes() +} diff --git a/cmd/blubberoid/main_test.go b/cmd/blubberoid/main_test.go index 7898e75..e01c0b1 100644 --- a/cmd/blubberoid/main_test.go +++ b/cmd/blubberoid/main_test.go @@ -12,7 +12,7 @@ import ( func TestBlubberoidYAMLRequest(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`--- + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`--- version: v3 base: foo variants: @@ -33,7 +33,7 @@ func TestBlubberoidYAMLRequest(t *testing.T) { func TestBlubberoidJSONRequest(t *testing.T) { t.Run("valid JSON syntax", func(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`{ + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`{ "version": "v3", "base": "foo", "variants": { @@ -55,7 +55,7 @@ func TestBlubberoidJSONRequest(t *testing.T) { t.Run("invalid JSON syntax", func(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(`{ + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(`{ version: "v3", base: "foo", variants: { @@ -70,13 +70,13 @@ func TestBlubberoidJSONRequest(t *testing.T) { body, _ := ioutil.ReadAll(resp.Body) assert.Equal(t, http.StatusBadRequest, resp.StatusCode) - assert.Equal(t, string(body), "'application/json' media type given but request contains invalid JSON\n") + assert.Equal(t, string(body), "Failed to read 'application/json' config from request body. Error: invalid character 'v' looking for beginning of object key string\n") }) } func TestBlubberoidUnsupportedMediaType(t *testing.T) { rec := httptest.NewRecorder() - req := httptest.NewRequest("POST", "/test", strings.NewReader(``)) + req := httptest.NewRequest("POST", "/v1/test", strings.NewReader(``)) req.Header.Set("Content-Type", "application/foo") blubberoid(rec, req) diff --git a/cmd/blubberoid/openapi.go b/cmd/blubberoid/openapi.go new file mode 100644 index 0000000..632c447 --- /dev/null +++ b/cmd/blubberoid/openapi.go @@ -0,0 +1,222 @@ +// Code generated by ../../scripts/generate-const.sh DO NOT EDIT. +package main + +//go:generate ../../scripts/generate-const.sh openAPISpecTemplate ../../api/openapi-spec/blubberoid.yaml +const openAPISpecTemplate = `--- +openapi: '3.0.0' +info: + title: Blubberoid + description: > + Blubber is a highly opinionated abstraction for container build + configurations. + version: {{ .Version }} +paths: + /v1/{variant}: + post: + summary: > + Generates a valid Dockerfile based on Blubber YAML configuration + provided in the request body and the given variant name. + requestBody: + description: A valid Blubber configuration. + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/v3.Config' + application/yaml: + schema: + type: string + application/x-yaml: + schema: + type: string + responses: + '200': + description: OK. Response body should be a valid Dockerfile. + content: + text/plain: + schema: + type: string + '400': + description: Bad request. The YAML request body failed to parse. + '404': + description: No variant name was provided in the request path. + '422': + description: Provided Blubber config parsed correctly but failed validation. + '5XX': + description: An unexpected service-side error. + + x-amples: + - title: Mathoid test variant + request: + params: + variant: test + headers: + Content-Type: application/json + body: { + "version": "v3", + "base": "docker-registry.wikimedia.org/nodejs-slim", + "apt": { "packages": ["librsvg2-2"] }, + "lives": { "in": "/srv/service" }, + "variants": { + "build": { + "base": "docker-registry.wikimedia.org/nodejs-devel", + "apt": { + "packages": ["librsvg2-dev", "git", "pkg-config", "build-essential"] + }, + "node": { "requirements": ["package.json"] }, + "runs": { "environment": { "LINK": "g++" } } + }, + "test": { "includes": ["build"], "entrypoint": ["npm", "test"] } + } + } + response: + status: 200 + headers: + content-type: text/plain + body: /^FROM docker-registry.wikimedia.org\/nodejs-devel/ + +components: + schemas: + v3.Config: + title: Top-level blubber configuration (version v3) + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + required: [version, variants] + version: + type: string + description: Blubber configuration version + variants: + type: object + description: Configuration variants (e.g. development, test, production) + additionalProperties: true + # OpenAPI 3.0 supports only v4 of the JSON Schema draft spec and + # cannot define schema for object properties with arbitrary + # names, but the following commented section is included to be + # useful to humans for the time being. It patiently awaits v6 + # json schema draft support before its uncommenting. + # + # patternProperties: + # "^[a-zA-Z][a-zA-Z0-9\-\.]+[a-zA-Z0-9]$": + # $ref: '#/components/schemas/v3.VariantConfig' + + v3.CommonConfig: + type: object + properties: + base: + type: string + description: Base image reference + apt: + type: object + properties: + packages: + type: array + description: Packages to install from APT sources of base image + items: + type: string + node: + type: object + properties: + env: + type: string + description: Node environment (e.g. production, etc.) + requirements: + type: array + description: Files needed for Node package installation (e.g. package.json, package-lock.json) + items: + type: string + python: + type: object + properties: + version: + type: string + description: Python binary present in the system (e.g. python3) + requirements: + type: array + description: Files needed for Python package installation (e.g. requirements.txt, etc.) + items: + type: string + builder: + type: object + properties: + command: + type: array + description: Command and arguments of an arbitrary build command + items: + type: string + requirements: + type: array + description: Files needed by the build command (e.g. Makefile, ./src/, etc.) + items: + type: string + lives: + type: object + properties: + as: + type: string + description: Owner (name) of application files within the container + uid: + type: integer + description: Owner (UID) of application files within the container + gid: + type: integer + description: Group owner (GID) of application files within the container + in: + type: string + description: Application working directory within the container + runs: + type: object + properties: + as: + type: string + description: Runtime process owner (name) of application entrypoint + uid: + type: integer + description: Runtime process owner (UID) of application entrypoint + gid: + type: integer + description: Runtime process group (GID) of application entrypoint + environment: + type: object + description: Environment variables and values to be set before entrypoint execution + additionalProperties: true + insecurely: + type: boolean + description: Skip dropping of priviledge to the runtime process owner before entrypoint execution + entrypoint: + type: array + description: Runtime entry point command and arguments + items: + type: string + v3.VariantConfig: + allOf: + - $ref: '#/components/schemas/v3.CommonConfig' + - type: object + properties: + includes: + type: array + description: Names of other variants to inherit configuration from + items: + description: Variant name + type: string + copies: + type: string + description: Name of variant from which to copy application files, resulting in a multi-stage build + artifacts: + type: array + items: + type: object + description: Artifacts to copy from another variant, resulting in a multi-stage build + required: [from, source, destination] + properties: + from: + type: string + description: Variant name + source: + type: string + description: Path of files/directories to copy + destination: + type: string + description: Destination path +` diff --git a/cmd/blubberoid/openapi_test.go b/cmd/blubberoid/openapi_test.go new file mode 100644 index 0000000..559f681 --- /dev/null +++ b/cmd/blubberoid/openapi_test.go @@ -0,0 +1,16 @@ +package main + +import ( + "io/ioutil" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestBlubberoidOpenAPISpecTemplateMatchesFile(t *testing.T) { + specFile, err := ioutil.ReadFile("../../api/openapi-spec/blubberoid.yaml") + + if assert.NoError(t, err) { + assert.Equal(t, string(specFile), openAPISpecTemplate) + } +} diff --git a/scripts/generate-const.sh b/scripts/generate-const.sh new file mode 100755 index 0000000..f049389 --- /dev/null +++ b/scripts/generate-const.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# +# Generates a Go const from the contents of a static file, and appends it at +# the line following the go:generate directive that calls this script. +# +set -euo pipefail + +sed -i.sed.bak -e "$((GOLINE+1)),\$d" "$GOFILE" +rm "$GOFILE.sed.bak" + +(echo -n "const $1 = \`"; cat "$2"; echo "\`") >> "$GOFILE" -- cgit v1.2.1