From a85e7d15c57c11d8286656531c7394681fb855d9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 17 Feb 2021 08:49:38 +0200 Subject: refactor: move echo and muck examples under new examples/ directory --- echo.bib | 7 - echo.md | 52 ----- echo.sh | 50 ----- echo.yaml | 20 -- examples/echo/echo.bib | 7 + examples/echo/echo.md | 52 +++++ examples/echo/echo.sh | 50 +++++ examples/echo/echo.yaml | 20 ++ examples/muck/muck.md | 513 ++++++++++++++++++++++++++++++++++++++++++++++++ examples/muck/muck.py | 2 + examples/muck/muck.yaml | 47 +++++ muck.md | 513 ------------------------------------------------ muck.py | 2 - muck.yaml | 47 ----- 14 files changed, 691 insertions(+), 691 deletions(-) delete mode 100644 echo.bib delete mode 100644 echo.md delete mode 100644 echo.sh delete mode 100644 echo.yaml create mode 100644 examples/echo/echo.bib create mode 100644 examples/echo/echo.md create mode 100644 examples/echo/echo.sh create mode 100644 examples/echo/echo.yaml create mode 100644 examples/muck/muck.md create mode 100644 examples/muck/muck.py create mode 100644 examples/muck/muck.yaml delete mode 100644 muck.md delete mode 100644 muck.py delete mode 100644 muck.yaml diff --git a/echo.bib b/echo.bib deleted file mode 100644 index 05083fb..0000000 --- a/echo.bib +++ /dev/null @@ -1,7 +0,0 @@ -@book{foo2020, - author = "James Random", - title = "The Foo book", - publisher = "The Internet", - year = 2020, - address = "World Wide Web", -} \ No newline at end of file diff --git a/echo.md b/echo.md deleted file mode 100644 index c335f53..0000000 --- a/echo.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -title: "**echo**(1) acceptance tests" -author: The Subplot project -template: bash -bindings: echo.yaml -functions: echo.sh -bibliography: echo.bib -... - -Introduction -============================================================================= - -**echo**(1) is a Unix command line tool, which writes its command line -arguments to the standard output. This is a simple acceptance test -suite for the `/bin/echo` implementation. - -For more information, see [@foo2020]. - -No arguments -============================================================================= - -Run `/bin/echo` without arguments. - -```scenario -when user runs echo without arguments -then exit code is 0 -then standard output contains a newline -then standard error is empty -``` - -Hello, world -============================================================================= - -This scenario runs `/bin/echo` to produce the output "hello, world". - -```scenario -when user runs echo with arguments hello, world -then exit code is 0 -then standard output contains "hello, world" -then standard error is empty -``` - - -Test file - -~~~~{.file #foo.dat} -This is a test file. -Two lines. -~~~~ - - -# References diff --git a/echo.sh b/echo.sh deleted file mode 100644 index 0564d42..0000000 --- a/echo.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -_run() -{ - if "$@" < /dev/null > stdout 2> stderr - then - ctx_set exit 0 - else - ctx_set exit "$?" - fi - ctx_set stdout "$(cat stdout)" - ctx_set stderr "$(cat stderr)" -} - -run_echo_without_args() -{ - _run echo -} - -run_echo_with_args() -{ - args="$(cap_get args)" - _run echo "$args" -} - -exit_code_is() -{ - actual_exit="$(ctx_get exit)" - wanted_exit="$(cap_get exit_code)" - assert_eq "$actual_exit" "$wanted_exit" -} - -stdout_is_a_newline() -{ - stdout="$(ctx_get stdout)" - assert_eq "$stdout" "$(printf '\n')" -} - -stdout_is_text() -{ - stdout="$(ctx_get stdout)" - text="$(cap_get text)" - assert_contains "$stdout" "$text" -} - -stderr_is_empty() -{ - stderr="$(ctx_get stderr)" - assert_eq "$stderr" "" -} diff --git a/echo.yaml b/echo.yaml deleted file mode 100644 index 7be6e96..0000000 --- a/echo.yaml +++ /dev/null @@ -1,20 +0,0 @@ -- when: user runs echo without arguments - function: run_echo_without_args - -- when: user runs echo with arguments (?P.+) - function: run_echo_with_args - regex: true - -- then: exit code is (?P\d+) - function: exit_code_is - regex: true - -- then: standard output contains a newline - function: stdout_is_a_newline - -- then: standard output contains "(?P.*)" - function: stdout_is_text - regex: true - -- then: standard error is empty - function: stderr_is_empty diff --git a/examples/echo/echo.bib b/examples/echo/echo.bib new file mode 100644 index 0000000..05083fb --- /dev/null +++ b/examples/echo/echo.bib @@ -0,0 +1,7 @@ +@book{foo2020, + author = "James Random", + title = "The Foo book", + publisher = "The Internet", + year = 2020, + address = "World Wide Web", +} \ No newline at end of file diff --git a/examples/echo/echo.md b/examples/echo/echo.md new file mode 100644 index 0000000..c335f53 --- /dev/null +++ b/examples/echo/echo.md @@ -0,0 +1,52 @@ +--- +title: "**echo**(1) acceptance tests" +author: The Subplot project +template: bash +bindings: echo.yaml +functions: echo.sh +bibliography: echo.bib +... + +Introduction +============================================================================= + +**echo**(1) is a Unix command line tool, which writes its command line +arguments to the standard output. This is a simple acceptance test +suite for the `/bin/echo` implementation. + +For more information, see [@foo2020]. + +No arguments +============================================================================= + +Run `/bin/echo` without arguments. + +```scenario +when user runs echo without arguments +then exit code is 0 +then standard output contains a newline +then standard error is empty +``` + +Hello, world +============================================================================= + +This scenario runs `/bin/echo` to produce the output "hello, world". + +```scenario +when user runs echo with arguments hello, world +then exit code is 0 +then standard output contains "hello, world" +then standard error is empty +``` + + +Test file + +~~~~{.file #foo.dat} +This is a test file. +Two lines. +~~~~ + + +# References diff --git a/examples/echo/echo.sh b/examples/echo/echo.sh new file mode 100644 index 0000000..0564d42 --- /dev/null +++ b/examples/echo/echo.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +_run() +{ + if "$@" < /dev/null > stdout 2> stderr + then + ctx_set exit 0 + else + ctx_set exit "$?" + fi + ctx_set stdout "$(cat stdout)" + ctx_set stderr "$(cat stderr)" +} + +run_echo_without_args() +{ + _run echo +} + +run_echo_with_args() +{ + args="$(cap_get args)" + _run echo "$args" +} + +exit_code_is() +{ + actual_exit="$(ctx_get exit)" + wanted_exit="$(cap_get exit_code)" + assert_eq "$actual_exit" "$wanted_exit" +} + +stdout_is_a_newline() +{ + stdout="$(ctx_get stdout)" + assert_eq "$stdout" "$(printf '\n')" +} + +stdout_is_text() +{ + stdout="$(ctx_get stdout)" + text="$(cap_get text)" + assert_contains "$stdout" "$text" +} + +stderr_is_empty() +{ + stderr="$(ctx_get stderr)" + assert_eq "$stderr" "" +} diff --git a/examples/echo/echo.yaml b/examples/echo/echo.yaml new file mode 100644 index 0000000..7be6e96 --- /dev/null +++ b/examples/echo/echo.yaml @@ -0,0 +1,20 @@ +- when: user runs echo without arguments + function: run_echo_without_args + +- when: user runs echo with arguments (?P.+) + function: run_echo_with_args + regex: true + +- then: exit code is (?P\d+) + function: exit_code_is + regex: true + +- then: standard output contains a newline + function: stdout_is_a_newline + +- then: standard output contains "(?P.*)" + function: stdout_is_text + regex: true + +- then: standard error is empty + function: stderr_is_empty diff --git a/examples/muck/muck.md b/examples/muck/muck.md new file mode 100644 index 0000000..d5470a0 --- /dev/null +++ b/examples/muck/muck.md @@ -0,0 +1,513 @@ +--- +title: Muck JSON storage server and API +author: Lars Wirzenius +date: work in progress +bindings: muck.yaml +functions: muck.py +template: python +... + +Introduction +============================================================================= + +Muck is intended for storing relatively small pieces of data securely, +and accessing them quickly. Intended uses cases are: + +* storing user, client, application, and related data for an OpenID + Connect authenatication server +* storing personally identifiable information of data subjects (in the + GDPR sense) in a way that they can access and update, assuming + integration with a suitable authantication and authorization server +* in general, storage for web applications of data that isn't large + and fits easily into RAM + +Muck is a JSON store, with an access controlled RESTful HTTP API. Data +stored in Muck is persistent, but kept in memory for fast access. Data +is represented as JSON objects. + +Access is granted based on signed JWT bearer tokens. An OpenID Connect +or OAuth2 identity provider is expected to give such tokens to Muck +clients. The tokens must be signed with a public key that Muck is +configured to accept. + +Access control is simplistic. Each resource is assigned an owner +upon creation, and each user can access (see, update, delete) only +their own resources. A use with "super" powers can access, update, and +delete resources they don't own, but can't create resources for other. +This will be improved later. + +Architecture +----------------------------------------------------------------------------- + +Muck stores data persistently in its local file system. It provides an +HTTP API for clients. Muck itself does not communicate otherwise with +external entities. + +```dot +digraph "architecture" { +muck [shape=box label="Muck"]; +storage [shape=tab label="Persistent \n storage"]; +client [shape=ellipse label="API client"]; +idp [shape=ellipse label="OAuth2/OIDC server"]; + +storage -> muck [label="Read at \n startup"]; +muck -> storage [label="Write \n changes"]; +client -> muck [label="API read/write \n (HTTP)"]; +client -> idp [label="Get access token"]; +idp -> muck [label="Token signing key"]; +} +``` + + +Authentication +----------------------------------------------------------------------------- + +[OAuth2]: https://oauth.net/ +[OpenID Connect]: https://openid.net/connect/ +[JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token + +Muck uses [OAuth2][] or [OpenID Connect][] bearer tokens as access +tokens. The tokens are granted by some form of authentication service, +are [JWT][] tokens, and signed using public-key cryptography. The +authentication service is outside the scope of this document; any +standard implementation should work. + +Muck will be configured with one public key for validating the tokens. +For Muck to access a token: + +* its signature must be valid according to the public key +* it to must be used while it's valid (after the validity starts, but + before if expires) +* its audience must be the specific Muck instance +* its scope claim contains the specified scopes needed for the + attempted operation +* it specified an end-user (data subject) + +Every request to the Muck API must include a token, in the +`Authorizatin` header as a bearer token. The request is denied if the +token does not pass all the above checks. + +Requirements +============================================================================= + +This chapter lists high level requirements for Muck. + +Each requirement here is given a unique mnemnoic id for easier +reference in discussions. + +**SimpleOps** + +: Muck must be simple to install and operate. Installation should be + installing a .deb package, configuration by setting the public key + for token signing of the authentication server. + +**Fast** + +: Muck must be fast. The speed requirement is that Muck must be able + to handle at least 100 concurrent clients, creating 1000 objects + each, and then retrieving each object, and then deleting each + object, and all of this must happen in no more than ten minutes + (600 seconds). Muck and the clients should run on different + virtual machines. + +**Secure** + +: Muck must allow access only by an authenticated client + representing a data subject, and must only allow that client to + access objects owned by the data subject, unless the client has + super privileges. The data subject specifies, via the access + token, what operations the client is allowed to do: whether they + read, update, or delete objects. + + +HTTP API +============================================================================= + +The Muck HTTP API has one endpoint – `/res` – that's used +for all objects. The objects are called resources by Muck. + +The JSON objects Muck operates on must be valid, but their structure +does not matter to Muck. + +Metadata +----------------------------------------------------------------------------- + +Each JSON object stored in Muck is associated with metadata, which is +represented as the following HTTP headers: + +* **Muck-Id** – the resource id +* **Muck-Revision** – the resource revision + +The id is assiged by Muck at object creation time. The revision is +assigned by Muck when the object is created or modified. + + +API requests +----------------------------------------------------------------------------- + +The RESTful API requests are POST, PUT, GET, and DELETE. + +* **POST /res** – create a new object +* **PUT /res** – update an existing object +* **GET /res** – retrieve a existing object +* **DELETE /res** – delete an existing object + +Although it is usual for RESTful HTTP APIs to encode resource +identifiers in the URL, Muck uses headers (Muck-Id, Muck-Revision) for +consistency, and to provide for later expansion. Muck is not intended +to be used manually, but by programmatic clients. + +Additionally, the "sub" claim in the token is used to assign and check +ownership of the object. If the scope contains "super", the sub claim +is ignored, except for creation. + +The examples in this chapter use HTTP/1.1, but should provide the +necessary information for other versions of HTTP. Also, only the +headers relevant to Muck are shown. For example, HTTP/1.1 requires +also a Host header, but this is not shown in the examples. + + + +### Creating an object: POST /res + +Creating requires: + +* "create" in the scope claim +* a non-empty "sub" claim, which will be stored by Muck as the owner + of the created object + +The creation request looks like this: + +~~~{.numberLines} +POST /res HTTP/1.1 +Content-Type: application/ +Authorization: Bearer TOKEN + +{"foo": "bar"} +~~~ + +Note that the creation request does not include Muck-Id or +Muck-Revision headers. + +A successful response looks like this: + +~~~{.numberLines} +201 Created +Content-Type: application/json +Muck-Id: ID +Muck-Revision: REV1 +~~~ + +Note that the response does not contain a copy of the resource. + + + +### Updating an object: PUT /res + +Updating requires: + +* "update" in the scope claim +* one of the following: + - "super" in the scope claim + - "sub" claim matches owner of object Muck; super user can update + any resource, but otherwise data subjects can only update their own + objects +* Muck-Revision matches the current revision in Muck; this functions + as a simplistic guard against conflicting updates from different + clients. + +The update request looks like this: + +~~~{.numberLines} +PUT /res HTTP/1.1 +Authorization: Bearer TOKEN +Content-Type: application/json +Muck-Id: ID +Muck-Revision: REV1 + +{"foo": "yo"} +~~~ + +In the request, ID identifies the object, and REV1 is its revision. + +The successful response: + +~~~{.numberLines} +200 OK +Content-Type: application/json +Muck-Id: ID +Muck-Revision: REV2 +~~~ + +Note that the update response also doesn't contain the object. The +client should remember the new revision, or retrieve the object get +the latest revision before the next update. + + +### Retrieving an object: GET /res + +A request requires: + +* "show" in the scope claim +* one of the following: + - "super" in the scope claim + - "sub" claim matches owner of object Muck; super user can retrieve + any resource, but otherwise data subjects can only update their own + objects + +The request to retrieve a response: + +~~~{.numberLines} +GET /res HTTP/1.1 +Authorization: Bearer TOKEN +Muck-Id: ID +~~~ + +A successful response: + +~~~{.numberLines} +200 OK +Content-Type: application/json +Muck-Id: ID +Muck-Revision: REV2 + +{"foo": "yo"} +~~~ + +Note that the response does NOT indicate the owner of the resource. + + + +Acceptance criteria for Muck +============================================================================= + +This chapter details the acceptance criteria for Muck, and how they're +verified. + + +Basic object handling +----------------------------------------------------------------------------- + +First, we need a new Muck server. It will initially have no objects. +We also need a test user, whom we'll call Tomjon. + +~~~scenario +given a fresh Muck server +given I am Tomjon +~~~ + +Tomjon can create an object. + +~~~scenario +when I do POST /res with {"foo": "bar"} +then response code is 201 +then header Muck-Id is ID +then header Muck-Revision is REV1 +~~~ + +Tomjon can then retrieve the object. It has the same revision and +body. + +~~~scenario +when I do GET /res with Muck-Id: {ID} +then response code is 200 +then header Muck-Revision matches {REV1} +then body matches {"foo": "bar"} +~~~ + +Tomjon can update the object, and the update has the same id, but a +new revision and body. + +~~~scenario +when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} +then response code is 200 +then header Muck-Revision is {REV2} +then revisions {REV1} and {REV2} are different +~~~ + +If Tomjon tries to update with the old revision, it fails. + +~~~scenario +when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} +then response code is 409 +~~~ + +After the failed update, the object or its revision haven't changed. + +~~~scenario +when I do GET /res with Muck-Id: {ID} +then response code is 200 +then header Muck-Revision matches {REV2} +then body matches {"foo": "yo"} +~~~ + +We can delete the resource, and then it's gone. + +~~~scenario +when I do DELETE /res with Muck-Id: {ID} +then response code is 200 +when I do GET /res with Muck-Id: {ID} +then response code is 404 +~~~ + + +Restarting Muck +----------------------------------------------------------------------------- + +Muck should store data persistently. For this we need our test user to +have the "super" capability. + +~~~scenario +given a fresh Muck server +given I am Tomjon, with super capability +when I do POST /res with {"foo": "bar"} +then header Muck-Id is ID +then header Muck-Revision is REV1 +~~~ + +So far, so good. Nothing new here. Now we restart Muck. The resource +just created must still be there. + +~~~scenario +when I restart Muck +when I do GET /res with Muck-Id: {ID} +then response code is 200 +then header Muck-Revision matches {REV1} +then body matches {"foo": "bar"} +~~~ + + +Super user access +----------------------------------------------------------------------------- + +Check here that if we have super scope, we can retrieve, update, and +delete someone else's resources, but if we create a resourec, it's +ours. + +Invalid requests +----------------------------------------------------------------------------- + +There are a number of ways in which a request might be rejected. This +section verifies all of them. + +### Accessing someone else's data + +~~~scenario +given a fresh Muck server +given I am Tomjon +when I do POST /res with {"foo": "bar"} +then header Muck-Id is ID +then header Muck-Revision is REV1 +when I do GET /res with Muck-Id: {ID} +then response code is 200 +then header Muck-Revision matches {REV1} +then body matches {"foo": "bar"} +~~~ + +After this, we morph into another test user. + +~~~scenario +given I am Verence +when I do GET /res with Muck-Id: {ID} +then response code is 404 +~~~ + +Note that we get a "not found" error and not a "access denied" error +so that Verence doesn't know if the resource exists or not. + + +### Updating someone else's data + +This is similar to retrieving it, but we try to update instead. + +~~~scenario +given a fresh Muck server +given I am Tomjon +when I do POST /res with {"foo": "bar"} +then header Muck-Id is ID +then header Muck-Revision is REV1 +given I am Verence +when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} +then response code is 404 +~~~ + + +### Deleting someone else's data + +This is similar to retrieving it, but we try to delete it instead. + +~~~scenario +given a fresh Muck server +given I am Tomjon +when I do POST /res with {"foo": "bar"} +then header Muck-Id is ID +then header Muck-Revision is REV1 +given I am Verence +when I do DELETE /res with Muck-Id: {ID} +then response code is 404 +~~~ + +### Bad signature + +### Not valid yet + +### Not valid anymore + +### Not for our instance + +### Lack scope for creation + +### Lack scope for retrieval + +### Lack scope for updating + +### Lack scope for deletion + +### No subject when creating + +### No subject when retrieving + +### No subject when updating + +### No subject when deleting + +### Invalid JSON when creating + +### Invalid JSON when updating + + +# Possible future changes + +* There is no way to list all the resources a user has, or search for + resource. This should be doable in some way. With a search, a + listing operation is not strictly necessary. + +* It's going to be inconvenient to only be able to access one's own + resources. It would be good to support groups. A resource could be + owned by a group, and end-users / subjects could belong to any + number of groups. Also, groups should be able to belong to groups. + Each resource should be able to specify for each group what access + members of that group should have (retrieve, update, delete). There + should be no limits to how many group access control rules there are + per resource. + + This would allow setups such as each resource representing a stored + file, and some groups would be granted read access, or read-write + access, or read-delete access to the files. + +* Also, it might be good to be able to grant other groups access to + controll a resource's access control rules. + +* It might be good support schemas for resources? + +* It might be good to have a configurable maximum size of a resource. + Possibly per-user quotas. + +* It would be good to support replication, sharding, and fault + tolerance. + +* Monitoring, logging, other ops requirements? + +* Encryption of resources, so that Muck doesn't see the contents? + +* Should Muck sign the resources it returns, with it's own key? diff --git a/examples/muck/muck.py b/examples/muck/muck.py new file mode 100644 index 0000000..6ff62f0 --- /dev/null +++ b/examples/muck/muck.py @@ -0,0 +1,2 @@ +def fixme(ctx, **kwargs): + pass diff --git a/examples/muck/muck.yaml b/examples/muck/muck.yaml new file mode 100644 index 0000000..21fe303 --- /dev/null +++ b/examples/muck/muck.yaml @@ -0,0 +1,47 @@ +- given: "a fresh Muck server" + function: fixme + +- given: "I am {name}" + function: fixme + +- given: "I am {name}, with super capability" + function: fixme + +- when: "I do POST /res with (?P\\{.*\\})" + function: fixme + regex: true + +- when: "I do PUT /res with Muck-Id: \\{(?P\\S+)\\}, Muck-Revision: \\{(?P\\S+)\\}, and body (?P\\{.*\\})" + function: fixme + regex: true + +- when: "I do GET /res with Muck-Id: \\{(?P\\S+)\\}" + function: fixme + regex: true + +- when: "I do DELETE /res with Muck-Id: \\{(?P\\S+)\\}" + function: fixme + regex: true + +- when: "I restart Muck" + function: fixme + regex: true + +- then: "response code is (?P\\d+)" + function: fixme + regex: true + +- then: "header {header} is {name}" + function: fixme + +- then: "header (?P
\\S+) matches \\{(?P\\S+)\\}" + function: fixme + regex: true + +- then: "body matches (?P\\{.*\\})" + function: fixme + regex: true + +- then: "revisions \\{(?P\\S+)\\} and \\{(?P\\S+)\\} are different" + function: fixme + regex: true diff --git a/muck.md b/muck.md deleted file mode 100644 index d5470a0..0000000 --- a/muck.md +++ /dev/null @@ -1,513 +0,0 @@ ---- -title: Muck JSON storage server and API -author: Lars Wirzenius -date: work in progress -bindings: muck.yaml -functions: muck.py -template: python -... - -Introduction -============================================================================= - -Muck is intended for storing relatively small pieces of data securely, -and accessing them quickly. Intended uses cases are: - -* storing user, client, application, and related data for an OpenID - Connect authenatication server -* storing personally identifiable information of data subjects (in the - GDPR sense) in a way that they can access and update, assuming - integration with a suitable authantication and authorization server -* in general, storage for web applications of data that isn't large - and fits easily into RAM - -Muck is a JSON store, with an access controlled RESTful HTTP API. Data -stored in Muck is persistent, but kept in memory for fast access. Data -is represented as JSON objects. - -Access is granted based on signed JWT bearer tokens. An OpenID Connect -or OAuth2 identity provider is expected to give such tokens to Muck -clients. The tokens must be signed with a public key that Muck is -configured to accept. - -Access control is simplistic. Each resource is assigned an owner -upon creation, and each user can access (see, update, delete) only -their own resources. A use with "super" powers can access, update, and -delete resources they don't own, but can't create resources for other. -This will be improved later. - -Architecture ------------------------------------------------------------------------------ - -Muck stores data persistently in its local file system. It provides an -HTTP API for clients. Muck itself does not communicate otherwise with -external entities. - -```dot -digraph "architecture" { -muck [shape=box label="Muck"]; -storage [shape=tab label="Persistent \n storage"]; -client [shape=ellipse label="API client"]; -idp [shape=ellipse label="OAuth2/OIDC server"]; - -storage -> muck [label="Read at \n startup"]; -muck -> storage [label="Write \n changes"]; -client -> muck [label="API read/write \n (HTTP)"]; -client -> idp [label="Get access token"]; -idp -> muck [label="Token signing key"]; -} -``` - - -Authentication ------------------------------------------------------------------------------ - -[OAuth2]: https://oauth.net/ -[OpenID Connect]: https://openid.net/connect/ -[JWT]: https://en.wikipedia.org/wiki/JSON_Web_Token - -Muck uses [OAuth2][] or [OpenID Connect][] bearer tokens as access -tokens. The tokens are granted by some form of authentication service, -are [JWT][] tokens, and signed using public-key cryptography. The -authentication service is outside the scope of this document; any -standard implementation should work. - -Muck will be configured with one public key for validating the tokens. -For Muck to access a token: - -* its signature must be valid according to the public key -* it to must be used while it's valid (after the validity starts, but - before if expires) -* its audience must be the specific Muck instance -* its scope claim contains the specified scopes needed for the - attempted operation -* it specified an end-user (data subject) - -Every request to the Muck API must include a token, in the -`Authorizatin` header as a bearer token. The request is denied if the -token does not pass all the above checks. - -Requirements -============================================================================= - -This chapter lists high level requirements for Muck. - -Each requirement here is given a unique mnemnoic id for easier -reference in discussions. - -**SimpleOps** - -: Muck must be simple to install and operate. Installation should be - installing a .deb package, configuration by setting the public key - for token signing of the authentication server. - -**Fast** - -: Muck must be fast. The speed requirement is that Muck must be able - to handle at least 100 concurrent clients, creating 1000 objects - each, and then retrieving each object, and then deleting each - object, and all of this must happen in no more than ten minutes - (600 seconds). Muck and the clients should run on different - virtual machines. - -**Secure** - -: Muck must allow access only by an authenticated client - representing a data subject, and must only allow that client to - access objects owned by the data subject, unless the client has - super privileges. The data subject specifies, via the access - token, what operations the client is allowed to do: whether they - read, update, or delete objects. - - -HTTP API -============================================================================= - -The Muck HTTP API has one endpoint – `/res` – that's used -for all objects. The objects are called resources by Muck. - -The JSON objects Muck operates on must be valid, but their structure -does not matter to Muck. - -Metadata ------------------------------------------------------------------------------ - -Each JSON object stored in Muck is associated with metadata, which is -represented as the following HTTP headers: - -* **Muck-Id** – the resource id -* **Muck-Revision** – the resource revision - -The id is assiged by Muck at object creation time. The revision is -assigned by Muck when the object is created or modified. - - -API requests ------------------------------------------------------------------------------ - -The RESTful API requests are POST, PUT, GET, and DELETE. - -* **POST /res** – create a new object -* **PUT /res** – update an existing object -* **GET /res** – retrieve a existing object -* **DELETE /res** – delete an existing object - -Although it is usual for RESTful HTTP APIs to encode resource -identifiers in the URL, Muck uses headers (Muck-Id, Muck-Revision) for -consistency, and to provide for later expansion. Muck is not intended -to be used manually, but by programmatic clients. - -Additionally, the "sub" claim in the token is used to assign and check -ownership of the object. If the scope contains "super", the sub claim -is ignored, except for creation. - -The examples in this chapter use HTTP/1.1, but should provide the -necessary information for other versions of HTTP. Also, only the -headers relevant to Muck are shown. For example, HTTP/1.1 requires -also a Host header, but this is not shown in the examples. - - - -### Creating an object: POST /res - -Creating requires: - -* "create" in the scope claim -* a non-empty "sub" claim, which will be stored by Muck as the owner - of the created object - -The creation request looks like this: - -~~~{.numberLines} -POST /res HTTP/1.1 -Content-Type: application/ -Authorization: Bearer TOKEN - -{"foo": "bar"} -~~~ - -Note that the creation request does not include Muck-Id or -Muck-Revision headers. - -A successful response looks like this: - -~~~{.numberLines} -201 Created -Content-Type: application/json -Muck-Id: ID -Muck-Revision: REV1 -~~~ - -Note that the response does not contain a copy of the resource. - - - -### Updating an object: PUT /res - -Updating requires: - -* "update" in the scope claim -* one of the following: - - "super" in the scope claim - - "sub" claim matches owner of object Muck; super user can update - any resource, but otherwise data subjects can only update their own - objects -* Muck-Revision matches the current revision in Muck; this functions - as a simplistic guard against conflicting updates from different - clients. - -The update request looks like this: - -~~~{.numberLines} -PUT /res HTTP/1.1 -Authorization: Bearer TOKEN -Content-Type: application/json -Muck-Id: ID -Muck-Revision: REV1 - -{"foo": "yo"} -~~~ - -In the request, ID identifies the object, and REV1 is its revision. - -The successful response: - -~~~{.numberLines} -200 OK -Content-Type: application/json -Muck-Id: ID -Muck-Revision: REV2 -~~~ - -Note that the update response also doesn't contain the object. The -client should remember the new revision, or retrieve the object get -the latest revision before the next update. - - -### Retrieving an object: GET /res - -A request requires: - -* "show" in the scope claim -* one of the following: - - "super" in the scope claim - - "sub" claim matches owner of object Muck; super user can retrieve - any resource, but otherwise data subjects can only update their own - objects - -The request to retrieve a response: - -~~~{.numberLines} -GET /res HTTP/1.1 -Authorization: Bearer TOKEN -Muck-Id: ID -~~~ - -A successful response: - -~~~{.numberLines} -200 OK -Content-Type: application/json -Muck-Id: ID -Muck-Revision: REV2 - -{"foo": "yo"} -~~~ - -Note that the response does NOT indicate the owner of the resource. - - - -Acceptance criteria for Muck -============================================================================= - -This chapter details the acceptance criteria for Muck, and how they're -verified. - - -Basic object handling ------------------------------------------------------------------------------ - -First, we need a new Muck server. It will initially have no objects. -We also need a test user, whom we'll call Tomjon. - -~~~scenario -given a fresh Muck server -given I am Tomjon -~~~ - -Tomjon can create an object. - -~~~scenario -when I do POST /res with {"foo": "bar"} -then response code is 201 -then header Muck-Id is ID -then header Muck-Revision is REV1 -~~~ - -Tomjon can then retrieve the object. It has the same revision and -body. - -~~~scenario -when I do GET /res with Muck-Id: {ID} -then response code is 200 -then header Muck-Revision matches {REV1} -then body matches {"foo": "bar"} -~~~ - -Tomjon can update the object, and the update has the same id, but a -new revision and body. - -~~~scenario -when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} -then response code is 200 -then header Muck-Revision is {REV2} -then revisions {REV1} and {REV2} are different -~~~ - -If Tomjon tries to update with the old revision, it fails. - -~~~scenario -when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} -then response code is 409 -~~~ - -After the failed update, the object or its revision haven't changed. - -~~~scenario -when I do GET /res with Muck-Id: {ID} -then response code is 200 -then header Muck-Revision matches {REV2} -then body matches {"foo": "yo"} -~~~ - -We can delete the resource, and then it's gone. - -~~~scenario -when I do DELETE /res with Muck-Id: {ID} -then response code is 200 -when I do GET /res with Muck-Id: {ID} -then response code is 404 -~~~ - - -Restarting Muck ------------------------------------------------------------------------------ - -Muck should store data persistently. For this we need our test user to -have the "super" capability. - -~~~scenario -given a fresh Muck server -given I am Tomjon, with super capability -when I do POST /res with {"foo": "bar"} -then header Muck-Id is ID -then header Muck-Revision is REV1 -~~~ - -So far, so good. Nothing new here. Now we restart Muck. The resource -just created must still be there. - -~~~scenario -when I restart Muck -when I do GET /res with Muck-Id: {ID} -then response code is 200 -then header Muck-Revision matches {REV1} -then body matches {"foo": "bar"} -~~~ - - -Super user access ------------------------------------------------------------------------------ - -Check here that if we have super scope, we can retrieve, update, and -delete someone else's resources, but if we create a resourec, it's -ours. - -Invalid requests ------------------------------------------------------------------------------ - -There are a number of ways in which a request might be rejected. This -section verifies all of them. - -### Accessing someone else's data - -~~~scenario -given a fresh Muck server -given I am Tomjon -when I do POST /res with {"foo": "bar"} -then header Muck-Id is ID -then header Muck-Revision is REV1 -when I do GET /res with Muck-Id: {ID} -then response code is 200 -then header Muck-Revision matches {REV1} -then body matches {"foo": "bar"} -~~~ - -After this, we morph into another test user. - -~~~scenario -given I am Verence -when I do GET /res with Muck-Id: {ID} -then response code is 404 -~~~ - -Note that we get a "not found" error and not a "access denied" error -so that Verence doesn't know if the resource exists or not. - - -### Updating someone else's data - -This is similar to retrieving it, but we try to update instead. - -~~~scenario -given a fresh Muck server -given I am Tomjon -when I do POST /res with {"foo": "bar"} -then header Muck-Id is ID -then header Muck-Revision is REV1 -given I am Verence -when I do PUT /res with Muck-Id: {ID}, Muck-Revision: {REV1}, and body {"foo":"yo"} -then response code is 404 -~~~ - - -### Deleting someone else's data - -This is similar to retrieving it, but we try to delete it instead. - -~~~scenario -given a fresh Muck server -given I am Tomjon -when I do POST /res with {"foo": "bar"} -then header Muck-Id is ID -then header Muck-Revision is REV1 -given I am Verence -when I do DELETE /res with Muck-Id: {ID} -then response code is 404 -~~~ - -### Bad signature - -### Not valid yet - -### Not valid anymore - -### Not for our instance - -### Lack scope for creation - -### Lack scope for retrieval - -### Lack scope for updating - -### Lack scope for deletion - -### No subject when creating - -### No subject when retrieving - -### No subject when updating - -### No subject when deleting - -### Invalid JSON when creating - -### Invalid JSON when updating - - -# Possible future changes - -* There is no way to list all the resources a user has, or search for - resource. This should be doable in some way. With a search, a - listing operation is not strictly necessary. - -* It's going to be inconvenient to only be able to access one's own - resources. It would be good to support groups. A resource could be - owned by a group, and end-users / subjects could belong to any - number of groups. Also, groups should be able to belong to groups. - Each resource should be able to specify for each group what access - members of that group should have (retrieve, update, delete). There - should be no limits to how many group access control rules there are - per resource. - - This would allow setups such as each resource representing a stored - file, and some groups would be granted read access, or read-write - access, or read-delete access to the files. - -* Also, it might be good to be able to grant other groups access to - controll a resource's access control rules. - -* It might be good support schemas for resources? - -* It might be good to have a configurable maximum size of a resource. - Possibly per-user quotas. - -* It would be good to support replication, sharding, and fault - tolerance. - -* Monitoring, logging, other ops requirements? - -* Encryption of resources, so that Muck doesn't see the contents? - -* Should Muck sign the resources it returns, with it's own key? diff --git a/muck.py b/muck.py deleted file mode 100644 index 6ff62f0..0000000 --- a/muck.py +++ /dev/null @@ -1,2 +0,0 @@ -def fixme(ctx, **kwargs): - pass diff --git a/muck.yaml b/muck.yaml deleted file mode 100644 index 21fe303..0000000 --- a/muck.yaml +++ /dev/null @@ -1,47 +0,0 @@ -- given: "a fresh Muck server" - function: fixme - -- given: "I am {name}" - function: fixme - -- given: "I am {name}, with super capability" - function: fixme - -- when: "I do POST /res with (?P\\{.*\\})" - function: fixme - regex: true - -- when: "I do PUT /res with Muck-Id: \\{(?P\\S+)\\}, Muck-Revision: \\{(?P\\S+)\\}, and body (?P\\{.*\\})" - function: fixme - regex: true - -- when: "I do GET /res with Muck-Id: \\{(?P\\S+)\\}" - function: fixme - regex: true - -- when: "I do DELETE /res with Muck-Id: \\{(?P\\S+)\\}" - function: fixme - regex: true - -- when: "I restart Muck" - function: fixme - regex: true - -- then: "response code is (?P\\d+)" - function: fixme - regex: true - -- then: "header {header} is {name}" - function: fixme - -- then: "header (?P
\\S+) matches \\{(?P\\S+)\\}" - function: fixme - regex: true - -- then: "body matches (?P\\{.*\\})" - function: fixme - regex: true - -- then: "revisions \\{(?P\\S+)\\} and \\{(?P\\S+)\\} are different" - function: fixme - regex: true -- cgit v1.2.1 From ba04e71db20d47e021982fc99950443a9350ffe3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 17 Feb 2021 08:54:53 +0200 Subject: refactor: rewrite check in Python as check.py The old shell script became too hard to understand and maintain. This should be clearer and also more robust. --- check | 397 +++++++++++++++++++++++++----------- subplotlib/tests/files.rs | 507 ++++++++++++++++++++++++++++------------------ 2 files changed, 592 insertions(+), 312 deletions(-) diff --git a/check b/check index d41c9ae..2573a9e 100755 --- a/check +++ b/check @@ -1,113 +1,284 @@ -#!/bin/sh - -set -eu - -verbose=false -if [ "$#" -gt 0 ] -then - case "$1" in - verbose | -v | --verbose) - verbose=true - ;; - esac -fi - -hideok= -if command -v chronic > /dev/null -then - hideok=chronic -fi -quiet=-q -if $verbose -then - quiet= - hideok= -fi - -TOPDIR=$(pwd) - -_codegen() { - $hideok cargo run $quiet --package subplot --bin sp-codegen -- \ - "$1" --output "$2" --resources "${TOPDIR}/share" -} - -codegen() { - _codegen "$1" "$2" - rm -f test.log - template="$(sed -n '/^template: /s///p' "$1" | tail -n1)" - case "$template" in - python) $hideok python3 "$2" --log test.log ;; - bash) $hideok bash "$2" ;; - *) echo "Don't know interpreter for $2" ; exit 1 ;; - esac -} - -docgen() { - cargo run $quiet --package subplot --bin sp-docgen -- "$1" --output "$2" --resources "${TOPDIR}/share" -} - -# Run unit tests for the Python template. -(set -eu - cd share/python/template - for x in *_tests.py - do - $hideok echo "Unit tests: $x" - $hideok python3 "$x" - $hideok echo - done) - -if command -v flake8 > /dev/null -then - $hideok flake8 share/python/template share/python/lib/*.py -fi - -if command -v shellcheck > /dev/null -then - shellcheck check ./*.sh - find share/bash/template -name '*.sh' -exec shellcheck '{}' + -fi - -$hideok cargo build --all-targets -if cargo --list | awk '{ print $1 }' | grep 'clippy$' > /dev/null -then - cargo clippy $quiet -fi -$hideok cargo test $quiet - -if command -v rustfmt > /dev/null -then - cargo fmt --all -- --check -fi - -if command -v black > /dev/null -then - $hideok find . -type f -name '*.py' ! -name template.py ! -name test.py \ - -exec black --check '{}' + -fi - -(cd subplotlib; - $hideok cargo test --lib - for md in [!CR]*.md; do - $hideok echo "subplotlib/$md =====================================" - $hideok mkdir -p tests - _codegen "$md" "tests/$(basename "$md" .md).rs" "" - # This formatting is fine because we checked --all earlier - # so all it'll do is tidy up the test suite - cargo fmt - docgen "$md" "$(basename "$md" .md).pdf" - docgen "$md" "$(basename "$md" .md).html" - $hideok cargo test --test "$(basename "$md" .md)" - $hideok echo - done -) - -for md in [!CR]*.md share/python/lib/*.md -do - $hideok echo "$md =====================================" - codegen "$md" test.py - docgen "$md" "$(basename "$md" .md).pdf" - docgen "$md" "$(basename "$md" .md).html" - $hideok echo -done - -echo "Everything seems to be in order." +#!/usr/bin/env python3 + +import argparse +import glob +import os +import sys + +from subprocess import PIPE, DEVNULL, STDOUT + + +class Runcmd: + """Run external commands in various ways""" + + def __init__(self, verbose, progress): + self._verbose = verbose + self._progress = progress + + def _write_msg(self, msg): + sys.stdout.write(f"{msg}\n") + sys.stdout.flush() + + def msg(self, msg): + if self._verbose: + self._write_msg(msg) + + def title(self, title): + if self._verbose: + self._write_msg("") + self._write_msg("=" * len(title)) + self._write_msg(title) + self._write_msg("=" * len(title)) + elif self._progress: + self._write_msg(title) + + def _runcmd_unchecked(self, argv, **kwargs): + """Run a command (generic version) + + Return a subcommand.CompletedProcess. It's the caller's duty + check that the command succeeded. + + All actual execution of other programs happens via this method. + However, only methods of this class should ever call this + method. + """ + + # Import "run" here so that no other part of the code can see the + # symbol. + from subprocess import run + + self.msg(f"RUN: {argv} {kwargs}") + + if not self._verbose: + kwargs["stdout"] = PIPE + kwargs["stderr"] = STDOUT + + return run(argv, **kwargs) + + def runcmd(self, argv, **kwargs): + """Run a command (generic version) + + If the command fails, terminate the program. On success, return + a subprocess.CompletedProcess. + """ + + p = self._runcmd_unchecked(argv, **kwargs) + if p.returncode != 0: + sys.stdout.write((p.stdout or b"").decode("UTF-8")) + sys.stderr.write((p.stderr or b"").decode("UTF-8")) + sys.stderr.write(f"Command {argv} failed\n") + sys.exit(p.returncode) + return p + + def runcmd_maybe(self, argv, **kwargs): + """Run a command it's availbe, or quietly do no nothing""" + if self.got_command(argv[0]): + self.runcmd(argv, **kwargs) + + def got_command(self, name): + """Is a command of a given name available?""" + p = self._runcmd_unchecked(["which", name], stdout=DEVNULL) + return p.returncode == 0 + + def cargo(self, args, **kwargs): + """Run cargo with arguments.""" + self.runcmd(["cargo"] + args, **kwargs) + + def cargo_maybe(self, args, **kwargs): + """Run cargo if the desired subcommand is available""" + if self.got_cargo(args[0]): + self.runcmd(["cargo"] + args, **kwargs) + + def got_cargo(self, subcommand): + """Is a cargo subcommand available?""" + p = self.runcmd(["cargo", "--list"], check=True, stdout=PIPE) + lines = [line.strip() for line in p.stdout.decode("UTF-8").splitlines()] + return subcommand in lines + + def codegen(self, md, output, **kwargs): + """Run the Subplot code generator and the test program it produces""" + self.cargo( + [ + "run", + "--package=subplot", + "--bin=sp-codegen", + "--", + md, + f"--output={output}", + f"--resources={os.path.abspath('share')}", + ], + **kwargs, + ) + + def docgen(self, md, output, **kwargs): + """Run the Subplot document generator""" + self.cargo( + [ + "run", + "--package=subplot", + "--bin=sp-docgen", + "--", + md, + f"--output={output}", + f"--resources={os.path.abspath('share')}", + ], + **kwargs, + ) + + +def find_files(pattern, pred): + """Find files recursively, if they are accepted by a predicate function""" + return [f for f in glob.glob(pattern, recursive=True) if pred(f)] + + +def check_python(r): + """Run all checks for Python code""" + r.title("checking Python code") + + # Find all Python files anywhere, except those we know aren't proper Python. + py = find_files( + "**/*.py", lambda f: os.path.basename(f) not in ("template.py", "test.py") + ) + + # Test with flake8 if available. Flake8 finds Python files itself. + r.runcmd_maybe(["flake8", "check"] + py) + + # Check formatting with Black. We need to provide the files to Python + # ourselves. + r.runcmd_maybe(["black", "--check"] + py) + + # Find and run unit tests. + tests = find_files("**/*_tests.py", lambda f: True) + for test in tests: + dirname = os.path.dirname(test) + test = os.path.basename(test) + r.runcmd(["python3", test], cwd=dirname) + + +def check_shell(r): + """Run all checks for shell code""" + r.title("checking shell code") + + # Find all shell files anywhere, except generated test programs. + sh = find_files("**/*.sh", lambda f: os.path.basename(f) != "test.sh") + + r.runcmd_maybe(["shellcheck"] + sh) + + +def check_rust(r): + """Run all checks for Rust code""" + r.title("checking Rust code") + + r.runcmd(["cargo", "build", "--all-targets"]) + if r.got_cargo("clippy"): + r.runcmd(["cargo", "clippy"]) + r.runcmd(["cargo", "test"]) + r.runcmd(["cargo", "fmt"]) + + +def check_subplots(r): + """Run all Subplots and generate documents for them""" + mds = find_files("**/*.md", lambda f: f == f.lower() and "subplotlib" not in f) + for md0 in mds: + r.title(f"checking subplot {md0}") + + dirname = os.path.dirname(md0) or "." + md = os.path.basename(md0) + + template = get_template(md0) + if template == "python": + # Remove test log from previous run, if any. + test_log = os.path.join(dirname, "test.log") + if os.path.exists(test_log): + os.remove(test_log) + + r.codegen(md, "test.py", cwd=dirname) + r.runcmd(["python3", "test.py", "--log", "test.log"], cwd=dirname) + os.remove(test_log) + os.remove(os.path.join(dirname, "test.py")) + elif template == "bash": + r.codegen(md, "test.sh", cwd=dirname) + r.runcmd(["bash", "-x", "test.sh"], cwd=dirname) + os.remove(os.path.join(dirname, "test.sh")) + else: + sys.exit(f"unknown template {template} in {md0}") + + base, _ = os.path.splitext(md) + r.docgen(md, base + ".pdf", cwd=dirname) + r.docgen(md, base + ".html", cwd=dirname) + + +def check_subplotlib(r): + """Run all checks for subplotlib""" + r.title("checking subplotlib code") + + # Run Rust tests for the subplotlib library. + r.runcmd(["cargo", "test", "--lib"], cwd="subplotlib") + + # Find subplots for subplotlib. + mds = find_files("subplotlib/*.md", lambda f: True) + + os.makedirs("subplotlib/tests", exist_ok=True) + for md0 in mds: + r.title(f"checking subplot {md0}") + dirname = os.path.dirname(md0) + md = os.path.basename(md0) + base, _ = os.path.splitext(md) + test_rs = os.path.join("tests", base + ".rs") + r.codegen(md, test_rs, cwd=dirname) + r.cargo(["fmt"], cwd=dirname) + r.cargo(["test", "--test", base], cwd=dirname) + r.docgen(md, base + ".html", cwd=dirname) + r.docgen(md, base + ".pdf", cwd=dirname) + + +def get_template(filename): + prefix = "template: " + with open(filename) as f: + data = f.read() + for line in data.splitlines(): + if line.startswith(prefix): + line = line[len(prefix) :] + return line + sys.exit(f"{filename} does not specify a template") + + +def parse_args(): + """Parse command line arguments to this script""" + p = argparse.ArgumentParser() + p.add_argument("-v", dest="verbose", action="store_true", help="be verbose") + p.add_argument( + "-p", dest="progress", action="store_true", help="print some progress output" + ) + + all_whats = ["python", "shell", "rust", "subplots", "subplotlib"] + p.add_argument( + "what", nargs="*", default=all_whats, help=f"what to test: {all_whats}" + ) + return p.parse_args() + + +def main(): + """Main program""" + args = parse_args() + + r = Runcmd(args.verbose, args.progress) + + for what in args.what: + if what == "python": + check_python(r) + elif what == "shell": + check_shell(r) + elif what == "rust": + check_rust(r) + elif what == "subplots": + check_subplots(r) + elif what == "subplotlib": + check_subplotlib(r) + else: + sys.exit(f"Unknown test {what}") + + sys.stdout.write("Everything seems to be in order.\n") + + +main() diff --git a/subplotlib/tests/files.rs b/subplotlib/tests/files.rs index a8fce3d..2f0bc6c 100644 --- a/subplotlib/tests/files.rs +++ b/subplotlib/tests/files.rs @@ -1,70 +1,90 @@ use subplotlib::prelude::*; + + // -------------------------------- lazy_static! { - static ref SUBPLOT_EMBEDDED_FILES: Vec = - vec![SubplotDataFile::new("aGVsbG8udHh0", "aGVsbG8sIHdvcmxkCg=="),]; + static ref SUBPLOT_EMBEDDED_FILES: Vec = vec![ + + SubplotDataFile::new("aGVsbG8udHh0", + "aGVsbG8sIHdvcmxkCg=="), + + ]; } + + // --------------------------------- // Create on-disk files from embedded files #[test] fn create_on_disk_files_from_embedded_files() { - let mut scenario = Scenario::new(&base64_decode( - "Q3JlYXRlIG9uLWRpc2sgZmlsZXMgZnJvbSBlbWJlZGRlZCBmaWxlcw==", - )); - + let mut scenario = Scenario::new(&base64_decode("Q3JlYXRlIG9uLWRpc2sgZmlsZXMgZnJvbSBlbWJlZGRlZCBmaWxlcw==")); + let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default() - .embedded_file({ - use std::path::PathBuf; - // hello.txt - let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); - SUBPLOT_EMBEDDED_FILES - .iter() - .find(|df| df.name() == target_name) - .expect("Unable to find file at runtime") - .clone() - }) - .build(); + .embedded_file( + + { + use std::path::PathBuf; + // hello.txt + let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); + SUBPLOT_EMBEDDED_FILES + .iter() + .find(|df| df.name() == target_name) + .expect("Unable to find file at runtime") + .clone() + } + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_exists::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_contains::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .data( + ) + .data( + // "hello, world" - &base64_decode("aGVsbG8sIHdvcmxk"), + &base64_decode("aGVsbG8sIHdvcmxk" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_does_not_exist::Builder::default() - .filename( + .filename( + // "other.txt" - &base64_decode("b3RoZXIudHh0"), + &base64_decode("b3RoZXIudHh0" + ) ) - .build(); + .build(); scenario.add_step(step, None); - - let step = - subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default() - .filename_on_disk( - // "other.txt" - &base64_decode("b3RoZXIudHh0"), - ) - .embedded_file({ + + let step = subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default() + .filename_on_disk( + + // "other.txt" + &base64_decode("b3RoZXIudHh0" + ) + ) + .embedded_file( + + { use std::path::PathBuf; // hello.txt let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); @@ -73,293 +93,369 @@ fn create_on_disk_files_from_embedded_files() { .find(|df| df.name() == target_name) .expect("Unable to find file at runtime") .clone() - }) - .build(); + } + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_exists::Builder::default() - .filename( + .filename( + // "other.txt" - &base64_decode("b3RoZXIudHh0"), + &base64_decode("b3RoZXIudHh0" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_match::Builder::default() - .filename1( + .filename1( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" + ) ) - .filename2( + .filename2( + // "other.txt" - &base64_decode("b3RoZXIudHh0"), + &base64_decode("b3RoZXIudHh0" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::only_these_exist::Builder::default() - .filenames( + .filenames( + // "hello.txt, other.txt" - &base64_decode("aGVsbG8udHh0LCBvdGhlci50eHQ="), + &base64_decode("aGVsbG8udHh0LCBvdGhlci50eHQ=" ) - .build(); + ) + .build(); scenario.add_step(step, None); + scenario.run().unwrap(); } + // --------------------------------- // File metadata #[test] fn file_metadata() { let mut scenario = Scenario::new(&base64_decode("RmlsZSBtZXRhZGF0YQ==")); - + let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default() - .embedded_file({ - use std::path::PathBuf; - // hello.txt - let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); - SUBPLOT_EMBEDDED_FILES - .iter() - .find(|df| df.name() == target_name) - .expect("Unable to find file at runtime") - .clone() - }) - .build(); + .embedded_file( + + { + use std::path::PathBuf; + // hello.txt + let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); + SUBPLOT_EMBEDDED_FILES + .iter() + .find(|df| df.name() == target_name) + .expect("Unable to find file at runtime") + .clone() + } + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::remember_metadata::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::has_remembered_metadata::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::create_from_text::Builder::default() - .text( + .text( + // "yo" - &base64_decode("eW8="), + &base64_decode("eW8=" ) - .filename( + ) + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::has_different_metadata::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" + ) ) - .build(); + .build(); scenario.add_step(step, None); + scenario.run().unwrap(); } + // --------------------------------- // File modification time #[test] fn file_modification_time() { let mut scenario = Scenario::new(&base64_decode("RmlsZSBtb2RpZmljYXRpb24gdGltZQ==")); - + let step = subplotlib::steplibrary::files::touch_with_timestamp::Builder::default() - .filename( + .filename( + // "foo.dat" - &base64_decode("Zm9vLmRhdA=="), + &base64_decode("Zm9vLmRhdA==" ) - .mtime( + ) + .mtime( + // "1970-01-02 03:04:05" - &base64_decode("MTk3MC0wMS0wMiAwMzowNDowNQ=="), + &base64_decode("MTk3MC0wMS0wMiAwMzowNDowNQ==" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::mtime_is_ancient::Builder::default() - .filename( + .filename( + // "foo.dat" - &base64_decode("Zm9vLmRhdA=="), + &base64_decode("Zm9vLmRhdA==" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::touch::Builder::default() - .filename( + .filename( + // "foo.dat" - &base64_decode("Zm9vLmRhdA=="), + &base64_decode("Zm9vLmRhdA==" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::mtime_is_recent::Builder::default() - .filename( + .filename( + // "foo.dat" - &base64_decode("Zm9vLmRhdA=="), + &base64_decode("Zm9vLmRhdA==" ) - .build(); + ) + .build(); scenario.add_step(step, None); + scenario.run().unwrap(); } + // --------------------------------- // File contents #[test] fn file_contents() { let mut scenario = Scenario::new(&base64_decode("RmlsZSBjb250ZW50cw==")); - + let step = subplotlib::steplibrary::files::create_from_embedded::Builder::default() - .embedded_file({ - use std::path::PathBuf; - // hello.txt - let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); - SUBPLOT_EMBEDDED_FILES - .iter() - .find(|df| df.name() == target_name) - .expect("Unable to find file at runtime") - .clone() - }) - .build(); + .embedded_file( + + { + use std::path::PathBuf; + // hello.txt + let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); + SUBPLOT_EMBEDDED_FILES + .iter() + .find(|df| df.name() == target_name) + .expect("Unable to find file at runtime") + .clone() + } + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_contains::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .data( + ) + .data( + // "hello, world" - &base64_decode("aGVsbG8sIHdvcmxk"), + &base64_decode("aGVsbG8sIHdvcmxk" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_matches_regex::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" + ) ) - .regex( + .regex( + // "hello, .*" - &base64_decode("aGVsbG8sIC4q"), + &base64_decode("aGVsbG8sIC4q" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::file_matches_regex::Builder::default() - .filename( + .filename( + // "hello.txt" - &base64_decode("aGVsbG8udHh0"), + &base64_decode("aGVsbG8udHh0" ) - .regex( + ) + .regex( + // "hello, .*" - &base64_decode("aGVsbG8sIC4q"), + &base64_decode("aGVsbG8sIC4q" + ) ) - .build(); + .build(); scenario.add_step(step, None); + scenario.run().unwrap(); } + // --------------------------------- // Directories #[test] fn directories() { let mut scenario = Scenario::new(&base64_decode("RGlyZWN0b3JpZXM=")); - + let step = subplotlib::steplibrary::files::make_directory::Builder::default() - .path( + .path( + // "first" - &base64_decode("Zmlyc3Q="), + &base64_decode("Zmlyc3Q=" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_exists::Builder::default() - .path( + .path( + // "first" - &base64_decode("Zmlyc3Q="), + &base64_decode("Zmlyc3Q=" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_is_empty::Builder::default() - .path( + .path( + // "first" - &base64_decode("Zmlyc3Q="), + &base64_decode("Zmlyc3Q=" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::remove_directory::Builder::default() - .path( + .path( + // "first" - &base64_decode("Zmlyc3Q="), + &base64_decode("Zmlyc3Q=" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default() - .path( + .path( + // "first" - &base64_decode("Zmlyc3Q="), + &base64_decode("Zmlyc3Q=" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::make_directory::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_exists::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_is_empty::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); - - let step = - subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default() - .filename_on_disk( - // "second/third/hello.txt" - &base64_decode("c2Vjb25kL3RoaXJkL2hlbGxvLnR4dA=="), - ) - .embedded_file({ + + let step = subplotlib::steplibrary::files::create_from_embedded_with_other_name::Builder::default() + .filename_on_disk( + + // "second/third/hello.txt" + &base64_decode("c2Vjb25kL3RoaXJkL2hlbGxvLnR4dA==" + ) + ) + .embedded_file( + + { use std::path::PathBuf; // hello.txt let target_name: PathBuf = base64_decode("aGVsbG8udHh0").into(); @@ -368,49 +464,62 @@ fn directories() { .find(|df| df.name() == target_name) .expect("Unable to find file at runtime") .clone() - }) - .build(); + } + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_is_not_empty::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" ) - .build(); + ) + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_exists::Builder::default() - .path( + .path( + // "second/third" - &base64_decode("c2Vjb25kL3RoaXJk"), + &base64_decode("c2Vjb25kL3RoaXJk" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_is_not_empty::Builder::default() - .path( + .path( + // "second/third" - &base64_decode("c2Vjb25kL3RoaXJk"), + &base64_decode("c2Vjb25kL3RoaXJk" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::remove_directory::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); - + let step = subplotlib::steplibrary::files::path_does_not_exist::Builder::default() - .path( + .path( + // "second" - &base64_decode("c2Vjb25k"), + &base64_decode("c2Vjb25k" + ) ) - .build(); + .build(); scenario.add_step(step, None); + scenario.run().unwrap(); } + -- cgit v1.2.1