Support collection-level markdown documentation

.github/workflows/ci.yml

@@ -71,12 +71,18 @@ jobs:
           path: test/e2e/path
           edition: ${{ matrix.edition.name }}
 
-      - name: E2E (enterprise)
+      - name: E2E (enterprise/html)
         uses: ./.github/actions/e2e
         with:
           path: enterprise/e2e/html
           edition: ${{ matrix.edition.name }}
         if: matrix.edition.name == 'enterprise'
+      - name: E2E (enterprise/headless)
+        uses: ./.github/actions/e2e
+        with:
+          path: enterprise/e2e/headless
+          edition: ${{ matrix.edition.name }}
+        if: matrix.edition.name == 'enterprise'
 
       # Public instance
       - run: docker build .  --file public/Dockerfile --progress plain

Makefile

@@ -78,6 +78,7 @@ docker: docker-build
 	$(MAKE) -C test/e2e/chaos EDITION=$(EDITION)
 ifeq ($(ENTERPRISE),ON)
 	$(MAKE) -C enterprise/e2e/html EDITION=$(EDITION)
+	$(MAKE) -C enterprise/e2e/headless EDITION=$(EDITION)
 endif
 
 .PHONY: docker-benchmark

collections/self/v1/schemas/api/list/response.json

@@ -125,6 +125,9 @@
     },
     "website": {
       "$ref": "https://schemas.sourcemeta.com/sourcemeta/std/v0/ietf/uri/url"
+    },
+    "documentation": {
+      "$ref": "https://schemas.sourcemeta.com/sourcemeta/std/v0/ietf/uri/uri-relative"
     }
   },
   "additionalProperties": false

collections/self/v1/schemas/configuration/collection.json

@@ -30,6 +30,9 @@
     "x-sourcemeta-one:provenance": {
       "$ref": "./rpath.json"
     },
+    "x-sourcemeta-one:documentation": {
+      "$ref": "https://schemas.sourcemeta.com/sourcemeta/std/v0/ieee/posix/2017/path"
+    },
     "baseUri": {
       "$ref": "https://schemas.sourcemeta.com/sourcemeta/std/v0/ietf/uri/uri-reference"
     },

docs/api.md

@@ -82,6 +82,7 @@ navigation and discovery purposes.
     | `/email` | String | No | The e-mail address associated with the directory |
     | `/github` | String | No | The GitHub organisation or repository associated with the directory |
     | `/website` | String | No | The external URL associated with the directory  |
+    | `/documentation` | String | No | For directories that correspond to a collection with documentation configured, a relative URL to the static file containing the raw Markdown content. Only available in the [Enterprise](commercial.md) edition |
     | `/schemas` | Integer | Yes | The recursive count of schemas in this directory |
     | `/entries` | Array | Yes | The entries inside the directory |
     | `/entries/*/type` | String | Yes | The type of the entry (`schema` or `directory`) |
@@ -106,6 +107,27 @@ navigation and discovery purposes.
 
     The directory does not exist.
 
+## Static Files
+
+### Fetch
+
+*This endpoint serves a static file by its hash identifier.*
+
+```
+GET {base_path}/self/v1/static/{hash}
+```
+
+Static files are external assets (such as documentation Markdown files)
+that have been indexed into the registry.
+
+=== "200"
+
+    The raw file content.
+
+=== "404"
+
+    The static file does not exist.
+
 ## Schemas
 
 ### Fetch

docs/commercial.md

@@ -25,9 +25,10 @@ Sourcemeta One is available in two editions:
   release, the code transitions to AGPL-3.0.
 
 - **Enterprise**: Includes the [Standard
-  Library](https://github.com/sourcemeta/std), additional features, and supply
-  chain security capabilities not available in the Community edition. Requires
-  a [commercial
+  Library](https://github.com/sourcemeta/std), additional features such as
+  [custom linter rules](configuration.md#linter), [collection
+  documentation](configuration.md#documentation), and supply chain security
+  capabilities not available in the Community edition. Requires a [commercial
   license](https://github.com/sourcemeta/one/blob/main/LICENSE-COMMERCIAL)
   from Sourcemeta.
 

docs/configuration.md

@@ -147,6 +147,7 @@ contain the actual schema definitions that power your instance.
 | `/ignore`      | Array  | No  | None | An array of file paths (relative to the configuration file location) to exclude from the schema collection. See the [JSON Schema CLI configuration](https://github.com/sourcemeta/jsonschema/blob/main/docs/configuration.markdown) for more information |
 | `/x-sourcemeta-one:evaluate`      | Boolean  | No  | `true` | When set to `false`, disable the evaluation API for this schema collection. This is useful if you will never make use of the [evaluation API](api.md) and want to speed up the generation of the instance |
 | `/x-sourcemeta-one:alert`      | String  | No  | N/A | When set, provide a human-readable alert on both the API and the HTML explorer for every schema in the collection. This is useful to provide any important message to consumers. The web explorer renders this as Markdown |
+| `/x-sourcemeta-one:documentation` (**Enterprise**) | String  | No  | N/A | Path (relative to the configuration file) to a Markdown documentation file for this collection. Rendered in the web explorer below the file listing. See the [Documentation](#documentation) section for more information |
 
 !!! warning
 
@@ -295,6 +296,44 @@ Rule file paths are relative to the configuration file location. You can list
 multiple rules in the array to enforce several constraints at once. Rule names
 must be unique across all rules in a collection.
 
+### Documentation
+
+!!! success "Enterprise"
+
+    Collection documentation is only available in the
+    [Enterprise](commercial.md) edition. Learn more about [commercial
+    licensing](commercial.md).
+
+The `x-sourcemeta-one:documentation` property lets you attach a Markdown file
+to a schema collection. The content is rendered in the web explorer below the
+file listing, similar to how GitHub renders a README.
+
+```json hl_lines="6" title="one.json"
+{
+  "url": "https://schemas.example.com",
+  "contents": {
+    "my-collection": {
+      "path": "./schemas",
+      "x-sourcemeta-one:documentation": "./docs/my-collection.md"
+    }
+  }
+}
+```
+
+The path is relative to the configuration file location. The documentation
+file must exist at indexing time.
+
+The Markdown content is rendered using GitHub Flavored Markdown (GFM),
+supporting tables, autolinks, strikethrough, task lists, and footnotes.
+Unsafe HTML (such as `<script>` or `<iframe>` tags) is automatically
+filtered for security.
+
+!!! note
+
+    Documentation files are limited to 1 MB in size. For larger documents,
+    consider splitting them into smaller files or linking to external
+    documentation from within the Markdown content.
+
 ## Pages
 
 A page functions as an organizational container within the instance.  Unlike

enterprise/e2e/headless/Dockerfile

@@ -0,0 +1,8 @@
+FROM one
+COPY one.json .
+COPY schemas schemas
+COPY docs docs
+RUN sourcemeta one.json --profile
+RUN set -e && test -d "$SOURCEMETA_ONE_WORKDIR" && \
+  test -z "$(ls -A "$SOURCEMETA_ONE_WORKDIR")"
+RUN rm -rf "$SOURCEMETA_ONE_WORKDIR"

enterprise/e2e/headless/Makefile

@@ -0,0 +1 @@
+include ../../../test/e2e/common.mk

enterprise/e2e/headless/compose.yml

@@ -0,0 +1,11 @@
+services:
+  sandbox:
+    build:
+      context: .
+      dockerfile: Dockerfile
+      args:
+        SOURCEMETA_ONE_SANDBOX_EDITION: ${EDITION}
+    environment:
+      - SOURCEMETA_ONE_PORT=8001
+    ports:
+      - "${PORT}:8001"

enterprise/e2e/headless/docs/test-readme.md

@@ -0,0 +1,3 @@
+# Headless Documentation
+
+Documentation for the **test** collection in headless mode.

enterprise/e2e/headless/hurl/documentation.hurl

@@ -0,0 +1,149 @@
+# Collection with documentation has the documentation URL in the full list response
+GET {{base}}/self/v1/api/list/test
+HTTP 200
+Content-Type: application/json
+Access-Control-Allow-Origin: *
+Link: </self/v1/schemas/api/list/response>; rel="describedby"
+[Captures]
+last_response: body
+schema_path: header "Link" regex "<([^>]+)>"
+documentation_url: jsonpath "$.documentation"
+[Asserts]
+header "ETag" exists
+header "Last-Modified" exists
+jsonpath "$.path" == "/test"
+jsonpath "$.url" == "{{base}}/test"
+jsonpath "$.schemas" == 1
+jsonpath "$.health" == 0
+jsonpath "$.breadcrumb" count == 1
+jsonpath "$.breadcrumb[0].name" == "test"
+jsonpath "$.breadcrumb[0].path" == "/test/"
+jsonpath "$.documentation" matches "^/self/v1/static/[a-f0-9]{64}$"
+jsonpath "$.entries" count == 1
+jsonpath "$.entries[0].name" == "string"
+jsonpath "$.entries[0].type" == "schema"
+jsonpath "$.entries[0].path" == "/test/string"
+jsonpath "$.entries[0].identifier" == "{{base}}/test/string"
+jsonpath "$.entries[0].bytes" == 129
+jsonpath "$.entries[0].baseDialect" == "https://json-schema.org/draft/2020-12/schema"
+jsonpath "$.entries[0].dialect" == "https://json-schema.org/draft/2020-12/schema"
+jsonpath "$.entries[0].health" == 0
+jsonpath "$.entries[0].dependencies" == 0
+jsonpath "$.entries[0].alert" == null
+jsonpath "$.entries[0].provenance" == null
+jsonpath "$.entries[0].documentation" not exists
+
+POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}}
+```
+{{last_response}}
+```
+HTTP 200
+Link: </self/v1/schemas/api/schemas/evaluate/response>; rel="describedby"
+[Asserts]
+jsonpath "$.valid" == true
+
+# The static file endpoint serves the raw markdown content in headless mode
+GET {{base}}{{documentation_url}}
+HTTP 200
+[Asserts]
+header "ETag" exists
+header "Last-Modified" exists
+body contains "# Headless Documentation"
+body contains "**test**"
+
+# Collection without documentation does not have the documentation field
+GET {{base}}/self/v1/api/list/other
+HTTP 200
+Content-Type: application/json
+Access-Control-Allow-Origin: *
+Link: </self/v1/schemas/api/list/response>; rel="describedby"
+[Captures]
+last_response: body
+schema_path: header "Link" regex "<([^>]+)>"
+[Asserts]
+header "ETag" exists
+header "Last-Modified" exists
+jsonpath "$.path" == "/other"
+jsonpath "$.url" == "{{base}}/other"
+jsonpath "$.schemas" == 1
+jsonpath "$.health" == 0
+jsonpath "$.breadcrumb" count == 1
+jsonpath "$.breadcrumb[0].name" == "other"
+jsonpath "$.breadcrumb[0].path" == "/other/"
+jsonpath "$.documentation" not exists
+jsonpath "$.entries" count == 1
+jsonpath "$.entries[0].name" == "integer"
+jsonpath "$.entries[0].type" == "schema"
+jsonpath "$.entries[0].path" == "/other/integer"
+jsonpath "$.entries[0].identifier" == "{{base}}/other/integer"
+jsonpath "$.entries[0].bytes" == 132
+jsonpath "$.entries[0].baseDialect" == "https://json-schema.org/draft/2020-12/schema"
+jsonpath "$.entries[0].dialect" == "https://json-schema.org/draft/2020-12/schema"
+jsonpath "$.entries[0].health" == 0
+jsonpath "$.entries[0].dependencies" == 0
+jsonpath "$.entries[0].alert" == null
+jsonpath "$.entries[0].provenance" == null
+jsonpath "$.entries[0].documentation" not exists
+
+POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}}
+```
+{{last_response}}
+```
+HTTP 200
+Link: </self/v1/schemas/api/schemas/evaluate/response>; rel="describedby"
+[Asserts]
+jsonpath "$.valid" == true
+
+# No HTML pages in headless mode
+GET {{base}}/test
+Accept: text/html
+HTTP 404
+
+# Parent listing does not include documentation on child entries
+GET {{base}}/self/v1/api/list
+HTTP 200
+Content-Type: application/json
+Access-Control-Allow-Origin: *
+Link: </self/v1/schemas/api/list/response>; rel="describedby"
+[Captures]
+last_response: body
+schema_path: header "Link" regex "<([^>]+)>"
+[Asserts]
+header "ETag" exists
+header "Last-Modified" exists
+jsonpath "$.path" == "/"
+jsonpath "$.url" == "{{base}}"
+jsonpath "$.breadcrumb" count == 0
+jsonpath "$.health" == 33
+jsonpath "$.schemas" == 33
+jsonpath "$.documentation" not exists
+jsonpath "$.entries" count == 3
+jsonpath "$.entries[0].name" == "other"
+jsonpath "$.entries[0].type" == "directory"
+jsonpath "$.entries[0].health" == 0
+jsonpath "$.entries[0].schemas" == 1
+jsonpath "$.entries[0].path" == "/other/"
+jsonpath "$.entries[0].documentation" not exists
+jsonpath "$.entries[1].name" == "self"
+jsonpath "$.entries[1].type" == "directory"
+jsonpath "$.entries[1].health" == 100
+jsonpath "$.entries[1].schemas" == 31
+jsonpath "$.entries[1].path" == "/self/"
+jsonpath "$.entries[1].title" == "Self"
+jsonpath "$.entries[1].description" == "The schemas that define the current version of this instance"
+jsonpath "$.entries[1].documentation" not exists
+jsonpath "$.entries[2].name" == "test"
+jsonpath "$.entries[2].type" == "directory"
+jsonpath "$.entries[2].health" == 0
+jsonpath "$.entries[2].schemas" == 1
+jsonpath "$.entries[2].path" == "/test/"
+jsonpath "$.entries[2].documentation" not exists
+
+POST {{base}}/self/v1/api/schemas/evaluate{{schema_path}}
+```
+{{last_response}}
+```
+HTTP 200
+Link: </self/v1/schemas/api/schemas/evaluate/response>; rel="describedby"
+[Asserts]
+jsonpath "$.valid" == true

enterprise/e2e/headless/one.json

@@ -0,0 +1,16 @@
+{
+  "url": "http://localhost:8000",
+  "html": false,
+  "extends": [ "@self/v1" ],
+  "contents": {
+    "test": {
+      "baseUri": "https://example.com/schemas",
+      "path": "./schemas/test",
+      "x-sourcemeta-one:documentation": "./docs/test-readme.md"
+    },
+    "other": {
+      "baseUri": "https://example.com/other",
+      "path": "./schemas/other"
+    }
+  }
+}

enterprise/e2e/headless/schemas/other/integer.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://example.com/other/integer",
+  "type": "integer"
+}

enterprise/e2e/headless/schemas/test/string.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://example.com/schemas/string",
+  "type": "string"
+}

enterprise/e2e/html/Dockerfile

@@ -2,6 +2,7 @@ FROM one
 COPY one.json .
 COPY schemas schemas
 COPY rules rules
+COPY docs docs
 RUN sourcemeta one.json --profile
 RUN set -e && test -d "$SOURCEMETA_ONE_WORKDIR" && \
   test -z "$(ls -A "$SOURCEMETA_ONE_WORKDIR")"

enterprise/e2e/html/docs/test-readme.md

@@ -0,0 +1,17 @@
+# Test Collection
+
+This is a collection of **test schemas** used for validation.
+
+## Getting Started
+
+Check out the [example guide](https://example.com/guide) for more details.
+
+- Item one with *emphasis*
+- Item two with `inline code`
+- Item three
+
+## Security Test
+
+The following should be filtered: <script>alert('xss')</script>
+
+And this too: <iframe src="evil"></iframe>