Skip to content

Commit

Permalink
feat: add specs and user taxonomies
Browse files Browse the repository at this point in the history
  • Loading branch information
laurentsenta authored Sep 12, 2023
2 parents 5587ae7 + 07a5a32 commit 765f656
Show file tree
Hide file tree
Showing 21 changed files with 295 additions and 26 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Metadata logging used to associate tests with custom data like versions, specs identifiers, etc.
- Output Github's workflow URL with metadata. [PR](https://github.com/ipfs/gateway-conformance/pull/145)
- Basic Dashboard Output with content generation. [PR](https://github.com/ipfs/gateway-conformance/pull/152)
- Test Group Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/156)
- Specs Metadata on Tests. [PR](https://github.com/ipfs/gateway-conformance/pull/159)

## [0.3.0] - 2023-07-31
### Added
Expand Down
84 changes: 75 additions & 9 deletions aggregate-into-table.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const fs = require("fs");

const TestMetadata = "TestMetadata";
const METADATA_TEST_GROUP = "group";

// retrieve the list of input files from the command line
const files = process.argv.slice(2);
Expand All @@ -10,24 +11,85 @@ const inputs = files.map((file) => {
return JSON.parse(fs.readFileSync(file, 'utf8'));
});

// merge all the unique keys from all the inputs
let keys = new Set();
// merge all the unique keys & metadata from all the inputs
const metadata = {}
inputs.forEach((input) => {
Object.keys(input).forEach((key) => {
keys.add(key);
metadata[key] = { ...metadata[key], ...input[key]["meta"] || {} };
});
});
keys.delete(TestMetadata); // Extract TestMetadata which is a special case
keys = Array.from(keys).sort();
delete metadata[TestMetadata]; // Extract TestMetadata which is a special case

// generate groups: an array of {group, key} objects
// where group is the group name (or undefined), and key is the test key name (or undefined)
// It represents the table leftmost column.
//
// Group1
// Group1 - Test1
// Group1 - Test2
// Group2
// ...
const groups = []
const groupsAdded = new Set();
Object.entries(metadata).forEach(([key, value]) => {
const group = value[METADATA_TEST_GROUP] || undefined;

if (!groupsAdded.has(group)) {
groups.push({ group, key: undefined });
groupsAdded.add(group);
}

groups.push({ group, key });
});

// sort the groups so that the tests are ordered by group, then by key.
// undefined groups are always at the end.
groups.sort((a, b) => {
if (a.group === b.group) {
if (a.key === undefined) {
return -1;
}
if (b.key === undefined) {
return 1;
}
return a.key.localeCompare(b.key);
}

if (a.group === undefined) {
return 1;
}

if (b.group === undefined) {
return -1;
}

return a.group.localeCompare(b.group);
});

// generate a table
const columns = [];

// add the leading column ("gateway", "version", "key1", "key2", ... "keyN")
// add the leading column ("gateway", "version", "group1", "test11", ... "test42")
const leading = ["gateway", "version"];
keys.forEach((key) => {
groups.forEach(({ group, key }) => {
if (key === undefined) {
leading.push(`**${group || 'Other'}**`);
return;
}

// Skip the "Test" prefix
const niceKey = key.replace(/^Test/, '');
let niceKey = key.replace(/^Test/, '');

const m = metadata[key];
if (m.specs && m.specs.length > 0) {
if (m.specs.length === 1) {
niceKey = `[${niceKey}](https://${m.specs[0]})`
} else {
const urls = m.specs.map((url, index) => `[${index}](https://${url})`);
niceKey = `${niceKey} (${urls.join(', ')})`;
}
}

leading.push(niceKey);
});
columns.push(leading);
Expand Down Expand Up @@ -73,7 +135,11 @@ inputs.forEach((input, index) => {
const col = [name, versionCell];

// extract results
keys.forEach((key) => {
groups.forEach(({ group, key }) => {
if (key === undefined) {
col.push(null);
return;
}
col.push(cellRender(input[key] || null));
});
columns.push(col);
Expand Down
44 changes: 41 additions & 3 deletions aggregate.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ lines = lines.filter((line) => {
// # extract test metadata
// action is output, and starts with ".* --- META: (.*)"
// see details in https://github.com/ipfs/gateway-conformance/pull/125
const getMetadata = (line) => {
const extractMetadata = (line) => {
const { Action, Output } = line;

if (Action !== "output") {
Expand All @@ -34,7 +34,7 @@ const getMetadata = (line) => {
}

lines = lines.map((line) => {
const metadata = getMetadata(line);
const metadata = extractMetadata(line);

if (!metadata) {
return line;
Expand Down Expand Up @@ -87,6 +87,44 @@ lines.forEach((line) => {
});
})

// prepare metadata up the tree
const metadataTree = {};

// sort lines so that the one with the longest path is processed first
const sortedLines = lines.sort((a, b) => {
return b.Path.length - a.Path.length;
});

sortedLines.forEach((line) => {
const { Path, Action, Metadata } = line;
let current = metadataTree;

if (Action !== "meta") {
return;
}

Path.forEach((path) => {
if (!current[path]) {
current[path] = {};
}
current = current[path];
current["meta"] = { ...current["meta"], ...Metadata };
});
});

const getMetadata = (path) => {
let current = metadataTree;

path.forEach((path) => {
if (!current[path]) {
return null;
}
current = current[path];
});

return current["meta"];
}

// # Drop all lines where the Test "Path" does not point to a leaf
// if the test has children then we don't really care about it's pass / fail / skip status,
// we'll aggregate its children results'
Expand Down Expand Up @@ -118,7 +156,7 @@ lines.forEach((line) => {
const key = path.join(" > ");

if (!current[key]) {
current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": {} };
current[key] = { Path: path, "pass": 0, "fail": 0, "skip": 0, "total": 0, "meta": getMetadata(path) || {} };
}
current = current[key];

Expand Down
3 changes: 3 additions & 0 deletions tests/dnslink_gateway_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"net/url"
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/car"
. "github.com/ipfs/gateway-conformance/tooling/check"
"github.com/ipfs/gateway-conformance/tooling/dnslink"
Expand All @@ -14,6 +15,8 @@ import (
)

func TestDNSLinkGatewayUnixFSDirectoryListing(t *testing.T) {
tooling.LogTestGroup(t, GroupDNSLink)

fixture := car.MustOpenUnixfsCar("dir_listing/fixtures.car")
file := fixture.MustGetNode("ą", "ę", "file-źł.txt")

Expand Down
24 changes: 23 additions & 1 deletion tests/metadata_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,32 @@ import (
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/test"
)

func logGatewayURL(t *testing.T) {
tooling.LogMetadata(t, struct {
GatewayURL string `json:"gateway_url"`
SubdomainGatewayURL string `json:"subdomain_gateway_url"`
}{
GatewayURL: test.GatewayURL,
SubdomainGatewayURL: test.SubdomainGatewayURL,
})
}

func TestMetadata(t *testing.T) {
tooling.LogVersion(t)
tooling.LogJobURL(t)
tooling.LogGatewayURL(t)
logGatewayURL(t)
}

const (
GroupSubdomains = "Subdomains"
GroupCORS = "CORS"
GroupIPNS = "IPNS"
GroupDNSLink = "DNSLink"
GroupJSONCbor = "JSON-CBOR"
GroupBlockCar = "Block-CAR"
GroupTar = "Tar"
GroupUnixFS = "UnixFS"
)
3 changes: 3 additions & 0 deletions tests/path_gateway_cors_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package tests
import (
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/specs"
. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestCors(t *testing.T) {
tooling.LogTestGroup(t, GroupCORS)

cidHello := "bafkqabtimvwgy3yk" // hello

tests := SugarTests{
Expand Down
17 changes: 16 additions & 1 deletion tests/path_gateway_dag_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package tests
import (
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/car"
. "github.com/ipfs/gateway-conformance/tooling/check"
"github.com/ipfs/gateway-conformance/tooling/ipns"
Expand All @@ -12,6 +13,8 @@ import (
)

func TestGatewayJsonCbor(t *testing.T) {
tooling.LogTestGroup(t, GroupJSONCbor)

fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")

fileJSON := fixture.MustGetNode("ą", "ę", "t.json")
Expand Down Expand Up @@ -39,6 +42,7 @@ func TestGatewayJsonCbor(t *testing.T) {
},
{
Name: "GET UnixFS file with JSON bytes is returned with application/json Content-Type - with headers",
Spec: "specs.ipfs.tech/http-gateways/path-gateway/#accept-request-header",
Hint: `
## Quick regression check for JSON stored on UnixFS:
## it has nothing to do with DAG-JSON and JSON codecs,
Expand Down Expand Up @@ -66,6 +70,8 @@ func TestGatewayJsonCbor(t *testing.T) {
// ## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON
// ## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format)
func TestDagPbConversion(t *testing.T) {
tooling.LogTestGroup(t, GroupJSONCbor)

fixture := car.MustOpenUnixfsCar("path_gateway_dag/gateway-json-cbor.car")

dir := fixture.MustGetRoot()
Expand Down Expand Up @@ -205,6 +211,8 @@ func TestDagPbConversion(t *testing.T) {
// # Requesting CID with plain json (0x0200) and cbor (0x51) codecs
// # (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec)
func TestPlainCodec(t *testing.T) {
tooling.LogTestGroup(t, GroupJSONCbor)

table := []struct {
Name string
Format string
Expand Down Expand Up @@ -308,6 +316,8 @@ func TestPlainCodec(t *testing.T) {

// ## Pathing, traversal over DAG-JSON and DAG-CBOR
func TestPathing(t *testing.T) {
tooling.LogTestGroup(t, GroupJSONCbor)

dagJSONTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-json-traversal.car").MustGetRoot()
dagCBORTraversal := car.MustOpenUnixfsCar("path_gateway_dag/dag-cbor-traversal.car").MustGetRoot()

Expand Down Expand Up @@ -381,6 +391,8 @@ func TestPathing(t *testing.T) {
// ## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71):
// ## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested
func TestNativeDag(t *testing.T) {
tooling.LogTestGroup(t, GroupJSONCbor)

missingCID := car.RandomCID()

table := []struct {
Expand Down Expand Up @@ -467,7 +479,7 @@ func TestNativeDag(t *testing.T) {
Response: Expect().
Headers(
Header("Content-Type").Hint("expected Content-Type").Equals("application/vnd.ipld.dag-{{format}}", row.Format),
Header("Content-Length").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
Header("Content-Length").Spec("specs.ipfs.tech/http-gateways/path-gateway/#content-disposition-response-header").Hint("includes Content-Length").Equals("{{length}}", len(dagTraversal.RawData())),
Header("Content-Disposition").Hint("includes Content-Disposition").Contains(`{{disposition}}; filename="{{cid}}.{{format}}"`, row.Disposition, dagTraversalCID, row.Format),
Header("X-Content-Type-Options").Hint("includes nosniff hint").Contains("nosniff"),
),
Expand Down Expand Up @@ -541,6 +553,7 @@ func TestNativeDag(t *testing.T) {
},
{
Name: Fmt("HEAD {{name}} with only-if-cached for missing block returns HTTP 412 Precondition Failed", row.Name),
Spec: "specs.ipfs.tech/http-gateways/path-gateway/#only-if-cached",
Request: Request().
Path("/ipfs/{{cid}}", missingCID).
Header("Cache-Control", "only-if-cached").
Expand Down Expand Up @@ -570,6 +583,8 @@ func TestNativeDag(t *testing.T) {
}

func TestGatewayJSONCborAndIPNS(t *testing.T) {
tooling.LogTestGroup(t, GroupIPNS)

ipnsIdDagJSON := "k51qzi5uqu5dhjghbwdvbo6mi40htrq6e2z4pwgp15pgv3ho1azvidttzh8yy2"
ipnsIdDagCBOR := "k51qzi5uqu5dghjous0agrwavl8vzl64xckoqzwqeqwudfr74kfd11zcyk3b7l"

Expand Down
3 changes: 3 additions & 0 deletions tests/path_gateway_ipns_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@ package tests
import (
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/specs"
. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestRedirectCanonicalIPNS(t *testing.T) {
tooling.LogTestGroup(t, GroupIPNS)

tests := SugarTests{
{
Name: "GET for /ipns/{b58-multihash-of-ed25519-key} redirects to /ipns/{cidv1-libp2p-key-base36}",
Expand Down
3 changes: 3 additions & 0 deletions tests/path_gateway_raw_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@ import (
"strings"
"testing"

"github.com/ipfs/gateway-conformance/tooling"
"github.com/ipfs/gateway-conformance/tooling/car"
"github.com/ipfs/gateway-conformance/tooling/specs"
. "github.com/ipfs/gateway-conformance/tooling/test"
)

func TestGatewayBlock(t *testing.T) {
tooling.LogTestGroup(t, GroupBlockCar)

fixture := car.MustOpenUnixfsCar("gateway-raw-block.car")

tests := SugarTests{
Expand Down
Loading

0 comments on commit 765f656

Please sign in to comment.