Merge pull request #79 from anweiss/cddl-control-00

Implementation of additional proposed control operators and wasm bindings for validator
anweiss · Sep 13, 2021 · 4a69948 · 4a69948
2 parents f271aa0 + 3a9d8d1
commit 4a69948
Show file tree

Hide file tree

Showing 27 changed files with 5,784 additions and 943 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -56,7 +56,7 @@ jobs:
   compilation-check:
     strategy:
       matrix:
-        rust_toolchain: [stable, beta, nightly]
+        rust_toolchain: [stable, beta]
         os: [ubuntu-latest, macOS-latest, windows-latest]
     name: Compilation check
     runs-on: ${{ matrix.os }}
@@ -83,17 +83,10 @@ jobs:
           command: check
           args: --all --bins --examples --tests --no-default-features
 
-      - name: Check compilation with all features
-        if: matrix.rust_toolchain == 'nightly'
-        uses: actions-rs/cargo@v1
-        with:
-          command: check
-          args: --all --bins --examples --tests --all-features
-
   wasm-compilation-check:
     strategy:
       matrix:
-        rust_toolchain: [stable, beta, nightly]
+        rust_toolchain: [stable, beta]
     name: Compilation check for wasm target
     runs-on: ubuntu-latest
     steps:
@@ -117,7 +110,7 @@ jobs:
   test-suite:
     strategy:
       matrix:
-        rust_toolchain: [stable, beta, nightly]
+        rust_toolchain: [stable, beta]
         os: [ubuntu-latest, macOS-latest, windows-latest]
     name: Test suite
     runs-on: ${{ matrix.os }}
@@ -141,7 +134,7 @@ jobs:
   style-linting:
     strategy:
       matrix:
-        rust_toolchain: [stable, beta]
+        rust_toolchain: [stable]
     name: Style linting
     runs-on: ubuntu-latest
     steps:
@@ -173,7 +166,7 @@ jobs:
   wasm-style-linting:
     strategy:
       matrix:
-        rust_toolchain: [stable, beta]
+        rust_toolchain: [stable]
     name: wasm style linting
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -3,51 +3,43 @@
 #
 # You may wish to alter this file to override the set of languages analyzed,
 # or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
 name: 'CodeQL'
 
 on:
   push:
     branches: [main]
-    paths:
-      - '.github/workflows/codeql-analysis.yml'
-      - 'www/**'
-      - 'cddl-lsp/**'
   pull_request:
     # The branches below must be a subset of the branches above
     branches: [main]
-    paths:
-      - '.github/workflows/codeql-analysis.yml'
-      - 'www/**'
-      - 'cddl-lsp/**'
   schedule:
-    - cron: '0 2 * * 4'
+    - cron: '35 17 * * 4'
 
 jobs:
   analyze:
     name: Analyze
     runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
 
     strategy:
       fail-fast: false
       matrix:
-        # Override automatic language detection by changing the below list
-        # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python']
         language: ['javascript']
-        # Learn more...
-        # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
-        with:
-          # We must fetch at least the immediate parents so that if this is
-          # a pull request then we can checkout the head.
-          fetch-depth: 2
-
-      # If this run was triggered by a pull request event, then checkout
-      # the head of the pull request instead of the merge commit.
-      - run: git checkout HEAD^2
-        if: ${{ github.event_name == 'pull_request' }}
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

diff --git a/.github/workflows/lsp-extension.yml b/.github/workflows/lsp-extension.yml
@@ -44,7 +44,7 @@ jobs:
           npm run webpack
 
       - name: Run unit tests and integration tests
-        uses: GabrielBB/xvfb-action@v1.2
+        uses: GabrielBB/xvfb-action@v1.5
         with:
           working-directory: ./cddl-lsp
           run: npm test
diff --git a/Cargo.toml b/Cargo.toml
@@ -5,16 +5,12 @@ repository = "https://github.com/anweiss/cddl"
 homepage = "https://cddl.anweiss.tech"
 categories = ["parser-implementations", "encoding", "development-tools", "wasm"]
 license = "MIT"
-version = "0.8.7"
+version = "0.9.0-beta.0"
 authors = ["Andrew Weiss <[email protected]>"]
 readme = "README.md"
 edition = "2018"
 exclude = ["cddl-lsp/**/*", "www/**/*", ".github/**/*", ".devcontainer/**/*", "pkg/**/*", ".dockerignore", "Dockerfile", "tests/**/*"]
 
-# Temporary workaround for https://github.com/rustwasm/wasm-pack/issues/886
-[package.metadata.wasm-pack.profile.release]
-wasm-opt = ["-Oz", "--enable-mutable-globals"]
-
 [lib]
 crate-type = ["cdylib", "rlib"]
 
@@ -34,13 +30,19 @@ serde_cbor = { version = "0.11.1", optional = true, default-features = false, fe
 serde_json = { version = "1.0.66", optional = true, default-features = false }
 uriparse = { version = "0.6.3", optional = true }
 base64-url = { version = "1.4.10", optional = true }
+abnf_to_pest = "0.5.0"
+pest_meta = "2.1.3"
+pest_vm = "2.1.0"
+displaydoc = { version = "0.2.3", default-features = false }
+log = "0.4.14"
+simplelog = "0.10.0"
 
 [dev-dependencies]
 indoc = "1.0.3"
 pretty_assertions = "0.7.2"
 
 [target.'cfg(not(target_arch = "wasm32"))'.dependencies]
-crossterm = { version = "0.20.0", optional = true }
+crossterm = { version = "0.21.0", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 console_error_panic_hook = "0.1.6"
@@ -51,13 +53,18 @@ wee_alloc = { version = "0.4.5", optional = true }
 wasm-bindgen-test = "0.3.25"
 
 [features]
-default = ["std"]
-std = ["serde_json", "serde_cbor", "serde", "chrono", "wasm-bindgen", "clap", "crossterm", "uriparse", "base64-url", "regex-syntax"]
+default = ["std", "ast-span", "ast-comments", "json", "cbor", "additional-controls"]
+std = ["base16/alloc", "base64/alloc", "serde_json", "serde_cbor", "serde", "chrono", "wasm-bindgen", "clap", "crossterm", "uriparse", "base64-url", "regex-syntax"]
 lsp = ["std"]
+additional-controls = []
+ast-span = []
+ast-comments = []
+json = ["std"]
+cbor = ["std"]
 
 [[bin]]
 name = "cddl"
-required-features = ["std"]
+required-features = ["std", "json", "cbor"]
 path = "src/bin/cli.rs"
 test = false
 
@@ -69,7 +76,7 @@ test = false
 
 [[test]]
 name = "cddl"
-required-features = ["std"]
+required-features = ["std", "ast-span", "ast-comments", "json", "cbor"]
 path = "tests/cddl.rs"
 
 [profile.release]

diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # cddl-rs
 
-[![crates.io](https://img.shields.io/crates/v/cddl.svg)](https://crates.io/crates/cddl) [![docs.rs](https://docs.rs/cddl/badge.svg)](https://docs.rs/cddl) [![Publish packages](https://github.com/anweiss/cddl/workflows/Publish%20packages/badge.svg?branch=0.8.7&event=release)](https://github.com/anweiss/cddl/actions?query=workflow%3A%22Publish+packages%22) [![Build and Test](https://github.com/anweiss/cddl/workflows/Build%20and%20Test/badge.svg)](https://github.com/anweiss/cddl/actions?query=workflow%3A%22Build+and+Test%22)
+[![crates.io](https://img.shields.io/crates/v/cddl.svg)](https://crates.io/crates/cddl) [![docs.rs](https://docs.rs/cddl/badge.svg)](https://docs.rs/cddl) [![Publish packages](https://github.com/anweiss/cddl/workflows/Publish%20packages/badge.svg?branch=0.9.0-beta.0&event=release)](https://github.com/anweiss/cddl/actions?query=workflow%3A%22Publish+packages%22) [![Build and Test](https://github.com/anweiss/cddl/workflows/Build%20and%20Test/badge.svg)](https://github.com/anweiss/cddl/actions?query=workflow%3A%22Build+and+Test%22)
 
 > This crate was originally developed as a personal learning exercise for getting acquainted with Rust and parsing in general. There are likely more performant and stable libraries out there for parsing CDDL. While there are some examples of this crate being used in production, careful consideration should be made prior to using this crate as such.
 
@@ -57,7 +57,7 @@ cargo install cddl
 docker pull ghcr.io/anweiss/cddl-cli:latest
 ```
 
-### Usage
+### CLI usage
 
 Instructions for using the tool can be viewed by executing the `help` subcommand:
 
@@ -102,7 +102,7 @@ docker run -i --rm -v $PWD:/cddl -w /cddl ghcr.io/anweiss/cddl-cli:latest valida
 
 You can also find a simple RFC 8610 conformance tool at https://cddl.anweiss.tech. This same codebase has been compiled for use in the browser via WebAssembly.
 
-## Visual Studio Code Extension
+## Visual Studio Code extension
 
 An extension for editing CDDL documents with Visual Studio Code has been published to the Marketplace [here](https://marketplace.visualstudio.com/items?itemName=anweiss.cddl-languageserver). You can find more information in the [README](cddl-lsp/README.md).
 
@@ -140,11 +140,35 @@ Simply add the dependency to `Cargo.toml`:
 
 ```toml
 [dependencies]
-cddl = "0.8"
+cddl = "0.9.0-beta.0"
 ```
 
 Both JSON and CBOR validation require `std`.
 
+### Feature flags
+
+A few convenience features have been included to make the AST more concise and for enabling additional functionality. You can build with `default-features = false` for a `no_std` build and selectively enable any of the features below.
+
+**`--feature ast-span`**
+
+Add the `Span` type to the AST for keeping track of the position of the lexer and parser. Enabled by default.
+
+**`--feature ast-comments`**
+
+Include comment strings in the AST. Enabled by default.
+
+**`--feature json`**
+
+Enable JSON validation. Enabled by default.
+
+**`--feature cbor`**
+
+Enable CBOR validation. Enabled by default.
+
+**`--feature additional-controls`**
+
+Enable validation support for the additional control operators proposed in [https://datatracker.ietf.org/doc/html/draft-ietf-cbor-cddl-control-05](https://datatracker.ietf.org/doc/html/draft-ietf-cbor-cddl-control-05). Enabled by default.
+
 ### Parsing CDDL
 
 ```rust
@@ -267,6 +291,32 @@ Below is the table of supported control operators:
 
 <a name="regex">3</a>: Due to Perl-Compatible Regular Expressions (PCREs) being more widely used than XSD regular expressions, this crate also provides support for the proposed `.pcre` control extension in place of the `.regexp` operator (see [Discussion](https://tools.ietf.org/html/rfc8610#section-3.8.3.2) and [CDDL-Freezer proposal](https://tools.ietf.org/html/draft-bormann-cbor-cddl-freezer-03#section-5.1)). Ensure that your regex string is properly JSON escaped when using this control.
 
+If you've enabled the `additional-controls` feature, the table of controls below is also available for use:
+
+| Control operator | Supported                                                                                                                                         |
+| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `.plus`          | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.cat`           | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.det`           | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.abnf`          | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.abnfb`         | Ignored when validating JSON                                                                                                                      |
+| `.feature`       | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+
+You can activate features during validation as follows:
+
+```rust
+use cddl::validate_json_from_str;
+
+let cddl = r#"
+  v = JC<"v", 2>
+  JC<J, C> =  C .feature "cbor" / J .feature "json"
+"#;
+
+let json = r#""v""#;
+
+assert!(validate_json_from_str(cddl, json, Some(&["json"])).is_ok())
+```
+
 #### Comparing with JSON schema and JSON schema language
 
 [CDDL](https://tools.ietf.org/html/rfc8610), [JSON schema](https://json-schema.org/) and [JSON schema language](https://tools.ietf.org/html/draft-json-schema-language-02) can all be used to define JSON data structures. However, the approaches taken to develop each of these are vastly different. A good place to find past discussions on the differences between these formats is the [IETF mail archive](https://mailarchive.ietf.org/arch/), specifically in the JSON and CBOR lists. The purpose of this crate is not to argue for the use of CDDL over any one of these formats, but simply to provide an example implementation in Rust.
@@ -306,13 +356,39 @@ The following tags are supported when validating CBOR:
 | `mime-message = #6.36(tstr)`             | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
 | `cbor-any = #6.55799(any)`               | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
 
+If you've enabled the `additional-controls` feature, the table of controls below is also available for use:
+
+| Control operator | Supported                                                                                                                                         |
+| ---------------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `.plus`          | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.cat`           | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.det`           | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.abnf`          | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.abnfb`         | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+| `.feature`       | <g-emoji class="g-emoji" alias="heavy_check_mark" fallback-src="https://github.githubassets.com/images/icons/emoji/unicode/2714.png">✔️</g-emoji> |
+
+You can activate features during validation by passing a slice of feature strings as follows:
+
+```rust
+use cddl::validate_cbor_from_slice;
+
+let cddl = r#"
+  v = JC<"v", 2>
+  JC<J, C> =  C .feature "cbor" / J .feature "json"
+"#;
+
+let cbor = b"\x02";
+
+assert!(validate_cbor_from_slice(cddl, cbor, Some(&["cbor"])).is_ok())
+```
+
 ## `no_std` support
 
 Only the lexer and parser can be used in a `no_std` context provided that a heap allocator is available. This can be enabled by opting out of the default features in your `Cargo.toml` file as follows:
 
 ```toml
 [dependencies]
-cddl = { version = "0.8", default-features = false }
+cddl = { version = "0.9.0-beta.0", default-features = false }
 ```
 
 Zero-copy parsing is implemented to the extent that is possible. Allocation is required for error handling and diagnostics.

diff --git a/cddl-lsp/client/package-lock.json b/cddl-lsp/client/package-lock.json
diff --git a/cddl-lsp/client/package.json b/cddl-lsp/client/package.json
@@ -13,10 +13,10 @@
     "vscode": "^1.43.0"
   },
   "dependencies": {
-    "vscode-languageclient": "^6.1.3"
+    "vscode-languageclient": "^6.1.4"
   },
   "devDependencies": {
     "@types/vscode": "1.43.0",
     "vscode-test": "^1.3.0"
   }
-}
+}