Merge pull request #1 from RasaHQ/init

other: Init commit
RasaHQ · Dec 15, 2020 · ed5887d · ed5887d
2 parents 07c7751 + 84dfe8c
commit ed5887d
Show file tree

Hide file tree

Showing 22 changed files with 1,304 additions and 1 deletion.
diff --git a/.github/test_files/config_rasa_demo.yml b/.github/test_files/config_rasa_demo.yml
@@ -0,0 +1,49 @@
+language: en
+pipeline:
+- name: WhitespaceTokenizer
+- name: RegexFeaturizer
+- name: LexicalSyntacticFeaturizer
+- name: CountVectorsFeaturizer
+  OOV_token: oov
+  token_pattern: (?u)\b\w+\b
+- name: CountVectorsFeaturizer
+  analyzer: char_wb
+  min_ngram: 1
+  max_ngram: 4
+- name: DIETClassifier
+  epochs: 5
+  ranking_length: 5
+- name: DucklingEntityExtractor
+  url: http://localhost:8000
+  dimensions:
+  - email
+  - number
+  - amount-of-money
+- name: EntitySynonymMapper
+- name: ResponseSelector
+  retrieval_intent: out_of_scope
+  scale_loss: false
+  epochs: 5
+- name: ResponseSelector
+  retrieval_intent: faq
+  scale_loss: false
+  epochs: 5
+- name: ResponseSelector
+  retrieval_intent: chitchat
+  scale_loss: false
+  epochs: 5
+- name: FallbackClassifier
+  threshold: 0.7
+policies:
+- name: TEDPolicy
+  max_history: 5
+  epochs: 10
+  batch_size:
+  - 32
+  - 64
+- max_history: 6
+  name: AugmentedMemoizationPolicy
+- name: RulePolicy
+  core_fallback_threshold: 0.3
+  core_fallback_action_name: "action_default_fallback"
+  enable_fallback_prediction: True
diff --git a/.github/test_files/config_rasa_x_demo.yml b/.github/test_files/config_rasa_x_demo.yml
@@ -0,0 +1,36 @@
+# Configuration for Rasa NLU.
+# https://rasa.com/docs/rasa/nlu/components/
+language: en
+
+pipeline:
+# # No configuration for the NLU pipeline was provided. The following default pipeline was used to train your model.
+# # If you'd like to customize it, uncomment and adjust the pipeline.
+# # See https://rasa.com/docs/rasa/tuning-your-model for more information.
+   - name: WhitespaceTokenizer
+   - name: RegexFeaturizer
+   - name: LexicalSyntacticFeaturizer
+   - name: CountVectorsFeaturizer
+   - name: CountVectorsFeaturizer
+     analyzer: char_wb
+     min_ngram: 1
+     max_ngram: 4
+   - name: DIETClassifier
+     epochs: 5
+   - name: EntitySynonymMapper
+   - name: ResponseSelector
+     epochs: 5
+   - name: FallbackClassifier
+     threshold: 0.3
+     ambiguity_threshold: 0.1
+
+# Configuration for Rasa Core.
+# https://rasa.com/docs/rasa/core/policies/
+policies:
+# # No configuration for policies was provided. The following default policies were used to train your model.
+# # If you'd like to customize them, uncomment and adjust the policies.
+# # See https://rasa.com/docs/rasa/policies for more information.
+   - name: MemoizationPolicy
+   - name: TEDPolicy
+     max_history: 5
+     epochs: 5
+   - name: RulePolicy
diff --git a/.github/test_files/report_to_compare.json b/.github/test_files/report_to_compare.json
@@ -0,0 +1,76 @@
+{
+    "default": {
+      "Rasa Demo - NLU": {
+        "entity_prediction": {
+          "macro_avg": {
+            "f1-score": 0.2378897757789324,
+            "precision": 0.7177262292340671,
+            "recall": 0.630076550644482,
+            "support": 1451
+          },
+          "micro_avg": {
+            "f1-score": 0.557902566009669,
+            "precision": 0.8231017770597738,
+            "recall": 0.7022742935906272,
+            "support": 1451
+          },
+          "weighted_avg": {
+            "f1-score": 0.6357964876470346,
+            "precision": 0.8108170836964997,
+            "recall": 0.7022742935906272,
+            "support": 1451
+          }
+        },
+        "intent_classification": {
+          "macro_avg": {
+            "f1-score": 0.78507804141552495,
+            "precision": 0.6249297611496761,
+            "recall": 0.22805363055392253,
+            "support": 4950
+          },
+          "micro_avg": {
+            "f1-score": 0.5612119662490412,
+            "precision": 0.9004178272980501,
+            "recall": 0.5224242424242425,
+            "support": 4950
+          },
+          "weighted_avg": {
+            "f1-score": 0.5845322720747205,
+            "precision": 0.8560612861880288,
+            "recall": 0.5224242424242425,
+            "support": 4950
+          }
+        },
+        "response_selection": {
+          "accuracy": 0.657748611812216,
+          "macro_avg": {
+            "f1-score": 0.4511394032023792,
+            "precision": 0.541429844814056,
+            "recall": 0.47505801532174247,
+            "support": 1981
+          },
+          "weighted_avg": {
+            "f1-score": 0.6172915897390211,
+            "precision": 0.6469121522031172,
+            "recall": 0.657748611812216,
+            "support": 1981
+          }
+        },
+        "story_prediction": {
+          "accuracy": 0.51005291005291,
+          "macro_avg": {
+            "f1-score": 0.5328584995251662,
+            "precision": 0.8363425925925926,
+            "recall": 0.8482142857142857,
+            "support": 189
+          },
+          "weighted_avg": {
+            "f1-score": 0.5091936869714647,
+            "precision": 0.9186067019400354,
+            "recall": 0.91005291005291,
+            "support": 189
+          }
+        }
+      }
+    }
+  }
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,161 @@
+name: Continuous Integration
+on:
+  push:
+    branches:
+    - master
+    tags:
+    - '*'
+  pull_request:
+
+env:
+  RASA_X_DEMO_VERSION: '0.33.0'
+  RASA_IMAGE_TAG: '2.1.3-full'
+
+jobs:
+  test_action_init:
+    runs-on: ubuntu-latest
+    name: Test GH action - rasa init
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Create a new project
+      run: |
+        mkdir rasa-project
+        chmod 777 rasa-project
+        docker run --rm -v ${{ github.workspace }}/rasa-project:/rasa-project rasa/rasa:${{ env.RASA_IMAGE_TAG }} init --no-prompt --init-dir /rasa-project
+
+    - name: Download Rasa X model
+      run: |
+        wget https://github.com/RasaHQ/rasa-x-demo/blob/0.33.0/models/model.tar.gz?raw=true \
+          -O test_model.tar.gz
+
+    - id: action
+      name: Rasa Test & Train - rasa init
+      uses: ./
+      with:
+        rasa_version: ${{ env.RASA_IMAGE_TAG }}
+        workspace: ${{ github.workspace }}/rasa-project
+        test_type: all
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        configuration_name: 'rasa-init'
+        publish_summary: 'false'
+
+    - name: Checkout output
+      run: |
+        echo ${{ fromJSON(steps.action.outputs.report).default['rasa-init'].intent_classification.accuracy }}
+
+    - name: Checkout output - if
+      if: fromJSON(steps.action.outputs.report).default['rasa-init'].intent_classification.accuracy <= 0.5
+      run: |
+        echo "::error::${{ fromJSON(steps.action.outputs.report).default['rasa-init'].intent_classification.accuracy }}"
+        exit 1
+
+  # Test the GH action on Rasa X Demo data
+  # https://github.com/RasaHQ/rasa-x-demo
+  test_action_x:
+    runs-on: ubuntu-latest
+    name: Test GH action - Rasa X Demo
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Checkout Rasa X Demo
+      uses: actions/checkout@v2
+      with:
+        repository: RasaHQ/rasa-x-demo
+        path: rasa-x-demo
+        ref: ${{ env.RASA_X_DEMO_VERSION }}
+
+    - name: Download Rasa X model
+      run: |
+        wget https://github.com/RasaHQ/rasa-x-demo/blob/0.33.0/models/model.tar.gz?raw=true \
+          -O test_model.tar.gz
+
+    - id: action
+      name: Rasa Test & Train - Rasa X Demo
+      uses: ./
+      with:
+        rasa_version: '2.0.0-full'
+        workspace: ${{ github.workspace }}/rasa-x-demo
+        test_type: all
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        model: github_workspace/test_model.tar.gz
+        configuration: github_workspace/test_files/config_rasa_x_demo.yml
+        configuration_name: 'Rasa X Demo'
+        publish_summary: 'false'
+
+    - name: Checkout output
+      run: |
+        echo ${{ fromJSON(steps.action.outputs.report).default['Rasa X Demo'].intent_classification.accuracy }}
+
+    - name: Checkout output - if
+      if: fromJSON(steps.action.outputs.report).default['Rasa X Demo'].intent_classification.accuracy <= 0.5
+      run: |
+        echo "::error::${{ fromJSON(steps.action.outputs.report).default['Rasa X Demo'].intent_classification.accuracy }}"
+        exit 1
+
+  # Test the GH action on Rasa Demo data
+  # https://github.com/RasaHQ/rasa-x-demo
+  # Test type: NLU
+  test_action:
+    runs-on: ubuntu-latest
+    name: Test GH action - Rasa Demo - NLU
+    needs: [test_action_x]
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Checkout Rasa Demo
+      uses: actions/checkout@v2
+      with:
+        repository: RasaHQ/rasa-demo
+        path: rasa-demo
+
+    - name: Use test configuration
+      run: |
+        cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
+
+    - id: action
+      name: Rasa Test & Train - Rasa Demo
+      uses: ./
+      with:
+        rasa_version: '2.0.0-full'
+        configuration_name: 'Rasa Demo - NLU'
+        workspace: ${{ github.workspace }}/rasa-demo
+        test_type: nlu
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        test_args: '-f 2'
+        # Use cross validation
+        cross_validation: 'true'
+        # Compare results to the test report
+        compare_report: .github/test_files/report_to_compare.json
+
+  # Test the GH action on Rasa Demo data
+  # https://github.com/RasaHQ/rasa-x-demo
+  # Test type: ALL
+  test_action_all:
+    runs-on: ubuntu-latest
+    name: Test GH action - Rasa Demo - ALL
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Checkout Rasa Demo
+      uses: actions/checkout@v2
+      with:
+        repository: RasaHQ/rasa-demo
+        path: rasa-demo
+
+    - name: Use test configuration
+      run: |
+        cp .github/test_files/config_rasa_demo.yml rasa-demo/config.yml
+
+    - id: action
+      name: Rasa Test & Train - Rasa Demo
+      uses: ./
+      with:
+        rasa_version: '2.0.0-full'
+        configuration_name: 'Rasa Demo'
+        workspace: ${{ github.workspace }}/rasa-demo
+        github_token: ${{ secrets.GITHUB_TOKEN }}