action.yml

name: 'Rasa Train-Test Model GitHub Action'
description: 'The GitHub action to run the rasa train and rasa test commands'
inputs:
  rasa_version:
    description: 'The Rasa version used to run test and train'
    required: true
    default: 'latest-full'
  rasa_image:
    description: 'Custom Docker image'
    required: false
    default: ''
  rasa_train:
    description: 'Run rasa train'
    required: true
    default: 'true'
  rasa_test:
    description: 'Run rasa test'
    required: true
    default: 'true'
  fine_tune:
    description: 'Fine-tune model'
    required: false
    default: 'false'
  data_validate:
    description: 'Validates domain and data files to check for possible mistakes'
    required: true
    default: 'true'
  data_validate_args:
    description: 'Additional arguments passed to the rasa data validate command'
    required: false
    default: ''
  workspace:
    description: 'The root directory'
    required: false
    default: '${{ github.workspace }}'
  train_type:
    description: 'The types of training (available types: core/nlu/all)'
    required: false
    default: 'all'
  train_args:
    description: 'Additional arguments passed to the rasa train command'
    required: false
    default: ''
  test_nlu_args:
    description: 'Additional arguments passed to the rasa test nlu command'
    required: false
    default: ''
  test_core_args:
    description: 'Additional arguments passed to the rasa test core command'
    required: false
    default: ''
  test_type:
    description: 'The types of tests to run (available types: core/nlu/all)'
    required: true
    default: 'all'
  publish_summary:
    description: 'Publish tests summary as a PR comment'
    required: false
    default: 'true'
  github_token:
    description: 'GitHub Token - required to add a comment with summary'
    required: false
    default: ''
  configuration:
    description: 'Model configuration file'
    required: true
    default: 'config.yml'
  model:
    description: 'Path to a file with a model. Use existing model instead of training a new one'
    required: false
    default: ''
  cross_validation:
    description: 'Switch on cross validation mode. Any provided model will be ignored'
    required: false
    default: 'false'
  configuration_name:
    description: 'Configuration name used in summary. If not provided a file name is used'
    required: false
    default: ''
  data_name:
    description: 'Data name used in summary. If not provided a directory name is used'
    required: false
    default: 'default'
  compare_report:
    description: 'Path to a report that will be used to compare results'
    required: false
    default: ''
  result_directory:
    description: 'Directory name where results are stored in'
    required: false
    default: 'results'
  report_directory:
    description: 'Directory name where reports are stored in'
    required: false
    default: 'reports'
  gomplate_image:
    description: 'Custom gomplate image. Useful if you use custom gomplate image'
    required: false
    default: 'hairyhenderson/gomplate'
  tmpfs_directory:
    description: 'The directory location where tmpfs mounts'
    required: false
    default: '/.config'
outputs:
  report:
    description: "Return report as JSON"
    value: ${{ steps.report_output.outputs.report }}

branding:
  icon: 'layers'
  color: 'green'

runs:
  using: "composite"
  steps:
    - name: Set environment
      shell: bash
      run: |-
        mkdir -p ${{ inputs.workspace }}/models
        chmod 777 ${{ inputs.workspace }}/models
        mkdir -p ${{ inputs.workspace }}/${{ inputs.result_directory }}
        chmod 777 ${{ inputs.workspace }}/${{ inputs.result_directory }}

        case "${{ inputs.train_type }}" in
          all)
            echo TRAIN_TYPE="" >> $GITHUB_ENV
            ;;
          nlu)
            echo TRAIN_TYPE=nlu >> $GITHUB_ENV
            echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV
            ;;
          core)
            echo TRAIN_TYPE=core >> $GITHUB_ENV
            ;;
          *)
            echo "::error::Unknown training type: ${{ inputs.train_type }} (use: core/nlu/all)"
            exit 1
        esac

        case "${{ inputs.test_type }}" in
          all)
            echo TEST_NLU="true" >> $GITHUB_ENV
            echo TEST_CORE="true" >> $GITHUB_ENV
            ;;
          nlu)
            echo TEST_NLU="true" >> $GITHUB_ENV
            echo DEFAULT_ARGS="--config /app/${{ inputs.configuration }}" >> $GITHUB_ENV
            ;;
          core)
            echo TEST_CORE="true" >> $GITHUB_ENV
            ;;
          *)
            echo "::error::Unknown test type: ${{ inputs.test_type }} (use: core/nlu/all)"
            exit 1
        esac

        # Set a model path
        if [[ -n "${{ inputs.model }}" ]];then
          echo MODEL_ARGS="--model /app/${{ inputs.model }}" >> $GITHUB_ENV
        fi

        # Set cross validation
        if [[ "${{ inputs.cross_validation }}" == "true" ]];then
          echo CROSS_VALIDATION_ARGS="--cross-validation" >> $GITHUB_ENV
        fi

        # Set docker args
        echo DOCKER_ARGS="--rm --tmpfs ${{ inputs.tmpfs_directory }} -e HOME=/app -e MPLCONFIGDIR=/tmp/.mplconfigdir \
          -e CI=${CI} -e RASA_TELEMETRY_ENABLED=${{ env.RASA_TELEMETRY_ENABLED }} -v ${{ github.workspace }}:/app/github_workspace \
          -v ${{ inputs.workspace }}:/app -v ${{ inputs.workspace }}/models:/app/models \
          -v ${{ inputs.workspace }}/${{ inputs.result_directory }}:/app/results" >> $GITHUB_ENV

        # Set environment variables for generating a report
        echo "SUMMARY_FILE=${{ inputs.workspace }}/reports/report.json" >> $GITHUB_ENV

        if [[ -n "${{ inputs.configuration_name }}" ]]; then
          echo "CONFIG=${{ inputs.configuration_name }}" >> $GITHUB_ENV
        else
          echo "CONFIG=${{ inputs.configuration }}" >> $GITHUB_ENV
        fi

        echo "DATASET_NAME=${{ inputs.data_name }}" >> $GITHUB_ENV
        echo "RESULT_DIR=${{ inputs.workspace }}/${{ inputs.result_directory }}" >> $GITHUB_ENV

    - name: Set the Rasa version and image
      shell: bash
      run: |
        # Set custom rasa image
        if [[ -n "${{ inputs.rasa_image }}" ]]; then
          echo "RASA_IMAGE=${{ inputs.rasa_image }}" >> $GITHUB_ENV
          # Don't continue if a custom rasa image was used
          exit 0
        fi

        DOCKERHUB_TAGS_URL="https://registry.hub.docker.com/v2/repositories/rasa/rasa/tags?page_size=10000"
        # Get the latest version
        if [[ "${{ inputs.rasa_version }}" == "latest-full" ]]; then
          LATEST_RASA_VERSION=$(curl -s ${DOCKERHUB_TAGS_URL} | jq -r '.results[].name' | grep -E 'full' | grep -vE 'latest' | sort -Vr | head -n1)
          RASA_VERSION=${LATEST_RASA_VERSION}
        else
          # Validate Rasa version
          CHECK_VERSION=$((DOCKER_CLI_EXPERIMENTAL=enabled docker manifest inspect rasa/rasa:${{ inputs.rasa_version }} &> /dev/null && echo true || echo false) || true)
          if [[ "$CHECK_VERSION" != "true" ]]; then
            echo "::error::Rasa version ${{ inputs.rasa_version }} doesn't exist. Check if the given Rasa version is valid, https://hub.docker.com/r/rasa/rasa/tags" && exit 1
          fi
          RASA_VERSION=${{ inputs.rasa_version }}
        fi

        echo "RASA_IMAGE=rasa/rasa:${RASA_VERSION}" >> $GITHUB_ENV

    - name: Data Validation
      shell: bash
      run: |
        if [[ "${{ inputs.data_validate }}" == "true" ]]; then
          echo "Run data validation"
          echo
          echo docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} data validate ${{ inputs.data_validate_args }}
          docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} data validate ${{ inputs.data_validate_args }}
        else
          echo "::warning::Data validation is disabled. To turn on data validation set the data_validate parameter to 'true'."
        fi

    - name: Fine Tune
      shell: bash
      run: |
        if [[ "${{ inputs.fine_tune }}" == "true" ]]; then
          echo "Fine tuning model is enabled."
          echo FINE_TUNE="--finetune" >> $GITHUB_ENV
        else
          echo "::warning::Fine tune is disabled. To turn on fine-tune, set the fine_tune parameter to 'true'."
        fi

    - name: Run the rasa train command
      shell: bash
      run: |
        if [[ -n "${{ inputs.model }}" ]]; then
          echo
          echo "Use model ${{ inputs.model }}. Skipping training"
          echo
        elif [[ "${{ inputs.rasa_train }}" == "true" ]]; then
          echo "Run the rasa train"
          echo ""
          docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} train ${{ env.TRAIN_TYPE }} ${{ env.DEFAULT_ARGS }} ${{ env.FINE_TUNE }} ${{ inputs.train_args }}
        else
          echo "::warning::rasa train is disabled. To turn on the rasa train set the rasa_train parameter to 'true'."
        fi

    - name: Run the test command
      shell: bash
      run: |
        if [[ "${{ inputs.rasa_test }}" == "true" ]]; then
          echo "Run the rasa test"
          echo ""
          if [[ "${{ env.TEST_NLU }}" == "true" ]]; then
            docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test nlu ${{ env.MODEL_ARGS }} ${{ env.CROSS_VALIDATION_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_nlu_args }}
          fi
          if [[ "${{ env.TEST_CORE }}" == "true" ]]; then
            docker run ${{ env.DOCKER_ARGS }} ${{ env.RASA_IMAGE }} test core ${{ env.MODEL_ARGS }} ${{ env.DEFAULT_ARGS }} ${{ inputs.test_core_args }}
          fi
        else
          echo "::warning::rasa test is disabled. To turn on the rasa train set the rasa_test parameter to 'true'."
        fi

    - name: Generate a report
      id: report_output
      shell: bash
      run: |
        mkdir -p ${{ inputs.workspace }}/${{ inputs.report_directory }}
        python3 ${{ github.action_path }}/scripts/generate_results_report.py

        # Copy a report from the current one if the report to compare to wasn't provided
        cp ${{ inputs.workspace }}/${{ inputs.report_directory }}/report.json ${{ inputs.workspace }}/${{ inputs.report_directory }}/report_main.json

        if [[ -n "${{ inputs.compare_report }}" && -e "${{ inputs.compare_report }}" ]]; then
          cp ${{ inputs.compare_report }} ${{ inputs.workspace }}/${{ inputs.report_directory }}/report_main.json
        elif [[ -n "${{ inputs.compare_report }}" && ! -e "${{ inputs.compare_report }}" ]]; then
          echo "::warning::The report ${{ inputs.compare_report }} to compare doesn't exist."
        fi

        echo "::set-output name=report::$(docker run --rm -v ${{ github.action_path }}/templates:/templates \
          -v ${{ inputs.workspace }}/${{ inputs.report_directory }}/report.json:/report.json ${{ inputs.gomplate_image }} -d report=/report.json \
          -f /templates/report_to_json.tmpl)"

        # Convert JSON to CSV
        python3 ${{ github.action_path }}/scripts/json_to_csv.py ${{ inputs.workspace }}/${{ inputs.report_directory }}/report.json ${{ inputs.workspace }}/${{ inputs.report_directory }}/report.csv

    - name: Publish summary as a comment in a PR
      shell: bash
      run: |
        if [[ -z "${{ inputs.github_token }}" && "${{ inputs.publish_summary }}" == "true" && "${{ github.event_name }}" == "pull_request" ]]; then
          echo "::warning::The GITHUB_TOKEN is required. Set the 'github_token' input parameter in order to publish summary."
        elif [[ "${{ inputs.publish_summary }}" == "true" && "${{ github.event_name }}" == "pull_request" ]]; then
          docker run --rm  \
            -v ${{ github.action_path }}/templates:/templates \
            -v ${{ inputs.workspace }}/${{ inputs.report_directory }}:/reports hairyhenderson/gomplate -d data=/reports/report.json -d results_main=/reports/report_main.json \
            -f /templates/comment_summary.tmpl > comment_summary.md

          OUTPUT=$(cat comment_summary.md)

          if [[ "${{ inputs.cross_validation }}" == "true" ]]; then
            pip3 install pytablewriter
            OUTPUT="${OUTPUT}\n$(python3 ${{ github.action_path }}/scripts/cross_validation_results.py)"
          fi

          if [[ -f "${{ inputs.workspace }}/${{ inputs.result_directory }}/failed_test_stories.yml" ]]; then
            TEST_FAILED=$(grep 'None of the test stories failed' README.md > /dev/null; echo $?)
            if [[ $TEST_FAILED -eq 0 ]]; then
              OUTPUT="${OUTPUT}\n```yml\n$(cat ${{ inputs.workspace }}/${{ inputs.result_directory }}/failed_test_stories.yml)```"
            else
              OUTPUT="${OUTPUT}\n\n#$(cat ${{ inputs.workspace }}/${{ inputs.result_directory }}/failed_test_stories.yml)"
            fi
          fi

          OUTPUT="${OUTPUT//$'\n'/'\n'}"

          curl -X POST -s -H "Authorization: token ${{ inputs.github_token }}" -H "Accept: application/vnd.github.v3+json"  \
          https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.number }}/comments -d "{\"body\":\"Commit: ${{ github.sha }}\n$OUTPUT\"}"

        elif [[ "${{ github.event_name }}" != "pull_request" ]]; then
          echo
          echo "::warning::Publish Summary as a PR comment is available only for a pull request."
          echo
        else
          echo
          echo "Publish Summary as a PR comment is disabled. To enabled it set the 'publish_summary' input parameter to 'true'."
          echo
        fi

    - name: Clean
      shell: bash
      run: |
        rm -rf ${{ inputs.workspace }}/${{ inputs.report_directory }}/report_main.json