Added `run_tokenizer` method in `caikit_nlp/modules/text_generation/text_generation_local.py` #1150

	# Copyright The Caikit Authors
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: Build Caikit NLP Library

	on:
	push:
	branches: ["main", "release-*"]
	pull_request:
	branches: ["main", "release-*"]

	jobs:
	build:
	runs-on: ubuntu-latest
	strategy:
	matrix:
	python-version:
	# NOTE: TGIS does not support 3.8 or 3.11
	- setup: "3.9"
	tox: "py39"
	- setup: "3.10"
	tox: "py310"

	steps:
	- uses: actions/checkout@v3
	- name: Set up Python ${{ matrix.python-version.setup }}
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version.setup }}
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	python -m pip install -r setup_requirements.txt
	- name: Build and test with tox
	run: tox -e ${{ matrix.python-version.tox }}

Provide feedback