Added pretokenization notebook for dolly, metamath #231

	name: Unittests & Auto-publish

	# Allow to trigger the workflow manually (e.g. when deps changes)
	on: [push, workflow_dispatch]

	jobs:
	pytest-job:
	runs-on: ubuntu-latest
	timeout-minutes: 30

	concurrency:
	group: ${{ github.workflow }}-${{ github.ref }}
	cancel-in-progress: true

	steps:
	- uses: actions/checkout@v3

	# Install deps
	- uses: actions/setup-python@v4
	with:
	python-version: 3.9
	cache: pip
	cache-dependency-path: '**/pyproject.toml'

	- run: pip --version
	- run: pip install -e .[dev]
	- run: pip freeze

	# Run tests (in parallel)
	- name: Run core tests
	run: pytest -vv -n auto

	# Auto-publish when version is increased
	publish-job:
	# Only try to publish if:
	# * Repo is self (prevents running from forks)
	# * Branch is `main`
	if: \|
	github.repository == 'google-research/precondition'
	&& github.ref == 'refs/heads/main'
	needs: pytest-job # Only publish after tests are successful
	runs-on: ubuntu-latest
	permissions:
	contents: write
	timeout-minutes: 30
	environment: pypi publish
	steps:
	# Publish the package (if local `__version__` > pip version)
	- uses: etils-actions/pypi-auto-publish@v1
	with:
	pypi-token: ${{ secrets.PYPI_API_TOKEN }}
	gh-token: ${{ secrets.GITHUB_TOKEN }}
	parse-changelog: true

Provide feedback