Connector Tests #371

Workflow file for this run

.github/workflows/pr-python-connector-tests.yml at a808b47

	name: Connector Tests

	on:
	merge_group:
	pull_request:
	branches: [main]
	schedule:
	# This cron expression runs the job daily at 16:00 UTC (9am PT)
	- cron: "0 16 * * *"

	env:
	# AWS
	AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
	AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.AWS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}

	# Cloudflare R2
	R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS: ${{ vars.R2_ACCOUNT_ID_DAILY_CONNECTOR_TESTS }}
	R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
	R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.R2_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}

	# Google Cloud Storage
	GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_ACCESS_KEY_ID_DAILY_CONNECTOR_TESTS }}
	GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS: ${{ secrets.GCS_SECRET_ACCESS_KEY_DAILY_CONNECTOR_TESTS }}

	# Confluence
	CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }}
	CONFLUENCE_TEST_SPACE: ${{ vars.CONFLUENCE_TEST_SPACE }}
	CONFLUENCE_TEST_PAGE_ID: ${{ secrets.CONFLUENCE_TEST_PAGE_ID }}
	CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }}
	CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }}
	CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }}

	# Jira
	JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
	JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}
	JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }}
	JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }}

	# Gong
	GONG_ACCESS_KEY: ${{ secrets.GONG_ACCESS_KEY }}
	GONG_ACCESS_KEY_SECRET: ${{ secrets.GONG_ACCESS_KEY_SECRET }}

	# Google
	GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_DRIVE_SERVICE_ACCOUNT_JSON_STR }}
	GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR_TEST_USER_1 }}
	GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_DRIVE_OAUTH_CREDENTIALS_JSON_STR }}
	GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR: ${{ secrets.GOOGLE_GMAIL_SERVICE_ACCOUNT_JSON_STR }}
	GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR: ${{ secrets.GOOGLE_GMAIL_OAUTH_CREDENTIALS_JSON_STR }}

	# Slab
	SLAB_BOT_TOKEN: ${{ secrets.SLAB_BOT_TOKEN }}

	# Zendesk
	ZENDESK_SUBDOMAIN: ${{ secrets.ZENDESK_SUBDOMAIN }}
	ZENDESK_EMAIL: ${{ secrets.ZENDESK_EMAIL }}
	ZENDESK_TOKEN: ${{ secrets.ZENDESK_TOKEN }}

	# Salesforce
	SF_USERNAME: ${{ secrets.SF_USERNAME }}
	SF_PASSWORD: ${{ secrets.SF_PASSWORD }}
	SF_SECURITY_TOKEN: ${{ secrets.SF_SECURITY_TOKEN }}

	# Hubspot
	HUBSPOT_ACCESS_TOKEN: ${{ secrets.HUBSPOT_ACCESS_TOKEN }}

	# IMAP
	IMAP_HOST: ${{ vars.IMAP_HOST }}
	IMAP_USERNAME: ${{ vars.IMAP_USERNAME }}
	IMAP_PASSWORD: ${{ secrets.IMAP_PASSWORD }}
	IMAP_MAILBOXES: ${{ vars.IMAP_MAILBOXES }}

	# Airtable
	AIRTABLE_TEST_BASE_ID: ${{ vars.AIRTABLE_TEST_BASE_ID }}
	AIRTABLE_TEST_TABLE_ID: ${{ vars.AIRTABLE_TEST_TABLE_ID }}
	AIRTABLE_TEST_TABLE_NAME: ${{ vars.AIRTABLE_TEST_TABLE_NAME }}
	AIRTABLE_ACCESS_TOKEN: ${{ secrets.AIRTABLE_ACCESS_TOKEN }}

	# Sharepoint
	SHAREPOINT_CLIENT_ID: ${{ vars.SHAREPOINT_CLIENT_ID }}
	SHAREPOINT_CLIENT_SECRET: ${{ secrets.SHAREPOINT_CLIENT_SECRET }}
	SHAREPOINT_CLIENT_DIRECTORY_ID: ${{ vars.SHAREPOINT_CLIENT_DIRECTORY_ID }}
	SHAREPOINT_SITE: ${{ vars.SHAREPOINT_SITE }}

	# Github
	ACCESS_TOKEN_GITHUB: ${{ secrets.ACCESS_TOKEN_GITHUB }}

	# Gitlab
	GITLAB_ACCESS_TOKEN: ${{ secrets.GITLAB_ACCESS_TOKEN }}

	# Gitbook
	GITBOOK_SPACE_ID: ${{ secrets.GITBOOK_SPACE_ID }}
	GITBOOK_API_KEY: ${{ secrets.GITBOOK_API_KEY }}

	# Notion
	NOTION_INTEGRATION_TOKEN: ${{ secrets.NOTION_INTEGRATION_TOKEN }}

	# Highspot
	HIGHSPOT_KEY: ${{ secrets.HIGHSPOT_KEY }}
	HIGHSPOT_SECRET: ${{ secrets.HIGHSPOT_SECRET }}

	# Slack
	SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

	# Teams
	TEAMS_APPLICATION_ID: ${{ secrets.TEAMS_APPLICATION_ID }}
	TEAMS_DIRECTORY_ID: ${{ secrets.TEAMS_DIRECTORY_ID }}
	TEAMS_SECRET: ${{ secrets.TEAMS_SECRET }}

	# Bitbucket
	BITBUCKET_WORKSPACE: ${{ secrets.BITBUCKET_WORKSPACE }}
	BITBUCKET_REPOSITORIES: ${{ secrets.BITBUCKET_REPOSITORIES }}
	BITBUCKET_PROJECTS: ${{ secrets.BITBUCKET_PROJECTS }}
	BITBUCKET_EMAIL: ${{ vars.BITBUCKET_EMAIL }}
	BITBUCKET_API_TOKEN: ${{ secrets.BITBUCKET_API_TOKEN }}

	# Fireflies
	FIREFLIES_API_KEY: ${{ secrets.FIREFLIES_API_KEY }}

	jobs:
	connectors-check:
	# See https://runs-on.com/runners/linux/
	runs-on: [runs-on, runner=8cpu-linux-x64, "run-id=${{ github.run_id }}"]

	env:
	PYTHONPATH: ./backend

	steps:
	- name: Checkout code
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.11"
	cache: "pip"
	cache-dependency-path: \|
	backend/requirements/default.txt
	backend/requirements/dev.txt

	- name: Install Dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install --retries 5 --timeout 30 -r backend/requirements/default.txt
	pip install --retries 5 --timeout 30 -r backend/requirements/dev.txt
	playwright install chromium
	playwright install-deps chromium

	- name: Detect Connector changes
	id: changes
	uses: dorny/paths-filter@v3
	with:
	filters: \|
	hubspot:
	- 'backend/onyx/connectors/hubspot/**'
	- 'backend/tests/daily/connectors/hubspot/**'
	salesforce:
	- 'backend/onyx/connectors/salesforce/**'
	- 'backend/tests/daily/connectors/salesforce/**'
	github:
	- 'backend/onyx/connectors/github/**'
	- 'backend/tests/daily/connectors/github/**'
	file_processing:
	- 'backend/onyx/file_processing/**'

	- name: Run Tests (excluding HubSpot, Salesforce, and GitHub)
	shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
	run: \|
	py.test \
	-n 8 \
	--dist loadfile \
	--durations=8 \
	-o junit_family=xunit2 \
	-xv \
	--ff \
	backend/tests/daily/connectors \
	--ignore backend/tests/daily/connectors/hubspot \
	--ignore backend/tests/daily/connectors/salesforce \
	--ignore backend/tests/daily/connectors/github

	- name: Run HubSpot Connector Tests
	if: ${{ github.event_name == 'schedule' \|\| steps.changes.outputs.hubspot == 'true' \|\| steps.changes.outputs.file_processing == 'true' }}
	shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
	run: \|
	py.test \
	-n 8 \
	--dist loadfile \
	--durations=8 \
	-o junit_family=xunit2 \
	-xv \
	--ff \
	backend/tests/daily/connectors/hubspot

	- name: Run Salesforce Connector Tests
	if: ${{ github.event_name == 'schedule' \|\| steps.changes.outputs.salesforce == 'true' \|\| steps.changes.outputs.file_processing == 'true' }}
	shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
	run: \|
	py.test \
	-n 8 \
	--dist loadfile \
	--durations=8 \
	-o junit_family=xunit2 \
	-xv \
	--ff \
	backend/tests/daily/connectors/salesforce

	- name: Run GitHub Connector Tests
	if: ${{ github.event_name == 'schedule' \|\| steps.changes.outputs.github == 'true' \|\| steps.changes.outputs.file_processing == 'true' }}
	shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
	run: \|
	py.test \
	-n 8 \
	--dist loadfile \
	--durations=8 \
	-o junit_family=xunit2 \
	-xv \
	--ff \
	backend/tests/daily/connectors/github

	- name: Alert on Failure
	if: failure() && github.event_name == 'schedule'
	env:
	SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
	run: \|
	curl -X POST \
	-H 'Content-type: application/json' \
	--data '{"text":"Scheduled Connector Tests failed! Check the run at: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}' \
	$SLACK_WEBHOOK

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

Connector Tests #371

Workflow file

Connector Tests #371

Uh oh!

Workflow file for this run