603 better oauth datasources

webapp/src/components/CreateDatasourceForm.tsx

@@ -31,6 +31,7 @@ import DatasourceChunkingForm from 'components/DatasourceChunkingForm';
 import { StreamsList } from 'components/DatasourceStream';
 import ToolTip from 'components/shared/ToolTip';
 import FormContext from 'context/connectorform';
+import OauthSecretProviderFactory from 'lib/oauthsecret';
 import { defaultChunkingOptions } from 'misc/defaultchunkingoptions';
 import { usePostHog } from 'posthog-js/react';
 import submittingReducer from 'utils/submittingreducer';
@@ -59,7 +60,9 @@ export default function CreateDatasourceForm({
 	initialStep = 0,
 	fetchDatasources,
 	spec,
-	setSpec
+	setSpec,
+	provider,
+	token
 }: {
 	models?: any[];
 	compact?: boolean;
@@ -70,6 +73,8 @@ export default function CreateDatasourceForm({
 	fetchDatasources?: Function;
 	spec?: any;
 	setSpec?: Function;
+	provider?: string;
+	token?: string;
 }) {
 	//TODO: fix any types
 
@@ -97,6 +102,7 @@ export default function CreateDatasourceForm({
 	const [chunkingConfig, setChunkingConfig] = useReducer(submittingReducer, {
 		...defaultChunkingOptions
 	});
+	const [oauthRedirectUrl, setOauthRedirectUrl] = useState(false);
 
 	//TODO: move into RetrievalStrategyComponent, keep the setters passed as props
 	const [toolRetriever, setToolRetriever] = useState(Retriever.SELF_QUERY);
@@ -349,6 +355,181 @@ export default function CreateDatasourceForm({
 		}
 	}
 
+	async function hubspotDatasourcePost(token: string) {
+		const data = {
+			credentials: {
+				credentials_title: 'OAuth Credentials',
+				client_id: process.env.OAUTH_HUBSPOT_CLIENT_ID,
+				client_secret: process.env.OAUTH_HUBSPOT_CLIENT_SECRET,
+				refresh_token: token
+			}
+		};
+
+		setSubmitting(true);
+		setError(null);
+		const posthogEvent = step === 2 ? 'testDatasource' : 'createDatasource';
+		try {
+			if (step === 2) {
+				const body = {
+					sourceConfig: data,
+					_csrf: csrf,
+					connectorId: connector.value,
+					connectorName: connector.label,
+					resourceSlug,
+					scheduleType,
+					timeUnit,
+					units,
+					cronExpression,
+					datasourceName,
+					datasourceDescription,
+					embeddingField
+				};
+
+				console.log('post body: ', body);
+				//step 2, getting schema and testing connection
+				await API.testDatasource(
+					body,
+					stagedDatasource => {
+						posthog.capture(posthogEvent, {
+							datasourceName,
+							connectorId: connector?.value,
+							connectorName: connector?.label,
+							syncSchedule: scheduleType
+						});
+						if (stagedDatasource) {
+							setDatasourceId(stagedDatasource.datasourceId);
+							setDiscoveredSchema(stagedDatasource.discoveredSchema);
+							setStreamProperties(stagedDatasource.streamProperties);
+							setStep(3);
+						} else {
+							setError('Datasource connection test failed.'); //TODO: any better way to get error?
+						}
+						// nothing to toast here
+					},
+					res => {
+						posthog.capture(posthogEvent, {
+							datasourceName,
+							connectorId: connector?.value,
+							connectorName: connector?.label,
+							syncSchedule: scheduleType,
+							error: res
+						});
+						setError(res);
+					},
+					compact ? null : router
+				);
+				// callback && stagedDatasource && callback(stagedDatasource._id);
+			} else {
+				//step 4, saving datasource
+				const filteredStreamState = Object.fromEntries(
+					Object.entries(streamState).filter(
+						(e: [string, StreamConfig]) => e[1].checkedChildren.length > 0
+					)
+				);
+				const body = {
+					_csrf: csrf,
+					datasourceId: datasourceId,
+					resourceSlug,
+					scheduleType,
+					timeUnit,
+					units,
+					modelId,
+					cronExpression,
+					streamConfig: filteredStreamState,
+					datasourceName,
+					datasourceDescription,
+					embeddingField,
+					retriever: toolRetriever,
+					retriever_config: {
+						timeWeightField: toolTimeWeightField,
+						decay_rate: toolDecayRate,
+						k: topK
+					},
+					chunkingConfig,
+					enableConnectorChunking
+				};
+				const addedDatasource: any = await API.addDatasource(
+					body,
+					() => {
+						posthog.capture(posthogEvent, {
+							datasourceName,
+							connectorId: connector?.value,
+							connectorName: connector?.label,
+							numStreams: Object.keys(streamState)?.length,
+							syncSchedule: scheduleType
+						});
+						toast.success('Added datasource');
+					},
+					res => {
+						posthog.capture(posthogEvent, {
+							datasourceName,
+							connectorId: connector?.value,
+							connectorName: connector?.label,
+							syncSchedule: scheduleType,
+							numStreams: Object.keys(streamState)?.length,
+							error: res
+						});
+						toast.error(res);
+					},
+					compact ? null : router
+				);
+				callback && addedDatasource && callback(addedDatasource._id);
+			}
+		} catch (e) {
+			posthog.capture(posthogEvent, {
+				datasourceName,
+				connectorId: connector?.value,
+				connectorName: connector?.label,
+				syncSchedule: scheduleType,
+				numStreams: Object.keys(streamState)?.length,
+				error: e?.message || e
+			});
+			console.error(e);
+		} finally {
+			await new Promise(res => setTimeout(res, 750));
+			setSubmitting(false);
+		}
+	}
+
+	const [oauthProvider, setOauthProvider] = useState(provider);
+	const [oauthToken, setOauthToken] = useState(token);
+
+	//OAUTH LOGIC
+	useEffect(() => {
+		setOauthProvider(provider);
+		setOauthToken(token);
+		setDatasourceName('Hubspot');
+		setDatasourceDescription('Hubspot OAuth Datasource');
+		console.log('Form token and provider', oauthProvider, oauthToken);
+	}, [provider, token]);
+
+	//once provider and token have been set this should run once
+	useEffect(() => {
+		if (provider !== null && token !== null) {
+			setStep(2);
+			switch (provider) {
+				case 'hubspot':
+					console.log('Posting with OAuth credentials');
+					setConnector({
+						airbyte_platform: 'oss',
+						connector_definition_id: '36c891d9-4bd9-43ac-bad2-10e12756272c',
+						connector_name: 'HubSpot',
+						connector_type: 'source',
+						connector_version: '4.2.22',
+						disabled: false,
+						docker_repository: 'airbyte/source-hubspot',
+						icon: 'https://connectors.airbyte.com/files/metadata/airbyte/source-hubspot/latest/icon.svg',
+						label: 'HubSpot',
+						planAvailable: true,
+						sync_success_rate: 'high',
+						usage: 'high',
+						value: '36c891d9-4bd9-43ac-bad2-10e12756272c'
+					});
+					hubspotDatasourcePost(token);
+			}
+		}
+	}, []);
+
 	function getStepSection(_step) {
 		//TODO: make steps enum
 		switch (_step) {
@@ -501,6 +682,7 @@ export default function CreateDatasourceForm({
 										return setSubscriptionModalOpen(v.label);
 									}
 									setLoading(v != null);
+									console.log('v', v);
 									setConnector(v);
 									if (v) {
 										getSpecification(v.value);
@@ -614,6 +796,9 @@ export default function CreateDatasourceForm({
 														schema={spec.schema.connectionSpecification}
 														datasourcePost={datasourcePost}
 														error={error}
+														name={connector.label}
+														icon={connector.icon}
+														redirectUrl={oauthRedirectUrl}
 													/>
 												</FormContext>
 											</>

webapp/src/components/connectorform/DynamicConnectorForm.tsx

@@ -1,9 +1,11 @@
 import ButtonSpinner from 'components/ButtonSpinner';
 import ErrorAlert from 'components/ErrorAlert';
 import dayjs from 'dayjs';
+import Link from 'next/link';
 import { useEffect, useState } from 'react';
 import { FieldValues, useFormContext } from 'react-hook-form';
 import { Schema } from 'struct/form';
+import { AIRBYTE_OAUTH_PROVIDERS } from 'struct/oauth';
 
 import AdditionalFields from './AdditionalFields';
 import FormSection from './FormSection';
@@ -12,6 +14,10 @@ interface DynamicFormProps {
 	schema: Schema;
 	datasourcePost: (arg: any) => Promise<void>;
 	error?: string;
+	name?: string;
+	icon?: any;
+	oauthPost?: boolean;
+	redirectUrl?: boolean;
 }
 
 const ISODatePattern = '^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$';
@@ -100,7 +106,15 @@ function updateDateStrings(
 	});
 }
 
-const DynamicConnectorForm = ({ schema, datasourcePost, error }: DynamicFormProps) => {
+const DynamicConnectorForm = ({
+	schema,
+	datasourcePost,
+	error,
+	name,
+	icon,
+	oauthPost,
+	redirectUrl
+}: DynamicFormProps) => {
 	const { handleSubmit } = useFormContext();
 	const [submitting, setSubmitting] = useState(false);
 
@@ -120,24 +134,42 @@ const DynamicConnectorForm = ({ schema, datasourcePost, error }: DynamicFormProp
 	}, [schema]);
 
 	return (
-		<form onSubmit={handleSubmit(onSubmit)}>
-			<FormSection properties={schema.properties} requiredFields={schema.required} />
-			{schema.additionalProperties && <AdditionalFields />}
+		<>
+			{name.toUpperCase() in AIRBYTE_OAUTH_PROVIDERS ? (
+				<div className='flex flex-col'>
+					<a //when the user hits this button then redirect them to the authentication link in a new window/tab
+						className='max-w-[25%] rounded-md disabled:bg-slate-400 bg-indigo-600 mx-3 my-5 px-5 py-3 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 mt-3'
+						href={`/auth/${name.toLowerCase()}/free`}
+						target='_blank'
+						rel='noopener noreferrer'
+					>
+						{icon && <img src={icon} loading='lazy' className='inline-flex me-2 w-6 w-6' />}
+						Log in with {name}
+					</a>
 
-			{error && (
-				<div className='mb-4'>
-					<ErrorAlert error={error} />
+					{redirectUrl && <p>penis</p>}
 				</div>
+			) : (
+				<form onSubmit={handleSubmit(onSubmit)}>
+					<FormSection properties={schema.properties} requiredFields={schema.required} />
+					{schema.additionalProperties && <AdditionalFields />}
+
+					{error && (
+						<div className='mb-4'>
+							<ErrorAlert error={error} />
+						</div>
+					)}
+					<button
+						disabled={submitting}
+						type='submit'
+						className='w-full rounded-md disabled:bg-slate-400 bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 mt-3'
+					>
+						{submitting && <ButtonSpinner />}
+						{submitting ? 'Testing connection...' : 'Submit'}
+					</button>
+				</form>
 			)}
-			<button
-				disabled={submitting}
-				type='submit'
-				className='w-full rounded-md disabled:bg-slate-400 bg-indigo-600 px-3 py-2 text-sm font-semibold text-white shadow-sm hover:bg-indigo-500 focus-visible:outline focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-indigo-600 mt-3'
-			>
-				{submitting && <ButtonSpinner />}
-				{submitting ? 'Testing connection...' : 'Submit'}
-			</button>
-		</form>
+		</>
 	);
 };
 

webapp/src/controllers/airbyte.ts

@@ -2,7 +2,7 @@
 
 import { dynamicResponse } from '@dr';
 import { io } from '@socketio';
-import getAirbyteApi, { AirbyteApiType } from 'airbyte/api';
+import getAirbyteApi, { AirbyteApiType, getAirbyteAuthToken } from 'airbyte/api';
 import getAirbyteInternalApi from 'airbyte/internal';
 import {
 	getDatasourceByConnectionId,
@@ -339,3 +339,26 @@ export async function handleSuccessfulEmbeddingWebhook(req, res, next) {
 
 	return dynamicResponse(req, res, 200, {});
 }
+
+//this is an old implementation of airbyte oauth. If we end up going to airbyte cloud then this will be useful but until then so long as we override the oauth credentials then we'll need to use a normal passport.js auth
+// export async function getOAuthRedirectLink(req, res, next) {
+// 	//generate a link to redirect the user to, use this api spec: https://reference.airbyte.com/reference/initiateoauth
+// 	const { sourceType } = req.body;
+// 	const redirectUrl = `https://app.agentcloud.dev/welcome`; //TODO: Set up this endpoint and redirect to it (maybe store the secret in persistent storage? But this could pose a security risk)
+// 	console.log("sourceType: ", sourceType);
+// 	const workspaceId = process.env.AIRBYTE_ADMIN_WORKSPACE_ID
+
+// 	const sourcesApi = await getAirbyteApi(AirbyteApiType.SOURCES);
+// 	const body = {
+// 		sourceType: sourceType,
+// 		redirectUrl: redirectUrl,
+// 		workspaceId: workspaceId
+// 	}; //body for the fetch request
+
+// 	const oauthRedirect = await sourcesApi.initiateOAuth(body).then(({data}) => log(data));
+// 	return oauthRedirect;
+// }
+
+// export async function handleOAuthWebhook(req, res, next) {
+// 	//airbyte hits the oauth callback with a secretId in the body
+// }

webapp/src/controllers/datasource.ts

@@ -21,6 +21,7 @@ import { addTool, deleteToolsForDatasource, editToolsForDatasource } from 'db/to
 import debug from 'debug';
 import dotenv from 'dotenv';
 import { convertCronToQuartz, convertUnitToCron } from 'lib/airbyte/cronconverter';
+import OauthSecretProviderFactory from 'lib/oauthsecret';
 import { chainValidations } from 'lib/utils/validationutils';
 import VectorDBProxyClient from 'lib/vectorproxy/client';
 import { isVectorLimitReached } from 'lib/vectorproxy/limit';
@@ -126,7 +127,11 @@ export async function testDatasourceApi(req, res, next) {
 
 	const currentPlan = res.locals?.subscription?.stripePlan;
 	const allowedPeriods = pricingMatrix[currentPlan]?.cronProps?.allowedPeriods || [];
+	const { clientId, clientSecret } = OauthSecretProviderFactory.getSecretProvider('hubspot');
+	sourceConfig.credentials.client_id = clientId;
+	sourceConfig.credentials.client_secret = clientSecret;
 
+	log('Source config for test API: ', sourceConfig);
 	let validationError = chainValidations(
 		req.body,
 		[