From 8b8d6b7d2f6a01b55390d95d95825c4eaff89824 Mon Sep 17 00:00:00 2001 From: Juntao Wang Date: Tue, 11 Jun 2024 09:30:59 -0400 Subject: [PATCH] Remove DS pipeline endpoint hack --- .../__tests__/utils.spec.ts | 27 ++----------------- .../content/configurePipelinesServer/utils.ts | 7 +---- 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts b/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts index 4aed8ca986..82da6d349d 100644 --- a/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts +++ b/frontend/src/concepts/pipelines/content/configurePipelinesServer/__tests__/utils.spec.ts @@ -68,7 +68,7 @@ describe('configure pipeline server utils', () => { ]; const spec = createDSPipelineResourceSpec(config, secretsResponse); expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-1.amazonaws.com'); + expect(spec.objectStorage.externalStorage?.host).toBe('s3.amazonaws.com'); }); it('should parse S3 endpoint without scheme', () => { @@ -77,30 +77,7 @@ describe('configure pipeline server utils', () => { config.objectStorage.newValue = [{ key: AwsKeys.S3_ENDPOINT, value: 's3.amazonaws.com' }]; const spec = createDSPipelineResourceSpec(config, secretsResponse); expect(spec.objectStorage.externalStorage?.scheme).toBe('https'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-1.amazonaws.com'); - }); - - it('should convert S3 endpoint with region', () => { - const config = createPipelineServerConfig(); - const secretsResponse = createSecretsResponse(); - config.objectStorage.newValue = [ - { key: AwsKeys.S3_ENDPOINT, value: 'http://s3.amazonaws.com' }, - { key: AwsKeys.DEFAULT_REGION, value: 'us-east-2' }, - ]; - const spec = createDSPipelineResourceSpec(config, secretsResponse); - expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.us-east-2.amazonaws.com'); - }); - - it('should not convert endpoint when it is not S3', () => { - const config = createPipelineServerConfig(); - const secretsResponse = createSecretsResponse(); - config.objectStorage.newValue = [ - { key: AwsKeys.S3_ENDPOINT, value: 'http://s3.not-amazonaws.com' }, - ]; - const spec = createDSPipelineResourceSpec(config, secretsResponse); - expect(spec.objectStorage.externalStorage?.scheme).toBe('http'); - expect(spec.objectStorage.externalStorage?.host).toBe('s3.not-amazonaws.com'); + expect(spec.objectStorage.externalStorage?.host).toBe('s3.amazonaws.com'); }); it('should include bucket', () => { diff --git a/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts b/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts index 53ac0d89d1..41584d343f 100644 --- a/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts +++ b/frontend/src/concepts/pipelines/content/configurePipelinesServer/utils.ts @@ -108,7 +108,7 @@ export const createDSPipelineResourceSpec = ( dspVersion: 'v2', objectStorage: { externalStorage: { - host: convertAWSHostForRegion(externalStorageHost, externalStorageRegion), + host: externalStorageHost, scheme: externalStorageScheme || 'https', bucket: awsRecord.AWS_S3_BUCKET || '', region: externalStorageRegion, @@ -157,8 +157,3 @@ export const getLabelName = (index: string): string => { const field = PIPELINE_AWS_FIELDS.find((currentField) => currentField.key === index); return field ? field.label : ''; }; - -const convertAWSHostForRegion = (endpoint: string, region: string): string => { - const host = endpoint.replace(/\/$/, '') || ''; - return host === 's3.amazonaws.com' && region !== '' ? `s3.${region}.amazonaws.com` : host; -};