Skip to content

Commit 0b2563f

Browse files
vogievetsky317brian
vogievetsky
and
317brian
authored
Web console: adding format notice for CSV and TSV (#14783)
* adding format notice for CSV and TSV * Update web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx Co-authored-by: 317brian <[email protected]> * Update web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx Co-authored-by: 317brian <[email protected]> * Update web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx Co-authored-by: 317brian <[email protected]> * fix tests --------- Co-authored-by: 317brian <[email protected]>
1 parent 8fa7859 commit 0b2563f

File tree

10 files changed

+187
-54
lines changed

10 files changed

+187
-54
lines changed

Diff for: ‎licenses.yaml

+50
Original file line numberDiff line numberDiff line change
@@ -5319,6 +5319,16 @@ license_file_path: licenses/bin/color-name.MIT
53195319

53205320
---
53215321

5322+
name: "commander"
5323+
license_category: binary
5324+
module: web-console
5325+
license_name: MIT License
5326+
copyright: TJ Holowaychuk
5327+
version: 2.20.0
5328+
license_file_path: licenses/bin/commander.MIT
5329+
5330+
---
5331+
53225332
name: "constant-case"
53235333
license_category: binary
53245334
module: web-console
@@ -5419,6 +5429,16 @@ license_file_path: licenses/bin/d3-color.BSD3
54195429

54205430
---
54215431

5432+
name: "d3-dsv"
5433+
license_category: binary
5434+
module: web-console
5435+
license_name: BSD-3-Clause License
5436+
copyright: Mike Bostock
5437+
version: 2.0.0
5438+
license_file_path: licenses/bin/d3-dsv.BSD3
5439+
5440+
---
5441+
54225442
name: "d3-format"
54235443
license_category: binary
54245444
module: web-console
@@ -5787,6 +5807,16 @@ license_file_path: licenses/bin/hoist-non-react-statics.BSD3
57875807

57885808
---
57895809

5810+
name: "iconv-lite"
5811+
license_category: binary
5812+
module: web-console
5813+
license_name: MIT License
5814+
copyright: Alexander Shtuchkin
5815+
version: 0.4.24
5816+
license_file_path: licenses/bin/iconv-lite.MIT
5817+
5818+
---
5819+
57905820
name: "import-fresh"
57915821
license_category: binary
57925822
module: web-console
@@ -6336,6 +6366,16 @@ license_file_path: licenses/bin/resolve.MIT
63366366

63376367
---
63386368

6369+
name: "rw"
6370+
license_category: binary
6371+
module: web-console
6372+
license_name: BSD-3-Clause License
6373+
copyright: Mike Bostock
6374+
version: 1.3.3
6375+
license_file_path: licenses/bin/rw.BSD3
6376+
6377+
---
6378+
63396379
name: "safe-buffer"
63406380
license_category: binary
63416381
module: web-console
@@ -6346,6 +6386,16 @@ license_file_path: licenses/bin/safe-buffer.MIT
63466386

63476387
---
63486388

6389+
name: "safer-buffer"
6390+
license_category: binary
6391+
module: web-console
6392+
license_name: MIT License
6393+
copyright: Nikita Skovoroda
6394+
version: 2.1.2
6395+
license_file_path: licenses/bin/safer-buffer.MIT
6396+
6397+
---
6398+
63496399
name: "scheduler"
63506400
license_category: binary
63516401
module: web-console

Diff for: ‎web-console/package-lock.json

+59-10
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Diff for: ‎web-console/package.json

+2
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
"core-js": "^3.10.1",
7979
"d3-array": "^2.12.1",
8080
"d3-axis": "^2.1.0",
81+
"d3-dsv": "^2.0.0",
8182
"d3-scale": "^3.3.0",
8283
"d3-selection": "^2.0.0",
8384
"echarts": "^5.4.1",
@@ -114,6 +115,7 @@
114115
"@types/classnames": "^2.2.9",
115116
"@types/d3-array": "^2.12.3",
116117
"@types/d3-axis": "^2.1.3",
118+
"@types/d3-dsv": "^2.0.0",
117119
"@types/d3-scale": "^3.3.2",
118120
"@types/d3-selection": "^2.0.1",
119121
"@types/enzyme": "^3.10.3",

Diff for: ‎web-console/src/components/auto-form/auto-form.tsx

+1
Original file line numberDiff line numberDiff line change
@@ -366,6 +366,7 @@ export class AutoForm<T extends Record<string, any>> extends React.PureComponent
366366
disabled={AutoForm.evaluateFunctor(field.disabled, model, false)}
367367
intent={required && modelValue == null ? AutoForm.REQUIRED_INTENT : undefined}
368368
multiline={AutoForm.evaluateFunctor(field.multiline, model, false)}
369+
height={field.height}
369370
/>
370371
);
371372
}

Diff for: ‎web-console/src/components/formatted-input/formatted-input.tsx

+2
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ export const FormattedInput = React.memo(function FormattedInput(props: Formatte
4848
intent,
4949
placeholder,
5050
multiline,
51+
height,
5152
...rest
5253
} = props;
5354

@@ -105,6 +106,7 @@ export const FormattedInput = React.memo(function FormattedInput(props: Formatte
105106
onBlur={myOnBlur}
106107
intent={myIntent}
107108
placeholder={placeholder}
109+
style={height ? { height } : undefined}
108110
/>
109111
) : (
110112
<InputGroup

Diff for: ‎web-console/src/druid-models/ingestion-spec/ingestion-spec.tsx

+41-20
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import { Code } from '@blueprintjs/core';
2020
import { range } from 'd3-array';
21+
import { csvParseRows, tsvParseRows } from 'd3-dsv';
2122
import type { JSX } from 'react';
2223
import React from 'react';
2324

@@ -2135,33 +2136,53 @@ export function updateIngestionType(
21352136
return newSpec;
21362137
}
21372138

2139+
function findValueWithNewline(rows: string[][]): string | undefined {
2140+
return findMap(rows, row => findMap(row, value => (value.includes('\n') ? value : undefined)));
2141+
}
2142+
21382143
export function issueWithSampleData(
2139-
sampleData: SampleResponse,
2140-
spec: Partial<IngestionSpec>,
2144+
sampleLines: string[],
2145+
isStreaming: boolean,
21412146
): JSX.Element | undefined {
2142-
if (isStreamingSpec(spec)) return;
2147+
if (!sampleLines.length) return;
21432148

2144-
const firstData: string = findMap(sampleData.data, l => l.input?.raw);
2145-
if (firstData) return;
2149+
const firstLine = sampleLines[0];
2150+
if (!isStreaming) {
2151+
if (firstLine === '{') {
2152+
return (
2153+
<>
2154+
This data looks like a multi-line formatted JSON object. For Druid to parse a text file,
2155+
it must have one row per event. Consider reformatting your data as{' '}
2156+
<ExternalLink href="https://jsonlines.org">JSON Lines</ExternalLink>.
2157+
</>
2158+
);
2159+
}
21462160

2147-
if (firstData === '{') {
2148-
return (
2149-
<>
2150-
This data looks like multi-line formatted JSON object. For Druid to parse a text file it
2151-
must have one row per event. Consider reformatting your data as{' '}
2152-
<ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink>.
2153-
</>
2154-
);
2161+
if (oneOf(firstLine, '[', '[]')) {
2162+
return (
2163+
<>
2164+
This data looks like a multi-line JSON array. For Druid to parse a text file, it must have
2165+
one row per event. Consider reformatting your data as{' '}
2166+
<ExternalLink href="https://jsonlines.org">JSON Lines</ExternalLink>.
2167+
</>
2168+
);
2169+
}
21552170
}
21562171

2157-
if (oneOf(firstData, '[', '[]')) {
2158-
return (
2159-
<>
2160-
This data looks like a multi-line JSON array. For Druid to parse a text file it must have
2161-
one row per event. Consider reformatting your data as{' '}
2162-
<ExternalLink href="http://ndjson.org/">newline delimited JSON</ExternalLink>.
2163-
</>
2172+
const format = guessSimpleInputFormat(sampleLines, isStreaming);
2173+
const text = sampleLines.join('\n');
2174+
if (oneOf(format.type, 'csv', 'tsv')) {
2175+
const valueWithNewline = findValueWithNewline(
2176+
format.type === 'csv' ? csvParseRows(text) : tsvParseRows(text),
21642177
);
2178+
if (valueWithNewline) {
2179+
const formatLabel = format.type.toUpperCase();
2180+
return (
2181+
<>
2182+
{`This ${formatLabel} data has values that contain new lines. Druid requires ${formatLabel} files to have one event per line, so ${formatLabel} values can not contain new lines. Consider encoding new lines in the values of your ${formatLabel} with some special delimiter.`}
2183+
</>
2184+
);
2185+
}
21652186
}
21662187

21672188
return;

Diff for: ‎web-console/src/druid-models/input-source/input-source.tsx

+3
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ import { deepGet, deepSet, nonEmptyArray, typeIsKnown } from '../../utils';
2525

2626
export const FILTER_SUGGESTIONS: string[] = [
2727
'*',
28+
'*.jsonl',
29+
'*.jsonl.gz',
2830
'*.json',
2931
'*.json.gz',
3032
'*.csv',
@@ -179,6 +181,7 @@ export const INPUT_SOURCE_FIELDS: Field<InputSource>[] = [
179181
required: true,
180182
placeholder: 'Paste your data here',
181183
multiline: true,
184+
height: '400px',
182185
info: <p>Put you inline data here</p>,
183186
},
184187

0 commit comments

Comments
 (0)