diff --git a/src/vs/editor/common/services/semanticTokensProviderStyling.ts b/src/vs/editor/common/services/semanticTokensProviderStyling.ts
index ca9026d66fe6d..63a7409c4a284 100644
--- a/src/vs/editor/common/services/semanticTokensProviderStyling.ts
+++ b/src/vs/editor/common/services/semanticTokensProviderStyling.ts
@@ -22,6 +22,7 @@ export class SemanticTokensProviderStyling {
 	private _hasWarnedOverlappingTokens = false;
 	private _hasWarnedInvalidLengthTokens = false;
 	private _hasWarnedInvalidEditStart = false;
+	private _hasWarnedInvalidEditDeleteCount = false;
 
 	constructor(
 		private readonly _legend: SemanticTokensLegend,
@@ -125,6 +126,13 @@ export class SemanticTokensProviderStyling {
 		}
 	}
 
+	public warnInvalidEditDeleteCount(previousResultId: string | undefined, resultId: string | undefined, srcLength: number, deltaLength: number): void {
+		if (!this._hasWarnedInvalidEditDeleteCount) {
+			this._hasWarnedInvalidEditDeleteCount = true;
+			this._logService.warn(`Invalid semantic tokens edit detected (previousResultId: ${previousResultId}, resultId: ${resultId}): The edits delete more tokens than the previous data contains (previous length ${srcLength}, computed length ${srcLength + deltaLength}).`);
+		}
+	}
+
 }
 
 const enum SemanticColoringConstants {
diff --git a/src/vs/editor/contrib/semanticTokens/browser/documentSemanticTokens.ts b/src/vs/editor/contrib/semanticTokens/browser/documentSemanticTokens.ts
index 95348d44230af..3fcf8d92134bf 100644
--- a/src/vs/editor/contrib/semanticTokens/browser/documentSemanticTokens.ts
+++ b/src/vs/editor/contrib/semanticTokens/browser/documentSemanticTokens.ts
@@ -323,9 +323,17 @@ class ModelSemanticColoring extends Disposable {
 		}
 
 		if (isSemanticTokensEdits(tokens)) {
+			const resultId = tokens.resultId;
+			const rejectInvalidEdits = (requestFullRefresh: boolean) => {
+				provider.releaseDocumentSemanticTokens(resultId);
+				this._model.tokenization.setSemanticTokens(null, true);
+				if (requestFullRefresh) {
+					this._fetchDocumentSemanticTokens.schedule(0);
+				}
+			};
 			if (!currentResponse) {
 				// not possible!
-				this._model.tokenization.setSemanticTokens(null, true);
+				rejectInvalidEdits(false);
 				return;
 			}
 			if (tokens.edits.length === 0) {
@@ -341,7 +349,13 @@ class ModelSemanticColoring extends Disposable {
 				}
 
 				const srcData = currentResponse.data;
-				const destData = new Uint32Array(srcData.length + deltaLength);
+				const destDataLength = srcData.length + deltaLength;
+				if (destDataLength < 0) {
+					styling.warnInvalidEditDeleteCount(currentResponse.resultId, resultId, srcData.length, deltaLength);
+					rejectInvalidEdits(true);
+					return;
+				}
+				const destData = new Uint32Array(destDataLength);
 
 				let srcLastStart = srcData.length;
 				let destLastStart = destData.length;
@@ -349,9 +363,8 @@ class ModelSemanticColoring extends Disposable {
 					const edit = tokens.edits[i];
 
 					if (edit.start > srcData.length) {
-						styling.warnInvalidEditStart(currentResponse.resultId, tokens.resultId, i, edit.start, srcData.length);
-						// The edits are invalid and there's no way to recover
-						this._model.tokenization.setSemanticTokens(null, true);
+						styling.warnInvalidEditStart(currentResponse.resultId, resultId, i, edit.start, srcData.length);
+						rejectInvalidEdits(true);
 						return;
 					}
 
diff --git a/src/vs/editor/contrib/semanticTokens/test/browser/documentSemanticTokens.test.ts b/src/vs/editor/contrib/semanticTokens/test/browser/documentSemanticTokens.test.ts
index 571b8ca9dd513..f32d842a7e8b1 100644
--- a/src/vs/editor/contrib/semanticTokens/test/browser/documentSemanticTokens.test.ts
+++ b/src/vs/editor/contrib/semanticTokens/test/browser/documentSemanticTokens.test.ts
@@ -185,6 +185,106 @@ suite('ModelSemanticColoring', () => {
 		});
 	});
 
+	test('issue #322571: semantic token edits cannot delete more data than the previous result contains', async () => {
+		await runWithFakedTimers({}, async () => {
+
+			disposables.add(languageService.registerLanguage({ id: 'testMode' }));
+
+			const lastResultIds: (string | null)[] = [];
+			const releasedResultIds: (string | undefined)[] = [];
+			const recoveredResultProvided = new Barrier();
+
+			disposables.add(languageFeaturesService.documentSemanticTokensProvider.register('testMode', new class implements DocumentSemanticTokensProvider {
+				getLegend(): SemanticTokensLegend {
+					return { tokenTypes: ['class'], tokenModifiers: [] };
+				}
+				async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokens | SemanticTokensEdits | null> {
+					lastResultIds.push(lastResultId);
+					if (lastResultIds.length === 1) {
+						return {
+							resultId: 'full',
+							data: new Uint32Array([0, 0, 1, 0, 0])
+						};
+					}
+					if (lastResultIds.length === 2) {
+						return {
+							resultId: 'invalid-edit',
+							edits: [{ start: 0, deleteCount: 6 }]
+						};
+					}
+					if (lastResultIds.length === 3) {
+						recoveredResultProvided.open();
+						return {
+							resultId: 'recovered-full',
+							data: new Uint32Array([0, 0, 1, 0, 0])
+						};
+					}
+					assert.fail('Unexpected call');
+				}
+				releaseDocumentSemanticTokens(resultId: string | undefined): void {
+					releasedResultIds.push(resultId);
+				}
+			}));
+
+			const textModel = disposables.add(modelService.createModel('a', languageService.createById('testMode')));
+			textModel.onBeforeAttached();
+
+			await Event.toPromise(textModel.onDidChangeTokens);
+			textModel.applyEdits([{ range: new Range(1, 1, 1, 1), text: 'b' }]);
+			await recoveredResultProvided.wait();
+
+			assert.deepStrictEqual({
+				lastResultIds,
+				releasedResultIds
+			}, {
+				lastResultIds: [null, 'full', null],
+				releasedResultIds: ['full', 'invalid-edit']
+			});
+		});
+	});
+
+	test('invalid semantic token edits without a previous result are not retried', async () => {
+		await runWithFakedTimers({}, async () => {
+
+			disposables.add(languageService.registerLanguage({ id: 'testMode' }));
+
+			let requestCount = 0;
+			const releasedResultIds: (string | undefined)[] = [];
+			const resultProvided = new Barrier();
+
+			disposables.add(languageFeaturesService.documentSemanticTokensProvider.register('testMode', new class implements DocumentSemanticTokensProvider {
+				getLegend(): SemanticTokensLegend {
+					return { tokenTypes: ['class'], tokenModifiers: [] };
+				}
+				async provideDocumentSemanticTokens(model: ITextModel, lastResultId: string | null, token: CancellationToken): Promise<SemanticTokensEdits> {
+					requestCount++;
+					resultProvided.open();
+					return {
+						resultId: 'invalid-edit',
+						edits: [{ start: 0, deleteCount: 1 }]
+					};
+				}
+				releaseDocumentSemanticTokens(resultId: string | undefined): void {
+					releasedResultIds.push(resultId);
+				}
+			}));
+
+			const textModel = disposables.add(modelService.createModel('a', languageService.createById('testMode')));
+			textModel.onBeforeAttached();
+
+			await resultProvided.wait();
+			await timeout(1000);
+
+			assert.deepStrictEqual({
+				requestCount,
+				releasedResultIds
+			}, {
+				requestCount: 1,
+				releasedResultIds: ['invalid-edit']
+			});
+		});
+	});
+
 	test('issue #161573: onDidChangeSemanticTokens doesn\'t consistently trigger provideDocumentSemanticTokens', async () => {
 		await runWithFakedTimers({}, async () => {