diff --git a/README.md b/README.md
index 3de20ce..effa773 100644
--- a/README.md
+++ b/README.md
@@ -53,10 +53,10 @@ Must be a maximal number, rather than a range.
 
 ## Outputs
 
-|      Name      |  Type  |                                             Description                                              |
-| -------------- | ------ | ---------------------------------------------------------------------------------------------------- |
-| `new_label`    | string | The new label's name to be added.                                                                    |
-| `stale_labels` | string | The stale labels' name to be removed. If there're multiple labels, they're separated by line breaks. |
+|      Name      |  Type  |                                                                          Description                                                                           |
+| -------------- | ------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `new_label`    | string | The new label's name to be added. If there's no label to be added, it the value is `''`.                                                                       |
+| `stale_labels` | string | The stale labels' name to be removed. If there're multiple labels, they're separated by line breaks. If there's no labels to be removed, it the value is `''`. |
 
 ## Example
 
diff --git a/dist/index.js b/dist/index.js
index 2614fe4..5d46b3f 100644
--- a/dist/index.js
+++ b/dist/index.js
@@ -5055,12 +5055,14 @@ class Processor {
     }
     process() {
         const changes = Processor.getChangedLines();
-        const newLabel = this.determineLabel(changes);
+        const desiredLabel = this.determineLabel(changes);
+        const currentLabels = this.getCurrentLabels();
+        const newLabel = currentLabels.includes(desiredLabel) ? '' : desiredLabel;
+        const staleLabel = currentLabels.filter(label => label !== desiredLabel);
         core.setOutput('new_label', newLabel);
-        const staleLabels = this.getCurrentSizeLabels();
-        core.setOutput('stale_labels', staleLabels.join('\n'));
+        core.setOutput('stale_labels', staleLabel.join('\n'));
     }
-    getCurrentSizeLabels() {
+    getCurrentLabels() {
         const payload = github.context
             .payload;
         return payload.pull_request.labels
diff --git a/src/Processor.ts b/src/Processor.ts
index 3e60b24..4590d12 100644
--- a/src/Processor.ts
+++ b/src/Processor.ts
@@ -30,14 +30,17 @@ export class Processor {
   process() {
     const changes = Processor.getChangedLines();
 
-    const newLabel = this.determineLabel(changes);
-    core.setOutput('new_label', newLabel);
+    const desiredLabel = this.determineLabel(changes);
+    const currentLabels = this.getCurrentLabels();
+
+    const newLabel = currentLabels.includes(desiredLabel) ? '' : desiredLabel;
+    const staleLabel = currentLabels.filter(label => label !== desiredLabel);
 
-    const staleLabels = this.getCurrentSizeLabels();
-    core.setOutput('stale_labels', staleLabels.join('\n'));
+    core.setOutput('new_label', newLabel);
+    core.setOutput('stale_labels', staleLabel.join('\n'));
   }
 
-  private getCurrentSizeLabels(): string[] {
+  private getCurrentLabels(): string[] {
     const payload = github.context
       .payload as Webhooks.WebhookPayloadPullRequest;