diff --git a/Pinta.Core/Classes/DocumentSelection.cs b/Pinta.Core/Classes/DocumentSelection.cs
index f577527861..7b3eccba82 100644
--- a/Pinta.Core/Classes/DocumentSelection.cs
+++ b/Pinta.Core/Classes/DocumentSelection.cs
@@ -42,8 +42,15 @@ internal DocumentSelection ()
 	public List<List<IntPoint>> SelectionPolygons { get; set; } = [];
 	public Clipper SelectionClipper { get; } = new ();
 
-	public PointD Origin { get; set; }
-	public PointD End { get; set; }
+	/// <summary>
+	/// Bounding rectangle which is used by tools to display interactive
+	/// handles for manipulating the selection.
+	/// <remarks>
+	/// This might not reflect the bounds of the entire selection, for example
+	/// when creating a rectangle selection in "union" mode.
+	/// </remarks>
+	/// </summary>
+	public RectangleD HandleBounds { get; set; }
 
 	private bool visible = true;
 	public bool Visible {
@@ -94,8 +101,7 @@ public DocumentSelection Clone ()
 	{
 		return new () {
 			SelectionPolygons = [.. SelectionPolygons],
-			Origin = new PointD (Origin.X, Origin.Y),
-			End = new PointD (End.X, End.Y),
+			HandleBounds = HandleBounds,
 			visible = visible,
 		};
 	}
@@ -166,15 +172,15 @@ public DocumentSelection Transform (Matrix transform)
 			newPolygons.Add (newPolygon);
 		}
 
-		var origin = Origin;
-		var end = End;
-		transform.TransformPoint (ref origin);
-		transform.TransformPoint (ref end);
+		// Note this currently doesn't behave well with rotations, since we only
+		// store an axis-aligned bounding rectangle.
+		RectangleD transformedBounds = RectangleD.FromPoints (
+			transform.TransformPoint (HandleBounds.Location ()),
+			transform.TransformPoint (HandleBounds.EndLocation ()));
 
 		return new () {
 			SelectionPolygons = newPolygons,
-			Origin = origin,
-			End = end,
+			HandleBounds = transformedBounds,
 			visible = visible,
 		};
 	}
@@ -315,8 +321,7 @@ public void CreateRectangleSelection (RectangleD r)
 		SelectionPolygons.Clear ();
 		SelectionPolygons.Add (CreateRectanglePolygon (r));
 
-		Origin = new PointD (r.X, r.Y);
-		End = new PointD (r.Right, r.Bottom);
+		HandleBounds = r;
 
 		MarkDirty ();
 	}
@@ -400,8 +405,7 @@ private static List<IntPoint> CreateRectanglePolygon (RectangleD r)
 	public void Clear ()
 	{
 		SelectionPolygons.Clear ();
-		Origin = new PointD (0, 0);
-		End = new PointD (0, 0);
+		HandleBounds = RectangleD.Zero;
 		MarkDirty ();
 	}
 
diff --git a/Pinta.Core/Extensions/Cairo/CairoExtensions.Geometry.cs b/Pinta.Core/Extensions/Cairo/CairoExtensions.Geometry.cs
index 5e9854a554..92a791cf98 100644
--- a/Pinta.Core/Extensions/Cairo/CairoExtensions.Geometry.cs
+++ b/Pinta.Core/Extensions/Cairo/CairoExtensions.Geometry.cs
@@ -293,14 +293,14 @@ public static Matrix Clone (this Matrix m)
 		return result;
 	}
 
-	public static void TransformPoint (
+	public static PointD TransformPoint (
 		this Matrix m,
-		ref PointD p)
+		in PointD p)
 	{
 		double newX = p.X;
 		double newY = p.Y;
 		m.TransformPoint (ref newX, ref newY);
-		p = new PointD (newX, newY);
+		return new (newX, newY);
 	}
 
 	private static void GetRectangle (this Region region, int i, out CairoRectangleInt rect)
diff --git a/Pinta.Tools/Tools/SelectTool.cs b/Pinta.Tools/Tools/SelectTool.cs
index 1085c5817b..9b46109913 100644
--- a/Pinta.Tools/Tools/SelectTool.cs
+++ b/Pinta.Tools/Tools/SelectTool.cs
@@ -121,8 +121,7 @@ protected override void OnMouseUp (Document document, ToolMouseEventArgs e)
 
 			SelectionModeHandler.PerformSelectionMode (document, combine_mode, document.Selection.SelectionPolygons);
 
-			document.Selection.Origin = handle.Rectangle.Location ();
-			document.Selection.End = handle.Rectangle.EndLocation ();
+			document.Selection.HandleBounds = handle.Rectangle;
 			document.Workspace.Invalidate (last_dirty.Union (dirty));
 			last_dirty = dirty;
 
@@ -228,7 +227,7 @@ private void AfterSelectionChange (object? sender, EventArgs event_args)
 	private void LoadFromDocument (Document document)
 	{
 		DocumentSelection selection = document.Selection;
-		handle.Rectangle = RectangleD.FromPoints (selection.Origin, selection.End);
+		handle.Rectangle = selection.HandleBounds;
 		ShowHandles (document.Selection.Visible && tools.CurrentTool == this);
 	}
 }