From f48aa0a679530c40d2cc6683ed8b733bbd6620ad Mon Sep 17 00:00:00 2001
From: Hugh Sanderson <gamehaxe@gmail.com>
Date: Tue, 23 Apr 2024 14:57:48 +0800
Subject: [PATCH] Avoid overflow with AA and large image rendering.  Fix
 Point.normalized

---
 .../src/software-renderer/PolygonRender.cpp   | 26 ++++++++++++-------
 src/nme/geom/Point.hx                         |  2 +-
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/project/src/software-renderer/PolygonRender.cpp b/project/src/software-renderer/PolygonRender.cpp
index 2da417e8a..aba9377be 100644
--- a/project/src/software-renderer/PolygonRender.cpp
+++ b/project/src/software-renderer/PolygonRender.cpp
@@ -597,31 +597,37 @@ bool PolygonRender::Render(const RenderTarget &inTarget, const RenderState &inSt
    
    if (visible_pixels.HasPixels())
    {
+      Transform trans = inState.mTransform;
+      int size = (int) std::max( std::max( fabs(extent.minX), fabs(extent.maxX) ),
+                                 std::max( fabs(extent.minY), fabs(extent.maxY) ) );
+      while(trans.mAAFactor>1 && trans.mAAFactor*size>16383)
+         trans.mAAFactor>>=1;
+
       // Check to see if AlphaMask is invalid...
       int tx = 0;
       int ty = 0;
-      
-      if (mAlphaMask && !mAlphaMask->Compatible(inState.mTransform, rect,visible_pixels, tx, ty))
+
+      if (mAlphaMask && !mAlphaMask->Compatible(trans, rect,visible_pixels, tx, ty))
       {
          mAlphaMask->Dispose();
          mAlphaMask = 0;
       }
-      
+
       if (!mAlphaMask)
       {
-         SetTransform(inState.mTransform);
-         
+         SetTransform(trans);
+
          // TODO: make visible_pixels a bit bigger ?
-         SpanRect span(visible_pixels, inState.mTransform.mAAFactor);
+         SpanRect span(visible_pixels, trans.mAAFactor);
          span.mWinding = GetWinding();
          mSpanRect = &span;
-         
-         int alpha_factor = Iterate(itCreateRenderer, *inState.mTransform.mMatrix);
+
+         int alpha_factor = Iterate(itCreateRenderer, *trans.mMatrix);
          mAlphaMask = mSpanRect->CreateMask(mTransform, alpha_factor);
-         
+
          mSpanRect = 0;
       }
-      
+
       if (inTarget.mPixelFormat == pfAlpha)
       {
          mAlphaMask->RenderBitmap(tx, ty, inTarget, inState);
diff --git a/src/nme/geom/Point.hx b/src/nme/geom/Point.hx
index 290b005c0..267a308ab 100644
--- a/src/nme/geom/Point.hx
+++ b/src/nme/geom/Point.hx
@@ -95,7 +95,7 @@ class Point
    inline public function normalized(inplace=false):Point
    {
       var result = this;
-      var len = x*x+y*y;
+      var len = Math.sqrt(x*x+y*y);
       var scale  =  Math.abs(len)>1e-7 ? 1.0/len : 0.0;
       if (inplace)
       {