Compiler multi threading support

.github/workflows/arduino.yml

@@ -0,0 +1,28 @@
+# This is a basic workflow to help you get started with Actions
+
+name: Arduino_Compiler_CI
+
+# Controls when the workflow will run
+on:
+  # Triggers the workflow on push or pull request events
+  push:
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+  pull_request:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  # This workflow contains a single job called "build"
+  build:
+    # The type of runner that the job will run on
+    runs-on: windows-latest
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      - name: Checkout
+        uses: actions/checkout@master
+
+      - run: |
+          eng/ArduinoCsCI.cmd %USERPROFILE% Debug
+

azure-pipelines.yml

@@ -68,6 +68,10 @@ stages:
         sourceFolder: $(Build.SourcesDirectory)/artifacts/packages/$(BuildConfiguration)/Shipping
         targetFolder: $(Build.ArtifactStagingDirectory)/Packages
 
+    - task: PublishTestResults@2
+      inputs:
+        mergeTestResults: true
+
     - publish: $(Build.ArtifactStagingDirectory)/Packages
       displayName: Publish Build Artifacts
       artifact: BuildPackages
@@ -91,28 +95,6 @@ stages:
       displayName: 'Execute markdown-link-check'
       condition: eq(variables['build.reason'], 'PullRequest')
 
-  # - job: Windows_ArduinoIntegration
-  #   displayName: Arduino Integration Tests
-  #   timeoutInMinutes: 120
-  #   pool:
-  #     vmImage: windows-2022
-
-  #   strategy:
-  #     matrix:
-  #       Build_Release:
-  #         BuildConfiguration: Release
-  #       Build_Debug:
-  #         BuildConfiguration: Debug
-
-  #   steps:
-  #   - script: build.cmd -ci 
-  #       -configuration $(BuildConfiguration)
-  #       -prepareMachine
-  #     displayName: Build Iot
-
-  #   - script: eng\ArduinoCsCI.cmd $(UserProfile) $(BuildConfiguration)
-  #     displayName: Build and run Arduino Integration Tests
-
   - job: Linux
     displayName: Linux Build
     container: LinuxContainer

eng/ArduinoCsCI.cmd

@@ -5,7 +5,7 @@ REM Second argument is either "Debug" or "Release"
 if %1!==! goto :usage
 
 REM Defines the revision to check out in the ExtendedConfigurableFirmata repo
-set FIRMATA_SIMULATOR_CHECKOUT_REVISION=e2cfb5223aeb71e3a0756d67619db6c238b6acb5
+set FIRMATA_SIMULATOR_CHECKOUT_REVISION=4a3b895c062c8e48685b9018d642d2c5ea84c354
 set RUN_COMPILER_TESTS=FALSE
 
 choco install -y --no-progress arduino-cli
@@ -16,13 +16,18 @@ arduino-cli config init
 arduino-cli config add board_manager.additional_urls https://raw.githubusercontent.com/espressif/arduino-esp32/gh-pages/package_esp32_dev_index.json
 arduino-cli core update-index
 
+REM directly execute PS, we can ignore any test errors.
+powershell -ExecutionPolicy ByPass -command "%~dp0common\Build.ps1" -restore -build -ci -configuration %2 -preparemachine
+
 set ArduinoRootDir=%1\Documents\Arduino
 set acspath=%~dp0\..\tools\ArduinoCsCompiler\Frontend\bin\%2\net6.0\acs.exe
 
 git clone https://github.com/firmata/ConfigurableFirmata %ArduinoRootDir%\libraries\ConfigurableFirmata
 git clone https://github.com/pgrawehr/ExtendedConfigurableFirmata %ArduinoRootDir%\ExtendedConfigurableFirmata
 arduino-cli core install esp32:esp32
 
+git fetch --all
+git branch --list
 REM Check whether any compiler files have changed - if so, enable the (long running) compiler tests
 git diff --name-status origin/main | find /C /I "tools/ArduinoCsCompiler"
 REM Find returns 1 when the string was NOT found, we want to set the variable to true when it does find something

src/devices/Arduino/ArduinoBoard.cs

@@ -329,6 +329,27 @@ public void AddCommandHandler<T>(T newCommandHandler)
             }
         }
 
+        /// <summary>
+        /// Unregisters the given command handler
+        /// </summary>
+        /// <typeparam name="T">A type derived from <see cref="ExtendedCommandHandler"/></typeparam>
+        /// <param name="commandHandler">The instance</param>
+        /// <remarks>This is intended mostly for unit test scenarios, where the command handlers are recreated. It does not
+        /// remove the modes supported by the handler</remarks>
+        public void RemoveCommandHandler<T>(T commandHandler)
+            where T : ExtendedCommandHandler
+        {
+            _commandHandlersLock.EnterWriteLock();
+            try
+            {
+                _extendedCommandHandlers.Remove(commandHandler);
+            }
+            finally
+            {
+                _commandHandlersLock.ExitWriteLock();
+            }
+        }
+
         /// <summary>
         /// Gets the command handler with the provided type. An exact type match is performed.
         /// </summary>

src/devices/Arduino/FirmataDevice.cs

@@ -30,7 +30,7 @@ internal sealed class FirmataDevice : IDisposable
         private const byte FIRMATA_PROTOCOL_MAJOR_VERSION = 2;
         private const byte FIRMATA_PROTOCOL_MINOR_VERSION = 5; // 2.5 works, but 2.6 is recommended
         private const int FIRMATA_INIT_TIMEOUT_SECONDS = 2;
-        internal static readonly TimeSpan DefaultReplyTimeout = TimeSpan.FromMilliseconds(2000);
+        internal static readonly TimeSpan DefaultReplyTimeout = TimeSpan.FromMilliseconds(3000);
 
         private byte _firmwareVersionMajor;
         private byte _firmwareVersionMinor;
@@ -1209,7 +1209,7 @@ internal void DisableAnalogReporting(int pinNumber, int analogChannel)
                     pwmCommandSequence.WriteByte((byte)0);
                     pwmCommandSequence.WriteByte((byte)FirmataCommand.END_SYSEX);
                     SendCommand(pwmCommandSequence);
-                }
+        }
             }
         }
 

src/devices/CharacterLcd/LcdInterface.I2c4Bit.cs

@@ -24,11 +24,13 @@ private sealed class I2c4Bit : LcdInterface
 
             private readonly I2cDevice _i2cDevice;
             private bool _backlightOn;
+            private int _backlightFlag;
 
             public I2c4Bit(I2cDevice i2cDevice)
             {
                 _i2cDevice = i2cDevice ?? throw new ArgumentNullException(nameof(i2cDevice));
                 _backlightOn = true;
+                _backlightFlag = LCD_BACKLIGHT;
                 InitDisplay();
             }
 
@@ -44,26 +46,12 @@ public override bool BacklightOn
                 set
                 {
                     _backlightOn = value;
+                    _backlightFlag = value ? LCD_BACKLIGHT : 0;
                     // Need to send a command to make this happen immediately.
                     SendCommandAndWait(0);
                 }
             }
 
-            private byte BacklightFlag
-            {
-                get
-                {
-                    if (BacklightOn)
-                    {
-                        return LCD_BACKLIGHT;
-                    }
-                    else
-                    {
-                        return 0;
-                    }
-                }
-            }
-
             private void InitDisplay()
             {
                 // This sequence (copied from a python example) completely resets the display (if it was
@@ -89,8 +77,8 @@ public override void SendCommand(byte command)
 
             private void Write4Bits(byte command)
             {
-                _i2cDevice.WriteByte((byte)(command | ENABLE | BacklightFlag));
-                _i2cDevice.WriteByte((byte)((command & ~ENABLE) | BacklightFlag));
+                _i2cDevice.WriteByte((byte)(command | ENABLE | _backlightFlag));
+                _i2cDevice.WriteByte((byte)((command & ~ENABLE) | _backlightFlag));
             }
 
             public override void SendCommands(ReadOnlySpan<byte> commands)

tools/ArduinoCsCompiler/ArduinoCsCompiler.csproj

@@ -29,4 +29,10 @@
     <ProjectReference Include="..\..\src\System.Device.Gpio\System.Device.Gpio.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <AssemblyAttribute Include="System.Runtime.CompilerServices.InternalsVisibleTo">
+      <_Parameter1>$(AssemblyName).Tests</_Parameter1>
+    </AssemblyAttribute>
+  </ItemGroup>
+
 </Project>

tools/ArduinoCsCompiler/ArduinoImplementationAttribute.cs

@@ -107,6 +107,15 @@ public bool MergeGenericImplementations
             set;
         }
 
+        /// <summary>
+        /// Set to true for methods that are only called by the runtime (e.g. thread start callbacks)
+        /// </summary>
+        public bool InternalCall
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Computes a hash code for a string that stays consistent over different architectures and between program runs.
         /// </summary>

tools/ArduinoCsCompiler/ArduinoMethodDeclaration.cs

@@ -40,13 +40,13 @@ public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMe
             Name = $"{MethodBase.MethodSignature()} (Token 0x{Token:X})";
         }
 
-        public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMethodDeclaration? requestedBy, IlCode code)
+        public ArduinoMethodDeclaration(int token, EquatableMethod methodBase, ArduinoMethodDeclaration? requestedBy, IlCode code, MethodFlags extraFlags)
         {
             Index = -1;
             MethodBase = methodBase;
             RequestedBy = requestedBy;
             Code = code;
-            Flags = MethodFlags.None;
+            Flags = extraFlags;
             Token = token;
 
             var attribs = methodBase.GetCustomAttributes(typeof(ArduinoImplementationAttribute)).Cast<ArduinoImplementationAttribute>().ToList();

tools/ArduinoCsCompiler/ClassDeclaration.cs

@@ -7,7 +7,7 @@
 #pragma warning disable CS1591
 namespace ArduinoCsCompiler
 {
-    public class ClassDeclaration
+    public class ClassDeclaration : IEquatable<ClassDeclaration>
     {
         private readonly List<ClassMember> _members;
         private readonly List<Type> _interfaces;
@@ -70,6 +70,57 @@ public bool SuppressInit
             }
         }
 
+        public bool Equals(ClassDeclaration? other)
+        {
+            if (ReferenceEquals(null, other))
+            {
+                return false;
+            }
+
+            if (ReferenceEquals(this, other))
+            {
+                return true;
+            }
+
+            // Here, Type and MiniType must be distinct.
+            return NewToken == other.NewToken && Name == other.Name;
+        }
+
+        public override bool Equals(object? obj)
+        {
+            if (ReferenceEquals(null, obj))
+            {
+                return false;
+            }
+
+            if (ReferenceEquals(this, obj))
+            {
+                return true;
+            }
+
+            if (obj.GetType() != GetType())
+            {
+                return false;
+            }
+
+            return Equals((ClassDeclaration)obj);
+        }
+
+        public override int GetHashCode()
+        {
+            return NewToken;
+        }
+
+        public static bool operator ==(ClassDeclaration? left, ClassDeclaration? right)
+        {
+            return Equals(left, right);
+        }
+
+        public static bool operator !=(ClassDeclaration? left, ClassDeclaration? right)
+        {
+            return !Equals(left, right);
+        }
+
         public void AddClassMember(ClassMember member)
         {
             if (ReadOnly)

tools/ArduinoCsCompiler/CompilerCommandHandler.cs

@@ -196,7 +196,7 @@ private bool ParseReply(byte[] data, ExecutorCommand expectedCommand, ref Comman
                     }
                     else if (data[2] == (byte)ExecutorCommand.ConditionalBreakpointHit || data[2] == (byte)ExecutorCommand.Variables)
                     {
-                        _compiler.OnCompilerCallback(data[3] | (data[4] << 7), MethodState.Debugging, data);
+                        _compiler.OnCompilerCallback(data[4] | (data[5] << 7), MethodState.Debugging, data);
                     }
                 }
                 else
@@ -344,7 +344,7 @@ public void AddMethodDeclarations(IList<FirmataCommandSequence> sequences, int d
         {
             FirmataIlCommandSequence sequence = new FirmataIlCommandSequence(ExecutorCommand.DeclareMethod);
             sequence.SendInt32(declarationToken);
-            sequence.WriteByte((byte)methodFlags);
+            sequence.SendUInt14((ushort)methodFlags);
             sequence.WriteByte(maxStack);
             sequence.WriteByte(argCount);
             sequence.SendInt32((int)nativeMethod);

tools/ArduinoCsCompiler/CompilerSettings.cs

@@ -19,6 +19,7 @@ public CompilerSettings()
             AdditionalSuppressions = new List<string>();
             LaunchProgramFromFlash = false;
             MaxMemoryUsage = 256 * 1024;
+            UsePreviewFeatures = false;
         }
 
         /// <summary>
@@ -93,6 +94,8 @@ public bool ForceFlashWrite
             set;
         }
 
+        public bool UsePreviewFeatures { get; set; }
+
         object ICloneable.Clone()
         {
             return MemberwiseClone();