feat(record): add Linux support

* oa_pynput -> pynput

* add window/_linux.py

* multiprocessing_utils; xcffib

* global monitor_width/monitor_height

* add capture._linux

* cleanup

* get_double_click_interval_seconds/pixels on linux

* get_xinput_property

Author: abrichr

Diff for: openadapt/build.py

@@ -16,7 +16,7 @@
 import urllib.request
 
 import gradio_client
-import oa_pynput
+import pynput
 import pycocotools
 import pydicom
 import pyqttoast
@@ -34,7 +34,7 @@
 def build_pyinstaller() -> None:
     """Build the application using PyInstaller."""
     additional_packages_to_install = [
-        oa_pynput,
+        pynput,
         pydicom,
         spacy_alignments,
         gradio_client,
@@ -275,6 +275,8 @@ def main() -> None:
         create_macos_dmg()
     elif sys.platform == "win32":
         create_windows_installer()
+    else:
+        print(f"WARNING: openadapt.build is not yet supported on {sys.platform=}")
 
 
 if __name__ == "__main__":

Diff for: openadapt/build_utils.py

@@ -17,13 +17,15 @@ def get_root_dir_path() -> pathlib.Path:
         if not path.exists():
             path.mkdir(parents=True, exist_ok=True)
         return path
-    else:
+    elif sys.platform == "win32":
         # if windows, get the path to the %APPDATA% directory and set the path
         # for all user preferences
         path = pathlib.Path.home() / "AppData" / "Roaming" / "openadapt"
         if not path.exists():
             path.mkdir(parents=True, exist_ok=True)
         return path
+    else:
+        print(f"WARNING: openadapt.build_utils is not yet supported on {sys.platform=}")
 
 
 def is_running_from_executable() -> bool:

Diff for: openadapt/capture/__init__.py

@@ -9,6 +9,8 @@
     from . import _macos as impl
 elif sys.platform == "win32":
     from . import _windows as impl
+elif sys.platform.startswith("linux"):
+    from . import _linux as impl
 else:
     raise Exception(f"Unsupported platform: {sys.platform}")
 

Diff for: openadapt/capture/_linux.py

@@ -0,0 +1,140 @@
+import subprocess
+import os
+from datetime import datetime
+from sys import platform
+import pyaudio
+import wave
+
+from openadapt.config import CAPTURE_DIR_PATH
+
+
+class Capture:
+    """Capture the screen, audio, and camera on Linux."""
+
+    def __init__(self) -> None:
+        """Initialize the capture object."""
+        if not platform.startswith("linux"):
+            assert platform == "linux", platform
+
+        self.is_recording = False
+        self.audio_out = None
+        self.video_out = None
+        self.audio_stream = None
+        self.audio_frames = []
+
+        # Initialize PyAudio
+        self.audio = pyaudio.PyAudio()
+
+    def get_screen_resolution(self) -> tuple:
+        """Get the screen resolution dynamically using xrandr."""
+        try:
+            # Get screen resolution using xrandr
+            output = subprocess.check_output(
+                "xrandr | grep '*' | awk '{print $1}'", shell=True
+            )
+            resolution = output.decode("utf-8").strip()
+            width, height = resolution.split("x")
+            return int(width), int(height)
+        except subprocess.CalledProcessError as e:
+            raise RuntimeError(f"Failed to get screen resolution: {e}")
+
+    def start(self, audio: bool = True, camera: bool = False) -> None:
+        """Start capturing the screen, audio, and camera.
+
+        Args:
+            audio (bool, optional): Whether to capture audio (default: True).
+            camera (bool, optional): Whether to capture the camera (default: False).
+        """
+        if self.is_recording:
+            raise RuntimeError("Recording is already in progress")
+
+        self.is_recording = True
+        capture_dir = CAPTURE_DIR_PATH
+        if not os.path.exists(capture_dir):
+            os.mkdir(capture_dir)
+
+        # Get the screen resolution dynamically
+        screen_width, screen_height = self.get_screen_resolution()
+
+        # Start video capture using ffmpeg
+        video_filename = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".mp4"
+        self.video_out = os.path.join(capture_dir, video_filename)
+        self._start_video_capture(screen_width, screen_height)
+
+        # Start audio capture
+        if audio:
+            audio_filename = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + ".wav"
+            self.audio_out = os.path.join(capture_dir, audio_filename)
+            self._start_audio_capture()
+
+    def _start_video_capture(self, width: int, height: int) -> None:
+        """Start capturing the screen using ffmpeg with the dynamic resolution."""
+        cmd = [
+            "ffmpeg",
+            "-f",
+            "x11grab",  # Capture X11 display
+            "-video_size",
+            f"{width}x{height}",  # Use dynamic screen resolution
+            "-framerate",
+            "30",  # Set frame rate
+            "-i",
+            ":0.0",  # Capture from display 0
+            "-c:v",
+            "libx264",  # Video codec
+            "-preset",
+            "ultrafast",  # Speed/quality tradeoff
+            "-y",
+            self.video_out,  # Output file
+        ]
+        self.video_proc = subprocess.Popen(cmd)
+
+    def _start_audio_capture(self) -> None:
+        """Start capturing audio using PyAudio."""
+        self.audio_stream = self.audio.open(
+            format=pyaudio.paInt16,
+            channels=2,
+            rate=44100,
+            input=True,
+            frames_per_buffer=1024,
+            stream_callback=self._audio_callback,
+        )
+        self.audio_frames = []
+        self.audio_stream.start_stream()
+
+    def _audio_callback(
+        self, in_data: bytes, frame_count: int, time_info: dict, status: int
+    ) -> tuple:
+        """Callback function to process audio data."""
+        self.audio_frames.append(in_data)
+        return (None, pyaudio.paContinue)
+
+    def stop(self) -> None:
+        """Stop capturing the screen, audio, and camera."""
+        if self.is_recording:
+            # Stop the video capture
+            self.video_proc.terminate()
+
+            # Stop audio capture
+            if self.audio_stream:
+                self.audio_stream.stop_stream()
+                self.audio_stream.close()
+                self.audio.terminate()
+                self.save_audio()
+
+            self.is_recording = False
+
+    def save_audio(self) -> None:
+        """Save the captured audio to a WAV file."""
+        if self.audio_out:
+            with wave.open(self.audio_out, "wb") as wf:
+                wf.setnchannels(2)
+                wf.setsampwidth(self.audio.get_sample_size(pyaudio.paInt16))
+                wf.setframerate(44100)
+                wf.writeframes(b"".join(self.audio_frames))
+
+
+if __name__ == "__main__":
+    capture = Capture()
+    capture.start(audio=True, camera=False)
+    input("Press enter to stop")
+    capture.stop()

Diff for: openadapt/capture/_macos.py

@@ -23,10 +23,7 @@ class Capture:
 
     def __init__(self) -> None:
         """Initialize the capture object."""
-        if platform != "darwin":
-            raise NotImplementedError(
-                "This is the macOS implementation, please use the Windows version"
-            )
+        assert platform == "darwin", platform
 
         objc.options.structs_indexable = True
 

Diff for: openadapt/capture/_windows.py

@@ -21,10 +21,8 @@ def __init__(self, pid: int = 0) -> None:
             pid (int, optional): The process ID of the window to capture.
             Defaults to 0 (the entire screen)
         """
-        if platform != "win32":
-            raise NotImplementedError(
-                "This is the Windows implementation, please use the macOS version"
-            )
+        assert platform == "win32", platform
+
         self.is_recording = False
         self.video_out = None
         self.audio_out = None

Diff for: openadapt/models.py

@@ -9,7 +9,7 @@
 import sys
 
 from bs4 import BeautifulSoup
-from oa_pynput import keyboard
+from pynput import keyboard
 from PIL import Image, ImageChops
 import numpy as np
 import sqlalchemy as sa
@@ -649,9 +649,9 @@ def to_prompt_dict(
                     "title",
                     "help",
                 ]
-                if sys.platform == "win32":
+                if sys.platform != "darwin":
                     logger.warning(
-                        "key_suffixes have not yet been defined on Windows."
+                        "key_suffixes have not yet been defined on {sys.platform=}."
                         "You can help by uncommenting the lines below and pasting "
                         "the contents of the window_dict into a new GitHub Issue."
                     )

Diff for: openadapt/playback.py

@@ -1,6 +1,6 @@
 """Utilities for playing back ActionEvents."""
 
-from oa_pynput import keyboard, mouse
+from pynput import keyboard, mouse
 
 from openadapt.common import KEY_EVENTS, MOUSE_EVENTS
 from openadapt.custom_logger import logger

Diff for: openadapt/plotting.py

@@ -435,8 +435,10 @@ def plot_performance(
         if view_file:
             if sys.platform == "darwin":
                 os.system(f"open {fpath}")
-            else:
+            elif sys.platform == "win32":
                 os.system(f"start {fpath}")
+            else:
+                os.system(f"xdg-open {fpath}")
     else:
         plt.savefig(BytesIO(), format="png")  # save fig to void
         if view_file: