[WIP] add Keyboard input

examples/shadertoy_glsl_keyboard.py

@@ -0,0 +1,107 @@
+from wgpu_shadertoy import Shadertoy, ShadertoyChannelKeyboard
+
+# shadertoy source: https://www.shadertoy.com/view/lsXGzf by iq CC-BY-NC-SA-3.0
+
+# TODO: maybe find a whole keyboard example too.
+
+shader_code = """
+// Created by inigo quilez - iq/2013
+
+
+// An example showing how to use the keyboard input.
+//
+// Row 0: contain the current state of the 256 keys. 
+// Row 1: contains Keypress.
+// Row 2: contains a toggle for every key.
+//
+// Texel positions correspond to ASCII codes. Press arrow keys to test.
+
+
+// See also:
+//
+// Input - Keyboard    : https://www.shadertoy.com/view/lsXGzf
+// Input - Microphone  : https://www.shadertoy.com/view/llSGDh
+// Input - Mouse       : https://www.shadertoy.com/view/Mss3zH
+// Input - Sound       : https://www.shadertoy.com/view/Xds3Rr
+// Input - SoundCloud  : https://www.shadertoy.com/view/MsdGzn
+// Input - Time        : https://www.shadertoy.com/view/lsXGz8
+// Input - TimeDelta   : https://www.shadertoy.com/view/lsKGWV
+// Inout - 3D Texture  : https://www.shadertoy.com/view/4llcR4
+
+
+const int KEY_LEFT  = 37;
+const int KEY_UP    = 38;
+const int KEY_RIGHT = 39;
+const int KEY_DOWN  = 40;
+
+void mainImage( out vec4 fragColor, in vec2 fragCoord )
+{
+    fragColor = vec4(texelFetch(iChannel0, ivec2((fragCoord.xy/iResolution.xy)*vec2(512.0, 3.0)), 0).xyz, 1.0);
+    return; // little shortcut for testing I guess.
+    vec2 uv = (-iResolution.xy + 2.0*fragCoord) / iResolution.y;
+    vec3 col = vec3(0.0);
+
+    // state
+    col = mix( col, vec3(1.0,0.0,0.0), 
+        (1.0-smoothstep(0.3,0.31,length(uv-vec2(-0.75,0.0))))*
+        (0.3+0.7*texelFetch( iChannel0, ivec2(KEY_LEFT,0), 0 ).x) );
+
+    col = mix( col, vec3(1.0,1.0,0.0), 
+        (1.0-smoothstep(0.3,0.31,length(uv-vec2(0.0,0.5))))*
+        (0.3+0.7*texelFetch( iChannel0, ivec2(KEY_UP,0), 0 ).x));
+
+    col = mix( col, vec3(0.0,1.0,0.0),
+        (1.0-smoothstep(0.3,0.31,length(uv-vec2(0.75,0.0))))*
+        (0.3+0.7*texelFetch( iChannel0, ivec2(KEY_RIGHT,0), 0 ).x));
+
+    col = mix( col, vec3(0.0,0.0,1.0),
+        (1.0-smoothstep(0.3,0.31,length(uv-vec2(0.0,-0.5))))*
+        (0.3+0.7*texelFetch( iChannel0, ivec2(KEY_DOWN,0), 0 ).x));
+
+
+    // keypress	
+    col = mix( col, vec3(1.0,0.0,0.0), 
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(-0.75,0.0))-0.35)))*
+        texelFetch( iChannel0, ivec2(KEY_LEFT,1),0 ).x);
+
+    col = mix( col, vec3(1.0,1.0,0.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.0,0.5))-0.35)))*
+        texelFetch( iChannel0, ivec2(KEY_UP,1),0 ).x);
+
+    col = mix( col, vec3(0.0,1.0,0.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.75,0.0))-0.35)))*
+        texelFetch( iChannel0, ivec2(KEY_RIGHT,1),0 ).x);
+
+    col = mix( col, vec3(0.0,0.0,1.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.0,-0.5))-0.35)))*
+        texelFetch( iChannel0, ivec2(KEY_DOWN,1),0 ).x);
+
+
+    // toggle	
+    col = mix( col, vec3(1.0,0.0,0.0), 
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(-0.75,0.0))-0.3)))*
+        texelFetch( iChannel0, ivec2(KEY_LEFT,2),0 ).x);
+
+    col = mix( col, vec3(1.0,1.0,0.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.0,0.5))-0.3)))*
+        texelFetch( iChannel0, ivec2(KEY_UP,2),0 ).x);
+
+    col = mix( col, vec3(0.0,1.0,0.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.75,0.0))-0.3)))*
+        texelFetch( iChannel0, ivec2(KEY_RIGHT,2),0 ).x);
+
+    col = mix( col, vec3(0.0,0.0,1.0),
+        (1.0-smoothstep(0.0,0.01,abs(length(uv-vec2(0.0,-0.5))-0.3)))*
+        texelFetch( iChannel0, ivec2(KEY_DOWN,2),0 ).x);
+
+    fragColor = vec4(col,1.0);
+}
+"""
+
+keyboard_channel = ShadertoyChannelKeyboard(filter="nearest", wrap="clamp", vflip=False)
+
+shader = Shadertoy(shader_code, resolution=(800, 450), inputs=[keyboard_channel])
+
+if __name__ == "__main__":
+    shader.show()
+

wgpu_shadertoy/__init__.py

@@ -1,6 +1,6 @@
-from .inputs import ShadertoyChannel, ShadertoyChannelBuffer, ShadertoyChannelTexture
+from .inputs import ShadertoyChannel, ShadertoyChannelBuffer, ShadertoyChannelTexture, ShadertoyChannelKeyboard
 from .passes import BufferRenderPass, ImageRenderPass
 from .shadertoy import Shadertoy
 
 __version__ = "0.2.0"
 version_info = tuple(map(int, __version__.split(".")))  # noqa

wgpu_shadertoy/api.py

@@ -86,7 +86,10 @@ def _download_media_channels(
             args = {
                 "buffer": "abcd"[int(inp["src"][-5])]
             }  # hack with the preview image to get the buffer index
+        elif inp["ctype"] in ("keyboard", ):  # + others that don't need any conversion
+            args = {}  # maybe that can be set as default?
         else:
+            # just for unsupported types now
             complete = False
             continue  # skip the below rows
         channel = ShadertoyChannel(

wgpu_shadertoy/inputs.py

@@ -109,6 +109,8 @@ def infer_subclass(self, *args_, **kwargs_):
             return ShadertoyChannelTexture(*args, **kwargs)
         elif self.ctype == "buffer":
             return ShadertoyChannelBuffer(*args, **kwargs)
+        elif self.ctype == "keyboard":
+            return ShadertoyChannelKeyboard(*args, **kwargs)
         else:
             raise NotImplementedError(f"Doesn't support {self.ctype=} yet")
 
@@ -193,8 +195,117 @@ def __repr__(self):
 
 # "Misc" input tab
 class ShadertoyChannelKeyboard(ShadertoyChannel):
-    pass
+    # isn't this basically a texture/video??
+    # can we have a GPU buffer and then just write it to texture?
+    # do we only update on keypresses or do we do it every frame (can you do multiple keypressese faster than a frame?)
+    # ref: https://www.shadertoy.com/view/lsXGzf
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.format = wgpu.TextureFormat.r8unorm #or r8sint?
+        self.data = np.zeros((3, 256), dtype=np.uint8)  # 3 rows, 256 keys and only one channel
+        self.dynamic = True  # could be named "needs_update" to be more clear
+        self.vflip = True #always true but we handle that manually, so don't really need the var!
+
+        # when we get this via the ._infer_subclass() method - we might already have a parent and can register the events now!
+        if self._parent is not None:
+            # not the best solution I feel like - but works for now.
+            self.parent.main._canvas.add_event_handler(self.on_key_down, "key_down")
+            self.parent.main._canvas.add_event_handler(self.on_key_up, "key_up")
+
+    # do we have to redo both parts?
+    @property
+    def parent(self):
+        """
+        Parent renderpass of this channel.
+        """
+        if self._parent is None:
+            raise AttributeError("Parent not set.")
+        return self._parent
+
+    @parent.setter
+    def parent(self, parent):
+        self._parent = parent
+        # register the events here?
+        self._parent.main._canvas.add_event_handler(self.on_key_down, "key_down")
+        self._parent.main._canvas.add_event_handler(self.on_key_up, "key_up")
+
+    def on_key_down(self, event):
+        try:
+            key_code = ord(event["key"])
+        except TypeError:
+            key_code = 0
+        # note: we vflip the rows here!
+        self.data[2, key_code] = 255
+        self.data[1, key_code] = 255 # this only stays for a little bit of time?
+        self.data[0, key_code] = 255 if self.data[0, key_code] == 0 else 0 # toggle pressed state
+        print(f"Key down: {event['key']} ({key_code})")
+        self.dynamic = True  # basically tell it to update for next frame
+
+    def on_key_up(self, event):
+        try:
+            key_code = ord(event["key"])
+        except TypeError:
+            key_code = 0
+        self.data[2, key_code] = 0 # up action only triggers the "state" row
+        self.dynamic = True
+
+
+    # TODO: keymap or something to match events back into ascii numbers?
+    # https://jupyter-rfb.readthedocs.io/en/stable/events.html#keys
+
+
+    # copied from ShadertoyChannelTexture, changed sizes (maybe it could be moved to the base class?)
+    def bind_texture(self, device: wgpu.GPUDevice) -> Tuple[list, list]:
+        """
+        prepares the texture and sampler. Returns it's binding layouts and bindgroup layout entries
+        """
+        # this gets called during the draw too... so every single time (via _setup_renderpipeline!)
+
+        # TODO: we don't really need to draw the initial texture? it should be all zeros anyway.
+        binding_layout = self._binding_layout()
+        self._texture = device.create_texture(
+            size=(256, 3, 1), # note it's columns, rows here
+            format=self.format,
+            usage=wgpu.TextureUsage.TEXTURE_BINDING | wgpu.TextureUsage.COPY_DST,
+        )
+
+        texture_view = self._texture.create_view()
+        device.queue.write_texture(
+            destination={
+                "texture": self._texture,
+            },
+            data=np.ascontiguousarray(self.data),
+            data_layout={
+                "bytes_per_row": 256, # int8 texture of 256
+                "rows_per_image": 3,
+            },
+            size=self._texture.size,
+        )
+
+        # works if we put it here ... but that isn't the solution!
+        self.data[1, :] = np.zeros(256, dtype=np.uint8)  # reset the second row to 0s after we uploaded that data (could be an issue if we reuse this channel...)
+        sampler = device.create_sampler(**self.sampler_settings)
 
+        bind_groups_layout_entry = self._bind_groups_layout_entries(
+            texture_view, sampler
+        )
+
+        return binding_layout, bind_groups_layout_entry
+
+    def update(self, device: wgpu.GPUDevice):
+        # to be called just before the draw call for this pass:
+        device.queue.write_texture(
+            destination={
+                "texture": self._texture,
+            },
+            data=np.ascontiguousarray(self.data),
+            data_layout={
+                "bytes_per_row": 256, # int8 texture of 256
+                "rows_per_image": 3,
+            },
+            size=self._texture.size,
+        )
+        self.dynamic = False # we don't need to update every frame... (but the 2nd row reset won't work if we wait for the next event...)
 
 class ShadertoyChannelWebcam(ShadertoyChannel):
     pass
@@ -235,6 +346,15 @@ def renderpass(self):  # -> BufferRenderPass:
         if self._renderpass is None:
             self._renderpass = self.parent.main.buffers[self.buffer_idx]
         return self._renderpass
+
+    def update(self, device: wgpu.GPUDevice):
+        # TODO: buffer passes get updated through a combination of the draw function swapping textures
+        # and the draw function also calling _setup_renderpipeline() to get the newer bindings...
+        # this seems wasteful and should be improved!
+
+        # not the solution as we call it too often from here...
+        # self.renderpass._setup_renderpipeline()
+        pass # to avoid warnings here.
 
     def bind_texture(self, device: wgpu.GPUDevice) -> Tuple[list, list]:
         """

wgpu_shadertoy/passes.py

@@ -272,6 +272,11 @@ def draw(self) -> wgpu.GPUCommandBuffer:
             size=self.main._uniform_data.nbytes,
         )
 
+        for channel in self.channels:
+            if channel is not None and channel.dynamic:
+                # update the texture for the channel
+                channel.update(self._device)
+
         command_encoder: wgpu.GPUCommandEncoder = self._device.create_command_encoder()
         current_texture: wgpu.GPUTexture = self.get_current_texture()
 

wgpu_shadertoy/shadertoy.py

@@ -150,6 +150,7 @@ def __init__(
 
         self.title += " $fps FPS"
 
+
         device_features = []
         if buffers:
             device_features.append(wgpu.FeatureName.float32_filterable)
@@ -345,6 +346,7 @@ def _draw_frame(self):
         self._device.queue.submit(render_encoders)
         self._canvas.request_draw()
 
+
     def show(self):
         self._canvas.request_draw(self._draw_frame)
         if self._offscreen: