From ddcff9bbcd0212549040f72282b27a47c9dc7671 Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Thu, 18 Jan 2018 11:52:42 -0500 Subject: [PATCH 01/11] shader wip --- gallium-live/src/Editor.js | 43 +++++++---- gallium-live/src/playback.js | 7 +- gallium-live/src/shader.js | 133 +++++++++++++++++++++++++++++++++++ 3 files changed, 167 insertions(+), 16 deletions(-) create mode 100644 gallium-live/src/shader.js diff --git a/gallium-live/src/Editor.js b/gallium-live/src/Editor.js index 5e532ca..b3c9687 100644 --- a/gallium-live/src/Editor.js +++ b/gallium-live/src/Editor.js @@ -15,6 +15,7 @@ import * as Playback from "./playback"; import * as AppActions from "./app_actions"; import { BPMSelector } from "./BPMSelector"; import { ToggleInvert } from "./ToggleInvert"; +import * as Shader from "./shader"; type OwnProps = {}; @@ -120,6 +121,13 @@ export class _Editor extends React.Component< this.textarea.focus(); }; + registerCanvas = (ref: HTMLCanvasElement) => { + if (!ref) { + return; + } + Shader.registerWebGL(ref); + }; + render() { const barStyle = this.state.error ? "dotted" : "solid"; return ( @@ -144,6 +152,7 @@ export class _Editor extends React.Component< innerRef={this.onTextareaRefLoad} barStyle={barStyle} /> + @@ -168,16 +177,13 @@ export const Container: React$ComponentType<{ }> = styled.div` width: 100%; height: 100%; - display: flex; - flex-direction: column; opacity: ${props => (props.isInitialized ? 1 : 0)}; transition: opacity 500ms ease-in-out; background-color: white; `; const Pane = styled.div` - flex: 0 1 auto; - min-height: 50px; + height: 10%; display: flex; justify-content: flex-end; align-items: center; @@ -194,31 +200,38 @@ const PaneChild = styled.div` `; const Content = styled.div` - padding: 10vh 10vw; - flex-grow: 1; - flex-shrink: 0; display: flex; - background-color: white; + background-color: transparent; + position: relative; + height: 80%; +`; + +const Canvas = styled.canvas` + position: absolute; + z-index: 0; + height: 100%; + width: 100%; + background-color: black; `; export const Textarea: React.ComponentType<{ barStyle: string }> = styled.textarea` - ${Styles.transition}; border: 0; font-size: 16px; - background-color: transparent; margin: 0; flex-grow: 1; + background-color: transparent; font-family: monospace; outline: none; padding: 0; padding-left: 0.2em; - border-left: 1px ${props => props.barStyle} #000; - opacity: 0.75; - &:focus { - opacity: 1; - } + border-left: 1px ${props => props.barStyle} #fff; + opacity: 1; + margin: 0 10vw; + color: white; + mix-blend-mode: difference; + z-index: 1; `; const Description = styled.div` diff --git a/gallium-live/src/playback.js b/gallium-live/src/playback.js index f4adc6e..3b204bc 100644 --- a/gallium-live/src/playback.js +++ b/gallium-live/src/playback.js @@ -23,7 +23,6 @@ export class Player { now + (event.start - this.state.beat) * getBeatLength(this.state.bpm); const timestampOff = now + (event.end - this.state.beat) * getBeatLength(this.state.bpm) - 1; - this.state.output.send( MIDIUtils.noteOn({ channel: event.value.channel, @@ -41,6 +40,12 @@ export class Player { }), timestampOff ); + if (event.value[1] === 127) { + window.kick = 1.0; + requestAnimationFrame(() => { + window.kick = 0.0; + }); + } } queryAndSend(): void { diff --git a/gallium-live/src/shader.js b/gallium-live/src/shader.js new file mode 100644 index 0000000..8a2c520 --- /dev/null +++ b/gallium-live/src/shader.js @@ -0,0 +1,133 @@ +// @flow + +const vert = ` +attribute vec4 aVertexPosition; +varying vec2 uv; + +void main() { + gl_Position = aVertexPosition; + uv = aVertexPosition.xy; +} +`; + +const frag = ` +precision mediump float; + +varying vec2 uv; +uniform float time; +uniform float kick; + +void main() { + float t = sin(time* 0.5); + float i = 1.0*(4.0 * uv.x) - sin(uv.x*10.0 * uv.x *uv.x* 100.0 + t); + gl_FragColor = vec4(i, i, i, 1.0) * (1.0 - kick); +} +`; + +export function registerWebGL(canvas: HTMLCanvasElement) { + const gl = canvas.getContext("webgl"); + + if (!gl) { + throw new Error("Your browser doesn't seem to support webgl"); + } + + const program = createProgram(gl, { + fragmentShader: frag, + vertexShader: vert + }); + + const programInfo = { + program, + attribLocations: { + vertexPosition: gl.getAttribLocation(program, "aVertexPosition") + }, + uniformLocations: { + time: gl.getUniformLocation(program, "time"), + kick: gl.getUniformLocation(program, "kick") + } + }; + + const buffers = initBuffers(gl); + + const animate = () => { + drawScene(gl, programInfo, buffers); + requestAnimationFrame(animate); + }; + + requestAnimationFrame(animate); +} + +const createShader = ( + gl: WebGLRenderingContext, + source: string, + shaderType: any +): WebGLShader => { + const shader = gl.createShader(shaderType); + gl.shaderSource(shader, source); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + console.error(gl.getShaderInfoLog(shader)); + gl.deleteShader(shader); + } + return shader; +}; + +const createProgram = ( + gl: WebGLRenderingContext, + { + fragmentShader, + vertexShader + }: { fragmentShader: string, vertexShader: string } +): WebGLProgram => { + const program = gl.createProgram(); + gl.attachShader(program, createShader(gl, vertexShader, gl.VERTEX_SHADER)); + gl.attachShader( + program, + createShader(gl, fragmentShader, gl.FRAGMENT_SHADER) + ); + gl.linkProgram(program); + if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { + console.error(gl.getProgramInfoLog(program)); + gl.deleteProgram(program); + } + return program; +}; + +function initBuffers(gl: WebGLRenderingContext) { + const positionBuffer: WebGLBuffer = gl.createBuffer(); + gl.bindBuffer(gl.ARRAY_BUFFER, positionBuffer); + const positions = new Float32Array([1, 1, -1, 1, 1, -1, -1, -1]); + gl.bufferData(gl.ARRAY_BUFFER, positions, gl.STATIC_DRAW); + return { + position: positionBuffer + }; +} + +function drawScene(gl: WebGLRenderingContext, programInfo, buffers) { + gl.clearColor(0.0, 0.0, 0.0, 1.0); + gl.clear(gl.COLOR_BUFFER_BIT); + + { + gl.bindBuffer(gl.ARRAY_BUFFER, buffers.position); + gl.vertexAttribPointer( + programInfo.attribLocations.vertexPosition, + 2, // numComponents: pull out 2 values per iteration + gl.FLOAT, // type: the data in the buffer is 32bit floats + false, // normalize + 0, // stride: how many bytes to get from one set of values to the next + 0 // offset: how many bytes inside the buffer to start from + ); + gl.enableVertexAttribArray(programInfo.attribLocations.vertexPosition); + } + + gl.useProgram(programInfo.program); + + gl.uniform1f(programInfo.uniformLocations.time, performance.now() / 1000); + gl.uniform1f(programInfo.uniformLocations.kick, window.kick || 0.0); + + { + const offset = 0; + const vertexCount = 4; + gl.drawArrays(gl.TRIANGLE_STRIP, offset, vertexCount); + } +} From 538098b804d97e39dacea64585444e8ea388c1c3 Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Thu, 18 Jan 2018 18:45:15 +0000 Subject: [PATCH 02/11] wip --- gallium-live/src/playback.js | 11 +++++++---- gallium-live/src/shader.js | 5 ++++- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/gallium-live/src/playback.js b/gallium-live/src/playback.js index 3b204bc..7e7e284 100644 --- a/gallium-live/src/playback.js +++ b/gallium-live/src/playback.js @@ -9,6 +9,9 @@ import * as MIDIUtils from "gallium/lib/midi_utils"; function getBeatLength(bpm: number): number { return 1000 * 60 / bpm; } +window.strobe = false; +window.kick = 0.0; + export class Player { +state: AppState; @@ -41,10 +44,10 @@ export class Player { timestampOff ); if (event.value[1] === 127) { - window.kick = 1.0; - requestAnimationFrame(() => { - window.kick = 0.0; - }); + window.kick = Math.random(); + /* setTimeout(() => { + * window.strobe = false; + * }, (timestampOff - timestampOn) /2);*/ } } diff --git a/gallium-live/src/shader.js b/gallium-live/src/shader.js index 8a2c520..27b5031 100644 --- a/gallium-live/src/shader.js +++ b/gallium-live/src/shader.js @@ -123,7 +123,10 @@ function drawScene(gl: WebGLRenderingContext, programInfo, buffers) { gl.useProgram(programInfo.program); gl.uniform1f(programInfo.uniformLocations.time, performance.now() / 1000); - gl.uniform1f(programInfo.uniformLocations.kick, window.kick || 0.0); + gl.uniform1f(programInfo.uniformLocations.kick, window.kick); + /* if (window.strobe) { + * window.kick = 1.0 - window.kick; + * }*/ { const offset = 0; From 1bde8378a67ac8361a74010c8afa7d31d9c9f5ca Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Thu, 18 Jan 2018 14:49:20 -0500 Subject: [PATCH 03/11] use timestamps! --- gallium-live/src/playback.js | 11 +++++------ gallium-live/src/shader.js | 35 +++++++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/gallium-live/src/playback.js b/gallium-live/src/playback.js index 7e7e284..84f268b 100644 --- a/gallium-live/src/playback.js +++ b/gallium-live/src/playback.js @@ -9,8 +9,9 @@ import * as MIDIUtils from "gallium/lib/midi_utils"; function getBeatLength(bpm: number): number { return 1000 * 60 / bpm; } -window.strobe = false; -window.kick = 0.0; + +//invariant: kickQueue is ordered by timestamp +window.kickQueue = []; export class Player { @@ -44,10 +45,8 @@ export class Player { timestampOff ); if (event.value[1] === 127) { - window.kick = Math.random(); - /* setTimeout(() => { - * window.strobe = false; - * }, (timestampOff - timestampOn) /2);*/ + window.kickQueue.push({ value: 1.0, timestamp: timestampOn}); + window.kickQueue.push({ value: 0.0, timestamp: timestampOn + 30}); } } diff --git a/gallium-live/src/shader.js b/gallium-live/src/shader.js index 27b5031..0ba82a0 100644 --- a/gallium-live/src/shader.js +++ b/gallium-live/src/shader.js @@ -18,9 +18,13 @@ uniform float time; uniform float kick; void main() { - float t = sin(time* 0.5); - float i = 1.0*(4.0 * uv.x) - sin(uv.x*10.0 * uv.x *uv.x* 100.0 + t); - gl_FragColor = vec4(i, i, i, 1.0) * (1.0 - kick); + float t = time; + float i = 1.*(4. * uv.x) - sin(uv.x*10. * uv.x *uv.x* 100. + t); + i = clamp(i,0.,1.); + i = (i*2. - 1.); + i = i * (kick*2. - 1.); + i = i/2. + 0.5; + gl_FragColor = vec4(i, i, i, 1.0); } `; @@ -122,11 +126,26 @@ function drawScene(gl: WebGLRenderingContext, programInfo, buffers) { gl.useProgram(programInfo.program); - gl.uniform1f(programInfo.uniformLocations.time, performance.now() / 1000); - gl.uniform1f(programInfo.uniformLocations.kick, window.kick); - /* if (window.strobe) { - * window.kick = 1.0 - window.kick; - * }*/ + const now = performance.now(); + gl.uniform1f(programInfo.uniformLocations.time, now/1000); + + let kick; + + //compute kick value + for (let i = 0; i < window.kickQueue.length; i += 1) { + const { timestamp, value } = window.kickQueue[i]; + if (timestamp > now) { + window.kickQueue = window.kickQueue.slice(i); + break; + } + kick = value; + } + + gl.uniform1f(programInfo.uniformLocations.kick, kick); + + if (window.strobe) { + window.kick = 1.0 - window.kick; + } { const offset = 0; From 204b71ff63f1e5be010fa95e9a23df6eb21df0fd Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Thu, 18 Jan 2018 18:09:29 -0500 Subject: [PATCH 04/11] resize --- gallium-live/src/playback.js | 2 +- gallium-live/src/shader.js | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/gallium-live/src/playback.js b/gallium-live/src/playback.js index 84f268b..7f38c45 100644 --- a/gallium-live/src/playback.js +++ b/gallium-live/src/playback.js @@ -46,7 +46,7 @@ export class Player { ); if (event.value[1] === 127) { window.kickQueue.push({ value: 1.0, timestamp: timestampOn}); - window.kickQueue.push({ value: 0.0, timestamp: timestampOn + 30}); + window.kickQueue.push({ value: 0.0, timestamp: timestampOn + Math.max((timestampOff - timestampOn) / 4, 30)}); } } diff --git a/gallium-live/src/shader.js b/gallium-live/src/shader.js index 0ba82a0..1569a69 100644 --- a/gallium-live/src/shader.js +++ b/gallium-live/src/shader.js @@ -18,8 +18,9 @@ uniform float time; uniform float kick; void main() { - float t = time; - float i = 1.*(4. * uv.x) - sin(uv.x*10. * uv.x *uv.x* 100. + t); + float t = time * .25; + float s = uv.x * uv.x; + float i = 1.*(4. * uv.x) - sin(s*25. + t) * sin(s*299. + t); i = clamp(i,0.,1.); i = (i*2. - 1.); i = i * (kick*2. - 1.); @@ -30,10 +31,11 @@ void main() { export function registerWebGL(canvas: HTMLCanvasElement) { const gl = canvas.getContext("webgl"); - if (!gl) { throw new Error("Your browser doesn't seem to support webgl"); } + resize(gl); + gl.viewport(0,0, gl.drawingBufferWidth, gl.drawingBufferHeight); const program = createProgram(gl, { fragmentShader: frag, @@ -153,3 +155,14 @@ function drawScene(gl: WebGLRenderingContext, programInfo, buffers) { gl.drawArrays(gl.TRIANGLE_STRIP, offset, vertexCount); } } +function resize(gl: WebGLRenderingContext) { + const realToCSSPixels = window.devicePixelRatio; + + const displayWidth = Math.floor(gl.canvas.clientWidth * realToCSSPixels); + const displayHeight = Math.floor(gl.canvas.clientHeight * realToCSSPixels); + + if (gl.canvas.width !== displayWidth || gl.canvas.height !== displayHeight) { + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + } +} From 599c6151e179c9a261150146b242fdef17e00f35 Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Sun, 11 Feb 2018 18:12:12 -0500 Subject: [PATCH 05/11] wipgp --- gallium-live/src/playback.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gallium-live/src/playback.js b/gallium-live/src/playback.js index 7f38c45..e68a15b 100644 --- a/gallium-live/src/playback.js +++ b/gallium-live/src/playback.js @@ -44,7 +44,7 @@ export class Player { }), timestampOff ); - if (event.value[1] === 127) { + if (event.value.pitch === 127) { window.kickQueue.push({ value: 1.0, timestamp: timestampOn}); window.kickQueue.push({ value: 0.0, timestamp: timestampOn + Math.max((timestampOff - timestampOn) / 4, 30)}); } From c331772c670c1f3086d4312c8ec0d415cf6799d5 Mon Sep 17 00:00:00 2001 From: Sean Lee Date: Tue, 13 Feb 2018 00:57:21 -0500 Subject: [PATCH 06/11] wip --- gallium-live/src/Editor.js | 60 +++++++++++++---- gallium-live/src/playback.js | 8 ++- gallium-live/src/shader.js | 123 ++++++++++++++++++++++++++--------- 3 files changed, 145 insertions(+), 46 deletions(-) diff --git a/gallium-live/src/Editor.js b/gallium-live/src/Editor.js index b3c9687..7daa3e7 100644 --- a/gallium-live/src/Editor.js +++ b/gallium-live/src/Editor.js @@ -45,7 +45,9 @@ export class _Editor extends React.Component< }; } - textarea: ?HTMLTextAreaElement; + textarea: HTMLTextAreaElement; + + textCanvas: HTMLCanvasElement; componentDidMount() { this.props.dispatch(AppActions.initialize()); @@ -58,10 +60,14 @@ export class _Editor extends React.Component< } onChange = (e: *) => { + const text = e.target.value; this.setState({ - text: e.target.value + text }); - this.updateABT(e.target.value); + this.updateABT(text); + if (this.textCanvas) { + this.drawText(text); + } }; updateABT(text: string) { @@ -113,21 +119,41 @@ export class _Editor extends React.Component< } }; - onTextareaRefLoad = (ref: HTMLTextAreaElement) => { - this.textarea = ref; - if (!this.textarea) { + registerContent = (ref: HTMLElement) => { + if (!ref) { return; } - this.textarea.focus(); + const canvas: HTMLCanvasElement = (ref.children[1]: any); + const textCanvas: HTMLCanvasElement = (ref.children[2]: any); + textCanvas.width = 256; + textCanvas.height = 256; + + this.textCanvas = textCanvas; + this.drawText(this.state.text); + Shader.registerWebGL({ canvas, textCanvas }); }; - registerCanvas = (ref: HTMLCanvasElement) => { + registerTextarea = (ref: HTMLTextAreaElement) => { if (!ref) { return; } - Shader.registerWebGL(ref); + ref.focus(); + this.textarea = ref; }; + drawText(text: string) { + const ctx = this.textCanvas.getContext("2d"); + ctx.clearRect(0, 0, this.textCanvas.width, this.textCanvas.height); + ctx.font = "12px Serif"; + const lineHeight = ctx.measureText("M").width * 1.2; + const lines = text.split("\n"); + let y = 0; + for (const line of lines) { + ctx.fillText(line, 0, y); + y += lineHeight; + } + } + render() { const barStyle = this.state.error ? "dotted" : "solid"; return ( @@ -143,16 +169,17 @@ export class _Editor extends React.Component< source - +