From 0a0619149799e5251fb8cee7f4984523c6a795db Mon Sep 17 00:00:00 2001 From: guest271314 Date: Sat, 24 Jul 2021 19:02:16 -0700 Subject: [PATCH 1/4] Update AudioStream.js - Verify last bytes of input stream are written to audio output stream before disconnecting audio nodes - Start MediaRecorder before first AudioData is written to audio output stream - Include error handling in disconnect() and abortHandler for abort() called immediately following start() for case of close() called on a closed stream - Check if data set in recorder dataavailable event is defined before calling data.arrayBuffer() --- chromium_extension/AudioStream.js | 68 +++++++++++++++++++------------ 1 file changed, 43 insertions(+), 25 deletions(-) diff --git a/chromium_extension/AudioStream.js b/chromium_extension/AudioStream.js index 6935a900..1001b660 100644 --- a/chromium_extension/AudioStream.js +++ b/chromium_extension/AudioStream.js @@ -1,3 +1,4 @@ + class AudioStream { constructor({ stdin, recorder = false }) { if (!/^espeak-ng/.test(stdin)) { @@ -73,7 +74,11 @@ class AudioStream { this.audioReadableSignal = audioReadableSignal; this.audioReadableSignal.onabort = (e) => console.log(e.type); this.abortHandler = async (e) => { - await this.disconnect(true); + try { + await this.disconnect(true); + } catch (err) { + console.warn(err.message); + } console.log( `readOffset:${this.readOffset}, duration:${this.duration}, ac.currentTime:${this.ac.currentTime}`, `generator.readyState:${this.generator.readyState}, audioWriter.desiredSize:${this.audioWriter.desiredSize}`, @@ -97,9 +102,13 @@ class AudioStream { this.osc.disconnect(); this.outputSource.disconnect(); this.track.stop(); - await this.audioWriter.close(); - await this.audioWriter.closed; - await this.inputReader.cancel(); + try { + await this.audioWriter.close(); + await this.audioWriter.closed; + await this.inputReader.cancel(); + } catch (err) { + throw err; + } this.generator.stop(); if (this.recorder && this.recorder.state === 'recording') { this.recorder.stop(); @@ -158,13 +167,13 @@ class AudioStream { this.init = true; i = 44; } - for (; - i < value.buffer.byteLength; - i++, this.readOffset++ - ) { + for (; i < value.buffer.byteLength; i++, this.readOffset++) { if (channelData.length === this.channelDataLength) { - this.inputController.enqueue(new Uint8Array(channelData)); - channelData.length = 0; + this.inputController.enqueue( + new Uint8Array( + channelData.splice(0, this.channelDataLength) + ) + ); } channelData.push(value[i]); } @@ -175,7 +184,7 @@ class AudioStream { close: async () => { console.log('Done writing input stream.'); if (channelData.length) { - this.inputController.enqueue(channelData); + this.inputController.enqueue(new Uint8Array(channelData)); } this.inputController.close(); this.source.postMessage('Done writing input stream.', '*'); @@ -186,17 +195,6 @@ class AudioStream { this.audioReadable.pipeTo( new WritableStream({ write: async ({ timestamp }) => { - if (this.inputController.desiredSize === 0) { - await this.disconnect(); - console.log( - `readOffset:${this.readOffset}, duration:${this.duration}, ac.currentTime:${this.ac.currentTime}`, - `generator.readyState:${this.generator.readyState}, audioWriter.desiredSize:${this.audioWriter.desiredSize}` - ); - return await Promise.all([ - new Promise((resolve) => (this.stream.oninactive = resolve)), - new Promise((resolve) => (this.ac.onstatechange = resolve)), - ]); - } const { value, done } = await this.inputReader.read(); if (done) { console.log({ done }); @@ -219,17 +217,35 @@ class AudioStream { buffer.getChannelData(0).set(floats); this.duration += buffer.duration; const frame = new AudioData({ - format:'FLTP', + format: 'FLTP', sampleRate: 22050, numberOfChannels: 1, numberOfFrames: 220, timestamp, data: buffer.getChannelData(0), }); - await this.audioWriter.write(frame); if (this.recorder && this.recorder.state === 'inactive') { this.recorder.start(); } + await this.audioWriter.write(frame); + if ( + value.length < this.channelDataLength && + (await this.inputReader.read()).done + ) { + try { + await this.disconnect(); + } catch (err) { + console.warn(err.message); + } + console.log( + `readOffset:${this.readOffset}, duration:${this.duration}, ac.currentTime:${this.ac.currentTime}`, + `generator.readyState:${this.generator.readyState}, audioWriter.desiredSize:${this.audioWriter.desiredSize}` + ); + return await Promise.all([ + new Promise((resolve) => (this.stream.oninactive = resolve)), + new Promise((resolve) => (this.ac.onstatechange = resolve)), + ]); + } }, abort(e) { console.error(e.message); @@ -242,7 +258,9 @@ class AudioStream { ), ]); this.resolve( - this.recorder ? await this.data.arrayBuffer() : 'Done streaming.' + this.recorder + ? this.data && (await this.data.arrayBuffer()) + : 'Done streaming.' ); return this.promise; } catch (err) { From 6874f6e9133e8bdc976b52a5f8498ebeed81189c Mon Sep 17 00:00:00 2001 From: guest271314 Date: Sat, 24 Jul 2021 20:03:03 -0700 Subject: [PATCH 2/4] Use read().done to verify all bytes are written Substitute using existing defined done of ReadableStreamDefaultReader.read() for value.length < this.channelDataLength && (await this.inputReader.read()).done to verify all input stream bytes are written --- chromium_extension/AudioStream.js | 33 +++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/chromium_extension/AudioStream.js b/chromium_extension/AudioStream.js index 1001b660..416c5708 100644 --- a/chromium_extension/AudioStream.js +++ b/chromium_extension/AudioStream.js @@ -1,4 +1,3 @@ - class AudioStream { constructor({ stdin, recorder = false }) { if (!/^espeak-ng/.test(stdin)) { @@ -197,7 +196,21 @@ class AudioStream { write: async ({ timestamp }) => { const { value, done } = await this.inputReader.read(); if (done) { + await this.inputReader.closed; console.log({ done }); + try { + await this.disconnect(); + } catch (err) { + console.warn(err.message); + } + console.log( + `readOffset:${this.readOffset}, duration:${this.duration}, ac.currentTime:${this.ac.currentTime}`, + `generator.readyState:${this.generator.readyState}, audioWriter.desiredSize:${this.audioWriter.desiredSize}` + ); + return await Promise.all([ + new Promise((resolve) => (this.stream.oninactive = resolve)), + new Promise((resolve) => (this.ac.onstatechange = resolve)), + ]); } const uint16 = new Uint16Array(value.buffer); // https://stackoverflow.com/a/35248852 @@ -228,24 +241,6 @@ class AudioStream { this.recorder.start(); } await this.audioWriter.write(frame); - if ( - value.length < this.channelDataLength && - (await this.inputReader.read()).done - ) { - try { - await this.disconnect(); - } catch (err) { - console.warn(err.message); - } - console.log( - `readOffset:${this.readOffset}, duration:${this.duration}, ac.currentTime:${this.ac.currentTime}`, - `generator.readyState:${this.generator.readyState}, audioWriter.desiredSize:${this.audioWriter.desiredSize}` - ); - return await Promise.all([ - new Promise((resolve) => (this.stream.oninactive = resolve)), - new Promise((resolve) => (this.ac.onstatechange = resolve)), - ]); - } }, abort(e) { console.error(e.message); From b148a7ad1f95ef59af5434687a3e7e348de991c1 Mon Sep 17 00:00:00 2001 From: guest271314 Date: Sat, 24 Jul 2021 21:02:03 -0700 Subject: [PATCH 3/4] Update AudioStream.js --- chromium_extension/AudioStream.js | 1 - 1 file changed, 1 deletion(-) diff --git a/chromium_extension/AudioStream.js b/chromium_extension/AudioStream.js index 416c5708..ac2568bf 100644 --- a/chromium_extension/AudioStream.js +++ b/chromium_extension/AudioStream.js @@ -197,7 +197,6 @@ class AudioStream { const { value, done } = await this.inputReader.read(); if (done) { await this.inputReader.closed; - console.log({ done }); try { await this.disconnect(); } catch (err) { From 3ec8db404bc299086f6f46533c7bc3afd8e16b16 Mon Sep 17 00:00:00 2001 From: guest271314 Date: Sat, 24 Jul 2021 21:44:46 -0700 Subject: [PATCH 4/4] Remove AudioBuffer use AudioData --- chromium_extension/AudioStream.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/chromium_extension/AudioStream.js b/chromium_extension/AudioStream.js index ac2568bf..a6c831d5 100644 --- a/chromium_extension/AudioStream.js +++ b/chromium_extension/AudioStream.js @@ -221,21 +221,15 @@ class AudioStream { int >= 0x8000 ? -(0x10000 - int) / 0x8000 : int / 0x7fff; floats[i] = float; } - const buffer = new AudioBuffer({ - numberOfChannels: this.numberOfChannels, - length: floats.length, - sampleRate: this.sampleRate, - }); - buffer.getChannelData(0).set(floats); - this.duration += buffer.duration; const frame = new AudioData({ format: 'FLTP', sampleRate: 22050, numberOfChannels: 1, numberOfFrames: 220, timestamp, - data: buffer.getChannelData(0), + data: floats, }); + this.duration += (frame.duration / 10**6); if (this.recorder && this.recorder.state === 'inactive') { this.recorder.start(); }