JS에서 로컬 PyAudio로 이음새없는 오디오를 보내고 보내주세요

Question

유닉스 소켓을 사용하여 전자 렌지 (Electron NodeJS 앱을 통해)에서 소켓에서 수신하는 Python 프로그램으로 오디오를 스트리밍하고 재생을 위해 PyAudio에 오디오를 보냅니다.

전자 응용 getUserMedia() => WAV 형식 => NodeJS 소켓 => 유닉스 소켓 => 파이썬 소켓 => PyAudio

나는 그것이 작동해야하지만 각 청크가 시작되거나 종료 될 때 일정한 딸깍 소리가있다. 어디에서 디버깅을 시작해야합니까?

NodeJS 응용 프로그램 (발신자) :

var net = require('net'); 
const nodeWav = require("node-wav"); 

var recorder = null; 
var volume = null; 
var audioInput = null; 
var sampleRate = null; 
var audioContext = null; 
var context = null; 
var outputElement = document.getElementById('output'); 
var outputString; 
var bufferSize = 1024; 

var mediaSourceIn; 

// callback for navigator.mediaDevices.getUserMedia() 
function audioReceiver(e) { 
    // creates Socket 
    mediaSourceIn = e; 
    initSocket(); 
} 

var audioSocket; 
function initSocket() { 
    audioSocket = net.connect('/tmp/audio_input', connected) 
    .catch(function(err) { 
    console.log("Could not connect..."); 
    console.log(err); 
    }); 
} 

function connected() { 
    console.log("CONNECTED TO UNIX SOCKET!"); 
    audioSocket = this; 
    createRecordingTask(); 
} 

function createRecordingTask() { 
    // creates the audio context 
    audioContext = window.AudioContext || window.webkitAudioContext; 
    context = new audioContext(); 

    // retrieve the current sample rate to be used for WAV packaging 
    sampleRate = context.sampleRate; 

    // creates a gain node 
    volume = context.createGain(); 

    // creates an audio node from the microphone incoming stream 
    audioInput = context.createMediaStreamSource(mediaSourceIn); 

    // connect the stream to the gain node 
    audioInput.connect(volume); 

    /* From the spec: This value controls how frequently the audioprocess event is 
    dispatched and how many sample-frames need to be processed each call. 
    Lower values for buffer size will result in a lower (better) latency. 
    Higher values will be necessary to avoid audio breakup and glitches */ 
    recorder = context.createScriptProcessor(bufferSize, 2, 2); 

    recorder.onaudioprocess = function(e){ 
     console.log ('recording'); 
     var left = e.inputBuffer.getChannelData (0); 
     var right = e.inputBuffer.getChannelData (1); 
     var bf = createAudioBuffer(
      new Float32Array (left), 
      new Float32Array (right)); 

     upload(bf); 
    } 

    // we connect the recorder 
    volume.connect (recorder); 
    recorder.connect (context.destination); 
} 

function mergeBuffers(channelBuffer){ 
    var result = new Float32Array(bufferSize); 
    result.set(channelBuffer); // make a copy? 
    return result; 
} 

function interleave(leftChannel, rightChannel){ 
    var length = leftChannel.length + rightChannel.length; 
    var result = new Float32Array(length); 

    var inputIndex = 0; 

    for (var index = 0; index < length;){ 
    result[index++] = leftChannel[inputIndex]; 
    result[index++] = rightChannel[inputIndex]; 
    inputIndex++; 
    } 
    return result; 
} 

function writeUTFBytes(view, offset, string){ 
    var lng = string.length; 
    for (var i = 0; i < lng; i++){ 
    view.setUint8(offset + i, string.charCodeAt(i)); 
    } 
} 

function createAudioBuffer(leftchannel, rightchannel) { 

    // we flat the left and right channels down 
    var leftBuffer = mergeBuffers (leftchannel, bufferSize); 
    var rightBuffer = mergeBuffers (rightchannel, bufferSize); 

    // we interleave both channels together 
    var interleaved = interleave (leftBuffer, rightBuffer); 

    // we create our wav file 
    var buffer = new ArrayBuffer(44 + interleaved.length * 2); 
    //var buffer = new ArrayBuffer(interleaved.length * 2); 
    var view = new DataView(buffer); 

    // RIFF chunk descriptor 
    writeUTFBytes(view, 0, 'RIFF'); 
    view.setUint32(4, 44 + interleaved.length * 2, true); 
    writeUTFBytes(view, 8, 'WAVE'); 
    // FMT sub-chunk 
    writeUTFBytes(view, 12, 'fmt '); 
    view.setUint32(16, 16, true); 
    view.setUint16(20, 1, true); 
    // stereo (2 channels) 
    view.setUint16(22, 2, true); 
    view.setUint32(24, sampleRate, true); 
    view.setUint32(28, sampleRate * 4, true); 
    view.setUint16(32, 4, true); 
    view.setUint16(34, 16, true); 
    // data sub-chunk 
    writeUTFBytes(view, 36, 'data'); 
    view.setUint32(40, interleaved.length * 2, true); 
    // write the PCM samples 
    var lng = interleaved.length; 
    //var index = 0; 
    var index = 44; 
    var volume = 0.6; 
    for (var i = 0; i < lng; i++){ 
     view.setInt16(index, interleaved[i] * (0x7FFF * volume), true); 
     index += 2; 
    } 
    // our final binary blob 
    return Buffer.from(view.buffer); 
} 


function upload(thatAudio) { 
    if (audioSocket.writable) { 
    audioSocket.write(thatAudio); 
    } else { 
    console.log("DISCONNECTED!"); 
    } 
} 

파이썬 프로그램 (수신기) : stream.write() 버퍼 언더런이 발생하는 경우

import socket 
import os 
import pyaudio 
from threading import Thread 

sockfile = "/tmp/audio_input" 

FORMAT = pyaudio.paInt16 
CHUNK = 1024 
CHANNELS = 2 
RATE = 44100 
frames = [] 

if os.path.exists(sockfile): 
    os.remove(sockfile) 

print("Opening socket...") 
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 
server.bind(sockfile) 
server.listen(5) 
conn, addr = server.accept() 

print("Creating PyAudio stream...") 
p = pyaudio.PyAudio() 

stream = p.open(format=FORMAT, 
       channels = CHANNELS, 
       rate = RATE, 
       output = True, 
       frames_per_buffer = CHUNK, 
       ) 

print "Listening..." 
singleChunkSizeBytes = 44 + (CHUNK * CHANNELS*2) 
print singleChunkSizeBytes, "bytes at a time" 
while True: 
    soundData = conn.recv(singleChunkSizeBytes) 
    if soundData: 
     stream.write(soundData, CHUNK) 

server.close() 
os.remove(sockfile) 

Answer 1

먼저 당신이 확인해야합니다 여기에 코드입니다. 이는 exception_on_underflow 옵션을 사용하여 수행 할 수 있습니다 (docs 참조). write() 기능의 비 던진 버전을 원할 경우 sounddevice 모듈을 사용해보십시오 (참조).

이 언더런 인 경우 소켓이 데이터를 충분히 빠르게 제공하지 못할 수 있습니다. 이 경우 수신자 측에서 일부 버퍼링을 구현해야합니다 (예 : queue.Queue을 사용하십시오. 더 언더런이없는 경우

는 오류가 exception_on_underflow 사실 = 때