libjingleでportaudioを使ったmediaengineをつくった

Posted on 10月 18, 2008
Filed Under portaudio, libjingle |

置いておこう。

portaudioが、マイクからデータが取れたとき、スピーカーに音を流せるとき、にcallbackを呼んでくれる形なので、
・マイクから音をとってくるところ
・スピーカーに音を流すところ
にバッファを持っている。

覚えたてのC++のSTLの明示的cast、queueを使っています。
C++おもろいなー

libjingleが確保してくれたP2Pセッションに音声データを流すときに、

PLAIN TEXT
CODE:
  1. network_interface_->SendPacket(const void*, size_t);

ってのを呼ぶんだけど、
queueをconst void*に変換するとこが気持ち悪い。こんなもんなのかなぁ

音声は生のwavを送っているので、遅延は
DirectAudioを使ってたらそこで50msくらい（らしい？）
＋バッファ分の遅延、
＋portaudioの遅延
＋通信の遅延
くらいなのかな。なんか体感かなり遅延してるのは後で調べる。

portaudiomediaengine.cc

PLAIN TEXT
C++:
  1. #include "talk/third_party/mediastreamer/mediastream.h"
  2. #include <fcntl .h>
  3. #include <iostream>
  4. #include "talk/base/logging.h"
  5. #include "talk/base/thread.h"
  6. #include "talk/session/phone/codec.h"
  7. #include "talk/session/phone/portaudiomediaengine.h"
  8.  
  9. #include <sstream>
  10. #include <iostream>
  11. #include <iomanip>
  12. #include <string>
  13. #include <queue>
  14. #include "portaudio.h"
  15. #include <time .h>
  16.  
  17. using namespace cricket;
  18.  
  19.  
  20. static int playbackCallback( const void *inputBuffer,
  21.                             void *outputBuffer,
  22.                             unsigned long framesPerBuffer,
  23.                             const PaStreamCallbackTimeInfo* timeInfo,
  24.                             PaStreamCallbackFlags statusFlags,
  25.                             void *userData
  26.                            )
  27. {
  28.     PortAudioMediaChannel* channel = static_cast<portaudiomediachannel *>(userData);
  29.     if ( inputBuffer!=0 ) {
  30.         channel->saveFromMicrophone( static_cast<const float*>(inputBuffer), framesPerBuffer );
  31.     }
  32.     channel->pushToSpeaker( static_cast<float *>(outputBuffer), framesPerBuffer );
  33.     channel->SignalReadFromMicEvent( channel );
  34.     return paContinue;
  35. }
  36.  
  37.  
  38. PortAudioMediaChannel::PortAudioMediaChannel(PortAudioMediaEngine* eng) :
  39.     pt_(-1),
  40.     audio_stream_(0),
  41.     engine_(eng)
  42. {
  43.     PaStreamParameters  inputParameters;
  44.     inputParameters.device = Pa_GetDefaultInputDevice(); /* default input device */
  45.     if (inputParameters.device == paNoDevice) {
  46.         fprintf(stderr,"Error: No default input device.\n");
  47.         //goto done;
  48.     }
  49.     inputParameters.channelCount     = 2;                    /* stereo input */
  50.     inputParameters.sampleFormat     = PA_SAMPLE_TYPE;
  51.     inputParameters.suggestedLatency = Pa_GetDeviceInfo( inputParameters.device )->defaultLowInputLatency;
  52.     inputParameters.hostApiSpecificStreamInfo = 0;
  53.  
  54.     PaStreamParameters  outputParameters;
  55.     outputParameters.device = Pa_GetDefaultOutputDevice(); /* default output device */
  56.     if (outputParameters.device == paNoDevice) {
  57.         fprintf(stderr,"Error: No default output device.\n");
  58.         //goto done;
  59.     }
  60.     outputParameters.channelCount     = 2;                     /* stereo output */
  61.     outputParameters.sampleFormat     =  PA_SAMPLE_TYPE;
  62.     outputParameters.suggestedLatency = Pa_GetDeviceInfo( outputParameters.device )->defaultLowOutputLatency;
  63.     outputParameters.hostApiSpecificStreamInfo = 0;
  64.  
  65.     SignalReadFromMicEvent.connect(this, &PortAudioMediaChannel::OnReadFromMic);
  66.  
  67.     // Initialize PortAudio
  68.     int err = Pa_OpenStream(
  69.         &stream_,
  70.         &inputParameters,
  71.         &outputParameters,
  72.         SAMPLE_RATE,        // 44100
  73.         paFramesPerBufferUnspecified,   // FRAMES_PER_BUFFER
  74.         paClipOff,
  75.         playbackCallback,
  76.         this
  77.     );
  78.     if (err != paNoError)
  79.         fprintf(stderr, "Error creating a PortAudio stream: %s\n", Pa_GetErrorText(err));
  80.  
  81. }
  82.  
  83. PortAudioMediaChannel::~PortAudioMediaChannel() {
  84.     if (stream_) {
  85.         Pa_CloseStream(stream_);
  86.     }
  87. }
  88.  
  89. void PortAudioMediaChannel::SetCodecs(const std::vector<codec> &codecs) {
  90.     bool first = true;
  91.     std::vector</codec><codec>::const_iterator i;
  92.  
  93.     for (i = codecs.begin(); i <codecs.end(); i++) {
  94.         if (!engine_->FindCodec(*i))
  95.             continue;
  96.         if (first) {
  97.           LOG(LS_INFO) <<"Using " <<i->name <<"/" <<i->clockrate;
  98.           pt_ = i->id;
  99.           first = false;
  100.         }
  101.     }
  102.  
  103.     if (first) {
  104.         // We're being asked to set an empty list of codecs. This will only happen when
  105.         // working with a buggy client; let's try PCMU.
  106.         LOG(LS_WARNING) <<"Received empty list of codces; using PCMU/8000";
  107.     }
  108. }
  109.  
  110. // マイクから新しいデータがとれたら、リモートに送る
  111. void PortAudioMediaChannel::OnReadFromMic( PortAudioMediaChannel* channel )
  112. {
  113.     //char *buf[max_size];
  114.     int size = PORTAUDIO_PACKET_LENGTH;
  115.     float buff[PORTAUDIO_PACKET_LENGTH/4];
  116.     int len;
  117.     talk_base::CritScope cs(&crit_microphone);
  118.  
  119.     int i=0;
  120.     for ( i=0; i<size/sizeof(float) && !qu_microphone.empty(); i++ ) {
  121.         buff[i] = qu_microphone.front();
  122.         qu_microphone.pop();
  123.     }
  124.     if ( network_interface_ && !mute_ ) {
  125.         const void *buf = static_cast<const void*>(buff);
  126.         network_interface_->SendPacket( buff, i * sizeof(float) );
  127.     }
  128.     char dateStr [9];
  129.     _strtime( dateStr );
  130.     std::cout <<"[" <<dateStr <<"][OnReadFromMic]qu_microphone.size(): " <<qu_microphone.size() <<std::endl;
  131. }
  132.  
  133. // 届いたパケットをスピーカーのバッファに送る
  134. void PortAudioMediaChannel::OnPacketReceived( const void *data, int len ) {
  135.     const float* reader = static_cast<const float*>(data);
  136.     talk_base::CritScope cs(&crit_speaker);
  137.  
  138.     LOG(INFO) <<"[PMC]OnPacketReceived data: " <<std::setprecision(3) <<reader[0] <<" len: " <<len <<"qu_speaker.size(): " <<qu_speaker.size();
  139.  
  140.     for ( int i=0; i<len/4; i++ ) {
  141.         qu_speaker.push( *reader );
  142.         reader++;
  143.     }
  144. }
  145.  
  146. void PortAudioMediaChannel::SetPlayout(bool playout) {
  147.  
  148.     if (!stream_)
  149.         return;
  150.  
  151.     if (play_ && !playout) {
  152.         if ( Pa_IsStreamActive(stream_) ) {
  153.             int err = Pa_StopStream(stream_);
  154.             if (err != paNoError) {
  155.                 fprintf(stderr, "Error stopping PortAudio stream: %s\n", Pa_GetErrorText(err));
  156.                 LOG(LS_INFO) <<"Error stopping PortAudio stream: %s\n", Pa_GetErrorText(err);
  157.                 return;
  158.             }
  159.         }
  160.         play_ = false;
  161.     }
  162.     else if (!play_ && playout) {
  163.         if ( !Pa_IsStreamActive(stream_) ) {
  164.             int err = Pa_StartStream(stream_);
  165.             if (err != paNoError) {
  166.                 fprintf(stderr, "Error starting PortAudio stream: %s\n", Pa_GetErrorText(err));
  167.                 LOG(LS_INFO) <<"Error starting PortAudio stream: %s\n", Pa_GetErrorText(err);
  168.                 return;
  169.             }
  170.         }
  171.         play_ = true;
  172.     }
  173.  
  174. }
  175.  
  176. void PortAudioMediaChannel::SetSend(bool send) {
  177.     mute_ = !send;
  178. }
  179.  
  180. int PortAudioMediaChannel::GetOutputLevel() {
  181.     return 1;
  182. }
  183.  
  184. // bufにlen分だけ書き込む => 音が出る
  185. // リモートから届いたのを書き込む
  186. void PortAudioMediaChannel::pushToSpeaker( float* buf, int len ) {
  187.     talk_base::CritScope cs(&crit_speaker);
  188.     int i=0;
  189.  
  190.     // 遅延が大きかったら古いの消す
  191.     while ( qu_speaker.size()> (MAX_SPEAKER_QUEUE_SIZE) ) {
  192.         qu_speaker.pop();
  193.     }
  194.  
  195.     for( i=0; i <len && qu_speaker.size()> 0; i++ ) {
  196.         *buf++ = qu_speaker.front();
  197.         qu_speaker.pop();
  198.         if( NUM_CHANNELS == 2 ){
  199.             *buf++ = qu_speaker.front();
  200.             qu_speaker.pop();
  201.         }
  202.     }
  203.     bool is_empty = false;
  204.     for( i=i; i <len; i++ ) {
  205.         is_empty = true;
  206.         *buf++ = SAMPLE_SILENCE;
  207.         if( NUM_CHANNELS == 2 ){
  208.             *buf++ = SAMPLE_SILENCE;
  209.         }
  210.     }
  211.     char dateStr [9];
  212.     _strtime( dateStr );
  213.     if ( is_empty ) {
  214.         std::cout <<"[" <<dateStr <<"][pushToSpeaker]empty!!" <<std::endl;
  215.     }
  216.     std::cout <<"[" <<dateStr <<"][pushToSpeaker]qu_speaker.size(): " <<qu_speaker.size() <<std::endl;
  217. }
  218.  
  219. // マイクから来たのをためとく
  220. void PortAudioMediaChannel::saveFromMicrophone( const float* buf, int len ) {
  221.     talk_base::CritScope cs(&crit_microphone);
  222.  
  223.     for( int i=0; i<len; i++ ) {
  224.         qu_microphone.push( *buf++ );
  225.         if( NUM_CHANNELS == 2 ){
  226.             qu_microphone.push( *buf++ );
  227.         }
  228.     }
  229.     char dateStr [9];
  230.     _strtime( dateStr );
  231.     std::cout <<"[" <<dateStr <<"][saveFromMicrophone]qu_microphone.size(): " <<qu_microphone.size() <<std::endl;
  232. }

portaudiomediaengine.h

PLAIN TEXT
C++:
  1. /*
  2. * Jingle call example
  3. * Copyright 2004--2005, Google Inc.
  4. *
  5. * This program is free software; you can redistribute it and/or modify
  6. * it under the terms of the GNU General Public License as published by
  7. * the Free Software Foundation; either version 2 of the License, or
  8. * (at your option) any later version.
  9. *
  10. * This program is distributed in the hope that it will be useful,
  11. * but WITHOUT ANY WARRANTY; without even the implied warranty of
  12. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13. * GNU General Public License for more details.
  14. *
  15. * You should have received a copy of the GNU General Public License
  16. * along with this program; if not, write to the Free Software
  17. * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  18. */
  19.  
  20. // PortAudioMediaEngine is a Linphone implementation of MediaEngine
  21.  
  22. #ifndef TALK_SESSION_PHONE_PORTAUDIOMEDIAENGINE_H_
  23. #define TALK_SESSION_PHONE_PORTAUDIOMEDIAENGINE_H_
  24.  
  25. #include "talk/third_party/mediastreamer/mediastream.h"
  26. #include "talk/base/asyncsocket.h"
  27. #include "talk/base/scoped_ptr.h"
  28. #include "talk/session/phone/mediaengine.h"
  29. #include "talk/base/criticalsection.h"
  30. #include "portaudio.h"
  31. #include <queue>
  32. #include <cmath>
  33.  
  34. // Engine settings
  35. #define ENGINE_BUFFER_SIZE 2048
  36.  
  37. // PortAudio settings
  38. #define PA_SAMPLE_TYPE     (paFloat32)    // 32 bit floating point output
  39. typedef float SAMPLE;
  40. #define SAMPLE_RATE        (44100)
  41. //#define SAMPLE_RATE        (11025)
  42. //#define FRAMES_PER_BUFFER  (1024)
  43. #define FRAMES_PER_BUFFER  (256)
  44. #define SAMPLE_SILENCE     (0.0f)
  45. #define NUM_CHANNELS       (2)
  46. //#define NUM_CHANNELS       (1)
  47.  
  48. //#define PORTAUDIO_PACKET_LENGTH (2048)
  49. #define PORTAUDIO_PACKET_LENGTH (40960) // 一度に送る音声パケットのサイズ,これなら聞ける
  50. #define MAX_ALLOWED_LATENCY     (0.5)   // 許容する音声の最大遅延[s]
  51. #define MAX_SPEAKER_QUEUE_SIZE  (MAX_ALLOWED_LATENCY * SAMPLE_RATE * NUM_CHANNELS)  // キューの最大サイズ。これを超えたら先頭のは破棄する
  52.  
  53. #ifndef M_PI
  54. #define M_PI  (3.14159265)
  55. #endif
  56.  
  57.  
  58.  
  59.  
  60. namespace cricket {
  61.  
  62.  
  63.  
  64.  
  65. #define TABLE_SIZE   (200)
  66. typedef struct
  67. {
  68.     float sine[TABLE_SIZE];
  69.     int left_phase;
  70.     int right_phase;
  71.     char message[20];
  72. }
  73. paTestData;
  74.  
  75.  
  76. typedef struct {
  77.     std::queue<sample> qu;
  78. }
  79. paTestData2;
  80.  
  81.  
  82. class PortAudioMediaEngine;
  83.  
  84. class PortAudioMediaChannel : public MediaChannel {
  85. public:
  86.     PortAudioMediaChannel(PortAudioMediaEngine *eng);
  87.     virtual ~PortAudioMediaChannel();
  88.  
  89.     virtual void SetCodecs(const std::vector<codec> &codecs);
  90.     virtual void OnPacketReceived(const void *data, int len);
  91.  
  92.     virtual void SetPlayout(bool playout);
  93.     virtual void SetSend(bool send);
  94.  
  95.     virtual int GetOutputLevel();
  96.     bool mute() {return mute_;}
  97.  
  98.     virtual void StartMediaMonitor(VoiceChannel * voice_channel, uint32 cms) {}
  99.     virtual void StopMediaMonitor() {}
  100.  
  101.     // portaudio
  102.     void  pushToSpeaker( float*, int );
  103.     void  saveFromMicrophone( const float*, int );
  104.     sigslot::signal1<portaudiomediachannel *> SignalReadFromMicEvent;  // ready to read
  105.  
  106.  
  107.     paTestData data;
  108.     std::queue<sample> qu_sine;
  109.  
  110.  
  111.     paTestData2 data2;
  112.     int totalFrames;
  113.  
  114.  
  115. protected:
  116.     // portaudio
  117.     void readBuffer(float*, float**, float*, float*, float*, int);
  118.     void writeBuffer(float*, float*, float**, float*, float*, int);
  119.  
  120. private:
  121.     PortAudioMediaEngine* engine_;
  122.     AudioStream*          audio_stream_;
  123.     talk_base::scoped_ptr<talk_base ::AsyncSocket> socket_;
  124.     void  OnReadFromMic(PortAudioMediaChannel*);
  125.  
  126.     int   pt_;
  127.     bool  mute_;
  128.     bool  play_;
  129.  
  130.     talk_base::CriticalSection crit_speaker;
  131.     talk_base::CriticalSection crit_microphone;
  132.  
  133.     // portaudio
  134.     PaStream* stream_;
  135.     std::queue<sample> qu_microphone;   // ローカルのマイク用
  136.     std::queue</sample><sample> qu_speaker; // リモートから届いたのをためておく
  137.  
  138. };
  139.  
  140. class PortAudioMediaEngine : public MediaEngine {
  141.  public:
  142.   PortAudioMediaEngine();
  143.   ~PortAudioMediaEngine();
  144.   virtual bool Init();
  145.   virtual void Terminate();
  146.  
  147.   virtual MediaChannel *CreateChannel();
  148.  
  149.   virtual int SetAudioOptions(int options);
  150.   virtual int SetSoundDevices(int wave_in_device, int wave_out_device);
  151.  
  152.   virtual float GetCurrentQuality();
  153.   virtual int GetInputLevel();
  154.  
  155.   virtual std::vector<codec , std::allocator<Codec>> codecs() {return codecs_;}
  156.   virtual bool FindCodec(const Codec&);
  157.  
  158.  private:
  159.   std::vector</codec><codec , std::allocator<Codec>> codecs_;
  160. };
  161.  
  162. }  // namespace cricket
  163.  
  164. #endif  // TALK_SESSION_PHONE_PORTAUDIOMEDIAENGINE_H_

やりたいのは電話でなく一方向のストリーミングなので、
そろそろ自前のxmppサーバをたてて、
libjingleのxmppclientをいじることになるのかな。
あとspeexコーデックをいれてみたい。

Comments

Leave a Reply