Agora Java API Reference for Android
AgoraMediaBase.h
1 //
2 // Agora Engine SDK
3 //
4 // Created by Sting Feng in 2017-11.
5 // Copyright (c) 2017 Agora.io. All rights reserved.
6 
7 #pragma once // NOLINT(build/header_guard)
8 
9 #include <cstring>
10 #include <stdint.h>
11 #include <limits>
12 #include <stddef.h>
13 
14 #ifndef OPTIONAL_ENUM_SIZE_T
15 #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
16 #define OPTIONAL_ENUM_SIZE_T enum : size_t
17 #else
18 #define OPTIONAL_ENUM_SIZE_T enum
19 #endif
20 #endif
21 
22 namespace agora {
23 namespace rtc {
24 
25 typedef unsigned int uid_t;
26 typedef unsigned int track_id_t;
27 typedef unsigned int conn_id_t;
28 
29 static const unsigned int DEFAULT_CONNECTION_ID = 0;
30 static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits<unsigned int>::max)();
31 
32 struct EncodedVideoFrameInfo;
33 
37 enum AudioRoute
38 {
42  ROUTE_DEFAULT = -1,
46  ROUTE_HEADSET,
50  ROUTE_EARPIECE,
54  ROUTE_HEADSETNOMIC,
58  ROUTE_SPEAKERPHONE,
62  ROUTE_LOUDSPEAKER,
66  ROUTE_HEADSETBLUETOOTH
67 };
68 
72 enum BYTES_PER_SAMPLE {
76  TWO_BYTES_PER_SAMPLE = 2,
77 };
78 
80  int sample_rate;
81  size_t channels;
82  size_t frames_per_buffer;
83 
85  : sample_rate(0),
86  channels(0),
87  frames_per_buffer(0) {}
88 };
89 
90 enum RAW_AUDIO_FRAME_OP_MODE_TYPE {
95  RAW_AUDIO_FRAME_OP_MODE_READ_ONLY = 0,
96 
101  RAW_AUDIO_FRAME_OP_MODE_READ_WRITE = 2,
102 };
103 
104 } // namespace rtc
105 
106 namespace media {
107 namespace base {
108 
109 typedef void* view_t;
110 
111 typedef const char* user_id_t;
112 
113 static const uint8_t kMaxCodecNameLength = 50;
114 
118 enum MAX_METADATA_SIZE_TYPE {
119  MAX_METADATA_SIZE_IN_BYTE = 1024
120 };
121 
130  uint32_t timestamp;
131  // Audio level indication.
132  uint8_t audioLevelIndication;
133  PacketOptions()
134  : timestamp(0),
135  audioLevelIndication(127) {}
136 };
137 
146  // Stereo, 32 kHz, 60 ms (2 * 32 * 60)
153  kMaxDataSizeSamples = 3840,
155  kMaxDataSizeBytes = kMaxDataSizeSamples * sizeof(int16_t),
156  };
157 
172  rtc::BYTES_PER_SAMPLE bytes_per_sample;
174  int16_t data_[kMaxDataSizeSamples];
175 
176  AudioPcmFrame& operator=(const AudioPcmFrame& src) {
177  if(this == &src) {
178  return *this;
179  }
180 
181  this->capture_timestamp = src.capture_timestamp;
182  this->samples_per_channel_ = src.samples_per_channel_;
183  this->sample_rate_hz_ = src.sample_rate_hz_;
184  this->bytes_per_sample = src.bytes_per_sample;
185  this->num_channels_ = src.num_channels_;
186 
187  size_t length = src.samples_per_channel_ * src.num_channels_;
188  if ( length > kMaxDataSizeSamples) {
189  length = kMaxDataSizeSamples;
190  }
191 
192  memcpy(this->data_, src.data_, length * sizeof(int16_t));
193 
194  return *this;
195  }
196 
197  AudioPcmFrame() :
200  sample_rate_hz_(0),
201  num_channels_(0),
202  bytes_per_sample(rtc::TWO_BYTES_PER_SAMPLE) {
203  memset(data_, 0, sizeof(data_));
204  }
205 };
206 
208  public:
217  virtual void onFrame(const AudioPcmFrame* frame) = 0;
218  virtual ~IAudioFrameObserver() {}
219 };
220 
224 enum VIDEO_PIXEL_FORMAT {
228  VIDEO_PIXEL_UNKNOWN = 0,
232  VIDEO_PIXEL_I420 = 1,
236  VIDEO_PIXEL_BGRA = 2,
240  VIDEO_PIXEL_NV21 = 3,
244  VIDEO_PIXEL_RGBA = 4,
248  VIDEO_PIXEL_NV12 = 8,
252  VIDEO_TEXTURE_2D = 10,
256  VIDEO_TEXTURE_OES = 11,
260  VIDEO_PIXEL_I422 = 16,
261 };
262 
266 enum RENDER_MODE_TYPE {
271  RENDER_MODE_HIDDEN = 1,
277  RENDER_MODE_FIT = 2,
282  RENDER_MODE_ADAPTIVE = 3,
283 };
284 
291  format(VIDEO_PIXEL_UNKNOWN),
292  buffer(NULL),
293  stride(0),
294  height(0),
295  cropLeft(0),
296  cropTop(0),
297  cropRight(0),
298  cropBottom(0),
299  rotation(0),
300  timestamp(0),
301  eglContext(NULL),
303  textureId(0),
304  metadata_buffer(NULL),
305  metadata_size(0){}
306 
319  };
320 
337  };
338 
346  VIDEO_PIXEL_FORMAT format;
350  void* buffer;
355  int stride;
359  int height;
364  int cropLeft;
369  int cropTop;
384  int rotation;
389  long long timestamp;
395  void *eglContext;
408  uint8_t* metadata_buffer;
414 };
415 
419 struct VideoFrame {
420  VideoFrame():
421  type(VIDEO_PIXEL_UNKNOWN),
422  width(0),
423  height(0),
424  yStride(0),
425  uStride(0),
426  vStride(0),
427  yBuffer(NULL),
428  uBuffer(NULL),
429  vBuffer(NULL),
430  rotation(0),
431  renderTimeMs(0),
432  avsync_type(0),
433  metadata_buffer(NULL),
434  metadata_size(0){}
435 
439  VIDEO_PIXEL_FORMAT type;
443  int width;
447  int height;
451  int yStride;
455  int uStride;
459  int vStride;
463  uint8_t* yBuffer;
467  uint8_t* uBuffer;
471  uint8_t* vBuffer;
475  int rotation;
482  int64_t renderTimeMs;
491  uint8_t* metadata_buffer;
497 };
498 
500  public:
508  virtual void onFrame(const VideoFrame* frame) = 0;
509  virtual ~IVideoFrameObserver() {}
510  virtual bool isExternal() { return true; }
511 };
512 
513 enum MEDIA_PLAYER_SOURCE_TYPE {
519  MEDIA_PLAYER_SOURCE_DEFAULT,
525  MEDIA_PLAYER_SOURCE_FULL_FEATURED,
530  MEDIA_PLAYER_SOURCE_SIMPLE,
531 };
532 
533 enum VIDEO_MODULE_POSITION {
534  POSITION_POST_CAPTURER = 1 << 0,
535  POSITION_PRE_RENDERER = 1 << 1,
536  POSITION_PRE_ENCODER = 1 << 2,
537  POSITION_POST_FILTERS = 1 << 3,
538 };
539 
540 } // namespace base
541 
546  public:
555  };
559  struct AudioFrame {
571  agora::rtc::BYTES_PER_SAMPLE bytesPerSample;
575  int channels;
583  void* buffer;
591  int64_t renderTimeMs;
592  int avsync_type;
593 
596  bytesPerSample(rtc::TWO_BYTES_PER_SAMPLE),
597  channels(0),
598  samplesPerSec(0),
599  buffer(NULL),
600  renderTimeMs(0),
601  avsync_type(0) {}
602  };
603 
604  public:
605  virtual ~IAudioFrameObserver() {}
606 
614  virtual bool onRecordAudioFrame(AudioFrame& audioFrame) = 0;
622  virtual bool onPlaybackAudioFrame(AudioFrame& audioFrame) = 0;
630  virtual bool onMixedAudioFrame(AudioFrame& audioFrame) = 0;
639  virtual bool onPlaybackAudioFrameBeforeMixing(unsigned int uid, AudioFrame& audioFrame) = 0;
640 };
641 
643  const float *audioSpectrumData;
644  int dataLength;
645 
646  AudioSpectrumData() : audioSpectrumData(NULL), dataLength(0) {}
647  AudioSpectrumData(const float *data, int length) :
648  audioSpectrumData(data), dataLength(length) {}
649 };
650 
655  agora::rtc::uid_t uid;
656  struct AudioSpectrumData spectrumData;
657 
658  UserAudioSpectrumInfo () : uid(0), spectrumData() {}
659  UserAudioSpectrumInfo(agora::rtc::uid_t _uid, const float *data, int length) :
660  uid(_uid) { spectrumData.audioSpectrumData = data; spectrumData.dataLength = length; }
661 };
662 
667 public:
668  virtual ~IAudioSpectrumObserver() {}
669 
683  virtual bool onLocalAudioSpectrum(const AudioSpectrumData& data) = 0;
684 
703  virtual bool onRemoteAudioSpectrum(const UserAudioSpectrumInfo * spectrums, unsigned int spectrumNumber) = 0;
704 };
705 
710  public:
721  virtual bool OnEncodedVideoFrame(rtc::uid_t uid, const uint8_t* imageBuffer, size_t length,
722  const rtc::EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
723 
724  virtual ~IVideoEncodedFrameObserver() {}
725 };
726 
731  public:
733 
734  enum VIDEO_FRAME_PROCESS_MODE {
735  PROCESS_MODE_READ_ONLY, // Observer works as a pure renderer and will not modify the original frame.
736  PROCESS_MODE_READ_WRITE, // Observer works as a filter that will process the video frame and affect the following frame processing in SDK.
737  };
738 
739  public:
740  virtual ~IVideoFrameObserver() {}
741 
757  virtual bool onCaptureVideoFrame(VideoFrame& videoFrame) = 0;
758 
759  virtual bool onSecondaryCameraCaptureVideoFrame(VideoFrame& videoFrame) = 0;
760 
776  virtual bool onScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
793  virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0;
794 
795  virtual bool onSecondaryScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
796 
814  virtual bool onRenderVideoFrame(rtc::uid_t uid, rtc::conn_id_t connectionId,
815  VideoFrame& videoFrame) = 0;
816 
817  virtual bool onTranscodedVideoFrame(VideoFrame& videoFrame) = 0;
818 
823  virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode() {
824  return PROCESS_MODE_READ_ONLY;
825  }
826 
827  /*
828  * Occurs each time needs to get preference video frame type.
829  *
830  * @return preference video pixel format.
831  */
832  virtual base::VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return base::VIDEO_PIXEL_I420; }
833 
839  virtual int getRotationApplied() { return 0; }
840 
848  virtual bool getMirrorApplied() { return false; }
849 
857  virtual bool isExternal() { return true; }
858 };
859 
860 } // namespace media
861 } // namespace agora
agora::media::IAudioFrameObserver::FRAME_TYPE_PCM16
@ FRAME_TYPE_PCM16
Definition: AgoraMediaBase.h:554
agora::media::base::VideoFrame
Definition: AgoraMediaBase.h:419
agora::media::base::ExternalVideoFrame::cropBottom
int cropBottom
Definition: AgoraMediaBase.h:379
agora::media::base::VideoFrame::metadata_size
int metadata_size
Definition: AgoraMediaBase.h:496
agora::media::base::VideoFrame::uStride
int uStride
Definition: AgoraMediaBase.h:455
agora::media::base::VideoFrame::height
int height
Definition: AgoraMediaBase.h:447
agora::media::base::AudioPcmFrame::data_
int16_t data_[kMaxDataSizeSamples]
Definition: AgoraMediaBase.h:174
agora::media::IAudioFrameObserver::AudioFrame::samplesPerSec
int samplesPerSec
Definition: AgoraMediaBase.h:579
agora::media::IAudioSpectrumObserver
Definition: AgoraMediaBase.h:666
agora::rtc::AudioParameters
Definition: AgoraMediaBase.h:79
agora::media::IAudioFrameObserver::AudioFrame::renderTimeMs
int64_t renderTimeMs
Definition: AgoraMediaBase.h:591
agora::media::base::IAudioFrameObserver::onFrame
virtual void onFrame(const AudioPcmFrame *frame)=0
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_RAW_DATA
@ VIDEO_BUFFER_RAW_DATA
Definition: AgoraMediaBase.h:328
agora::media::base::ExternalVideoFrame::buffer
void * buffer
Definition: AgoraMediaBase.h:350
agora::rtc::EncodedVideoFrameInfo
Definition: AgoraBase.h:1691
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_TYPE
VIDEO_BUFFER_TYPE
Definition: AgoraMediaBase.h:324
agora::media::IAudioFrameObserver::onPlaybackAudioFrame
virtual bool onPlaybackAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::IVideoFrameObserver
Definition: AgoraMediaBase.h:499
agora::media::IVideoFrameObserver::getVideoFrameProcessMode
virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode()
Definition: AgoraMediaBase.h:823
agora::media::IVideoFrameObserver
Definition: AgoraMediaBase.h:730
agora::media::IAudioFrameObserver::AudioFrame::buffer
void * buffer
Definition: AgoraMediaBase.h:583
agora::media::IAudioFrameObserver
Definition: AgoraMediaBase.h:545
agora::media::base::VideoFrame::rotation
int rotation
Definition: AgoraMediaBase.h:475
agora::media::base::IVideoFrameObserver::onFrame
virtual void onFrame(const VideoFrame *frame)=0
agora::media::base::AudioPcmFrame::OPTIONAL_ENUM_SIZE_T
OPTIONAL_ENUM_SIZE_T
Definition: AgoraMediaBase.h:145
agora::media::base::ExternalVideoFrame::eglContext
void * eglContext
Definition: AgoraMediaBase.h:395
agora::media::base::ExternalVideoFrame::cropRight
int cropRight
Definition: AgoraMediaBase.h:374
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_ARRAY
@ VIDEO_BUFFER_ARRAY
Definition: AgoraMediaBase.h:332
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_TEXTURE
@ VIDEO_BUFFER_TEXTURE
Definition: AgoraMediaBase.h:336
agora::media::base::VideoFrame::avsync_type
int avsync_type
Definition: AgoraMediaBase.h:486
agora::media::base::ExternalVideoFrame::cropLeft
int cropLeft
Definition: AgoraMediaBase.h:364
agora::media::IAudioSpectrumObserver::onRemoteAudioSpectrum
virtual bool onRemoteAudioSpectrum(const UserAudioSpectrumInfo *spectrums, unsigned int spectrumNumber)=0
agora::media::base::ExternalVideoFrame
Definition: AgoraMediaBase.h:288
agora::media::IVideoFrameObserver::getRotationApplied
virtual int getRotationApplied()
Definition: AgoraMediaBase.h:839
agora::media::IVideoEncodedFrameObserver::OnEncodedVideoFrame
virtual bool OnEncodedVideoFrame(rtc::uid_t uid, const uint8_t *imageBuffer, size_t length, const rtc::EncodedVideoFrameInfo &videoEncodedFrameInfo)=0
agora::media::base::AudioPcmFrame::capture_timestamp
uint32_t capture_timestamp
Definition: AgoraMediaBase.h:160
agora::media::base::VideoFrame::type
VIDEO_PIXEL_FORMAT type
Definition: AgoraMediaBase.h:439
agora::media::base::ExternalVideoFrame::timestamp
long long timestamp
Definition: AgoraMediaBase.h:389
agora::media::IVideoFrameObserver::onScreenCaptureVideoFrame
virtual bool onScreenCaptureVideoFrame(VideoFrame &videoFrame)=0
agora::media::base::AudioPcmFrame::sample_rate_hz_
int sample_rate_hz_
Definition: AgoraMediaBase.h:166
agora::media::IAudioFrameObserver::onMixedAudioFrame
virtual bool onMixedAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::ExternalVideoFrame::height
int height
Definition: AgoraMediaBase.h:359
agora::media::IAudioFrameObserver::AudioFrame::type
AUDIO_FRAME_TYPE type
Definition: AgoraMediaBase.h:563
agora::media::base::ExternalVideoFrame::rotation
int rotation
Definition: AgoraMediaBase.h:384
agora::media::base::PacketOptions
Definition: AgoraMediaBase.h:126
agora::media::base::PacketOptions::timestamp
uint32_t timestamp
Definition: AgoraMediaBase.h:130
agora::media::IAudioFrameObserver::onPlaybackAudioFrameBeforeMixing
virtual bool onPlaybackAudioFrameBeforeMixing(unsigned int uid, AudioFrame &audioFrame)=0
agora::media::base::VideoFrame::yBuffer
uint8_t * yBuffer
Definition: AgoraMediaBase.h:463
agora::media::base::AudioPcmFrame::bytes_per_sample
rtc::BYTES_PER_SAMPLE bytes_per_sample
Definition: AgoraMediaBase.h:172
agora::media::UserAudioSpectrumInfo
Definition: AgoraMediaBase.h:651
agora::media::IVideoFrameObserver::isExternal
virtual bool isExternal()
Definition: AgoraMediaBase.h:857
agora::media::base::VideoFrame::yStride
int yStride
Definition: AgoraMediaBase.h:451
agora::media::base::ExternalVideoFrame::metadata_buffer
uint8_t * metadata_buffer
Definition: AgoraMediaBase.h:408
agora::media::base::ExternalVideoFrame::EGL_CONTEXT14
@ EGL_CONTEXT14
Definition: AgoraMediaBase.h:318
agora::media::IAudioFrameObserver::AudioFrame::channels
int channels
Definition: AgoraMediaBase.h:575
agora::media::IAudioFrameObserver::AudioFrame
Definition: AgoraMediaBase.h:559
agora::media::base::ExternalVideoFrame::cropTop
int cropTop
Definition: AgoraMediaBase.h:369
agora::media::base::ExternalVideoFrame::type
VIDEO_BUFFER_TYPE type
Definition: AgoraMediaBase.h:342
agora::media::base::VideoFrame::vBuffer
uint8_t * vBuffer
Definition: AgoraMediaBase.h:471
agora::media::base::ExternalVideoFrame::EGL_CONTEXT_TYPE
EGL_CONTEXT_TYPE
Definition: AgoraMediaBase.h:310
agora::media::IAudioFrameObserver::AudioFrame::bytesPerSample
agora::rtc::BYTES_PER_SAMPLE bytesPerSample
Definition: AgoraMediaBase.h:571
agora::media::base::AudioPcmFrame::num_channels_
size_t num_channels_
Definition: AgoraMediaBase.h:169
agora::media::base::VideoFrame::metadata_buffer
uint8_t * metadata_buffer
Definition: AgoraMediaBase.h:491
agora::media::base::IAudioFrameObserver
Definition: AgoraMediaBase.h:207
agora::media::AudioSpectrumData
Definition: AgoraMediaBase.h:642
agora::media::base::VideoFrame::width
int width
Definition: AgoraMediaBase.h:443
agora::media::base::VideoFrame::renderTimeMs
int64_t renderTimeMs
Definition: AgoraMediaBase.h:482
agora::media::base::ExternalVideoFrame::metadata_size
int metadata_size
Definition: AgoraMediaBase.h:413
agora::media::base::AudioPcmFrame::samples_per_channel_
size_t samples_per_channel_
Definition: AgoraMediaBase.h:163
agora::media::IVideoFrameObserver::onRenderVideoFrame
virtual bool onRenderVideoFrame(rtc::uid_t uid, rtc::conn_id_t connectionId, VideoFrame &videoFrame)=0
agora::media::IVideoFrameObserver::onMediaPlayerVideoFrame
virtual bool onMediaPlayerVideoFrame(VideoFrame &videoFrame, int mediaPlayerId)=0
agora::media::base::ExternalVideoFrame::textureId
int textureId
Definition: AgoraMediaBase.h:403
agora::media::IVideoEncodedFrameObserver
Definition: AgoraMediaBase.h:709
agora::media::IVideoFrameObserver::getMirrorApplied
virtual bool getMirrorApplied()
Definition: AgoraMediaBase.h:848
agora::media::IAudioSpectrumObserver::onLocalAudioSpectrum
virtual bool onLocalAudioSpectrum(const AudioSpectrumData &data)=0
agora::media::base::VideoFrame::vStride
int vStride
Definition: AgoraMediaBase.h:459
agora::media::base::AudioPcmFrame
Definition: AgoraMediaBase.h:141
agora::media::IVideoFrameObserver::onCaptureVideoFrame
virtual bool onCaptureVideoFrame(VideoFrame &videoFrame)=0
agora::media::IAudioFrameObserver::AUDIO_FRAME_TYPE
AUDIO_FRAME_TYPE
Definition: AgoraMediaBase.h:550
agora::media::base::VideoFrame::uBuffer
uint8_t * uBuffer
Definition: AgoraMediaBase.h:467
agora::media::base::ExternalVideoFrame::EGL_CONTEXT10
@ EGL_CONTEXT10
Definition: AgoraMediaBase.h:314
agora::media::IAudioFrameObserver::AudioFrame::samplesPerChannel
int samplesPerChannel
Definition: AgoraMediaBase.h:567
agora::media::UserAudioSpectrumInfo::uid
agora::rtc::uid_t uid
Definition: AgoraMediaBase.h:655
agora::media::base::ExternalVideoFrame::format
VIDEO_PIXEL_FORMAT format
Definition: AgoraMediaBase.h:346
agora::media::base::ExternalVideoFrame::stride
int stride
Definition: AgoraMediaBase.h:355
agora::media::IAudioFrameObserver::onRecordAudioFrame
virtual bool onRecordAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::ExternalVideoFrame::eglType
EGL_CONTEXT_TYPE eglType
Definition: AgoraMediaBase.h:399