Agora Java API Reference for Android
AgoraMediaBase.h
1 //
2 // Agora Engine SDK
3 //
4 // Created by Sting Feng in 2017-11.
5 // Copyright (c) 2017 Agora.io. All rights reserved.
6 
7 #pragma once // NOLINT(build/header_guard)
8 
9 #include <cstring>
10 #include <stdint.h>
11 #include <limits>
12 #include <stddef.h>
13 
14 namespace agora {
15 namespace rtc {
16 
17 typedef unsigned int uid_t;
18 typedef unsigned int track_id_t;
19 typedef unsigned int conn_id_t;
20 
21 static const unsigned int DEFAULT_CONNECTION_ID = 0;
22 static const unsigned int DUMMY_CONNECTION_ID = (std::numeric_limits<unsigned int>::max)();
23 
27 enum AudioRoute
28 {
32  ROUTE_DEFAULT = -1,
36  ROUTE_HEADSET,
40  ROUTE_EARPIECE,
44  ROUTE_HEADSETNOMIC,
48  ROUTE_SPEAKERPHONE,
52  ROUTE_LOUDSPEAKER,
56  ROUTE_HEADSETBLUETOOTH
57 };
58 
62 enum BYTES_PER_SAMPLE {
66  TWO_BYTES_PER_SAMPLE = 2,
67 };
68 
70  int sample_rate;
71  size_t channels;
72  size_t frames_per_buffer;
73 
75  : sample_rate(0),
76  channels(0),
77  frames_per_buffer(0) {}
78 };
79 
80 } // namespace rtc
81 
82 namespace media {
83 namespace base {
84 
85 typedef void* view_t;
86 
87 typedef const char* user_id_t;
88 
89 static const uint8_t kMaxCodecNameLength = 50;
90 
91 
95  enum MAX_METADATA_SIZE_TYPE
96  {
97  MAX_METADATA_SIZE_IN_BYTE = 1024
98  };
99 
100 
101 
110  uint32_t timestamp;
111  // Audio level indication.
112  uint8_t audioLevelIndication;
113  PacketOptions()
114  : timestamp(0),
115  audioLevelIndication(127) {}
116 };
117 
125  enum : size_t {
126  // Stereo, 32 kHz, 60 ms (2 * 32 * 60)
136  };
137 
152  rtc::BYTES_PER_SAMPLE bytes_per_sample;
155 
156  AudioPcmFrame() :
159  sample_rate_hz_(0),
160  num_channels_(0),
161  bytes_per_sample(rtc::TWO_BYTES_PER_SAMPLE) {
162  memset(data_, 0, sizeof(data_));
163  }
164 };
165 
167  public:
176  virtual void onFrame(const AudioPcmFrame* frame) = 0;
177  virtual ~IAudioFrameObserver() {}
178 };
179 
183 enum VIDEO_PIXEL_FORMAT {
187  VIDEO_PIXEL_UNKNOWN = 0,
191  VIDEO_PIXEL_I420 = 1,
195  VIDEO_PIXEL_BGRA = 2,
199  VIDEO_PIXEL_NV21 = 3,
203  VIDEO_PIXEL_RGBA = 4,
207  VIDEO_PIXEL_NV12 = 8,
211  VIDEO_TEXTURE_2D = 10,
215  VIDEO_TEXTURE_OES = 11,
219  VIDEO_PIXEL_I422 = 16,
220 };
221 
225 enum RENDER_MODE_TYPE {
230  RENDER_MODE_HIDDEN = 1,
236  RENDER_MODE_FIT = 2,
241  RENDER_MODE_ADAPTIVE = 3,
242 };
243 
250  format(VIDEO_PIXEL_UNKNOWN),
251  buffer(NULL),
252  stride(0),
253  height(0),
254  cropLeft(0),
255  cropTop(0),
256  cropRight(0),
257  cropBottom(0),
258  rotation(0),
259  timestamp(0),
260  eglContext(NULL),
262  textureId(0),
263  metadata_buffer(NULL),
264  metadata_size(0){}
265 
278  };
279 
296  };
297 
305  VIDEO_PIXEL_FORMAT format;
309  void* buffer;
314  int stride;
318  int height;
323  int cropLeft;
328  int cropTop;
343  int rotation;
348  long long timestamp;
354  void *eglContext;
367  uint8_t* metadata_buffer;
373 };
374 
378 struct VideoFrame {
379  VideoFrame():
380  type(VIDEO_PIXEL_UNKNOWN),
381  width(0),
382  height(0),
383  yStride(0),
384  uStride(0),
385  vStride(0),
386  yBuffer(NULL),
387  uBuffer(NULL),
388  vBuffer(NULL),
389  rotation(0),
390  renderTimeMs(0),
391  avsync_type(0),
392  metadata_buffer(NULL),
393  metadata_size(0){}
394 
398  VIDEO_PIXEL_FORMAT type;
402  int width;
406  int height;
410  int yStride;
414  int uStride;
418  int vStride;
422  uint8_t* yBuffer;
426  uint8_t* uBuffer;
430  uint8_t* vBuffer;
434  int rotation;
441  int64_t renderTimeMs;
450  uint8_t* metadata_buffer;
456 };
457 
459  public:
467  virtual void onFrame(const VideoFrame* frame) = 0;
468  virtual ~IVideoFrameObserver() {}
469  virtual bool isExternal() { return true; }
470 };
471 
472 enum MEDIA_PLAYER_SOURCE_TYPE {
478  MEDIA_PLAYER_SOURCE_DEFAULT,
484  MEDIA_PLAYER_SOURCE_FULL_FEATURED,
489  MEDIA_PLAYER_SOURCE_SIMPLE,
490 };
491 
492 enum VIDEO_MODULE_POSITION {
493  POSITION_POST_CAPTURER = 1 << 0,
494  POSITION_PRE_RENDERER = 1 << 1,
495  POSITION_PRE_ENCODER = 1 << 2,
496  POSITION_POST_FILTERS = 1 << 3,
497 };
498 
499 } // namespace base
500 
505  public:
514  };
518  struct AudioFrame {
530  agora::rtc::BYTES_PER_SAMPLE bytesPerSample;
534  int channels;
542  void* buffer;
550  int64_t renderTimeMs;
551  int avsync_type;
552 
555  bytesPerSample(rtc::TWO_BYTES_PER_SAMPLE),
556  channels(0),
557  samplesPerSec(0),
558  buffer(NULL),
559  renderTimeMs(0),
560  avsync_type(0) {}
561  };
562 
563  public:
564  virtual ~IAudioFrameObserver() {}
565 
573  virtual bool onRecordAudioFrame(AudioFrame& audioFrame) = 0;
581  virtual bool onPlaybackAudioFrame(AudioFrame& audioFrame) = 0;
589  virtual bool onMixedAudioFrame(AudioFrame& audioFrame) = 0;
598  virtual bool onPlaybackAudioFrameBeforeMixing(unsigned int uid, AudioFrame& audioFrame) = 0;
599 };
604  public:
606 
607  enum VIDEO_FRAME_PROCESS_MODE {
608  PROCESS_MODE_READ_ONLY, // Observer works as a pure renderer and will not modify the original frame.
609  PROCESS_MODE_READ_WRITE, // Observer works as a filter that will process the video frame and affect the following frame processing in SDK.
610  };
611 
612  public:
613  virtual ~IVideoFrameObserver() {}
614 
630  virtual bool onCaptureVideoFrame(VideoFrame& videoFrame) = 0;
631 
632  virtual bool onSecondaryCameraCaptureVideoFrame(VideoFrame& videoFrame) = 0;
633 
649  virtual bool onScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
666  virtual bool onMediaPlayerVideoFrame(VideoFrame& videoFrame, int mediaPlayerId) = 0;
667 
668  virtual bool onSecondaryScreenCaptureVideoFrame(VideoFrame& videoFrame) = 0;
669 
687  virtual bool onRenderVideoFrame(rtc::uid_t uid, rtc::conn_id_t connectionId,
688  VideoFrame& videoFrame) = 0;
689 
690  virtual bool onTranscodedVideoFrame(VideoFrame& videoFrame) = 0;
691 
696  virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode() {
697  return PROCESS_MODE_READ_ONLY;
698  }
699 
700  /*
701  * Occurs each time needs to get preference video frame type.
702  *
703  * @return preference video pixel format.
704  */
705  virtual base::VIDEO_PIXEL_FORMAT getVideoPixelFormatPreference() { return base::VIDEO_PIXEL_I420; }
706 
712  virtual int getRotationApplied() { return 0; }
713 
721  virtual bool getMirrorApplied() { return false; }
722 
730  virtual bool isExternal() { return true; }
731 };
732 
733 } // namespace media
734 } // namespace agora
agora::media::IAudioFrameObserver::FRAME_TYPE_PCM16
@ FRAME_TYPE_PCM16
Definition: AgoraMediaBase.h:513
agora::media::base::VideoFrame
Definition: AgoraMediaBase.h:378
agora::media::base::ExternalVideoFrame::cropBottom
int cropBottom
Definition: AgoraMediaBase.h:338
agora::media::base::VideoFrame::metadata_size
int metadata_size
Definition: AgoraMediaBase.h:455
agora::media::base::VideoFrame::uStride
int uStride
Definition: AgoraMediaBase.h:414
agora::media::base::VideoFrame::height
int height
Definition: AgoraMediaBase.h:406
agora::media::base::AudioPcmFrame::data_
int16_t data_[kMaxDataSizeSamples]
Definition: AgoraMediaBase.h:154
agora::media::IAudioFrameObserver::AudioFrame::samplesPerSec
int samplesPerSec
Definition: AgoraMediaBase.h:538
agora::rtc::AudioParameters
Definition: AgoraMediaBase.h:69
agora::media::IAudioFrameObserver::AudioFrame::renderTimeMs
int64_t renderTimeMs
Definition: AgoraMediaBase.h:550
agora::media::base::IAudioFrameObserver::onFrame
virtual void onFrame(const AudioPcmFrame *frame)=0
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_RAW_DATA
@ VIDEO_BUFFER_RAW_DATA
Definition: AgoraMediaBase.h:287
agora::media::base::ExternalVideoFrame::buffer
void * buffer
Definition: AgoraMediaBase.h:309
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_TYPE
VIDEO_BUFFER_TYPE
Definition: AgoraMediaBase.h:283
agora::media::IAudioFrameObserver::onPlaybackAudioFrame
virtual bool onPlaybackAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::IVideoFrameObserver
Definition: AgoraMediaBase.h:458
agora::media::IVideoFrameObserver::getVideoFrameProcessMode
virtual VIDEO_FRAME_PROCESS_MODE getVideoFrameProcessMode()
Definition: AgoraMediaBase.h:696
agora::media::IVideoFrameObserver
Definition: AgoraMediaBase.h:603
agora::media::IAudioFrameObserver::AudioFrame::buffer
void * buffer
Definition: AgoraMediaBase.h:542
agora::media::IAudioFrameObserver
Definition: AgoraMediaBase.h:504
agora::media::base::VideoFrame::rotation
int rotation
Definition: AgoraMediaBase.h:434
agora::media::base::IVideoFrameObserver::onFrame
virtual void onFrame(const VideoFrame *frame)=0
agora::media::base::ExternalVideoFrame::eglContext
void * eglContext
Definition: AgoraMediaBase.h:354
agora::media::base::ExternalVideoFrame::cropRight
int cropRight
Definition: AgoraMediaBase.h:333
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_ARRAY
@ VIDEO_BUFFER_ARRAY
Definition: AgoraMediaBase.h:291
agora::media::base::ExternalVideoFrame::VIDEO_BUFFER_TEXTURE
@ VIDEO_BUFFER_TEXTURE
Definition: AgoraMediaBase.h:295
agora::media::base::VideoFrame::avsync_type
int avsync_type
Definition: AgoraMediaBase.h:445
agora::media::base::ExternalVideoFrame::cropLeft
int cropLeft
Definition: AgoraMediaBase.h:323
agora::media::base::ExternalVideoFrame
Definition: AgoraMediaBase.h:247
agora::media::IVideoFrameObserver::getRotationApplied
virtual int getRotationApplied()
Definition: AgoraMediaBase.h:712
agora::media::base::AudioPcmFrame::capture_timestamp
uint32_t capture_timestamp
Definition: AgoraMediaBase.h:140
agora::media::base::VideoFrame::type
VIDEO_PIXEL_FORMAT type
Definition: AgoraMediaBase.h:398
agora::media::base::ExternalVideoFrame::timestamp
long long timestamp
Definition: AgoraMediaBase.h:348
agora::media::IVideoFrameObserver::onScreenCaptureVideoFrame
virtual bool onScreenCaptureVideoFrame(VideoFrame &videoFrame)=0
agora::media::base::AudioPcmFrame::sample_rate_hz_
int sample_rate_hz_
Definition: AgoraMediaBase.h:146
agora::media::IAudioFrameObserver::onMixedAudioFrame
virtual bool onMixedAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::ExternalVideoFrame::height
int height
Definition: AgoraMediaBase.h:318
agora::media::IAudioFrameObserver::AudioFrame::type
AUDIO_FRAME_TYPE type
Definition: AgoraMediaBase.h:522
agora::media::base::ExternalVideoFrame::rotation
int rotation
Definition: AgoraMediaBase.h:343
agora::media::base::PacketOptions
Definition: AgoraMediaBase.h:106
agora::media::base::PacketOptions::timestamp
uint32_t timestamp
Definition: AgoraMediaBase.h:110
agora::media::IAudioFrameObserver::onPlaybackAudioFrameBeforeMixing
virtual bool onPlaybackAudioFrameBeforeMixing(unsigned int uid, AudioFrame &audioFrame)=0
agora::media::base::VideoFrame::yBuffer
uint8_t * yBuffer
Definition: AgoraMediaBase.h:422
agora::media::base::AudioPcmFrame::bytes_per_sample
rtc::BYTES_PER_SAMPLE bytes_per_sample
Definition: AgoraMediaBase.h:152
agora::media::IVideoFrameObserver::isExternal
virtual bool isExternal()
Definition: AgoraMediaBase.h:730
agora::media::base::AudioPcmFrame::kMaxDataSizeSamples
@ kMaxDataSizeSamples
Definition: AgoraMediaBase.h:133
agora::media::base::VideoFrame::yStride
int yStride
Definition: AgoraMediaBase.h:410
agora::media::base::ExternalVideoFrame::metadata_buffer
uint8_t * metadata_buffer
Definition: AgoraMediaBase.h:367
agora::media::base::ExternalVideoFrame::EGL_CONTEXT14
@ EGL_CONTEXT14
Definition: AgoraMediaBase.h:277
agora::media::IAudioFrameObserver::AudioFrame::channels
int channels
Definition: AgoraMediaBase.h:534
agora::media::IAudioFrameObserver::AudioFrame
Definition: AgoraMediaBase.h:518
agora::media::base::ExternalVideoFrame::cropTop
int cropTop
Definition: AgoraMediaBase.h:328
agora::media::base::ExternalVideoFrame::type
VIDEO_BUFFER_TYPE type
Definition: AgoraMediaBase.h:301
agora::media::base::VideoFrame::vBuffer
uint8_t * vBuffer
Definition: AgoraMediaBase.h:430
agora::media::base::ExternalVideoFrame::EGL_CONTEXT_TYPE
EGL_CONTEXT_TYPE
Definition: AgoraMediaBase.h:269
agora::media::IAudioFrameObserver::AudioFrame::bytesPerSample
agora::rtc::BYTES_PER_SAMPLE bytesPerSample
Definition: AgoraMediaBase.h:530
agora::media::base::AudioPcmFrame::num_channels_
size_t num_channels_
Definition: AgoraMediaBase.h:149
agora::media::base::VideoFrame::metadata_buffer
uint8_t * metadata_buffer
Definition: AgoraMediaBase.h:450
agora::media::base::IAudioFrameObserver
Definition: AgoraMediaBase.h:166
agora::media::base::VideoFrame::width
int width
Definition: AgoraMediaBase.h:402
agora::media::base::VideoFrame::renderTimeMs
int64_t renderTimeMs
Definition: AgoraMediaBase.h:441
agora::media::base::ExternalVideoFrame::metadata_size
int metadata_size
Definition: AgoraMediaBase.h:372
agora::media::base::AudioPcmFrame::samples_per_channel_
size_t samples_per_channel_
Definition: AgoraMediaBase.h:143
agora::media::IVideoFrameObserver::onRenderVideoFrame
virtual bool onRenderVideoFrame(rtc::uid_t uid, rtc::conn_id_t connectionId, VideoFrame &videoFrame)=0
agora::media::IVideoFrameObserver::onMediaPlayerVideoFrame
virtual bool onMediaPlayerVideoFrame(VideoFrame &videoFrame, int mediaPlayerId)=0
agora::media::base::ExternalVideoFrame::textureId
int textureId
Definition: AgoraMediaBase.h:362
agora::media::IVideoFrameObserver::getMirrorApplied
virtual bool getMirrorApplied()
Definition: AgoraMediaBase.h:721
agora::media::base::VideoFrame::vStride
int vStride
Definition: AgoraMediaBase.h:418
agora::media::base::AudioPcmFrame
Definition: AgoraMediaBase.h:121
agora::media::IVideoFrameObserver::onCaptureVideoFrame
virtual bool onCaptureVideoFrame(VideoFrame &videoFrame)=0
agora::media::base::AudioPcmFrame::kMaxDataSizeBytes
@ kMaxDataSizeBytes
Definition: AgoraMediaBase.h:135
agora::media::IAudioFrameObserver::AUDIO_FRAME_TYPE
AUDIO_FRAME_TYPE
Definition: AgoraMediaBase.h:509
agora::media::base::VideoFrame::uBuffer
uint8_t * uBuffer
Definition: AgoraMediaBase.h:426
agora::media::base::ExternalVideoFrame::EGL_CONTEXT10
@ EGL_CONTEXT10
Definition: AgoraMediaBase.h:273
agora::media::IAudioFrameObserver::AudioFrame::samplesPerChannel
int samplesPerChannel
Definition: AgoraMediaBase.h:526
agora::media::base::ExternalVideoFrame::format
VIDEO_PIXEL_FORMAT format
Definition: AgoraMediaBase.h:305
agora::media::base::ExternalVideoFrame::stride
int stride
Definition: AgoraMediaBase.h:314
agora::media::IAudioFrameObserver::onRecordAudioFrame
virtual bool onRecordAudioFrame(AudioFrame &audioFrame)=0
agora::media::base::ExternalVideoFrame::eglType
EGL_CONTEXT_TYPE eglType
Definition: AgoraMediaBase.h:358