Agora Java API Reference for Android
NGIAgoraMediaNode.h
1 #pragma once
2 #include "AgoraRefPtr.h"
3 #include "AgoraBase.h"
4 #include "IAgoraLog.h"
5 #include "NGIAgoraVideoFrame.h"
6 #include "AgoraExtensionVersion.h"
7 
8 #ifndef OPTIONAL_PROCESSRESULT_SPECIFIER
9 #if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
10 #define OPTIONAL_PROCESSRESULT_SPECIFIER ProcessResult::
11 #else
12 #define OPTIONAL_PROCESSRESULT_SPECIFIER
13 #endif
14 #endif
15 
16 namespace agora {
17 namespace rtc {
23  public:
24  // Return false if filter decide to drop the frame.
25  // Implementation should not change samples_per_channel_/sample_rate_hz_/num_channels_
26  // Implementation should just adapt data of audio frame.
35  virtual bool adaptAudioFrame(const media::base::AudioPcmFrame& inAudioFrame,
36  media::base::AudioPcmFrame& adaptedFrame) = 0;
37 
45  virtual int getPreferredSampleRate() = 0;
46 
55  virtual int getPreferredChannelNumbers() = 0;
56 
57  protected:
58  ~IAudioFilterBase() {}
59 };
60 
68  public:
75  virtual void setEnabled(bool enable) = 0;
82  virtual bool isEnabled() const = 0;
93  virtual int setProperty(const char* key, const void* buf, int buf_size) = 0;
104  virtual int getProperty(const char* key, void* buf, int buf_size) const = 0;
112  virtual const char * getName() const = 0;
113 
122  virtual int getPreferredSampleRate() { return 0; };
123 
132  virtual int getPreferredChannelNumbers() { return 0; };
133 
134  protected:
135  ~IAudioFilter() {}
136 };
137 
143  public:
154  virtual bool adaptVideoFrame(const media::base::VideoFrame& capturedFrame,
155  media::base::VideoFrame& adaptedFrame) = 0;
156 };
157 
158 // TODO(Bob): private inherit?
167  public:
175  virtual void setEnabled(bool enable) {}
182  virtual bool isEnabled() { return true; }
193  virtual int setProperty(const char* key, const void* buf, size_t buf_size) { return -1; }
204  virtual int getProperty(const char* key, void* buf, size_t buf_size) { return -1; }
212  virtual bool onDataStreamWillStart() { return true; }
217  virtual void onDataStreamWillStop() { }
225  virtual bool isExternal() { return true; }
233  virtual bool isExtensionFilter() { return false; }
234 };
235 
243  public:
244  enum ProcessMode {
245  kSync, // Indicates that video frame data will be exchanged via "adaptVideoFrame"
246  kAsync, // Indicates that video frame data will be exchanged via "pendVideoFrame" & "deliverVideoFrame"
247  };
248 
249  enum ProcessResult {
250  kSuccess, // Video frame data is successfully processed
251  kBypass, // Video frame data should bypass the current filter and flow to its successsors
252  kDrop, // Video Frame data should be discarded
253  };
254 
262  class Control : public RefCountInterface {
263  public:
270  virtual ProcessResult deliverVideoFrame(agora::agora_refptr<IVideoFrame> frame) = 0;
281  virtual int postEvent(const char* key, const char* value) = 0;
288  virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
294  virtual void disableMe(int error, const char* msg) = 0;
295  };
296 
307  virtual void getProcessMode(ProcessMode& mode, bool& independent_thread) = 0;
315  virtual int start(agora::agora_refptr<Control> control) = 0;
321  virtual int stop() = 0;
330  virtual void getVideoFormatWanted(VideoFrameData::Type& type, RawPixelBuffer::Format& format) = 0;
337  virtual ProcessResult pendVideoFrame(agora::agora_refptr<IVideoFrame> frame) {
338  return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
339  }
346  return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
347  }
348  /* Occurs each time needs to get rotation apply.
349  *
350  * @return Determines whether to rotate.
351  * - true: need to rotate.
352  * - false: no rotate.
353  */
354  virtual bool getRotationApplied() { return false; }
355 
356  // NOTE: The following two interfaces should never be overriden!
357  virtual bool isExtensionFilter() { return true; }
358  virtual bool adaptVideoFrame(
359  const media::base::VideoFrame& capturedFrame,
360  media::base::VideoFrame& adaptedFrame) {
361  return -ERR_NOT_SUPPORTED;
362  }
363 };
364 
369  public:
380  virtual int setProperty(const char* key, const void* buf, int buf_size) { return -1; }
391  virtual int getProperty(const char* key, void* buf, int buf_size) { return -1; }
396  virtual int onFrame(const media::base::VideoFrame& videoFrame) = 0;
401  virtual bool isExternalSink() { return true; }
409  virtual bool onDataStreamWillStart() { return true; }
414  virtual void onDataStreamWillStop() { }
421  virtual bool applyMirror() { return false; }
428  virtual bool applyRotation() { return false; }
429 };
430 
432 public:
433  virtual ~IMediaExtensionObserver() {}
434  virtual void onEvent(const char* provider_name, const char* ext_name, const char* key, const char* json_value) {}
435  virtual void onExtensionStopped(const char* provider_name, const char* ext_name) {}
436  virtual void onExtensionStarted(const char* provider_name, const char* ext_name) {}
437  virtual void onExtensionErrored(const char* provider_name, const char* ext_name, int error, const char* msg) {}
438 };
439 
446  public:
460  virtual int sendAudioPcmData(
461  const void* audio_data, uint32_t capture_timestamp,
462  const size_t samples_per_channel, // for 10ms Data, number_of_samples * 100 = sample_rate
463  const agora::rtc::BYTES_PER_SAMPLE bytes_per_sample, // 2
464  const size_t number_of_channels,
465  const uint32_t sample_rate) = 0; // sample_rate > 8000
466 
467  protected:
468  ~IAudioPcmDataSender() {}
469 };
470 
479  public:
492  virtual bool sendEncodedAudioFrame(const uint8_t* payload_data, size_t payload_size,
493  const EncodedAudioFrameInfo& audioFrameInfo) = 0;
494 
495  protected:
497 };
498 
500  public:
510  const uint8_t *packet, size_t length, const media::base::AudioEncodedFrameInfo& info) = 0;
511 
512  virtual ~IAudioEncodedFrameReceiver() {}
513 };
514 
520  public:
529  virtual bool onMediaPacketReceived(
530  const uint8_t *packet, size_t length, const agora::media::base::PacketOptions& options) = 0;
531 
532  virtual ~IMediaPacketReceiver() {}
533 };
534 
542  public:
550  virtual bool onMediaControlPacketReceived(uid_t uid, const uint8_t *packet, size_t length) = 0;
551 
552  virtual ~IMediaControlPacketReceiver() {}
553 };
554 
563  public:
575  virtual int sendMediaPacket(const uint8_t *packet, size_t length,
576  const media::base::PacketOptions &options) = 0;
577  protected:
578  ~IMediaPacketSender() {}
579 };
580 
589  public:
602  virtual int sendPeerMediaControlPacket(media::base::user_id_t userId,
603  const uint8_t *packet,
604  size_t length) = 0;
605 
616  virtual int sendBroadcastMediaControlPacket(const uint8_t *packet, size_t length) = 0;
617 
618  virtual ~IMediaControlPacketSender() {}
619 };
620 
626  public:
634  virtual bool onAudioFrame(const media::base::AudioPcmFrame& audioFrame) = 0;
635 
636  protected:
637  ~IAudioSinkBase() {}
638 };
639 
648  public:
658  virtual int sendVideoFrame(const media::base::ExternalVideoFrame& videoFrame) = 0;
659 
660  protected:
661  ~IVideoFrameSender() {}
662 };
663 
672  public:
684  virtual bool sendEncodedVideoImage(const uint8_t* imageBuffer, size_t length,
685  const EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
686 
687  protected:
689 };
690 
695  public:
699  struct BeautyOptions {
716  };
717 
723 
728 
734 
740 
741  BeautyOptions(LIGHTENING_CONTRAST_LEVEL contrastLevel, float lightening, float smoothness,
742  float redness)
743  : lighteningContrastLevel(contrastLevel),
744  lighteningLevel(lightening),
745  smoothnessLevel(smoothness),
746  rednessLevel(redness) {}
747 
748  BeautyOptions()
750  lighteningLevel(0),
751  smoothnessLevel(0),
752  rednessLevel(0) {}
753  };
764  virtual int setBeautyEffectOptions(bool enabled, BeautyOptions options) = 0;
765 };
766 
771  public:
779  virtual int setRenderMode(media::base::RENDER_MODE_TYPE renderMode) = 0;
788  virtual int setRenderMode(void* view, media::base::RENDER_MODE_TYPE renderMode) = 0;
798  virtual int setMirror(bool mirror) = 0;
809  virtual int setMirror(void* view, bool mirror) = 0;
817  virtual int setView(void* view) = 0;
826  virtual int addView(void* view, const Rectangle& cropArea) = 0;
833  virtual int unsetView() = 0;
840  virtual int removeView(void* view) = 0;
841 };
842 
843 static const int kDeviceIdSize = 128;
844 
845 class IVideoTrack;
850  public:
851  virtual int getTranscodingDelayMs() = 0;
852  virtual int addVideoTrack(agora_refptr<IVideoTrack> track) = 0;
853  virtual int removeVideoTrack(agora_refptr<IVideoTrack> track) = 0;
854 };
855 
856 }
857 }
agora::media::base::VideoFrame
Definition: AgoraMediaBase.h:653
agora::rtc::IMediaControlPacketSender::sendPeerMediaControlPacket
virtual int sendPeerMediaControlPacket(media::base::user_id_t userId, const uint8_t *packet, size_t length)=0
agora::rtc::IVideoSinkBase::onDataStreamWillStart
virtual bool onDataStreamWillStart()
Definition: NGIAgoraMediaNode.h:409
agora::rtc::IMediaControlPacketReceiver
Definition: NGIAgoraMediaNode.h:541
agora::rtc::IVideoFilter::onDataStreamWillStop
virtual void onDataStreamWillStop()
Definition: NGIAgoraMediaNode.h:217
agora::rtc::IMediaControlPacketSender::sendBroadcastMediaControlPacket
virtual int sendBroadcastMediaControlPacket(const uint8_t *packet, size_t length)=0
agora::rtc::IAudioEncodedFrameReceiver::onEncodedAudioFrameReceived
virtual bool onEncodedAudioFrameReceived(const uint8_t *packet, size_t length, const media::base::AudioEncodedFrameInfo &info)=0
agora::rtc::IVideoFrameSender::sendVideoFrame
virtual int sendVideoFrame(const media::base::ExternalVideoFrame &videoFrame)=0
agora::rtc::IVideoSinkBase::applyRotation
virtual bool applyRotation()
Definition: NGIAgoraMediaNode.h:428
agora::rtc::IVideoBeautyFilter
Definition: NGIAgoraMediaNode.h:694
agora::rtc::IAudioFilterBase
Definition: NGIAgoraMediaNode.h:22
agora::rtc::IVideoFilterBase::adaptVideoFrame
virtual bool adaptVideoFrame(const media::base::VideoFrame &capturedFrame, media::base::VideoFrame &adaptedFrame)=0
agora::rtc::IVideoTrack
Definition: NGIAgoraVideoTrack.h:37
agora::agora_refptr
Definition: AgoraRefPtr.h:44
agora::rtc::IVideoFrameTransceiver
Definition: NGIAgoraMediaNode.h:849
agora::rtc::IAudioFilterBase::adaptAudioFrame
virtual bool adaptAudioFrame(const media::base::AudioPcmFrame &inAudioFrame, media::base::AudioPcmFrame &adaptedFrame)=0
agora::rtc::IExtensionVideoFilter::isExtensionFilter
virtual bool isExtensionFilter()
Definition: NGIAgoraMediaNode.h:357
agora::rtc::IVideoRenderer::setView
virtual int setView(void *view)=0
agora::rtc::IVideoFilter::setProperty
virtual int setProperty(const char *key, const void *buf, size_t buf_size)
Definition: NGIAgoraMediaNode.h:193
agora::rtc::IVideoFilter::isExtensionFilter
virtual bool isExtensionFilter()
Definition: NGIAgoraMediaNode.h:233
agora::rtc::Rectangle
Definition: AgoraBase.h:2173
agora::rtc::IVideoSinkBase
Definition: NGIAgoraMediaNode.h:368
agora::rtc::IVideoSinkBase::applyMirror
virtual bool applyMirror()
Definition: NGIAgoraMediaNode.h:421
agora::rtc::EncodedVideoFrameInfo
Definition: AgoraBase.h:1870
agora::rtc::IVideoFilter::onDataStreamWillStart
virtual bool onDataStreamWillStart()
Definition: NGIAgoraMediaNode.h:212
agora::rtc::EncodedAudioFrameInfo
Definition: AgoraBase.h:1744
agora::rtc::IVideoRenderer::unsetView
virtual int unsetView()=0
agora::rtc::IVideoEncodedImageSender::sendEncodedVideoImage
virtual bool sendEncodedVideoImage(const uint8_t *imageBuffer, size_t length, const EncodedVideoFrameInfo &videoEncodedFrameInfo)=0
agora::rtc::IAudioPcmDataSender
Definition: NGIAgoraMediaNode.h:445
agora::rtc::IVideoRenderer::removeView
virtual int removeView(void *view)=0
agora::rtc::IMediaControlPacketReceiver::onMediaControlPacketReceived
virtual bool onMediaControlPacketReceived(uid_t uid, const uint8_t *packet, size_t length)=0
agora::rtc::IVideoEncodedImageSender
Definition: NGIAgoraMediaNode.h:671
agora::rtc::IExtensionVideoFilter::adaptVideoFrame
virtual bool adaptVideoFrame(const media::base::VideoFrame &capturedFrame, media::base::VideoFrame &adaptedFrame)
Definition: NGIAgoraMediaNode.h:358
agora::rtc::IAudioFilter::setProperty
virtual int setProperty(const char *key, const void *buf, int buf_size)=0
agora::rtc::IAudioFilter::isEnabled
virtual bool isEnabled() const =0
agora::rtc::IMediaPacketSender
Definition: NGIAgoraMediaNode.h:562
agora::rtc::IAudioFilter
Definition: NGIAgoraMediaNode.h:67
agora::rtc::IExtensionVideoFilter::stop
virtual int stop()=0
SDK will invoke this API when the data stream is about to stop. Filter can perform cleanup jobs in th...
agora::media::base::ExternalVideoFrame
Definition: AgoraMediaBase.h:515
agora::rtc::IAudioFilterBase::getPreferredSampleRate
virtual int getPreferredSampleRate()=0
agora::rtc::IVideoSinkBase::setProperty
virtual int setProperty(const char *key, const void *buf, int buf_size)
Definition: NGIAgoraMediaNode.h:380
agora::rtc::IVideoRenderer::addView
virtual int addView(void *view, const Rectangle &cropArea)=0
agora::rtc::IVideoSinkBase::onDataStreamWillStop
virtual void onDataStreamWillStop()
Definition: NGIAgoraMediaNode.h:414
agora::rtc::IVideoRenderer::setRenderMode
virtual int setRenderMode(void *view, media::base::RENDER_MODE_TYPE renderMode)=0
agora::rtc::IExtensionVideoFilter::Control::disableMe
virtual void disableMe(int error, const char *msg)=0
Ask SDK to disable the current filter if a fatal error is detected.
agora::rtc::IAudioEncodedFrameReceiver
Definition: NGIAgoraMediaNode.h:499
agora::rtc::IVideoBeautyFilter::BeautyOptions::LIGHTENING_CONTRAST_HIGH
@ LIGHTENING_CONTRAST_HIGH
Definition: NGIAgoraMediaNode.h:715
agora::rtc::IExtensionVideoFilter::getProcessMode
virtual void getProcessMode(ProcessMode &mode, bool &independent_thread)=0
SDK will invoke this API first to get the filter's requested process mode ProcessMode and threading m...
agora::rtc::IAudioEncodedFrameSender
Definition: NGIAgoraMediaNode.h:478
agora::media::base::PacketOptions
Definition: AgoraMediaBase.h:299
agora::rtc::IAudioEncodedFrameSender::sendEncodedAudioFrame
virtual bool sendEncodedAudioFrame(const uint8_t *payload_data, size_t payload_size, const EncodedAudioFrameInfo &audioFrameInfo)=0
agora::rtc::IAudioFilterBase::getPreferredChannelNumbers
virtual int getPreferredChannelNumbers()=0
agora::rtc::IVideoSinkBase::onFrame
virtual int onFrame(const media::base::VideoFrame &videoFrame)=0
agora::rtc::IVideoFilterBase
Definition: NGIAgoraMediaNode.h:142
agora::rtc::IExtensionVideoFilter::adaptVideoFrame
virtual ProcessResult adaptVideoFrame(agora::agora_refptr< IVideoFrame > in, agora::agora_refptr< IVideoFrame > &out)
SDK will invoke this API to send video frame to the filter if process mode is "Sync".
Definition: NGIAgoraMediaNode.h:345
agora::rtc::IMediaPacketReceiver
Definition: NGIAgoraMediaNode.h:519
agora::rtc::IVideoBeautyFilter::BeautyOptions::LIGHTENING_CONTRAST_LEVEL
LIGHTENING_CONTRAST_LEVEL
Definition: NGIAgoraMediaNode.h:703
agora::rtc::IAudioPcmDataSender::sendAudioPcmData
virtual int sendAudioPcmData(const void *audio_data, uint32_t capture_timestamp, const size_t samples_per_channel, const agora::rtc::BYTES_PER_SAMPLE bytes_per_sample, const size_t number_of_channels, const uint32_t sample_rate)=0
agora::rtc::IVideoRenderer
Definition: NGIAgoraMediaNode.h:770
agora::rtc::IVideoRenderer::setRenderMode
virtual int setRenderMode(media::base::RENDER_MODE_TYPE renderMode)=0
agora::rtc::IExtensionVideoFilter::getVideoFormatWanted
virtual void getVideoFormatWanted(VideoFrameData::Type &type, RawPixelBuffer::Format &format)=0
SDK will invoke this API every time before sending data to the filter. Filter can desigante the type ...
agora::rtc::IExtensionVideoFilter::Control
Definition: NGIAgoraMediaNode.h:262
agora::rtc::IExtensionVideoFilter::pendVideoFrame
virtual ProcessResult pendVideoFrame(agora::agora_refptr< IVideoFrame > frame)
SDK will invoke this API to send video frame to the filter if process mode is "Async"....
Definition: NGIAgoraMediaNode.h:337
agora::rtc::IVideoBeautyFilter::BeautyOptions::rednessLevel
float rednessLevel
Definition: NGIAgoraMediaNode.h:739
agora::rtc::IVideoBeautyFilter::BeautyOptions::LIGHTENING_CONTRAST_LOW
@ LIGHTENING_CONTRAST_LOW
Definition: NGIAgoraMediaNode.h:707
agora::rtc::IAudioSinkBase
Definition: NGIAgoraMediaNode.h:625
agora::rtc::IAudioSinkBase::onAudioFrame
virtual bool onAudioFrame(const media::base::AudioPcmFrame &audioFrame)=0
agora::rtc::IVideoFrameSender
Definition: NGIAgoraMediaNode.h:647
agora::rtc::IExtensionVideoFilter
Definition: NGIAgoraMediaNode.h:242
agora::rtc::IAudioFilter::getPreferredChannelNumbers
virtual int getPreferredChannelNumbers()
Definition: NGIAgoraMediaNode.h:132
agora::rtc::IExtensionVideoFilter::Control::getMemoryPool
virtual agora::agora_refptr< IVideoFrameMemoryPool > getMemoryPool()=0
Filter can invoke this function to get the IVideoFrameMemoryPool object if a new IVideoFrame data obj...
agora::RefCountInterface
Definition: AgoraRefPtr.h:31
agora::rtc::IVideoFilter::getProperty
virtual int getProperty(const char *key, void *buf, size_t buf_size)
Definition: NGIAgoraMediaNode.h:204
agora::rtc::IMediaPacketReceiver::onMediaPacketReceived
virtual bool onMediaPacketReceived(const uint8_t *packet, size_t length, const agora::media::base::PacketOptions &options)=0
agora::rtc::IVideoFilter::isEnabled
virtual bool isEnabled()
Definition: NGIAgoraMediaNode.h:182
agora::rtc::IVideoFilter::setEnabled
virtual void setEnabled(bool enable)
Definition: NGIAgoraMediaNode.h:175
agora::rtc::IMediaPacketSender::sendMediaPacket
virtual int sendMediaPacket(const uint8_t *packet, size_t length, const media::base::PacketOptions &options)=0
agora::rtc::IVideoBeautyFilter::BeautyOptions::smoothnessLevel
float smoothnessLevel
Definition: NGIAgoraMediaNode.h:733
agora::rtc::IExtensionVideoFilter::Control::deliverVideoFrame
virtual ProcessResult deliverVideoFrame(agora::agora_refptr< IVideoFrame > frame)=0
Filter can invoke this function to deliver the processed frame to SDK if the Process Mode is designat...
agora::rtc::IAudioFilter::setEnabled
virtual void setEnabled(bool enable)=0
agora::media::base::AudioEncodedFrameInfo
Definition: AgoraMediaBase.h:325
agora::rtc::IVideoBeautyFilter::setBeautyEffectOptions
virtual int setBeautyEffectOptions(bool enabled, BeautyOptions options)=0
agora::rtc::IVideoBeautyFilter::BeautyOptions::lighteningLevel
float lighteningLevel
Definition: NGIAgoraMediaNode.h:727
agora::rtc::IVideoFilter::isExternal
virtual bool isExternal()
Definition: NGIAgoraMediaNode.h:225
agora::rtc::IAudioFilter::getPreferredSampleRate
virtual int getPreferredSampleRate()
Definition: NGIAgoraMediaNode.h:122
agora::rtc::IMediaControlPacketSender
Definition: NGIAgoraMediaNode.h:588
agora::rtc::IVideoRenderer::setMirror
virtual int setMirror(void *view, bool mirror)=0
agora::rtc::IVideoBeautyFilter::BeautyOptions::lighteningContrastLevel
LIGHTENING_CONTRAST_LEVEL lighteningContrastLevel
Definition: NGIAgoraMediaNode.h:722
agora::rtc::IVideoBeautyFilter::BeautyOptions::LIGHTENING_CONTRAST_NORMAL
@ LIGHTENING_CONTRAST_NORMAL
Definition: NGIAgoraMediaNode.h:711
agora::rtc::IExtensionVideoFilter::Control::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
print log to the SDK.
agora::media::base::AudioPcmFrame
Definition: AgoraMediaBase.h:342
agora::rtc::IAudioFilter::getProperty
virtual int getProperty(const char *key, void *buf, int buf_size) const =0
agora::rtc::IVideoRenderer::setMirror
virtual int setMirror(bool mirror)=0
agora::rtc::IMediaExtensionObserver
Definition: NGIAgoraMediaNode.h:431
agora::rtc::IVideoFilter
Definition: NGIAgoraMediaNode.h:166
agora::rtc::IExtensionVideoFilter::start
virtual int start(agora::agora_refptr< Control > control)=0
SDK will invoke this API before feeding video frame data to the filter. Filter can perform its initia...
agora::rtc::IVideoSinkBase::isExternalSink
virtual bool isExternalSink()
Definition: NGIAgoraMediaNode.h:401
agora::rtc::IVideoBeautyFilter::BeautyOptions
Definition: NGIAgoraMediaNode.h:699
agora::rtc::IAudioFilter::getName
virtual const char * getName() const =0
agora::rtc::IVideoSinkBase::getProperty
virtual int getProperty(const char *key, void *buf, int buf_size)
Definition: NGIAgoraMediaNode.h:391
agora::rtc::IExtensionVideoFilter::Control::postEvent
virtual int postEvent(const char *key, const char *value)=0
Post an event and notify the end users.