Agora C++ API Reference for All Platforms

Loading...
Searching...
No Matches
AgoraExtensions.h
1#pragma once
2#include "NGIAgoraMediaNode.h"
3
4namespace agora {
5 namespace rtc {
12 class IAudioFilter : public IAudioFilterBase {
13 public:
20 virtual void setEnabled(bool enable) = 0;
27 virtual bool isEnabled() const = 0;
38 virtual int setProperty(const char* key, const void* buf, int buf_size) = 0;
49 virtual int getProperty(const char* key, void* buf, int buf_size) const = 0;
57 virtual const char * getName() const = 0;
58
67 virtual int getPreferredSampleRate() { return 0; };
68
77 virtual int getPreferredChannelNumbers() { return 0; };
78
79 protected:
80 ~IAudioFilter() {}
81};
82
83 class IAudioFilterV2 : public IAudioFilter {
84public:
85 class Control : public RefCountInterface {
86 public:
92 virtual int postEvent(const char* key, const char* value) = 0;
99 virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
100 };
101public:
106 virtual void setExtensionControl(agora::agora_refptr<IAudioFilterV2::Control> control) = 0;
107};
108
114 class IExtensionVideoFilter : public IVideoFilter {
115 public:
116 enum ProcessMode {
117 kSync, // Indicates that video frame data will be exchanged via "adaptVideoFrame"
118 kAsync, // Indicates that video frame data will be exchanged via "pendVideoFrame" & "deliverVideoFrame"
119 };
120
121 enum ProcessResult {
122 kSuccess, // Video frame data is successfully processed
123 kBypass, // Video frame data should bypass the current filter and flow to its successsors
124 kDrop, // Video Frame data should be discarded
125 };
126
134 class Control : public RefCountInterface {
135 public:
142 virtual ProcessResult deliverVideoFrame(agora::agora_refptr<IVideoFrame> frame) = 0;
147 virtual agora::agora_refptr<IVideoFrameMemoryPool> getMemoryPool() = 0;
153 virtual int postEvent(const char* key, const char* value) = 0;
160 virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
166 virtual void disableMe(int error, const char* msg) = 0;
167 };
168
176 virtual void getProcessMode(ProcessMode& mode, bool& independent_thread) = 0;
177
185 virtual int start(agora::agora_refptr<Control> control) = 0;
191 virtual int stop() = 0;
200 virtual void getVideoFormatWanted(VideoFrameData::Type& type, RawPixelBuffer::Format& format) = 0;
207 virtual ProcessResult pendVideoFrame(agora::agora_refptr<IVideoFrame> frame) {
208 return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
209 }
210
215 virtual ProcessResult adaptVideoFrame(agora::agora_refptr<IVideoFrame> in, agora::agora_refptr<IVideoFrame>& out) {
216 return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
217 }
218 /* Occurs each time needs to get rotation apply.
219 *
220 * @return Determines whether to rotate.
221 * - true: need to rotate.
222 * - false: no rotate.
223 */
224 virtual bool getRotationApplied() { return false; }
225
226 // NOTE: The following two interfaces should never be overriden!
227 virtual bool isExtensionFilter() { return true; }
228 virtual bool adaptVideoFrame(
229 const media::base::VideoFrame& capturedFrame,
230 media::base::VideoFrame& adaptedFrame) {
231 return -ERR_NOT_SUPPORTED;
232 }
233};
234
235 class IExtensionVideoSink : public IVideoSinkBase {
236};
237
238 class ILipSyncFilter : public RefCountInterface {
239 public:
240 enum ProcessResult {
241 kSuccess, // Video frame data is successfully processed
242 kBypass, // Video frame data should bypass the current filter and flow to its successsors
243 kDrop, // Video Frame data should be discarded
244 };
245
246 class Control : public RefCountInterface {
247 public:
253 virtual int postEvent(const char* key, const char* value) = 0;
260 virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
266 virtual void disableMe(int error, const char* msg) = 0;
272 virtual void ReportCounter(int32_t counter_id, int32_t value) = 0;
277 virtual int GetStats(int32_t counter_id) = 0;
278 };
279
280 virtual int start(agora::agora_refptr<Control> control) = 0;
281
282 virtual int stop() = 0;
283
284 virtual int setProperty(const char* key, const void* buf, size_t buf_size) { return -1; }
291 virtual ProcessResult convertAudioFrameToFaceInfo(const agora::media::base::AudioPcmFrame& inAudioFrame, char* outFaceInfo) {
292 return kBypass;
293 }
294};
295
296 class ExtensionVideoCodecSettings {
297 public:
298 // Structure to hold settings for the video encoder.
299 struct VideoEncoderSettings {
300 uint32_t framerate = 0; // The frame rate of the video in frames per second (fps).
301 uint32_t bitrate_kbps = 0; // The bitrate of the video in kilobits per second (kbps).
302 uint32_t key_frame_interval = 0; // The interval between key frames in seconds (time between I-frames).
303 };
304 // Structure to hold settings for the video decoder.
305 struct VideoDecoderSettings {
306 media::base::VIDEO_PIXEL_FORMAT prefer_data_format; // The preferred pixel format for the decoded video.
307 };
308
309 uint16_t width = 0; // The width of the video frame in pixels.
310 uint16_t height = 0; // The height of the video frame in pixels.
311 VideoEncoderSettings encoder_settings; // An instance of VideoEncoderSettings to configure the encoder.
312 VideoDecoderSettings decoder_settings; // An instance of VideoDecoderSettings to configure the decoder.
313};
314
315 class ExtensionEncodedImage {
316public:
317 size_t length = 0; // The length of the encoded image data (in bytes).
318 uint8_t* buffer = nullptr; // Pointer to the buffer holding the encoded image data (raw bytes).
319 VIDEO_FRAME_TYPE frame_type = VIDEO_FRAME_TYPE_DELTA_FRAME; // The type of the video frame (e.g., key frame, delta frame).
320 uint32_t pts_ms = 0; // Presentation timestamp (PTS) in milliseconds, indicating when the frame should be displayed.
321 uint32_t dts_ms = 0; // Decoding timestamp (DTS) in milliseconds, indicating when the frame should be decoded.
322 uint32_t width = 0; // The width of the video frame in pixels.
323 uint32_t height = 0; // The height of the video frame in pixels.
324};
325
326 class IExtensionDecoderCallback {
327 public:
328 // Destructor for the IExtensionDecoderCallback class.
329 virtual ~IExtensionDecoderCallback() {}
335 virtual void onDecodedFrame(agora_refptr<IVideoFrame> frame, const char* extended_info_json) = 0;
342 virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
343};
344
345 class IExtensionEncoderCallback {
346 public:
347 // Destructor for the IExtensionEncoderCallback class.
348 virtual ~IExtensionEncoderCallback() {}
354 virtual void onEncodedImage(const ExtensionEncodedImage* img, const char* extended_info_json) = 0;
361 virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
362};
363
364 class IExtensionVideoEncoder : public RefCountInterface {
365 public:
373 virtual int initVideoEncoder(ExtensionVideoCodecSettings codec_settings) = 0;
381 virtual int32_t registerEncoderCallback(IExtensionEncoderCallback* callback) = 0;
390 virtual int encode(agora_refptr<IVideoFrame> frame, VIDEO_FRAME_TYPE frame_type) = 0;
397 virtual int32_t releaseVideoEncoder() = 0;
406 virtual int32_t setRates(uint32_t bitrate_kbps, uint32_t framerate) = 0;
412 virtual void querySupportCodecDataFormat(media::base::VIDEO_PIXEL_FORMAT* data_format, int& size) = 0;
423 virtual int setAdvancedCodecProperty(const char* key, const char* json_value) = 0;
434 virtual int getAdvancedCodecProperty(const char* key, char* json_value, int& length) const = 0;
435};
436
437 class IExtensionVideoDecoder : public RefCountInterface {
438 public:
446 virtual int initVideoDecoder(const ExtensionVideoCodecSettings& codec_settings) = 0;
454 virtual int decode(const ExtensionEncodedImage& data) = 0;
462 virtual int32_t registerDecoderCallback(IExtensionDecoderCallback* callback) = 0;
469 virtual int32_t releaseVideoDecoder() = 0;
475 virtual void querySupportCodecDataFormat(media::base::VIDEO_PIXEL_FORMAT* data_format, int& size) = 0;
476
487 virtual int setAdvancedCodecProperty(const char* key, const char* json_value) = 0;
498 virtual int getAdvancedCodecProperty(const char* key, char* json_value, int& length) const = 0;
499};
500}
501}
agora::RefCountInterface
Definition AgoraRefPtr.h:31
agora::agora_refptr
Definition AgoraRefPtr.h:44
agora::rtc::ExtensionEncodedImage
Definition AgoraExtensions.h:315
agora::rtc::ExtensionEncodedImage::length
size_t length
Definition AgoraExtensions.h:317
agora::rtc::ExtensionEncodedImage::width
uint32_t width
Definition AgoraExtensions.h:322
agora::rtc::ExtensionEncodedImage::height
uint32_t height
Definition AgoraExtensions.h:323
agora::rtc::ExtensionEncodedImage::buffer
uint8_t * buffer
Definition AgoraExtensions.h:318
agora::rtc::ExtensionEncodedImage::dts_ms
uint32_t dts_ms
Definition AgoraExtensions.h:321
agora::rtc::ExtensionEncodedImage::pts_ms
uint32_t pts_ms
Definition AgoraExtensions.h:320
agora::rtc::ExtensionEncodedImage::frame_type
VIDEO_FRAME_TYPE frame_type
Definition AgoraExtensions.h:319
agora::rtc::ExtensionVideoCodecSettings
Definition AgoraExtensions.h:296
agora::rtc::ExtensionVideoCodecSettings::width
uint16_t width
Definition AgoraExtensions.h:309
agora::rtc::ExtensionVideoCodecSettings::height
uint16_t height
Definition AgoraExtensions.h:310
agora::rtc::ExtensionVideoCodecSettings::encoder_settings
VideoEncoderSettings encoder_settings
Definition AgoraExtensions.h:311
agora::rtc::ExtensionVideoCodecSettings::decoder_settings
VideoDecoderSettings decoder_settings
Definition AgoraExtensions.h:312
agora::rtc::IAudioFilterBase
Definition NGIAgoraMediaNode.h:27
agora::rtc::IAudioFilter
Definition AgoraExtensions.h:12
agora::rtc::IAudioFilter::setEnabled
virtual void setEnabled(bool enable)=0
agora::rtc::IAudioFilter::getName
virtual const char * getName() const =0
agora::rtc::IAudioFilter::getPreferredSampleRate
virtual int getPreferredSampleRate()
Definition AgoraExtensions.h:67
agora::rtc::IAudioFilter::setProperty
virtual int setProperty(const char *key, const void *buf, int buf_size)=0
agora::rtc::IAudioFilter::isEnabled
virtual bool isEnabled() const =0
agora::rtc::IAudioFilter::getPreferredChannelNumbers
virtual int getPreferredChannelNumbers()
Definition AgoraExtensions.h:77
agora::rtc::IAudioFilter::getProperty
virtual int getProperty(const char *key, void *buf, int buf_size) const =0
agora::rtc::IAudioFilter::~IAudioFilter
~IAudioFilter()
Definition AgoraExtensions.h:80
agora::rtc::IAudioFilterV2::Control
Definition AgoraExtensions.h:85
agora::rtc::IAudioFilterV2::Control::postEvent
virtual int postEvent(const char *key, const char *value)=0
Post an event and notify the end users.
agora::rtc::IAudioFilterV2::Control::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
print log to the SDK.
agora::rtc::IAudioFilterV2
Definition AgoraExtensions.h:83
agora::rtc::IAudioFilterV2::setExtensionControl
virtual void setExtensionControl(agora::agora_refptr< IAudioFilterV2::Control > control)=0
AgoraSDK set IAudioFilterV2::Control to filter.
agora::rtc::IExtensionDecoderCallback
Definition AgoraExtensions.h:326
agora::rtc::IExtensionDecoderCallback::onDecodedFrame
virtual void onDecodedFrame(agora_refptr< IVideoFrame > frame, const char *extended_info_json)=0
agora::rtc::IExtensionDecoderCallback::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
agora::rtc::IExtensionDecoderCallback::~IExtensionDecoderCallback
virtual ~IExtensionDecoderCallback()
Definition AgoraExtensions.h:329
agora::rtc::IExtensionEncoderCallback
Definition AgoraExtensions.h:345
agora::rtc::IExtensionEncoderCallback::~IExtensionEncoderCallback
virtual ~IExtensionEncoderCallback()
Definition AgoraExtensions.h:348
agora::rtc::IExtensionEncoderCallback::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
agora::rtc::IExtensionEncoderCallback::onEncodedImage
virtual void onEncodedImage(const ExtensionEncodedImage *img, const char *extended_info_json)=0
agora::rtc::IExtensionVideoDecoder
Definition AgoraExtensions.h:437
agora::rtc::IExtensionVideoDecoder::getAdvancedCodecProperty
virtual int getAdvancedCodecProperty(const char *key, char *json_value, int &length) const =0
agora::rtc::IExtensionVideoDecoder::releaseVideoDecoder
virtual int32_t releaseVideoDecoder()=0
agora::rtc::IExtensionVideoDecoder::initVideoDecoder
virtual int initVideoDecoder(const ExtensionVideoCodecSettings &codec_settings)=0
agora::rtc::IExtensionVideoDecoder::setAdvancedCodecProperty
virtual int setAdvancedCodecProperty(const char *key, const char *json_value)=0
agora::rtc::IExtensionVideoDecoder::decode
virtual int decode(const ExtensionEncodedImage &data)=0
agora::rtc::IExtensionVideoDecoder::registerDecoderCallback
virtual int32_t registerDecoderCallback(IExtensionDecoderCallback *callback)=0
agora::rtc::IExtensionVideoDecoder::querySupportCodecDataFormat
virtual void querySupportCodecDataFormat(media::base::VIDEO_PIXEL_FORMAT *data_format, int &size)=0
agora::rtc::IExtensionVideoEncoder
Definition AgoraExtensions.h:364
agora::rtc::IExtensionVideoEncoder::releaseVideoEncoder
virtual int32_t releaseVideoEncoder()=0
agora::rtc::IExtensionVideoEncoder::encode
virtual int encode(agora_refptr< IVideoFrame > frame, VIDEO_FRAME_TYPE frame_type)=0
agora::rtc::IExtensionVideoEncoder::querySupportCodecDataFormat
virtual void querySupportCodecDataFormat(media::base::VIDEO_PIXEL_FORMAT *data_format, int &size)=0
agora::rtc::IExtensionVideoEncoder::initVideoEncoder
virtual int initVideoEncoder(ExtensionVideoCodecSettings codec_settings)=0
agora::rtc::IExtensionVideoEncoder::registerEncoderCallback
virtual int32_t registerEncoderCallback(IExtensionEncoderCallback *callback)=0
agora::rtc::IExtensionVideoEncoder::getAdvancedCodecProperty
virtual int getAdvancedCodecProperty(const char *key, char *json_value, int &length) const =0
agora::rtc::IExtensionVideoEncoder::setAdvancedCodecProperty
virtual int setAdvancedCodecProperty(const char *key, const char *json_value)=0
agora::rtc::IExtensionVideoEncoder::setRates
virtual int32_t setRates(uint32_t bitrate_kbps, uint32_t framerate)=0
agora::rtc::IExtensionVideoFilter::Control
Definition AgoraExtensions.h:134
agora::rtc::IExtensionVideoFilter::Control::disableMe
virtual void disableMe(int error, const char *msg)=0
Ask SDK to disable the current filter if a fatal error is detected.
agora::rtc::IExtensionVideoFilter::Control::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
print log to the SDK.
agora::rtc::IExtensionVideoFilter::Control::getMemoryPool
virtual agora::agora_refptr< IVideoFrameMemoryPool > getMemoryPool()=0
Filter can invoke this function to get the IVideoFrameMemoryPool object if a new IVideoFrame data obj...
agora::rtc::IExtensionVideoFilter::Control::deliverVideoFrame
virtual ProcessResult deliverVideoFrame(agora::agora_refptr< IVideoFrame > frame)=0
Filter can invoke this function to deliver the processed frame to SDK if the Process Mode is designat...
agora::rtc::IExtensionVideoFilter::Control::postEvent
virtual int postEvent(const char *key, const char *value)=0
Post an event and notify the end users.
agora::rtc::IExtensionVideoFilter
Definition AgoraExtensions.h:114
agora::rtc::IExtensionVideoFilter::adaptVideoFrame
virtual bool adaptVideoFrame(const media::base::VideoFrame &capturedFrame, media::base::VideoFrame &adaptedFrame)
Definition AgoraExtensions.h:228
agora::rtc::IExtensionVideoFilter::getVideoFormatWanted
virtual void getVideoFormatWanted(VideoFrameData::Type &type, RawPixelBuffer::Format &format)=0
SDK will invoke this API every time before sending data to the filter. Filter can desigante the type ...
agora::rtc::IExtensionVideoFilter::isExtensionFilter
virtual bool isExtensionFilter()
Definition AgoraExtensions.h:227
agora::rtc::IExtensionVideoFilter::adaptVideoFrame
virtual ProcessResult adaptVideoFrame(agora::agora_refptr< IVideoFrame > in, agora::agora_refptr< IVideoFrame > &out)
SDK will invoke this API to send video frame to the filter if process mode is "Sync".
Definition AgoraExtensions.h:215
agora::rtc::IExtensionVideoFilter::pendVideoFrame
virtual ProcessResult pendVideoFrame(agora::agora_refptr< IVideoFrame > frame)
SDK will invoke this API to send video frame to the filter if process mode is "Async"....
Definition AgoraExtensions.h:207
agora::rtc::IExtensionVideoFilter::ProcessMode
ProcessMode
Definition AgoraExtensions.h:116
agora::rtc::IExtensionVideoFilter::kAsync
@ kAsync
Definition AgoraExtensions.h:118
agora::rtc::IExtensionVideoFilter::kSync
@ kSync
Definition AgoraExtensions.h:117
agora::rtc::IExtensionVideoFilter::stop
virtual int stop()=0
SDK will invoke this API when the data stream is about to stop. Filter can perform cleanup jobs in th...
agora::rtc::IExtensionVideoFilter::start
virtual int start(agora::agora_refptr< Control > control)=0
SDK will invoke this API before feeding video frame data to the filter. Filter can perform its initia...
agora::rtc::IExtensionVideoFilter::getRotationApplied
virtual bool getRotationApplied()
Definition AgoraExtensions.h:224
agora::rtc::IExtensionVideoFilter::ProcessResult
ProcessResult
Definition AgoraExtensions.h:121
agora::rtc::IExtensionVideoFilter::kSuccess
@ kSuccess
Definition AgoraExtensions.h:122
agora::rtc::IExtensionVideoFilter::kDrop
@ kDrop
Definition AgoraExtensions.h:124
agora::rtc::IExtensionVideoFilter::kBypass
@ kBypass
Definition AgoraExtensions.h:123
agora::rtc::IExtensionVideoFilter::getProcessMode
virtual void getProcessMode(ProcessMode &mode, bool &independent_thread)=0
SDK will invoke this API first to get the filter's requested process mode ProcessMode.
agora::rtc::IExtensionVideoSink
Definition AgoraExtensions.h:235
agora::rtc::ILipSyncFilter::Control
Definition AgoraExtensions.h:246
agora::rtc::ILipSyncFilter::Control::disableMe
virtual void disableMe(int error, const char *msg)=0
Ask SDK to disable the current filter if a fatal error is detected.
agora::rtc::ILipSyncFilter::Control::ReportCounter
virtual void ReportCounter(int32_t counter_id, int32_t value)=0
report counter to the SDK.
agora::rtc::ILipSyncFilter::Control::postEvent
virtual int postEvent(const char *key, const char *value)=0
Post an event and notify the end users.
agora::rtc::ILipSyncFilter::Control::printLog
virtual void printLog(commons::LOG_LEVEL level, const char *format,...)=0
print log to the SDK.
agora::rtc::ILipSyncFilter::Control::GetStats
virtual int GetStats(int32_t counter_id)=0
get stats to the SDK.
agora::rtc::ILipSyncFilter
Definition AgoraExtensions.h:238
agora::rtc::ILipSyncFilter::convertAudioFrameToFaceInfo
virtual ProcessResult convertAudioFrameToFaceInfo(const agora::media::base::AudioPcmFrame &inAudioFrame, char *outFaceInfo)
Definition AgoraExtensions.h:291
agora::rtc::ILipSyncFilter::start
virtual int start(agora::agora_refptr< Control > control)=0
agora::rtc::ILipSyncFilter::stop
virtual int stop()=0
agora::rtc::ILipSyncFilter::ProcessResult
ProcessResult
Definition AgoraExtensions.h:240
agora::rtc::ILipSyncFilter::kDrop
@ kDrop
Definition AgoraExtensions.h:243
agora::rtc::ILipSyncFilter::kSuccess
@ kSuccess
Definition AgoraExtensions.h:241
agora::rtc::ILipSyncFilter::kBypass
@ kBypass
Definition AgoraExtensions.h:242
agora::rtc::ILipSyncFilter::setProperty
virtual int setProperty(const char *key, const void *buf, size_t buf_size)
Definition AgoraExtensions.h:284
agora::rtc::IVideoFilter
Definition NGIAgoraMediaNode.h:94
agora::rtc::IVideoSinkBase
Definition NGIAgoraMediaNode.h:167
agora::commons::LOG_LEVEL
OPTIONAL_ENUM_CLASS LOG_LEVEL
The output log level of the SDK.
Definition IAgoraLog.h:33
agora::media::base::VIDEO_PIXEL_FORMAT
VIDEO_PIXEL_FORMAT
The video pixel format.
Definition AgoraMediaBase.h:519
agora::rtc::VIDEO_FRAME_TYPE
VIDEO_FRAME_TYPE
The video frame type.
Definition AgoraBase.h:1066
agora::rtc::VIDEO_FRAME_TYPE_DELTA_FRAME
@ VIDEO_FRAME_TYPE_DELTA_FRAME
Definition AgoraBase.h:1078
agora
Definition AgoraAtomicOps.h:21
agora::ERR_NOT_SUPPORTED
@ ERR_NOT_SUPPORTED
Definition AgoraBase.h:488
rtc
Definition video_node_i.h:28
agora::media::base::AudioPcmFrame
The parameters of the audio frame in PCM format.
Definition AgoraMediaBase.h:387
agora::media::base::VideoFrame
Configurations of the video frame.
Definition AgoraMediaBase.h:1024
agora::rtc::ExtensionVideoCodecSettings::VideoDecoderSettings
Definition AgoraExtensions.h:305
agora::rtc::ExtensionVideoCodecSettings::VideoDecoderSettings::prefer_data_format
media::base::VIDEO_PIXEL_FORMAT prefer_data_format
Definition AgoraExtensions.h:306
agora::rtc::ExtensionVideoCodecSettings::VideoEncoderSettings
Definition AgoraExtensions.h:299
agora::rtc::ExtensionVideoCodecSettings::VideoEncoderSettings::framerate
uint32_t framerate
Definition AgoraExtensions.h:300
agora::rtc::ExtensionVideoCodecSettings::VideoEncoderSettings::bitrate_kbps
uint32_t bitrate_kbps
Definition AgoraExtensions.h:301
agora::rtc::ExtensionVideoCodecSettings::VideoEncoderSettings::key_frame_interval
uint32_t key_frame_interval
Definition AgoraExtensions.h:302
agora::rtc::RawPixelBuffer::Format
OPTIONAL_ENUM_CLASS Format
Definition NGIAgoraVideoFrame.h:29
agora::rtc::VideoFrameData::Type
OPTIONAL_ENUM_CLASS Type
Definition NGIAgoraVideoFrame.h:94