NGIAgoraMediaNode.h

Go to the documentation of this file.

#pragma once
#include "AgoraRefPtr.h"
#include "AgoraBase.h"
#include "IAgoraLog.h"
#include "NGIAgoraVideoFrame.h"
#include "AgoraExtensionVersion.h"
 
#ifndef OPTIONAL_PROCESSRESULT_SPECIFIER
#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
#define OPTIONAL_PROCESSRESULT_SPECIFIER ProcessResult::
#else
#define OPTIONAL_PROCESSRESULT_SPECIFIER
#endif
#endif
 
namespace webrtc {
struct AudioFrameHandleInfo;
}  // namespace webrtc
 
namespace agora {
namespace rtc {
class IAudioFilterBase : public RefCountInterface {
 public:
  // Return false if filter decide to drop the frame.
  // Implementation should not change samples_per_channel_/sample_rate_hz_/num_channels_
  // Implementation should just adapt data of audio frame.
  virtual bool adaptAudioFrame(const media::base::AudioPcmFrame& inAudioFrame,
                               media::base::AudioPcmFrame& adaptedFrame) = 0;
 
  virtual int getPreferredSampleRate() = 0;
 
  virtual int getPreferredChannelNumbers() = 0;
 
 protected:
  ~IAudioFilterBase() {}
};
 
class IAudioFilter : public IAudioFilterBase {
 public:
  virtual void setEnabled(bool enable) = 0;
  virtual bool isEnabled() const = 0;
  virtual int setProperty(const char* key, const void* buf, int buf_size) = 0;
  virtual int getProperty(const char* key, void* buf, int buf_size) const = 0;
  virtual const char * getName() const = 0;
 
  virtual int getPreferredSampleRate() { return 0; };
 
  virtual int getPreferredChannelNumbers() { return 0; };
 
 protected:
  ~IAudioFilter() {}
};
 
class IVideoFilterBase : public RefCountInterface {
 public:
  virtual bool adaptVideoFrame(const media::base::VideoFrame& capturedFrame,
                               media::base::VideoFrame& adaptedFrame) = 0;
};
 
// TODO(Bob): private inherit?
class IVideoFilter : public IVideoFilterBase {
 public:
  virtual void setEnabled(bool enable) {}
  virtual bool isEnabled() { return true; }
  virtual int setProperty(const char* key, const void* buf, size_t buf_size) { return -1; }
  virtual int getProperty(const char* key, void* buf, size_t buf_size) { return -1; }
  virtual bool onDataStreamWillStart() { return true; }
  virtual void onDataStreamWillStop() { }
  virtual bool isExternal() { return true; }
  virtual bool isExtensionFilter() { return false; }
};
 
class IExtensionVideoFilter : public IVideoFilter {
 public:
  enum ProcessMode {
    kSync, // Indicates that video frame data will be exchanged via "adaptVideoFrame"
    kAsync, // Indicates that video frame data will be exchanged via "pendVideoFrame" & "deliverVideoFrame"
  };
 
  enum ProcessResult {
    kSuccess, // Video frame data is successfully processed
    kBypass,  // Video frame data should bypass the current filter and flow to its successsors
    kDrop, // Video Frame data should be discarded
  };
 
  class Control : public RefCountInterface {
   public:
    virtual ProcessResult deliverVideoFrame(agora::agora_refptr<IVideoFrame> frame) = 0;
    virtual agora::agora_refptr<IVideoFrameMemoryPool> getMemoryPool() = 0;
    virtual int postEvent(const char* key, const char* value) = 0;
    virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
    virtual void disableMe(int error, const char* msg) = 0;
  };
 
  virtual void getProcessMode(ProcessMode& mode, bool& independent_thread) = 0;
  virtual int start(agora::agora_refptr<Control> control) = 0;
  virtual int stop() = 0;
  virtual void getVideoFormatWanted(VideoFrameData::Type& type, RawPixelBuffer::Format& format) = 0;
  virtual ProcessResult pendVideoFrame(agora::agora_refptr<IVideoFrame> frame) {
    return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
  }
  virtual ProcessResult adaptVideoFrame(agora::agora_refptr<IVideoFrame> in, agora::agora_refptr<IVideoFrame>& out) {
    return OPTIONAL_PROCESSRESULT_SPECIFIER kBypass;
  }
  /* Occurs each time needs to get rotation apply.
   *
   * @return Determines whether to rotate.
   * - true: need to rotate.
   * - false: no rotate.
   */
  virtual bool getRotationApplied() { return false; }
 
  // NOTE: The following two interfaces should never be overriden!
  virtual bool isExtensionFilter() { return true; }
  virtual bool adaptVideoFrame(
      const media::base::VideoFrame& capturedFrame,
      media::base::VideoFrame& adaptedFrame) {
    return -ERR_NOT_SUPPORTED;
  }
};
 
class ILipSyncFilter : public RefCountInterface {
 public:
  enum ProcessResult {
    kSuccess, // Video frame data is successfully processed
    kBypass,  // Video frame data should bypass the current filter and flow to its successsors
    kDrop, // Video Frame data should be discarded
  };
  
  class Control : public RefCountInterface {
   public:
    virtual int postEvent(const char* key, const char* value) = 0;
    virtual void printLog(commons::LOG_LEVEL level, const char* format, ...) = 0;
    virtual void disableMe(int error, const char* msg) = 0;
    virtual void ReportCounter(int32_t counter_id, int32_t value) = 0;
    virtual int GetStats(int32_t counter_id) = 0;
  };
  
  virtual int start(agora::agora_refptr<Control> control) = 0;
  
  virtual int stop() = 0;
  
  virtual int setProperty(const char* key, const void* buf, size_t buf_size) { return -1; }
  virtual ProcessResult convertAudioFrameToFaceInfo(const agora::media::base::AudioPcmFrame& inAudioFrame, char* outFaceInfo) {
    return kBypass;
  }
};
 
class IVideoSinkBase : public RefCountInterface {
 public:
  virtual int setProperty(const char* key, const void* buf, int buf_size) { return -1; }
  virtual int getProperty(const char* key, void* buf, int buf_size) { return -1; }
  virtual int onFrame(const media::base::VideoFrame& videoFrame) = 0;
  virtual bool isExternalSink() { return true; }
  virtual bool onDataStreamWillStart() { return true; }
  virtual void onDataStreamWillStop() { }
  virtual bool applyMirror() { return false; }
  virtual bool applyRotation() { return false; }
};
 
class IMediaExtensionObserver : public RefCountInterface {
public:
  virtual ~IMediaExtensionObserver() {}
  virtual void onEvent(const char* provider, const char* extension, const char* key, const char* json_value) {}
  virtual void onExtensionStopped(const char* provider, const char* extension) {}
  virtual void onExtensionStarted(const char* provider, const char* extension) {}
  virtual void onExtensionError(const char* provider, const char* extension, int error, const char* message) {}
};
 
class IAudioPcmDataSender : public RefCountInterface {
 public:
  virtual int sendAudioPcmData(
      const void* audio_data, uint32_t capture_timestamp, int64_t presentation_ms,
      const size_t samples_per_channel,  // for 10ms Data, number_of_samples * 100 = sample_rate
      const agora::rtc::BYTES_PER_SAMPLE bytes_per_sample,     // 2
      const size_t number_of_channels,
      const uint32_t sample_rate) = 0; // sample_rate > 8000)
 
 protected:
  ~IAudioPcmDataSender() {}
};
 
class IAudioEncodedFrameSender : public RefCountInterface {
 public:
  virtual bool sendEncodedAudioFrame(const uint8_t* payload_data, size_t payload_size,
                                     const EncodedAudioFrameInfo& audioFrameInfo) = 0;
 
 protected:
  ~IAudioEncodedFrameSender() {}
};
 
class IAudioEncodedFrameReceiver {
 public:
  virtual bool onEncodedAudioFrameReceived (
      const uint8_t *packet, size_t length, const media::base::AudioEncodedFrameInfo& info) = 0;
 
  virtual ~IAudioEncodedFrameReceiver() {}
};
 
class IMediaPacketReceiver {
 public:
  virtual bool onMediaPacketReceived(
      const uint8_t *packet, size_t length, const agora::media::base::PacketOptions& options) = 0;
 
  virtual ~IMediaPacketReceiver() {}
};
 
class IMediaControlPacketReceiver {
 public:
  virtual bool onMediaControlPacketReceived(uid_t uid, const uint8_t *packet, size_t length) = 0;
 
  virtual ~IMediaControlPacketReceiver() {}
};
 
class IMediaPacketSender : public RefCountInterface {
 public:
  virtual int sendMediaPacket(const uint8_t *packet, size_t length,
                              const media::base::PacketOptions &options) = 0;
 protected:
  ~IMediaPacketSender() {}
};
 
class IMediaControlPacketSender {
 public:
  virtual int sendPeerMediaControlPacket(media::base::user_id_t userId,
                                         const uint8_t *packet,
                                         size_t length) = 0;
 
  virtual int sendBroadcastMediaControlPacket(const uint8_t *packet, size_t length) = 0;
 
  virtual ~IMediaControlPacketSender() {}
};
 
class IAudioSinkBase : public RefCountInterface {
 public:
  virtual bool onAudioFrame(const media::base::AudioPcmFrame& audioFrame) = 0;
 
 protected:
  ~IAudioSinkBase() {}
};
 
class IVideoFrameSender : public RefCountInterface {
 public:
  virtual int sendVideoFrame(const media::base::ExternalVideoFrame& videoFrame) = 0;
 
 protected:
  ~IVideoFrameSender() {}
};
 
class IVideoEncodedImageSender : public RefCountInterface {
 public:
  virtual bool sendEncodedVideoImage(const uint8_t* imageBuffer, size_t length,
                                     const EncodedVideoFrameInfo& videoEncodedFrameInfo) = 0;
 
 protected:
  ~IVideoEncodedImageSender() {}
};
 
class IVideoBeautyFilter : public IVideoFilter {
 public:
  struct BeautyOptions {
    enum LIGHTENING_CONTRAST_LEVEL {
      LIGHTENING_CONTRAST_LOW = 0,
      LIGHTENING_CONTRAST_NORMAL,
      LIGHTENING_CONTRAST_HIGH
    };
 
    LIGHTENING_CONTRAST_LEVEL lighteningContrastLevel;
 
    float lighteningLevel;
 
    float smoothnessLevel;
 
    float rednessLevel;
 
    BeautyOptions(LIGHTENING_CONTRAST_LEVEL contrastLevel, float lightening, float smoothness,
                  float redness)
        : lighteningContrastLevel(contrastLevel),
          lighteningLevel(lightening),
          smoothnessLevel(smoothness),
          rednessLevel(redness) {}
 
    BeautyOptions()
        : lighteningContrastLevel(LIGHTENING_CONTRAST_NORMAL),
          lighteningLevel(0),
          smoothnessLevel(0),
          rednessLevel(0) {}
  };
  virtual int setBeautyEffectOptions(bool enabled, BeautyOptions options) = 0;
};
 
class IVideoRenderer : public IVideoSinkBase {
 public:
  virtual int setRenderMode(media::base::RENDER_MODE_TYPE renderMode) = 0;
  virtual int setRenderMode(void* view, media::base::RENDER_MODE_TYPE renderMode) = 0;
 
  virtual int setRotation(agora::rtc::VIDEO_ORIENTATION rotation) = 0;
  virtual int setRotation(void* view, agora::rtc::VIDEO_ORIENTATION rotation) = 0;
 
  virtual int setMirror(bool mirror) = 0;
  virtual int setMirror(void* view, bool mirror) = 0;
  virtual int setView(void* view) = 0;
  virtual int addView(void* view, const Rectangle& cropArea) = 0;
  virtual int unsetView() = 0;
  virtual int removeView(void* view) = 0;
};
 
static const int kDeviceIdSize = 128;
 
class IVideoTrack;
class IVideoFrameTransceiver : public RefCountInterface {
 public:
  virtual int getTranscodingDelayMs() = 0;
  virtual int addVideoTrack(agora_refptr<IVideoTrack> track) = 0;
  virtual int removeVideoTrack(agora_refptr<IVideoTrack> track) = 0;
};
 
}
}