NGIAgoraLocalUser.h

Go to the documentation of this file.

//
//  Agora SDK
//
//  Created by Zheng Ender in 2018-01.
//  Copyright (c) 2018 Agora.io. All rights reserved.
//
 
#pragma once  // NOLINT(build/header_guard)
 
#include <cstring>
#include <vector>
#include "AgoraBase.h"
#include "AgoraOptional.h"
 
namespace agora {
namespace media {
class IAudioFrameObserver;
}
 
class ILocalDataChannel;
class IDataChannelObserver;
 
namespace rtc {
class IAudioEngineWrapper;
 
class ILocalUserObserver;
class ILocalVideoTrack;
class IRemoteVideoTrack;
class IVideoFrameObserver2;
 
class IMediaControlPacketSender;
class IMediaControlPacketReceiver;
struct AudioEncoderConfiguration;
struct VideoEncoderConfiguration;
 
class ILocalAudioTrack;
class IAudioMixerSource;
struct RemoteAudioTrackStats;
class IRemoteAudioTrack;
struct LocalVideoTrackStats;
struct RemoteVideoTrackStats;
class IMediaPacketReceiver;
class IVideoSinkBase;
 
class ILocalUser {
 public:
  struct ANAStats {
    agora::Optional<uint32_t> bitrate_action_counter;
    agora::Optional<uint32_t> channel_action_counter;
    agora::Optional<uint32_t> dtx_action_counter;
    agora::Optional<uint32_t> fec_action_counter;
    agora::Optional<uint32_t> frame_length_increase_counter;
    agora::Optional<uint32_t> frame_length_decrease_counter;
    agora::Optional<float> uplink_packet_loss_fraction;
  };
 
  struct AudioProcessingStats {
    agora::Optional<double> echo_return_loss;
    //
    agora::Optional<double> echo_return_loss_enhancement;
    agora::Optional<double> divergent_filter_fraction;
 
    agora::Optional<int32_t> delay_median_ms;
    agora::Optional<int32_t> delay_standard_deviation_ms;
 
    agora::Optional<double> residual_echo_likelihood;
    agora::Optional<double> residual_echo_likelihood_recent_max;
 
    agora::Optional<int32_t> delay_ms;
  };
    
  struct LocalAudioDetailedStats {
    uint32_t local_ssrc;
    int64_t bytes_sent;
    int32_t packets_sent;
    int32_t packets_lost;
    float fraction_lost;
    char codec_name[media::base::kMaxCodecNameLength];
    agora::Optional<int> codec_payload_type;
    int32_t ext_seqnum;
    int32_t jitter_ms;
    int64_t rtt_ms;
    int32_t audio_level;
    double total_input_energy;
    double total_input_duration;
    bool typing_noise_detected;
 
    ANAStats ana_statistics;
    AudioProcessingStats apm_statistics;
 
    LocalAudioDetailedStats() : local_ssrc(0), bytes_sent(0), packets_sent(0), packets_lost(-1), fraction_lost(-1.0f),
                                ext_seqnum(-1), jitter_ms(-1), rtt_ms(-1), audio_level(-1),
                                total_input_energy(0.0), total_input_duration(0.0), typing_noise_detected(false) {
      memset(codec_name, 0, sizeof(codec_name));
    }
  };
    
  enum NS_MODE {
      ElderNsStatistical = 0,    /* Elder Statistical Noise Suppression.*/
      NsNGStatistical = 1,  /* Next Generation Statistical Noise Suppression.*/
      NsNG = 2 /* Next Generation Noise Suppression.*/
  };
  enum NS_LEVEL {
      Soft = 0,/* Soft Noise Suppression.*/
      Aggressive = 1 /* Aggressiveness Noise Suppression.*/
  };
  enum NS_DELAY {
      HighQuality = 0,/* High Audio Quality with High Delay.*/
      Balance = 1,/* Balanced Audio Quality and Delay.*/
      LowDelay = 2/* Slight Low Audio Quality with Low Delay.*/
  };
 
 public:
  virtual ~ILocalUser() {}
 
  virtual void setUserRole(rtc::CLIENT_ROLE_TYPE role) = 0;
 
  virtual CLIENT_ROLE_TYPE getUserRole() = 0;
 
 
  virtual void setAudienceLatencyLevel(AUDIENCE_LATENCY_LEVEL_TYPE level) = 0;
 
  virtual AUDIENCE_LATENCY_LEVEL_TYPE getAudienceLatencyLevel() = 0;
 
  virtual int setAudioEncoderConfiguration(const rtc::AudioEncoderConfiguration& config) = 0;
 
  virtual int setAudioScenario(AUDIO_SCENARIO_TYPE scenario) = 0;
 
  virtual int setVideoScenario(VIDEO_APPLICATION_SCENARIO_TYPE scenarioType) = 0;
 
  virtual bool getLocalAudioStatistics(LocalAudioDetailedStats& stats) = 0;
 
  virtual int publishAudio(agora_refptr<ILocalAudioTrack> audioTrack) = 0;
 
  virtual int unpublishAudio(agora_refptr<ILocalAudioTrack> audioTrack) = 0;
 
  virtual int publishVideo(agora_refptr<ILocalVideoTrack> videoTrack) = 0;
 
  virtual int unpublishVideo(agora_refptr<ILocalVideoTrack> videoTrack) = 0;
 
  virtual int subscribeAudio(user_id_t userId) = 0;
 
  virtual int subscribeAllAudio() = 0;
 
  virtual int unsubscribeAudio(user_id_t userId) = 0;
 
  virtual int unsubscribeAllAudio() = 0;
 
  virtual int adjustPlaybackSignalVolume(int volume) = 0;
 
  virtual int getPlaybackSignalVolume(int* volume) = 0;
 
  /*
   * Adjust the playback volume of the user specified by uid.
   *
   * You can call this method to adjust the playback volume of the user specified by uid
   * in call. If you want to adjust playback volume of the multi user, you can call this
   * this method multi times.
   *
   * @note
   * Please call this method after join channel.
   * This method adjust the playback volume of specified user.
   *
   * @param userId The ID of the Remote user.
   * @param volume The playback volume of the specified remote user. The value ranges between 0 and 400, including the following:
   * 0: Mute.
   * 100: (Default) Original volume.
   * 400: Four times the original volume with signal-clipping protection.
   * @return
   * - 0: Success.
   * - < 0: Failure.
   */
  virtual int adjustUserPlaybackSignalVolume(user_id_t userId, int volume) = 0;
 
  virtual int getUserPlaybackSignalVolume(user_id_t userId, int* volume) = 0;
 
  virtual int enableSoundPositionIndication(bool enabled) = 0;
 
  virtual int setRemoteVoicePosition(user_id_t userId, double pan, double gain) = 0;
  
  virtual int enableSpatialAudio(bool enabled) = 0;
 
  virtual int setRemoteUserSpatialAudioParams(user_id_t userId, const agora::SpatialAudioParams& param) = 0;
 
  virtual int setPlaybackAudioFrameParameters(size_t numberOfChannels,
                                              uint32_t sampleRateHz,
                                              RAW_AUDIO_FRAME_OP_MODE_TYPE mode = RAW_AUDIO_FRAME_OP_MODE_READ_ONLY,
                                              int samplesPerCall = 0) = 0;
  virtual int setRecordingAudioFrameParameters(size_t numberOfChannels,
                                               uint32_t sampleRateHz,
                                               RAW_AUDIO_FRAME_OP_MODE_TYPE mode = RAW_AUDIO_FRAME_OP_MODE_READ_ONLY,
                                               int samplesPerCall = 0) = 0;
  virtual int setMixedAudioFrameParameters(size_t numberOfChannels,
                                           uint32_t sampleRateHz,
                                           int samplesPerCall = 0) = 0;
 
  virtual int setEarMonitoringAudioFrameParameters(bool enabled,
                                                   size_t numberOfChannels,
                                                   uint32_t sampleRateHz,
                                                   RAW_AUDIO_FRAME_OP_MODE_TYPE mode = RAW_AUDIO_FRAME_OP_MODE_READ_ONLY,
                                                   int samplesPerCall = 0) = 0;
 
  virtual int setPlaybackAudioFrameBeforeMixingParameters(size_t numberOfChannels,
                                                          uint32_t sampleRateHz) = 0;
 
  virtual int registerAudioFrameObserver(agora::media::IAudioFrameObserverBase * observer) = 0;
  virtual int unregisterAudioFrameObserver(agora::media::IAudioFrameObserverBase * observer) = 0;
 
  virtual int enableAudioSpectrumMonitor(int intervalInMS = 100) = 0;
  virtual int disableAudioSpectrumMonitor() = 0;
 
  virtual int registerAudioSpectrumObserver(agora::media::IAudioSpectrumObserver * observer, void (*safeDeleter)(agora::media::IAudioSpectrumObserver*)) = 0;
  virtual int unregisterAudioSpectrumObserver(agora::media::IAudioSpectrumObserver * observer) = 0;
 
  virtual int registerLocalVideoEncodedFrameObserver(agora::media::IVideoEncodedFrameObserver* observer) = 0;
  virtual int unregisterLocalVideoEncodedFrameObserver(agora::media::IVideoEncodedFrameObserver* observer) = 0;
  virtual int forceNextIntraFrame() = 0;
  virtual int registerVideoEncodedFrameObserver(agora::media::IVideoEncodedFrameObserver* observer) = 0;
  virtual int unregisterVideoEncodedFrameObserver(agora::media::IVideoEncodedFrameObserver* observer) = 0;
 
  virtual int registerVideoFrameObserver(IVideoFrameObserver2* observer) = 0;
  virtual int unregisterVideoFrameObserver(IVideoFrameObserver2* observer) = 0;
 
  virtual int setVideoSubscriptionOptions(user_id_t userId,
                                          const VideoSubscriptionOptions& options) = 0;
  
  virtual int setHighPriorityUserList(uid_t* vipList, int uidNum, int option) = 0;
 
  virtual int getHighPriorityUserList(std::vector<uid_t>& vipList, int& option) = 0;
 
  virtual int setRemoteSubscribeFallbackOption(int option) = 0;
 
   virtual int setSubscribeAudioBlocklist(user_id_t* userList, int userNumber) = 0;
 
   virtual int setSubscribeAudioAllowlist(user_id_t* userList, int userNumber) = 0;
 
   virtual int setSubscribeVideoBlocklist(user_id_t* userList, int userNumber) = 0;
 
   virtual int setSubscribeVideoAllowlist(user_id_t* userList, int userNumber) = 0;
 
  virtual int subscribeVideo(user_id_t userId,
                             const VideoSubscriptionOptions &subscriptionOptions) = 0;
 
  virtual int subscribeAllVideo(const VideoSubscriptionOptions &subscriptionOptions) = 0;
 
  virtual int unsubscribeVideo(user_id_t userId) = 0;
 
  virtual int unsubscribeAllVideo() = 0;
 
  virtual int setAudioVolumeIndicationParameters(int intervalInMS, int smooth, bool reportVad) = 0;
 
  virtual int registerLocalUserObserver(
      ILocalUserObserver* observer,
      void(*safeDeleter)(ILocalUserObserver*) = NULL) = 0;
 
  virtual int unregisterLocalUserObserver(ILocalUserObserver* observer) = 0;
 
  virtual IMediaControlPacketSender* getMediaControlPacketSender() = 0;
 
  virtual int registerMediaControlPacketReceiver(IMediaControlPacketReceiver* ctrlPacketReceiver) = 0;
 
  virtual int unregisterMediaControlPacketReceiver(IMediaControlPacketReceiver* ctrlPacketReceiver) = 0;
 
  virtual int sendIntraRequest(user_id_t userId) = 0;
 
  virtual int setAudioFilterable(bool filterable) = 0;
 
 virtual int enableRemoteAudioTrackFilter(user_id_t userId, const char* id, bool enable) = 0;
 
 virtual int setRemoteAudioTrackFilterProperty(user_id_t userId, const char* id, const char* key, const char* jsonValue) = 0;
 
 virtual int getRemoteAudioTrackFilterProperty(user_id_t userId, const char* id, const char* key, char* jsonValue, size_t bufSize) = 0;
  virtual int publishDataChannel(agora_refptr<ILocalDataChannel> channel) = 0;
  virtual int unpublishDataChannel(agora_refptr<ILocalDataChannel> channel) = 0;
  virtual int subscribeDataChannel(user_id_t userId, int channelId) = 0;
  virtual int unsubscribeDataChannel(user_id_t userId, int channelId) = 0;
  virtual int registerDataChannelObserver(IDataChannelObserver * observer) = 0;
  virtual int unregisterDataChannelObserver(IDataChannelObserver * observer) = 0;
  virtual int SetAudioNsMode(bool NsEnable, NS_MODE NsMode, NS_LEVEL NsLevel, NS_DELAY NsDelay) = 0;
  virtual int registerMixSource(agora_refptr<IAudioMixerSource> mixSource, int mixPolicy) = 0;
};
 
struct AudioVolumeInformation {
  user_id_t userId;
 
  unsigned int volume;
 
  /*
   * The activity status of remote users
   */
  unsigned int vad;
 
  double voicePitch;
 
  AudioVolumeInformation() : userId(NULL), volume(0), vad(0), voicePitch(0.0) {}
};
 
class ILocalUserObserver {
 public:
  virtual ~ILocalUserObserver() {}
 
  virtual void onAudioTrackPublishStart(agora_refptr<ILocalAudioTrack> audioTrack) = 0;
  virtual void onAudioTrackPublishSuccess(agora_refptr<ILocalAudioTrack> audioTrack) = 0;
 
  virtual void onAudioTrackUnpublished(agora_refptr<ILocalAudioTrack> audioTrack) = 0;
 
  virtual void onAudioTrackPublicationFailure(agora_refptr<ILocalAudioTrack> audioTrack,
                                              ERROR_CODE_TYPE error) = 0;
 
  virtual void onLocalAudioTrackStatistics(const LocalAudioStats& stats) = 0;
  virtual void onRemoteAudioTrackStatistics(agora_refptr<rtc::IRemoteAudioTrack> audioTrack, const RemoteAudioTrackStats& stats) = 0;
  virtual void onUserAudioTrackSubscribed(user_id_t userId,
                                          agora_refptr<rtc::IRemoteAudioTrack> audioTrack) = 0;
 
  virtual void onUserAudioTrackStateChanged(user_id_t userId,
                                            agora_refptr<rtc::IRemoteAudioTrack> audioTrack,
                                            REMOTE_AUDIO_STATE state,
                                            REMOTE_AUDIO_STATE_REASON reason,
                                            int elapsed) = 0;
 
  virtual void onVideoTrackPublishStart(agora_refptr<ILocalVideoTrack> videoTrack) = 0;
  virtual void onVideoTrackPublishSuccess(agora_refptr<ILocalVideoTrack> videoTrack) = 0;
 
  virtual void onVideoTrackPublicationFailure(agora_refptr<ILocalVideoTrack> videoTrack,
                                              ERROR_CODE_TYPE error) = 0;
 
  virtual void onVideoTrackUnpublished(agora_refptr<ILocalVideoTrack> videoTrack) = 0;
  virtual void onLocalVideoTrackStateChanged(agora_refptr<rtc::ILocalVideoTrack> videoTrack,
                                             LOCAL_VIDEO_STREAM_STATE state,
                                             LOCAL_VIDEO_STREAM_ERROR errorCode) = 0;
 
  virtual void onLocalVideoTrackStatistics(agora_refptr<rtc::ILocalVideoTrack> videoTrack,
                                           const LocalVideoTrackStats& stats) = 0;
 
  virtual void onUserVideoTrackSubscribed(user_id_t userId, VideoTrackInfo trackInfo,
                                          agora_refptr<rtc::IRemoteVideoTrack> videoTrack) = 0;
 
  virtual void onUserVideoTrackStateChanged(user_id_t userId,
                                            agora_refptr<rtc::IRemoteVideoTrack> videoTrack,
                                            REMOTE_VIDEO_STATE state,
                                            REMOTE_VIDEO_STATE_REASON reason,
                                            int elapsed) = 0;
 
  virtual void onFirstRemoteVideoFrameRendered(user_id_t userId, int width,
                                               int height, int elapsed) = 0;
 
 
  virtual void onRemoteVideoTrackStatistics(agora_refptr<rtc::IRemoteVideoTrack> videoTrack,
                                            const RemoteVideoTrackStats& stats) = 0;
 
  virtual void onAudioVolumeIndication(const AudioVolumeInformation* speakers, unsigned int speakerNumber,
                                       int totalVolume) = 0;
 
  virtual void onActiveSpeaker(user_id_t userId) = 0;
 
  virtual void onAudioSubscribeStateChanged(const char* channel, user_id_t userId, STREAM_SUBSCRIBE_STATE oldState, STREAM_SUBSCRIBE_STATE newState, int elapseSinceLastState) = 0;
 
  virtual void onVideoSubscribeStateChanged(const char* channel, user_id_t userId, STREAM_SUBSCRIBE_STATE oldState, STREAM_SUBSCRIBE_STATE newState, int elapseSinceLastState) = 0;
 
  virtual void onAudioPublishStateChanged(const char* channel, STREAM_PUBLISH_STATE oldState, STREAM_PUBLISH_STATE newState, int elapseSinceLastState) = 0;
 
  virtual void onVideoPublishStateChanged(const char* channel, STREAM_PUBLISH_STATE oldState, STREAM_PUBLISH_STATE newState, int elapseSinceLastState) = 0;
 
  virtual void onRemoteSubscribeFallbackToAudioOnly(user_id_t userId, bool isFallbackOrRecover) {
    (void)userId;
    (void)isFallbackOrRecover;
  }
 
  virtual void onFirstRemoteAudioFrame(user_id_t userId, int elapsed) = 0;
 
  virtual void onFirstRemoteAudioDecoded(user_id_t userId, int elapsed) = 0;
 
  virtual void onFirstRemoteVideoFrame(user_id_t userId, int width, int height, int elapsed) = 0;
 
  virtual void onFirstRemoteVideoDecoded(user_id_t userId, int width, int height, int elapsed) = 0;
 
  virtual void onVideoSizeChanged(user_id_t userId, int width, int height, int rotation) = 0;
 
  enum USER_MEDIA_INFO {
    USER_MEDIA_INFO_MUTE_AUDIO = 0,
    USER_MEDIA_INFO_MUTE_VIDEO = 1,
    USER_MEDIA_INFO_ENABLE_VIDEO = 4,
    USER_MEDIA_INFO_ENABLE_LOCAL_VIDEO = 8,
  };
 
  virtual void onUserInfoUpdated(user_id_t userId, USER_MEDIA_INFO msg, bool val) {
    (void)userId;
    (void)msg;
    (void)val;
  }
 
  virtual void onIntraRequestReceived() {}
 
  virtual void onStreamMessage(user_id_t userId, int streamId, const char* data, size_t length) {}
 
  virtual void onUserStateChanged(user_id_t userId, uint32_t state){}
 
  virtual void onVideoRenderingTracingResult(user_id_t user_id, MEDIA_TRACE_EVENT currentState, VideoRenderingTracingInfo tracingInfo) {}
};
 
class IVideoFrameObserver2 {
 public:
  virtual void onFrame(const char* channelId, user_id_t remoteUid, const media::base::VideoFrame* frame) = 0;
 
  virtual ~IVideoFrameObserver2() {}
};
 
}  // namespace rtc
}  // namespace agora