NGIAgoraAudioTrack.h

 
// Copyright (c) 2018 Agora.io. All rights reserved
 
// This program is confidential and proprietary to Agora.io.
// And may not be copied, reproduced, modified, disclosed to others, published
// or used, in whole or in part, without the express prior written permission
// of Agora.io.
 
#pragma once  // NOLINT(build/header_guard)
 
#include "AgoraBase.h"
#include <api/cpp/aosl_ares_class.h>
 
#ifndef OPTIONAL_OVERRIDE
#if __cplusplus >= 201103L || (defined(_MSC_VER) && _MSC_VER >= 1800)
#define OPTIONAL_OVERRIDE override
#else
#define OPTIONAL_OVERRIDE
#endif
#endif
 
// FIXME(Ender): use this class instead of AudioSendStream as local track
namespace agora {
namespace rtc {
class IAudioTrackStateObserver;
class IAudioFilter;
class IAudioSinkBase;
class IMediaPacketReceiver;
class IAudioEncodedFrameReceiver;
struct AudioSinkWants {
  int samplesPerSec;
 
  size_t channels;
 
  AudioSinkWants() : samplesPerSec(0),
                     channels(0) {}
  AudioSinkWants(int sampleRate, size_t chs) : samplesPerSec(sampleRate),
                                               channels(chs) {}
  AudioSinkWants(int sampleRate, size_t chs, int trackNum) : samplesPerSec(sampleRate), channels(chs) {}                                            
};
 
enum AudioTrackType {
  LOCAL_AUDIO_TRACK,
  REMOTE_AUDIO_TRACK,
};
 
class IAudioTrack : public RefCountInterface {
 public:
  enum AudioFilterPosition {
    RecordingLocalPlayback,
    PostAudioProcessing,
    RemoteUserPlayback,
    PcmSource,
    PcmSourceSending,
    PcmSourceLocalPlayback,
    RemoteMixedPlayback,
  };
 
 public:
  virtual int adjustPlayoutVolume(int volume, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int getPlayoutVolume(int* volume) = 0;
 
  virtual bool addAudioFilter(agora_refptr<IAudioFilter> filter, AudioFilterPosition position, ExtensionContext *extContext = NULL, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  virtual bool removeAudioFilter(agora_refptr<IAudioFilter> filter, AudioFilterPosition position, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int enableAudioFilter(const char* id, bool enable, AudioFilterPosition position, aosl_ref_t ares = AOSL_REF_INVALID) {
    (void)id;
    (void)enable;
    (void)position;
    return -1;
  }
 
  virtual int setFilterProperty(const char* id, const char* key, const char* jsonValue, AudioFilterPosition position, aosl_ref_t ares = AOSL_REF_INVALID) {
    (void)id;
    (void)key;
    (void)jsonValue;
    (void)position;
    return -1;
  }
 
  virtual int getFilterProperty(const char* id, const char* key, char* jsonValue, size_t bufSize, AudioFilterPosition position) {
    (void)id;
    (void)key;
    (void)jsonValue;
    (void)bufSize;
    (void)position;
    return -1;
  }
 
  virtual agora_refptr<IAudioFilter> getAudioFilter(const char *name, AudioFilterPosition position) const = 0;
 
  virtual bool addAudioSink(agora_refptr<IAudioSinkBase> sink, const AudioSinkWants& wants, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual bool removeAudioSink(agora_refptr<IAudioSinkBase> sink, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  virtual AudioTrackType getType() = 0;
 
};
 
class ILocalAudioTrackObserver {
 public:
  virtual ~ILocalAudioTrackObserver() {}
 
  virtual void onLocalAudioTrackStateChanged(LOCAL_AUDIO_STREAM_STATE state,
                                             LOCAL_AUDIO_STREAM_REASON reasonCode) = 0;
 
  virtual void onLocalAudioTrackError(int errorCode) = 0;
};
 
class ILocalAudioTrack : public IAudioTrack {
 public:
  struct LocalAudioTrackStats {
    uint32_t source_id;
    uint32_t buffered_pcm_data_list_size;
    uint32_t missed_audio_frames;
    uint32_t sent_audio_frames;
    uint32_t pushed_audio_frames;
    uint32_t dropped_audio_frames;
    uint32_t playout_audio_frames;
    uint32_t effect_type;
    uint32_t hw_ear_monitor;
    bool enabled;
    uint32_t audio_volume;  // [0,255]
 
    LocalAudioTrackStats() : source_id(0),
                             buffered_pcm_data_list_size(0),
                             missed_audio_frames(0),
                             sent_audio_frames(0),
                             pushed_audio_frames(0),
                             dropped_audio_frames(0),
                             playout_audio_frames(0),
                             effect_type(0),
                             hw_ear_monitor(0),
                             enabled(false),
                             audio_volume(0) {}
  };
 
 public:
  virtual int setEnabled(bool enable, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual bool isEnabled() const = 0;
 
  virtual LOCAL_AUDIO_STREAM_STATE getState() = 0;
 
  virtual LocalAudioTrackStats GetStats() = 0;
 
  virtual int adjustPublishVolume(int volume, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int getPublishVolume(int* volume) = 0;
 
  virtual int enableLocalPlayback(bool enable, bool sync = true, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  
  virtual int enableEarMonitor(bool enable, int includeAudioFilters, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  virtual int registerTrackObserver(ILocalAudioTrackObserver* observer, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  virtual int unregisterTrackObserver(ILocalAudioTrackObserver* observer) = 0;
 
  virtual void setMaxBufferedAudioFrameNumber(int number, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int ClearSenderBuffer() = 0;
  virtual AudioTrackType getType() OPTIONAL_OVERRIDE { return LOCAL_AUDIO_TRACK; }
 
 
 protected:
  ~ILocalAudioTrack() {}
};
 
struct RemoteAudioTrackStats {
  uid_t uid;
  int quality;
  int network_transport_delay;
  uint32_t jitter_buffer_delay;
  int audio_loss_rate;
  int num_channels;
  int received_sample_rate;
  int received_bitrate;
  int total_frozen_time;
  int frozen_rate;
  int64_t received_bytes;
  int mean_waiting_time;
  size_t expanded_speech_samples;
  size_t expanded_noise_samples;
  uint32_t timestamps_since_last_report;
  uint16_t min_sequence_number;
  uint16_t max_sequence_number;
  int32_t audio_level;
  uint32_t downlink_process_time_ms;
  uint32_t packet_expired_loss;
  uint32_t packet_max_expired_ms;
  uint32_t burst_peak_num;
  uint32_t burst_jitter;
  uint32_t target_level_base_ms;
  uint32_t target_level_prefered_ms;
  uint16_t accelerate_rate;
  uint16_t preemptive_expand_rate;
  uint16_t frozen_count_80_ms;
  uint16_t frozen_time_80_ms;
  uint16_t frozen_count_200_ms;
  uint16_t frozen_time_200_ms;
 
  uint16_t frozen_count_by_custom;
  uint16_t frozen_time_ms_by_custom;
  uint16_t full_frozen_time_80_ms;
  uint16_t full_frozen_time_200_ms;
  uint32_t delay_estimate_ms;
  uint32_t mos_value;
  uint32_t frozen_rate_by_custom_plc_count;
  uint32_t plc_count;
  int32_t fec_decode_ms;
  uint16_t frozen_count_10_ms;
  uint64_t total_active_time;
  uint64_t publish_duration;
 
  int32_t e2e_delay_ms;
 
  int32_t new_e2e_delay_ms;
  int qoe_quality;
  int32_t quality_changed_reason;
 
  int32_t downlink_effect_type;
 
  RemoteAudioTrackStats() :
    uid(0),
    quality(0),
    network_transport_delay(0),
    jitter_buffer_delay(0),
    audio_loss_rate(0),
    num_channels(0),
    received_sample_rate(0),
    received_bitrate(0),
    total_frozen_time(0),
    frozen_rate(0),
    received_bytes(0),
    mean_waiting_time(0),
    expanded_speech_samples(0),
    expanded_noise_samples(0),
    timestamps_since_last_report(0),
    min_sequence_number(0xFFFF),
    max_sequence_number(0),
    audio_level(0),
    downlink_process_time_ms(0),
    packet_expired_loss(0),
    packet_max_expired_ms(0),
    burst_peak_num(0),
    burst_jitter(0),
    target_level_base_ms(0),
    target_level_prefered_ms(0),
    accelerate_rate(0),
    preemptive_expand_rate(0),
    frozen_count_80_ms(0),
    frozen_time_80_ms(0),
    frozen_count_200_ms(0),
    frozen_time_200_ms(0),
    full_frozen_time_80_ms(0),
    full_frozen_time_200_ms(0),
    frozen_count_by_custom(0),
    frozen_time_ms_by_custom(0),
    delay_estimate_ms(0),
    mos_value(0),
    frozen_rate_by_custom_plc_count(0),
    plc_count(0),
    fec_decode_ms(-1),
    frozen_count_10_ms(0),
    total_active_time(0),
    publish_duration(0),
    e2e_delay_ms(0),
    new_e2e_delay_ms(0),
    qoe_quality(0),
    quality_changed_reason(0),
    downlink_effect_type(0) {}
};
 
 
struct AudioEncFrameRecvParams {
 
  enum ReceiveMode {
    ORIGINAL = 0,
 
    MATCHED_CODEC = 1,
 
    MATCHED_ALL = 2,
  };
 
  ReceiveMode receive_mode;
 
  AUDIO_CODEC_TYPE target_codec;  
  
  int32_t target_sample_rate;
  
  int32_t target_num_channels;
 
 
  AudioEncFrameRecvParams() :
    receive_mode(ORIGINAL),
    target_codec(AUDIO_CODEC_AACLC),
    target_sample_rate(0),
    target_num_channels(0) {}  
 
  AudioEncFrameRecvParams(const AudioEncFrameRecvParams& src_params) {
    receive_mode = src_params.receive_mode;
    target_codec = src_params.target_codec;
    target_sample_rate = src_params.target_sample_rate;
    target_num_channels = src_params.target_num_channels;    
  }  
};
 
class IRemoteAudioTrack : public IAudioTrack {
 public:
  virtual bool getStatistics(RemoteAudioTrackStats& stats) = 0;
 
  virtual REMOTE_AUDIO_STATE getState() = 0;
 
  virtual int registerMediaPacketReceiver(IMediaPacketReceiver* packetReceiver, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int unregisterMediaPacketReceiver(IMediaPacketReceiver* packetReceiver) = 0;
 
  virtual int registerAudioEncodedFrameReceiver(IAudioEncodedFrameReceiver* packetReceiver,
                                                const AudioEncFrameRecvParams& recvParams,
                                                aosl_ref_t ares = AOSL_REF_INVALID) = 0;
  virtual int unregisterAudioEncodedFrameReceiver(IAudioEncodedFrameReceiver* packetReceiver) = 0;
 
  virtual int setRemoteVoicePosition(float pan, float gain, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int adjustDecodedAudioVolume(int decoded_index, int volume, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int muteRemoteFromTimestamp(uint32_t timestamp) = 0;
  
  virtual int unmuteRemoteFromTimestamp(uint32_t timestamp) = 0;
  
  virtual int adjustAudioAcceleration(int percentage, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int adjustAudioDeceleration(int percentage, aosl_ref_t ares = AOSL_REF_INVALID) = 0;  
 
  virtual int enableAudioBurstAccelerate(bool enable, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int enableSpatialAudio(bool enabled, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual int setRemoteUserSpatialAudioParams(const agora::SpatialAudioParams& params, aosl_ref_t ares = AOSL_REF_INVALID) = 0;
 
  virtual AudioTrackType getType() OPTIONAL_OVERRIDE { return REMOTE_AUDIO_TRACK; }
  
};
 
}  // namespace rtc
}  // namespace agora