video_track_i.h

//
//  Agora Media SDK
//
//  Created by Rao Qi in 2019.
//  Copyright (c) 2019 Agora IO. All rights reserved.
//
#pragma once
 
#include <atomic>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <vector>
 
#include "AgoraBase.h"
#include "AgoraMediaBase.h"
#include "AgoraOptional.h"
#include "NGIAgoraVideoTrack.h"
#include <api/cpp/aosl_ares_class.h>
#include "api/transport/network_types.h"
#include "main/core/video/base/video_frame/video_content_type.h"
#include "call/rtp_config.h"
 
#include "rtc_connection_i.h"
#include "track_stat_i.h"
#include "video_config_i.h"
#include "common_defines.h"
#include "video_node_i.h"
 
#include "main/core/video/multi_stream_subscribe_interface.h"
#include "facilities/media_config/policy_chain/media_config_policy_chain.h"
#include "facilities/tools/weak_observers.h"
#include "facilities/media_config/policy_chain/general_val_policy_chain.h"
#include "main/core/video/strategy_framework/module_controller/video_module_control_aspect.h"
#include "main/core/video/stats_and_events/video_stats_events_base.h"
#include "main/core/video/video_frame_source_filter.h"
#include "main/core/video/strategy_framework/strategy_common_defines.h"
#include "engine_adapter/video/video_common_defines.h"
#include "main/core/video/strategy_framework/strategy_qoe_decision/strategy_qoe_decision_interface.h"
#include "modules/agora_video_codec/include/video_encoder_interface.h"
#include "main/core/video/strategy_framework/statistic_manager/statistic_defines.h"
 
namespace webrtc {
  class IFecMethodFactoryInterface;
  class ISmoothRender;
  class IRsfecCodecFactoryInterface;
  class IAutoAdjustHarq;
}
 
namespace agora {
namespace rtc {
 
class VideoNodeRtpSink;
class VideoNodeRtpSource;
class ProactiveCaller;
class IModuleControlPanel;
class IVideoStatsMetrics;
class StrategyQoeDecisionInterface;
 
enum class InternalVideoSourceType : unsigned {
  None = 0,
  Camera = 1,
  Custom = 2,
  Screen = 3,
  CustomYuvSource = 4,
  CustomEncodedImageSource = 5,
  CustomPacketSource = 6,
  MixedSource = 7,
  TranscodedSource = 8,
};
 
enum VideoModuleId {
  VideoModuleCapture = 1,
  VideoModulePreprocess,
  VideoModuleEncode,
  VideoModuleNetwork,
  VideoModuleDecode,
  VideoModulePostprocess,
  VideoModuleRender,
  VideoModulePipeline,
  VideoModuleQoE,
  VideoModuleExtension,
};
 
enum VideoAvailabilityLevel {
  VideoAvailabilityLevel1 = 1,  // Completely unusable.
  VideoAvailabilityLevel2,      // Usable but with very poor experience.
  VideoAvailabilityLevel3,      // Usable but with poor experience.
};
 
// Events report. New enum can be added but do not change the existing value!
enum VideoPipelineEvent {
  kVideoUplinkEventStaticFrames = 1,  // Continous static frames, maybe green/black picktures.
};
 
enum VideoQoeEvent {
  kVideoQoeCriticalDrop = 1,
  kVideoQoe600msFreezeDrop = 2,
  kVideoQoe200msFreezeDrop = 3,
  kVideoQoeFpsSubstandard = 4,
  kVideoTimestampException = 5,
  kVideoQoePipelineException = 6,
  kVideoGlitchDetection = 7,
  kVideoExtensionStatusDetection = 8,
};
 
// Events report. New enum can be added but do not change the existing value!
enum VideoProcessEvent {
  kVideoProcessEventNone = 0,
 
  // These events report will be throttled, refer to VideoEngine::doReportVideoEvent().
  kVideoProcessEventPreprocessEnqueueFailure = 1000,
  kVideoProcessEventPreprocessFrameFailure = 1001,
  kVideoProcessEventPreprocessNoIncomingFrame = 1002,  // No incoming frame for builtin VPM module
  kVideoProcessEventPreprocessCongested = 1003,
};
 
// report hardware codec availability event.
enum VideoCodecAvailableEvent {
  kVideoHwH265EncoderAvailable = 2000,
  kVideoHwH264EncoderHighProfileAvailable = 2001,
};
 
enum VideoDumpMode {
  VIDEO_DUMP_DEFAULT = 0,  // Dump all.
  VIDEO_DUMP_ALL = 0xFFFF, // Dump all.
  VIDEO_DUMP_CAPTURED_YUV = (1 << 0), // Dump YUV after video capturing.
  VIDEO_DUMP_FILTERED_YUV = (1 << 1), // Dump YUV before video encoding.
  VIDEO_DUMP_ENCODED_STREAM = (1 << 2), // Dump stream after video encoding.
  VIDEO_DUMP_RECEIVED_STREAM = (1 << 3), // Dump stream before video decoding.
  VIDEO_DUMP_DECODED_YUV = (1 << 4), // Dump YUV after video decoding.
  VIDEO_DUMP_RENDERED_YUV = (1 << 5), // Dump YUV before video rendering.
  VIDEO_DUMP_PRE_ENCODER_YUV = (1 << 6), // Dump YUV right before video encoding.
};
 
struct VideoAvailabilityIndicator {
  VideoAvailabilityLevel level;
  VideoModuleId module;
  int code;
  uid_t uid;
  int extra;
  std::vector<agora::rtc::QoEDropInfo> extra2;
  std::vector<agora::rtc::VideoTimestampExceptionInfo> ts_exception_info;
  std::vector<GlitchDetectionInfo> glitch_detection_infos;
  std::string extension_info;
};
 
struct VideoRendererInfo {
  int peer = 0;
  int streamDepth = 0;
  int isSupportHDR = 0;
  int isTexture = 0;
  int renderType = 0;
  int colorPrimary = 0;
  int colorTransfer = 0;
  int colorMatrix = 0;
  int colorRange = 0;
};
 
struct VideoQoEAnalyzerParameter {
  bool qoe_analyzer_enable = false;
  bool enable_video_diagnose_logger = true;
  int qoe_critical_report_max_times = 0;
  int qoe_high_report_max_times = 0;
  int qoe_normal_report_max_times = 0;
  int qoe_report_strategy = 0;
  int qoe_timing_strategy_report_period = 0;
};
 
struct YuvConvertStats {
  int width;
  int height;
  int method;
  int avgUs;
  int maxUs;
  std::string deviceId;
};
 
struct WatermarkBufferInner {
  std::vector<uint8_t> buffer;
  int width;
  int height;
  int length;
  media::base::VIDEO_PIXEL_FORMAT format;
};
 
struct WatermarkLiteralInner {
  std::string wm_literal;
  std::string font_file_path;
  int font_size;
  int stroke_width;
};
 
struct WatermarkTimestampInner {
  std::string font_file_path;
  int font_size;
  std::string format;
  int stroke_width;
};
 
struct WatermarkConfigInner {
  std::string id;
  WATERMARK_SOURCE_TYPE type;
  WatermarkBufferInner buffer;
  WatermarkLiteralInner literal;
  WatermarkTimestampInner timestamp;
  std::string image_url;
 
  WatermarkOptions options;
 
  ~WatermarkConfigInner() {
    if (type == WATERMARK_SOURCE_TYPE::BUFFER) {
      buffer.buffer.clear();
    }
  }
 
  WatermarkConfigInner(): id(""), type(WATERMARK_SOURCE_TYPE::IMAGE), image_url("") {
 
  }
 
  WatermarkConfigInner(const WatermarkConfig& other) {
    id = other.id;
    type = other.type;
    options = other.options;
    switch (other.type) {
    case WATERMARK_SOURCE_TYPE::BUFFER: {
      WatermarkBufferInner _buffer;
      _buffer.buffer = other.buffer.buffer ?
          std::vector<uint8_t>(other.buffer.buffer, other.buffer.buffer + other.buffer.length) : std::vector<uint8_t>(0);
      _buffer.width = other.buffer.width;
      _buffer.height = other.buffer.height;
      _buffer.length = other.buffer.length;
      _buffer.format = other.buffer.format;
      buffer = _buffer;
      break;
    }
    case WATERMARK_SOURCE_TYPE::LITERAL: {
      WatermarkLiteralInner _literal;
      _literal.wm_literal = other.literal.wmLiteral ? other.literal.wmLiteral : "";
      _literal.font_file_path = other.literal.fontFilePath ? other.literal.fontFilePath : "";
      _literal.font_size = other.literal.fontSize;
      _literal.stroke_width = other.literal.strokeWidth;
      literal = _literal;
      break;
    }
    case WATERMARK_SOURCE_TYPE::TIMESTAMPS: {
      WatermarkTimestampInner _timestamp;
      _timestamp.font_file_path = other.timestamp.fontFilePath ? other.timestamp.fontFilePath : "";
      _timestamp.font_size = other.timestamp.fontSize;
      _timestamp.format = other.timestamp.format ? other.timestamp.format : "";
      _timestamp.stroke_width = other.timestamp.strokeWidth;
      timestamp = _timestamp;
      break;
    }
    case WATERMARK_SOURCE_TYPE::IMAGE:
      image_url = other.imageUrl ? other.imageUrl : "";
      break;
    default:
      break;
    }
  }
};
 
class IVideoTrackObserver : public std::enable_shared_from_this<IVideoTrackObserver> {
 public:
  virtual ~IVideoTrackObserver() = default;
  virtual void onLocalVideoEvent(int id, LOCAL_VIDEO_EVENT_TYPE event) {}
  virtual void onLocalVideoStateChanged(int id,
                                        LOCAL_VIDEO_STREAM_STATE state,
                                        LOCAL_VIDEO_STREAM_REASON reason,
                                        int timestamp_ms) {}
 
  virtual void onRemoteVideoStateChanged(uid_t uid,
                                         REMOTE_VIDEO_STATE state,
                                         REMOTE_VIDEO_STATE_REASON reason,
                                         int timestamp_ms) {}
 
  virtual void onFirstVideoFrameRendered(int id, uid_t uid, int width, int height, int timestamp_ms) {}
 
  virtual void onFirstVideoFrameDecoded(std::string cid, uid_t uid, uint32_t ssrc, int width, int height, int timestamp_ms) {}
 
  virtual void onFirstVideoKeyFrameReceived(uid_t uid, uint64_t timestamp, const webrtc::FirstVideoFrameStreamInfo &streamInfo) {}
 
  virtual void onSourceVideoSizeChanged(uid_t uid,
                                        int width, int height,
                                        int rotation, int timestamp_ms) {}
  virtual void onSendSideDelay(int id, int send_delay) {}
  virtual void onRecvSideDelay(uid_t uid, int recv_delay) {}
  virtual void onRecvSideFps(uid_t uid, int fps) {}
  virtual void onEncoderConfigurationChanged(const std::unordered_map<int, VideoConfigurationEx>& config) {}
  virtual void onSvcLayerUpdated(SvcAABandwidthReportInfo info, int supported_max_layers,
                                 AdaptiveScenarioType scenario_type, int channel_size,
                                 const std::map<VideoStreamIndex, int>& be_subscribed_streams,
                                 const std::vector<int>& svc_layer_bitrate, SvcAASourceType source,
                                 const std::vector<SvcAAStrategyResult>& svc_aa_strategy_result_vec) {}
  virtual void onSvcaaEnabledStateUpdated(uint16_t enabled_state) {}
  virtual void onEncoderStrategySupportedSvcLayersUpdated(int max_layers) {}
  virtual void onVideoPipelineDataFormatChanged(int format) {}
  virtual void onCameraFacingChanged(int facing) {}
  virtual void onViewSizeChanged(uid_t uid, view_t view, int width, int height) {}
  virtual void OnSetRexferParams(bool fec_rexfer, float rexfer_alpha, int max_rexfer_times) {}
  virtual void OnRexferStatusUpdated(bool status, int32_t target_bitrate) {}
  virtual void OnNotifyDepartedFrame(uid_t uid, int picture_id) {}
  virtual void onCameraInfoListChanged(CameraInfoList cameraInfoList) {}
  virtual void onCameraCharacteristicProfileChanged(agora::rtc::CameraCharacteristicProfile profile) {}
  virtual void OnEncoderStatusUpdate(webrtc::VideoCodecType codec_type,
                                     webrtc::HW_ENCODER_ACCELERATING_STATUS hw_accelerate_status) {};
  virtual void OnVideoStatusUpdated(bool is_fallback, webrtc::MediaStreamId stream_id, int reason) {}
  virtual void onVideoAvailabilityIndicatorEvent(VideoAvailabilityIndicator indicator) {}
  virtual void onVideoRendererInfoEvent(VideoRendererInfo info) {}
  virtual void onVideoHWCodecSpecEvent(agora::rtc::VideoHWCodecSpec codecSpec) {}
  virtual void onVideoSizeChanged(int id, uid_t uid, int width, int height, int rotation) {}
  virtual void onYuvConvertStats(const YuvConvertStats& stats) {}
  virtual void onWatermarkStats(int32_t image_count, int32_t buffer_count, int32_t literal_count, int32_t timestamp_count) {}
 
  virtual void onLocalAddVideoFilter(int track_id, std::string filter_name, bool enabled){}
  virtual void onLocalFilterStatusChanged(int track_id, std::string filter_name, bool enabled){}
  virtual void onRemoteAddVideoFilter(std::string cid, uid_t uid, uint32_t ssrc, std::string filter_name, bool enabled){}
  virtual void onRemoteFilterStatusChanged(std::string cid, uid_t uid, uint32_t ssrc, std::string filter_name, bool enabled, bool isDisableMe = false){}
  virtual void onVideoContentChanged(uid_t uid, agora::VideoContentType newType,
                             agora::VideoContentSubType newSubtype) {}
  virtual void OnRequestKeyFrame(uid_t uid, VIDEO_STREAM_TYPE type) {}
  virtual void OnVideoCodingModuleInfosUpdated(webrtc::VideoCodingModuleInfos videoCodingInfos) {}
  virtual void OnVideoEncodeStrategyUpdated(agora::rtc::VideoEncodeStrategyInfo strategy) {}
  virtual void OnFirstDecodableFrameReceived(uint8_t stream_id) {}
  virtual void OnBcmDecodeFeedback(uid_t uid, std::string&& fb) {}
  virtual void OnBcmNetworkStatsFeeback(uid_t uid, std::string&& fb) {}
};
 
struct LocalVideoTrackStatsEx {
  LocalVideoTrackStats local_video_stats;
  int sent_loss_ratio;
  uint32_t total_bwe_bps;
  uint32_t total_video_send_target_bps;
  uint32_t media_send_bps;
  uint32_t qp;
};
 
class ILocalVideoTrackEx : public ILocalVideoTrack,
                           public VideoLocalTrackControlAspect {
 public:
  enum DetachReason { MANUAL, TRACK_DESTROY, NETWORK_DESTROY, CODEC_CHANGE};
 
  // keep the same as webrtc::RsfecConfig
  struct RsfecConfig {
    std::vector<int> fec_protection_factor;
    std::vector<std::vector<int>> fec_ratioLevel;
    std::vector<int> fec_rttThreshold;
    bool pec_enabled;
  };
 
  struct CaptureModeItem {
    int32_t mode = 0;
    int32_t scene = 0;
    int32_t policy = 0;
    std::string type = "";
  };
 
  struct AttachInfo {
    uint32_t uid;
    uint32_t cid;
    VideoNodeRtpSink* network;
    WeakPipelineBuilder builder;
    uint64_t stats_space;
    CongestionControlType cc_type;
    bool enable_two_bytes_extension;
    webrtc::RsfecConfig rsfec_config;
 
    // hardware encoder related
    std::string enable_hw_encoder;
 
    // video config
    VideoNodeEncoderEx::OPSParametersCollection ops_parameters;
    std::shared_ptr<webrtc::IAutoAdjustHarq> auto_adjust_harq;
    int harq_version;
    int32_t fec_outside_bandwidth_ratio;
    bool enable_minor_stream_vqc = false;
    bool enable_minor_stream_fec = false;
    bool enable_minor_stream_fec_outside_ratio = false;
    bool enable_minor_stream_intra_request = false;
 
    int fec_method;
    std::string switch_to_rq;
    bool enable_rq_classic_method;
 
    int fec_fix_rate;
    bool enable_check_for_disable_fec;
    bool enable_quick_intra_high_fec = false;
    absl::optional<int> max_inflight_frame_count_pre_processsing;
 
    // enable video diagnose
    bool enable_video_send_diagnose;
    // video codec and codec feature_caps
    VideoNegotiatedCodecCaps negotiated_video_codec_caps;
 
    int hw_capture_delay;
    uint32_t sync_peer_uid;
 
    Optional<SIMULCAST_STREAM_MODE> cfg_simulcast_stream_mode;
    bool support_higher_standard_bitrate;
    VideoQoEAnalyzerParameter qoe_analyzer_parameters;
    std::unordered_map<uint8_t, std::string> fec_capabilities;
    int prefer_fec_method = -1;
    int fec_mode = 0;
    Optional<bool> response_quick_intra_request;
  };
 
  struct DetachInfo {
    VideoNodeRtpSink* network;
    DetachReason reason;
  };
 
  ILocalVideoTrackEx() : id_(id_generator_++) {}
  virtual ~ILocalVideoTrackEx() {}
 
  virtual bool hasPublished() = 0;
 
  virtual int setVideoEncoderConfigurationEx(const VideoEncoderConfiguration& config, utils::ConfigPriority priority = utils::CONFIG_PRIORITY_USER) = 0;
 
  virtual int setSimulcastStreamModeEx(SIMULCAST_STREAM_MODE mode, const SimulcastConfigInternal& config, utils::ConfigPriority priority = utils::CONFIG_PRIORITY_USER) = 0;
 
  virtual int SetVideoConfigEx(int index, const VideoConfigurationEx& configEx, utils::ConfigPriority priority = utils::CONFIG_PRIORITY_USER) = 0;
 
  virtual int ResetVideoConfigEx(utils::ConfigPriority priority) = 0;
 
  virtual int GetConfigExs(std::unordered_map<int, VideoConfigurationEx>& configs, bool include_disable_config = false) = 0;
 
  virtual int GetVideoProfileAutoAdjust(std::string& config_video_profile, std::string& actual_video_profile) = 0;
 
  virtual int GetCaptureMode(CaptureModeItem& captureModeOut) = 0;
 
  virtual void RequestKeyFrame(VIDEO_STREAM_TYPE type, bool is_quick_intra_request, bool internal) {}
 
  virtual void AddVideoAvailabilityIndicatorEvents(VideoAvailabilityIndicator event) {}
 
  virtual void GetVideoAvailabilityIndicatorEvents(std::vector<VideoAvailabilityIndicator>& events) {}
 
  virtual int setUserId(uid_t uid) { user_id_ = uid; return 0; }
 
  virtual uid_t getUserId() { return user_id_; }
 
  virtual int GetActiveStreamsCount() = 0;
 
  virtual int prepareNodes(const char* id = nullptr) = 0;
 
  virtual bool attach(const AttachInfo& info) = 0;
  virtual bool detach(const DetachInfo& info) = 0;
  virtual bool registerTrackObserver(std::shared_ptr<IVideoTrackObserver> observer) {
    return false;
  }
  virtual bool unregisterTrackObserver(IVideoTrackObserver* observer) {
    return false;
  }
 
  virtual bool getStatisticsEx(LocalVideoTrackStatsEx& statsEx) { return false; }
  virtual bool getVideoFrameSourceFilterStats(agora::rtc::VideoFrameSourceFilter::Stats& stats) { return false; }
  virtual int32_t Width() const = 0;
  virtual int32_t Height() const = 0;
  virtual void getBillingVideoProfile(int32_t& w, int32_t& h, int32_t& frame_rate) {};
  virtual bool Enabled() const = 0;
 
  virtual int updateVideoWatermarks(const WatermarkConfig* configs, uint32_t count) { return -ERR_NOT_SUPPORTED; }
 
  virtual InternalVideoSourceType getInternalVideoSourceType() { return InternalVideoSourceType::None; }
 
  virtual rtc::VideoEncoderConfiguration getVideoEncoderConfiguration() { return {}; }
 
  virtual bool getVideoTextureCopyStatus(VideoTextureCopyParam& param) { return false; }
 
  virtual void getSimucastStreamConfig(SimulcastConfigInternal& simu_stream_config) {}
  virtual void getSimucastStreamStatus(SIMULCAST_STREAM_MODE& mode, bool& enable) {}
  virtual void getBillingVideoProfileWithSimulcast(bool& enable, SimulcastConfigInternal& simu_config) {}
 
  virtual int updateContentHint(VIDEO_CONTENT_HINT contentHint) { return -ERR_NOT_SUPPORTED; }
 
  virtual int updateScreenCaptureScenario(SCREEN_SCENARIO_TYPE screenScenario) { return -ERR_NOT_SUPPORTED; }
 
  int TrackId() const { return id_; }
 
  virtual int registerVideoEncodedFrameObserverLLApiInternal(media::IVideoEncodedFrameObserver* videoReceiver) = 0;
 
  virtual int unregisterVideoEncodedFrameObserverLLApiInternal(media::IVideoEncodedFrameObserver* videoReceiver) = 0;
 
  virtual int setLocalVideoSend(bool send) = 0;
 
  virtual bool ClearVideoConfigsOnLeaveChannel() { return false; }
 
  virtual int getCodecType() = 0;
 
  virtual int getVideoFrameBitDepth() = 0;
 
  virtual void setVideoDumpMode(int mode, bool enabled, int frame_cnt = -1) {}
 
  virtual bool NegotiateCodec(VideoNegotiatedCodecCaps negotiated_video_decode_caps) { return false; }
 
  virtual void onVideoModuleStatus(std::string node_name, int type) {}
 
  virtual int onRequestEnableSimulcastStream() { return 0; }
 
  virtual bool getIsAttachedToNetwork() { return false; }
 
  virtual bool isVideoFilterEnabled(const char* id) { return false; }
 
  virtual void ConfigVideoFec(const VideoFecConfig& fec_cap) {}
 
  virtual void ReconfigureCaptureDelayMs(int video_capture_delay_ms) {}
 
  virtual bool enableMinorStreamPeriodicKeyFrame()  { return false; }
 
  virtual void registerProactiveCaller(const std::shared_ptr<rtc::ProactiveCaller>& configurator) { return; }
  virtual void unregisterProactiveCaller() { return; }
 
  virtual void registerModuleControlPanel(std::shared_ptr<rtc::IModuleControlPanel> panel) { return; }
  virtual void unregisterModuleControlPanel() { return; }
 
  virtual void registerAAManager(std::shared_ptr<rtc::VideoAAManagerInterface> strategy_qoe_decision) { return; }
  virtual void unregisterAAManager(std::shared_ptr<rtc::VideoAAManagerInterface> strategy_qoe_decision) { return; }
 
  virtual void registerVideoTrackStatsMetrics(const std::shared_ptr<IVideoStatsMetrics>& metrics) { return; }
  virtual void unregisterVideoTrackStatsMetrics(const std::shared_ptr<IVideoStatsMetrics>& metrics) { return; }
  virtual bool NegotiateHDRSource(bool hdr_capture) { return false; }
 
  virtual int setEnabledLLApiInternal(bool enable, bool action_droppable = true) = 0;
  virtual LOCAL_VIDEO_STREAM_STATE getStateLLApiInternal() = 0;
  virtual int setSimulcastStreamModeLLApiInternal(SIMULCAST_STREAM_MODE mode, const SimulcastConfigInternal& simu_config) = 0;
  virtual int setVideoEncoderConfigurationLLApiInternal(const rtc::VideoEncoderConfiguration& config) = 0;
  virtual bool addVideoFilterLLApiInternal(agora_refptr<IVideoFilter> filter, media::base::VIDEO_MODULE_POSITION position = media::base::POSITION_POST_CAPTURER,
      const char* id = NULL) = 0;
  virtual bool removeVideoFilterLLApiInternal(agora_refptr<IVideoFilter> filter, media::base::VIDEO_MODULE_POSITION position = media::base::POSITION_POST_CAPTURER,
      const char* id = NULL) = 0;
  virtual bool hasVideoFilterLLApiInternal(const char* id, media::base::VIDEO_MODULE_POSITION position) = 0;
  virtual int setLocalRenderTargetFps(int target_fps) = 0;
  virtual bool addRendererLLApiInternal(agora_refptr<IVideoSinkBase> videoRenderer, media::base::VIDEO_MODULE_POSITION position) = 0;
  virtual bool removeRendererLLApiInternal(agora_refptr<IVideoSinkBase> videoRenderer, media::base::VIDEO_MODULE_POSITION position) = 0;
  virtual bool getStatisticsLLApiInternal(LocalVideoTrackStats& stats) = 0;
  virtual int enableVideoFilterLLApiInternal(const char* id, bool enable) {return -1;}
  virtual void enableSourceFilterLLApiInternal(const char* id, bool enable) { return; }
  virtual int setFilterPropertyLLApiInternal(const char* id, const char* key, const char* json_value) { return -1; };
  virtual int getFilterPropertyLLApiInternal(const char* id, const char* key, char* json_value, size_t buf_size) { return -1; };
  virtual void SetVideoCodecCaps(VideoNegotiatedCodecCaps negotiated_video_codec_caps) {};
  virtual void SetSvcAutoAdjusterEnabled(bool enabled) {}
  virtual void UpdateRemoteDlBweFeedback(int bwe) = 0;
  virtual void UpdateChannelBandwidthAndSubscription(AggregatedRemoteDlBweQuantile bwe_quantile,
                                                     std::map<VideoStreamIndex, int>&& be_subscribed_streams) = 0;
  virtual void UpdateAdaptiveScenarioTypeAndChannelSize(AdaptiveScenarioType scenario_type, int channel_size, bool sync_ability) = 0;
  virtual void SetQoeMetricsWeight(VideoStreamIndex video_stream_index, QoeMetricsWeight qoe_metrics_weight) { return; };
  virtual void OnVideoFecStatsChanged(const StreamsFecStatsMap& aut_fec_map) = 0;
 
 protected:
  const int id_;
  utils::WeakObserversFacility<IVideoTrackObserver> track_observers_;
  uid_t user_id_;
 
 private:
  static std::atomic<int> id_generator_;
};
 
struct RemoteVideoTrackStatsEx : RemoteVideoTrackStats {
  uint64_t firstDecodingTimeTickMs = 0;
  uint64_t firstVideoFrameRendered = 0;
  bool isHardwareCodec = false;
  int64_t totalFrozen200ms = 0;
  uint32_t last_frame_max = 0;
  uint32_t dec_in_num = 0;
  uint32_t render_in_num = 0;
  uint32_t render_out_num = 0;
  uint32_t fec_pkts_num = 0;
  uint32_t loss_af_fec = 0;
  int jitter_offset_ms = 0;
  int decode_level[10] = {0};
  uint64_t qp_sum = 0;
  VideoContentType content_type = VideoContentType::UNSPECIFIED;
  std::vector<VideoAvailabilityIndicator> video_availability;
};
 
class IRemoteVideoTrackEx : public IRemoteVideoTrack,
                            public VideoRemoteTrackControlAspect {
 public:
  enum DetachReason { MANUAL, TRACK_DESTROY, NETWORK_DESTROY };
  using RemoteVideoEvents = StateEvents<REMOTE_VIDEO_STATE, REMOTE_VIDEO_STATE_REASON>;
 
  struct AttachInfo {
    VideoNodeRtpSource* source;
    VideoNodeRtpSink* rtcp_sender;
    WeakPipelineBuilder builder;
    bool recv_media_packet = false;
    uint64_t stats_space = 0;
    bool enable_vpr = false;
    agora::rtc::VPR_POSITION vpr_position = agora::rtc::VPR_POSITION::POSITION_PRE_RENDER;
    std::shared_ptr<webrtc::IRsfecCodecFactoryInterface> rsfec_codec_factory;
    uint32_t video_threshhold_ms = 0;
    VideoQoEAnalyzerParameter qoe_analyzer_parameters;
  };
 
  struct DetachInfo {
    VideoNodeRtpSource* source;
    VideoNodeRtpSink* rtcp_sender;
    DetachReason reason;
  };
 
  IRemoteVideoTrackEx() = default;
 
  virtual ~IRemoteVideoTrackEx() {}
 
  virtual uint32_t getRemoteSsrc() = 0;
 
  virtual bool attach(const AttachInfo& info, REMOTE_VIDEO_STATE_REASON reason) = 0;
  virtual bool detach(const DetachInfo& info, REMOTE_VIDEO_STATE_REASON reason) = 0;
 
  virtual bool getStatisticsEx(RemoteVideoTrackStatsEx& statsex) { return false; }
 
  virtual bool registerTrackObserver(std::shared_ptr<IVideoTrackObserver> observer) {
    return false;
  }
  virtual bool unregisterTrackObserver(IVideoTrackObserver* observer) {
    return false;
  }
  virtual void registerProactiveCaller(const std::shared_ptr<ProactiveCaller>&) {}
  virtual void unregisterProactiveCaller() {}
  virtual void registerVideoTrackStatsMetrics(const std::shared_ptr<IVideoStatsMetrics>& metrics) { return; }
  virtual void unregisterVideoTrackStatsMetrics(const std::shared_ptr<IVideoStatsMetrics>& metrics) { return; }
 
  virtual REMOTE_VIDEO_STATE getStateLLApiInternal() = 0;
  virtual bool addVideoFilterLLApiInternal(agora_refptr<IVideoFilter> filter, media::base::VIDEO_MODULE_POSITION position = media::base::POSITION_POST_CAPTURER,
      const char* id = NULL) = 0;
  virtual bool removeVideoFilterLLApiInternal(agora_refptr<IVideoFilter> filter, media::base::VIDEO_MODULE_POSITION position = media::base::POSITION_POST_CAPTURER,
      const char* id = NULL) = 0;
  virtual bool hasVideoFilterLLApiInternal(const char* id, media::base::VIDEO_MODULE_POSITION position) = 0;
  virtual bool addRendererLLApiInternal(agora_refptr<IVideoSinkBase> videoRenderer, media::base::VIDEO_MODULE_POSITION position = media::base::POSITION_PRE_RENDERER) = 0;
  virtual bool removeRendererLLApiInternal(agora_refptr<IVideoSinkBase> videoRenderer, media::base::VIDEO_MODULE_POSITION position) = 0;
  virtual bool getStatisticsLLApiInternal(RemoteVideoTrackStats& stats) = 0;
  virtual bool getTrackInfoLLApiInternal(VideoTrackInfo& info) = 0;
  virtual int registerVideoEncodedFrameObserverLLApiInternal( agora::media::IVideoEncodedFrameObserver* encodedObserver) = 0;
  virtual int unregisterVideoEncodedFrameObserverLLApiInternal( agora::media::IVideoEncodedFrameObserver* encodedObserver) = 0;
  virtual int enableVideoFilterLLApiInternal(const char* id, bool enable){ return -1; };
  virtual int setFilterPropertyLLApiInternal(const char* id, const char* key, const char* json_value){ return -1; };
  virtual int registerMediaPacketReceiverLLApiInternal(IMediaPacketReceiver* packetReceiver) = 0;
  virtual int unregisterMediaPacketReceiverLLApiInternal(IMediaPacketReceiver* packetReceiver) = 0;
  virtual void OnUserRoleStatusUpdate(UserRoleStatus user_role_status) {};
 
 protected:
  utils::WeakObserversFacility<IVideoTrackObserver> track_observers_;
};
 
}  // namespace rtc
}  // namespace agora