Agora Java API Reference for Android
packet_i.h
1 // Agora RTC/MEDIA SDK
2 //
3 // Created by Pengfei Han in 2019-06.
4 // Copyright (c) 2019 Agora.io. All rights reserved.
5 //
6 #pragma once
7 
8 #include <cstdint>
9 #include <list>
10 #include <memory>
11 #include <string>
12 #include <vector>
13 
14 #include "AgoraBase.h"
15 
16 namespace agora {
17 namespace rtc {
18 
19 
20 static const uint8_t kVideoEngineFlagHasIntraRequest = 0x10;
21 static const uint8_t kVideoEngineFlagStdCodec = 0x8;
22 static const uint8_t kVideoEngineFlagNasa = 0x40;
23 static const uint8_t kVideoEngineFlagScalableDelta = 0x80;
24 static const uint8_t kVideoEngineFlagMajorStreamOnly = 0x01;
25 
26 static const uint8_t kAgoraHeaderLength = 3;
27 static const uint8_t kAgoraAudioExtendLength = 5;
28 
29 enum MEDIA_STREAM_TYPE {
30  MEDIA_STREAM_TYPE_AUDIO = (1 << 0),
31  MEDIA_STREAM_TYPE_VIDEO_LOW = (1 << 1),
32  MEDIA_STREAM_TYPE_VIDEO_HIGH = (1 << 2),
33  MEDIA_STREAM_TYPE_VIDEO = (MEDIA_STREAM_TYPE_VIDEO_LOW | MEDIA_STREAM_TYPE_VIDEO_HIGH),
34 };
35 
36 struct SMediaFrame {
37  uid_t uid_;
38  uint8_t flags_;
39  uint16_t seq_;
40  uint16_t ssrc_;
41  uint64_t packetSentTs_;
42  uint64_t sentTs_;
43  uint64_t receiveTs_;
44  std::string payload_;
45  SMediaFrame() : uid_(0), flags_(0), seq_(0), ssrc_(0), sentTs_(0), receiveTs_(0) {}
46 };
47 
48 struct SAudioFrame : public SMediaFrame {
49  uint8_t codec_;
50  uint32_t ts_;
51  int8_t vad_;
52  uint8_t internalFlags_;
53  uint8_t audio_fec_level_;
54  SAudioFrame() : SMediaFrame(), codec_(0), ts_(0), vad_(0), internalFlags_(0) {}
55 };
56 
57 using SharedSAudioFrame = std::shared_ptr<SAudioFrame>;
58 
59 struct SAudioPacket {
60  enum AUDIO_PACKET_TYPE {
61  AUDIO_PACKET_REXFERRED = 0x1,
62  AUDIO_PACKET_FROM_VOS = 0x2,
63  AUDIO_PACKET_FROM_P2P = 0x4,
64  };
65  int8_t vad_;
66  uint8_t codec_;
67  uint8_t internalFlags_;
68  uint16_t seq_;
69  uint16_t ssrc_;
70  uint16_t payloadLength_;
71  uint16_t latestFrameSeq_;
72  std::list<SharedSAudioFrame> frames_;
73  SAudioPacket()
74  : vad_(0),
75  codec_(0),
76  internalFlags_(0),
77  seq_(0),
78  ssrc_(0),
79  payloadLength_(0),
80  latestFrameSeq_(0) {}
81 };
82 
83 struct rtc_packet_t {
84  enum INTERNAL_FLAG_TYPE {
85  RTC_FLAG_REXFERRED = 0x1,
86  RTC_FLAG_FROM_VOS = 0x2,
87  RTC_FLAG_FROM_P2P = 0x4,
88  RTC_FLAG_UNIQUE_PACKET = 0x8,
89  VIDEO_FLAG_TIMESTAMP_SET = 0x10,
90  VIDEO_FLAG_CACHED = 0x20,
91  VIDEO_FLAG_VIDEO3 = 0x40,
92  };
93  uid_t uid;
94  uint32_t seq;
95  uint16_t payload_length; // should be the same as payload.length()
96  uint64_t sent_ts;
97  uint64_t recv_ts;
98  int link_id;
99  uint8_t internal_flags;
100  std::string payload;
101  rtc_packet_t()
102  : uid(0), seq(0), payload_length(0), sent_ts(0), recv_ts(0), link_id(-1), internal_flags(0) {}
103  virtual ~rtc_packet_t() {}
104 };
105 
107  bool quit;
108  bool rtcp;
109  broadcast_packet_t() : quit(false), rtcp(false) {}
110 };
111 
112 struct audio_packet_t : public rtc_packet_t {
113  uint32_t ts;
114  int8_t vad;
115  uint8_t codec;
116  int last_error; // error code set by last filter
117  uint32_t reqMs; // for calculating RTT only
118  audio_packet_t() : ts(0), vad(0), codec(0), last_error(0), reqMs(0) {}
119 };
120 
121 struct video_packet_t : public rtc_packet_t {
122  enum VIDEO_STREAM_TYPE {
123  VIDEO_STREAM_UNKNOWN = -1,
124  VIDEO_STREAM_HIGH = 0,
125  VIDEO_STREAM_LOW = 1,
126  VIDEO_STREAM_MEDIUM = 2,
127  VIDEO_STREAM_LIVE = 3,
128  VIDEO_STREAM_MIN = VIDEO_STREAM_HIGH,
129  VIDEO_STREAM_MAX = VIDEO_STREAM_LIVE,
130  };
131 
132  enum VIDEO_FLAG_TYPE {
133  // below is for video2 only, not used in video3
134  VIDEO_FLAG_KEY_FRAME = 0x80,
135  VIDEO_FLAG_FEC = 0x40,
136  VIDEO_FLAG_LIVE = 0x20,
137  VIDEO_FLAG_STD_CODEC = 0x8, // also for video3 to differentiate std stream and private stream
138  VIDEO_FLAG_B_FRAME = 0x10,
139  // below is for video3
140  VIDEO_FLAG_HARDWARE_ENCODE = 0x4,
141  };
142 
143  enum VIDEO_FRAME_TYPE {
144  KEY_FRAME = 0,
145  DELTA_FRAME = 1,
146  B_FRAME = 2,
147  };
148 
149  // TODO(Bob): This should be removed and use public API definitions.
150  enum VIDEO_CODEC_TYPE {
151  VIDEO_CODEC_VP8 = 1, // std VP8
152  VIDEO_CODEC_H264 = 2, // std H264
153  VIDEO_CODEC_EVP = 3, // VP8 with BCM
154  VIDEO_CODEC_E264 = 4, // H264 with BCM
155  };
156 
157  enum VIDEO_EXTRA_FLAG_TYPE {
158  // marks if the |req_ms| field of PVideoRexferRes_v4 is set
159  VIDEO_EXTRA_FLAG_TIMESTAMP_SET = 0x1,
160  };
161 
162  enum EXTENSION_VERSION {
163  EXTENSION_VERSION_0 = 0,
164  EXTENSION_VERSION_1 = 1,
165  EXTENSION_VERSION_2 = 2,
166  };
167 
168  struct Extension {
169  bool has_extension_ = false;
170  uint16_t tag_ = EXTENSION_VERSION_0;
171  std::vector<uint32_t> content_;
172  };
173 
174  uint32_t frameSeq;
175  uint8_t frameType;
176  uint8_t streamType;
177  uint16_t packets;
178  uint16_t subseq;
179  uint8_t fecPkgNum;
180  uint8_t codec;
181  uint8_t flags;
182  uint8_t protocolVersion;
183  uint32_t reqMs; // for calculating RTT only
184  uint32_t reserve1;
185  Extension extension;
186  int64_t transport_seq; // for transport-cc
187  int8_t cc_type;
188 
190  : frameSeq(0),
191  frameType(0),
192  streamType(0),
193  packets(0),
194  subseq(0),
195  fecPkgNum(0),
196  codec(0),
197  flags(0),
198  protocolVersion(0),
199  reqMs(0),
200  reserve1(0),
201  transport_seq(-1),
202  cc_type(0) {}
203 
204  union video3_flags {
205  struct {
206  uint8_t stream_type : 4;
207  uint8_t frame_type : 4;
208  };
209  uint8_t video_type;
210  };
211 
212  void fromVideType(uint8_t f) {
213  video3_flags t;
214  t.video_type = f;
215  streamType = t.stream_type;
216  frameType = t.frame_type;
217  }
218 
219  uint8_t toVideoType() const {
220  video3_flags t;
221  t.stream_type = streamType;
222  t.frame_type = frameType;
223  return t.video_type;
224  }
225 
226  bool hasReserveBit(uint16_t bit) { return (reserve1 & (1 << bit)) == (1 << bit); }
227 };
228 
230  rtc::uid_t uid = 0;
231  bool from_vos = false;
232  std::string payload;
233 };
234 
236  rtc::uid_t uid;
237  std::string payload;
238 };
239 
241  rtc::uid_t uid;
242  int type;
243  std::string user_id;
244  std::string payload;
245 };
246 
247 enum VideoStreamType {
248  MASTER_VIDEO_STREAM = 0,
249  LOW_BITRATE_VIDEO_STREAM = 1,
250  MEDIUM_BITRATE_VIDEO_STREAM = 2,
251  LIVE_VIDEO_STREAM = 3,
252 };
253 
254 
256 {
257 public:
258  virtual ~IVideoListener() {}
260  {
261  int width;
262  int height;
263  unsigned int prevFrameNumber;
264  int sentBytes;
265  int sentFrames;
266  int sentQP;
267  int sentPkgNumber;
268 
269  void reset() {
270  sentBytes = 0;
271  sentFrames = 0;
272  sentQP = 0;
273  sentPkgNumber = 0;
274  }
275 
277  prevFrameNumber = 0;
278  width = 640;
279  height = 360;
280  reset();
281  }
282 
283  };
285  {
286 
287  // local stat
288  LocalVideoStreamStat highStream;
289  LocalVideoStreamStat lowStream;
290  int sentRtt;
291  int sentLoss;
292  int sentTargetBitRate;
293  int sentTargetFrameRate;
294  int sentBitrateExcludeFec;
295  unsigned int encodeTimeMs;
296  unsigned int minEncodeTimeMs;
297  unsigned int maxEncodeTimeMs;
298  int captureWidth;
299  int captureHeight;
300  int captureFrames;
301  int encoderRecvFrames;
302  int encodedFrames;
303  int encodedKeyFrames;
304  unsigned int totalEncodedFrames;
305  int encodeFailFrames;
306  int encoderSkipFrames;
307  int renderedFrames;
308  int duration;
309  int fecLevel;
310  int estimateBandwidth;
311  unsigned int maxFrameOutInterval;
312  int uplinkFreezeCount;
313  int uplinkFreezeTime;
314  int cameraOpenStatus;
315  int captureType;
316  int renderType;
317  uint64_t render_buffer_size;
318  int beautyCostTime;
319  // compressed states
320  /*
321  bit 31~28: version of this structure. value 1
322  bit 27: video engine exists. 0: not exist; 1: exit
323  bit 26: texture encode. 0: yuv encode; 1: texture encode (android only)
324  bit 25: app rendering. 0: sdk rendering; 1: app rendering
325  // bit 25: sender disabled. 0: sender enabled; 1: sender disabled (duplicate with SDK Mute Status)
326  bit 24~21: encoder codec index.
327  // bit 24: hardware encode. 0: software encode; 1: hardware encode (duplicate with Video Hardware Encode)
328  bit 20~19: fec type.
329  bit 18: has_intra_request (no periodic key frame) parameter setting.
330  bit 17~13: reserved.
331  bit 12~0: camera id.
332  */
333  unsigned int bitFieldStates;
334 
335  void reset() {
336  highStream.reset();
337  lowStream.reset();
338  sentRtt = 0;
339  sentLoss = 0;
340  sentTargetBitRate = 0;
341  sentTargetFrameRate = 0;
342  sentBitrateExcludeFec = 0;
343  encodeTimeMs = 0;
344  minEncodeTimeMs = 0;
345  maxEncodeTimeMs = 0;
346  captureWidth = 0;
347  captureHeight = 0;
348  captureFrames = 0;
349  encoderRecvFrames = 0;
350  encodedFrames = 0;
351  encodeFailFrames = 0;
352  encoderSkipFrames = 0;
353  renderedFrames = 0;
354  duration = 0;
355  fecLevel = 0;
356  estimateBandwidth = 0;
357  maxFrameOutInterval = 0;
358  cameraOpenStatus = -1;
359  captureType = 0;
360  renderType = -1;
361  render_buffer_size = 0;
362  beautyCostTime = 0;
363  }
364  };
365 
367  {
368  // remote stat
369  int uid;
370  int delay;
371  int renderedFrames;
372  int receivedBytes;
373  int recvPkgNumber;
374  int width;
375  int height;
376  int lossAfterFec;
377  int decodeFailedFrames;
378  int decodeRejectedFrames;
379  int streamType;
380  int decodedFrames;
381  int rendererRecvFrames;
382  int decodeTimeMs;
383  int duration;
384  int decodedQP; // Add the QP reportor of decoder
385  unsigned int maxRenderInterval;
386  unsigned int lastRenderMs;
387  unsigned int minFrameNumber;
388  unsigned int maxFrameNumber;
389  unsigned int freezeCnt;
390  unsigned int freezeCnt300;
391  unsigned int freezeCnt500;
392  int freezeTimeMs;
393  int freezeTimeMs300;
394  int freezeTimeMs500;
395  bool isHardwareDecoding;
396  int decoderInFrames;
397  int renderType;
398  bool muted;
399  uint64_t render_buffer_size;
400  int decodeBgDroppedFrames;
401  bool isSuperResolutionEnabled;
402 
403  void reset() {
404  uid = 0;
405  delay = 0;
406  width = 0;
407  height = 0;
408  renderedFrames = 0;
409  receivedBytes = 0;
410  lossAfterFec = 0;
411  decodeFailedFrames = 0;
412  //streamType = 0;
413  decodedFrames = 0;
414  rendererRecvFrames = 0;
415  decodeRejectedFrames = 0;
416  decodeTimeMs = 0;
417  recvPkgNumber = 0;
418  duration = 0;
419  maxRenderInterval = 0;
420  decodedQP = 0;
421  minFrameNumber = 0xFFFFFFFF;
422  maxFrameNumber = 0;
423  freezeCnt300 = 0;
424  freezeTimeMs300 = 0;
425  freezeCnt500 = 0;
426  freezeTimeMs500 = 0;
427  freezeCnt = 0;
428  freezeTimeMs = 0;
429  isHardwareDecoding = false;
430  decoderInFrames = 0;
431  renderType = -1;
432  render_buffer_size = 0;
433  decodeBgDroppedFrames = 0;
434  isSuperResolutionEnabled = false;
435  }
436  };
437 
438  // #define VIDEO_ENGINE_FLAG_KEY_FRAME 0x00000080
439  // #define VIDEO_ENGINE_FLAG_FEC 0x00000040
440  // #define VIDEO_ENGINE_FLAG_LIVE 0x00000020
441  // #define VIDEO_ENGINE_FLAG_B_FRAME 0x00000010
442  #define VIDEO_ENGINE_FLAG_SCALABLE_DELTA 0x00000080
443  #define VIDEO_ENGINE_FLAG_NASA 0x00000040
444  #define VIDEO_ENGINE_FLAG_HAS_PISE 0x00000020
445  #define VIDEO_ENGINE_FLAG_HAS_INTRA_REQUEST 0x00000010
446  #define VIDEO_ENGINE_FLAG_STD_CODEC 0x00000008
447  #define VIDEO_ENGINE_FLAG_HARDWARE_ENCODE 0x00000004
448  #define VIDEO_ENGINE_FLAG_NEW_AVSYNC_TIMESTAMP 0x00000002
449  #define VIDEO_ENGINE_FLAG_MULTI_STREAM 0x00000001
450 
451  enum VideoFrameType {
452  KEY_FRAME = 0,
453  DELTA_FRAME = 1,
454  B_FRAME = 2,
455  PISE_FRAME = 3,
456  SCALABLE_DELTA_FRAME = 4,
457  };
458  enum VideoCodecType {
459  VIDEO_CODEC_VP8 = 1,
460  VIDEO_CODEC_H264 = 2,
461  VIDEO_CODEC_EVP = 3,
462  VIDEO_CODEC_E264 = 4,
463  };
464  enum VideoFeedbackPacketType {
465  VIDEO_FEEDBACK_INTRA_REQUEST = 1,
466  VIDEO_FEEDBACK_DISABLE_INTRA_FEC = 2,
467  VIDEO_FEEDBACK_INTRA_REQUEST_QUICK = 3,
468  VIDEO_FEEDBACK_DECODE_FB = 4,
469  VIDEO_FEEDBACK_BANDWIDTH_ESTIMATION = 5,
470  VIDEO_FEEDBACK_REQUEST_MULTISTREAM = 6,
471  VIDEO_FEEDBACK_AUT = 7,
472  };
473  struct PacketInfo
474  {
475  const void* packet;
476  unsigned short packetLen;
477  unsigned int frame_num;
478  uint16_t frame_pkg_num;
479  uint16_t pkg_seq_in_frame;
480  unsigned char fec_num;
481  unsigned char fec_method;
482  int codec_type;
483  VideoFrameType frame_type;
484  VideoStreamType stream_type;
485  unsigned int flag;
486  int version;
487  unsigned short reserve1;
488  };
489  struct AutStreamData {
490  uint16_t bandwidth;
491  uint8_t recv_stream_type;
492  uint8_t expected_stream_type;
493  };
495  uint16_t rtt;
496  uint16_t lost_ratio;
497  uint16_t bwe;
498  uint16_t padding;
499  uint16_t total_sent;
500  uint16_t queueing_time;
501  uint16_t active_uid_counts;
502  uint16_t allocated;
503  uint16_t mtu;
504  uint16_t jitter95;
505  std::list<AutStreamData> stream_info;
506  };
507 
508  virtual int sendVideoPacket(const PacketInfo& info) = 0;
509  virtual int sendVideoRtcpPacket(unsigned int uid, const void* packet, unsigned short packetLen, bool isToVos) = 0;
510 
511  // new interface for feedback message
512  virtual int sendVideoRtcpFeedbackPacket(const VideoFeedbackPacketType type, unsigned int toUid, const void* packet, int packetSize) = 0;
513 
514  virtual void onRemoteFirstFrameDrawed(int viewIndex, unsigned int uid, int width, int height) = 0;
515  virtual void onLocalFirstFrameDrawed(int width, int height) = 0;
516  virtual void onRemoteFirstFrameDecoded(unsigned int uid, int width, int height) = 0;
517  virtual void onRemoteVideoInterrupted(unsigned int uid, unsigned int elapse_time) = 0;
518  virtual void onVideoStat(const LocalVideoStat& localStat, RemoteVideoStat* remoteStat, int remoteCount) = 0;
519  virtual void onVideoProfile(unsigned int width, unsigned int height, unsigned int fps, unsigned int bitrate) = 0;
520  virtual void switchVideoStream(unsigned int uid, VideoStreamType stream) = 0;
521  virtual void onBandWidthLevelChanged(int level) = 0;
522  virtual void onCameraFocusAreaChanged(int x, int y, int width, int height) = 0;
523  virtual void onCameraExposureAreaChanged(int x, int y, int width, int height) = 0;
524  virtual void onVideoViewSizeChanged(int userID, int newWidth, int newHeight) {
525  (void)userID;
526  (void)newWidth;
527  (void)newHeight;
528  }
529 
530  virtual void onVideoFrameFrozen(unsigned int uid, bool frozen) = 0;
531 
532  virtual void onAppSetVideoStartBitRate(int value) = 0;
533 
534  virtual int onEncodeVideoSEI(char** info, int *len) = 0;
535  virtual void onVideoSizeChanged(unsigned int uid, int newWidth, int newHeight, int newRotation) {
536  (void)uid;
537  (void)newWidth;
538  (void)newHeight;
539  (void)newRotation;
540  }
541 
542  virtual void onSendVideoPaced(bool status) = 0;
543  virtual void onBWELevel(int level) = 0;
544  virtual void onVideoRexferStatus(bool status, int target_bitrate) = 0;
545  virtual void onStartCaptureSuccess() = 0;
546  // add for sei
547  virtual int onSendSEI(char **info, int *len, long long timeStampMs, int streamType, bool isDualStream) = 0;
548  virtual int onReceiveSEI(char *info, int len, unsigned int uid, long long timeStampMs) = 0;
549  virtual void OnVideoStreamBitrateRangeChanged(VideoStreamType type,
550  uint32_t max_kbps, uint32_t min_kbps) = 0;
551 };
552 
553 } // namespace rtc
554 } // namespace agora
agora::rtc::IVideoListener::PacketInfo
Definition: packet_i.h:474
agora::rtc::IVideoListener::AutStreamData
Definition: packet_i.h:489
agora::rtc::IVideoListener::LocalVideoStat
Definition: packet_i.h:285
agora::rtc::video_packet_t::Extension
Definition: packet_i.h:168
agora::rtc::IVideoListener::LocalVideoStreamStat
Definition: packet_i.h:260
agora::rtc::video_packet_t
Definition: packet_i.h:121
agora::rtc::video_rtcp_packet_t
Definition: packet_i.h:229
agora::rtc::SMediaFrame
Definition: packet_i.h:36
agora::rtc::SAudioFrame
Definition: packet_i.h:48
agora::rtc::broadcast_packet_t
Definition: packet_i.h:106
agora::rtc::audio_packet_t
Definition: packet_i.h:112
agora::rtc::IVideoListener::RemoteVideoStat
Definition: packet_i.h:367
agora::rtc::video_report_packet_t
Definition: packet_i.h:240
agora::rtc::video_custom_ctrl_broadcast_packet_t
Definition: packet_i.h:235
agora::rtc::rtc_packet_t
Definition: packet_i.h:83
agora::rtc::SAudioPacket
Definition: packet_i.h:59
agora::rtc::video_packet_t::video3_flags
Definition: packet_i.h:204
agora::rtc::IVideoListener
Definition: packet_i.h:256
agora::rtc::IVideoListener::AutFeedbackData
Definition: packet_i.h:494