Agora Java API Reference for Android
packet_i.h
1 // Agora RTC/MEDIA SDK
2 //
3 // Created by Pengfei Han in 2019-06.
4 // Copyright (c) 2019 Agora.io. All rights reserved.
5 //
6 #pragma once
7 
8 #include <cstdint>
9 #include <list>
10 #include <memory>
11 #include <string>
12 #include <vector>
13 
14 #include "AgoraBase.h"
15 
16 namespace agora {
17 namespace rtc {
18 
19 
20 static const uint8_t kVideoEngineFlagHasIntraRequest = 0x10;
21 static const uint8_t kVideoEngineFlagStdCodec = 0x8;
22 static const uint8_t kVideoEngineFlagNasa = 0x40;
23 static const uint8_t kVideoEngineFlagScalableDelta = 0x80;
24 static const uint8_t kVideoEngineFlagMajorStreamOnly = 0x01;
25 
26 static const uint8_t kAgoraHeaderLength = 3;
27 static const uint8_t kAgoraAudioExtendLength = 5;
28 
29 enum MEDIA_STREAM_TYPE {
30  MEDIA_STREAM_TYPE_AUDIO = (1 << 0),
31  MEDIA_STREAM_TYPE_VIDEO_LOW = (1 << 1),
32  MEDIA_STREAM_TYPE_VIDEO_HIGH = (1 << 2),
33  MEDIA_STREAM_TYPE_VIDEO = (MEDIA_STREAM_TYPE_VIDEO_LOW | MEDIA_STREAM_TYPE_VIDEO_HIGH),
34 };
35 
36 struct SMediaFrame {
37  uid_t uid_;
38  uint8_t flags_;
39  uint16_t seq_;
40  uint16_t ssrc_;
41  uint64_t packetSentTs_;
42  uint64_t sentTs_;
43  uint64_t receiveTs_;
44  std::string payload_;
45  SMediaFrame() : uid_(0), flags_(0), seq_(0), ssrc_(0), sentTs_(0), receiveTs_(0) {}
46 };
47 
48 struct SAudioFrame : public SMediaFrame {
49  uint8_t codec_;
50  uint32_t ts_;
51  int8_t vad_;
52  uint8_t internalFlags_;
53  uint8_t audio_fec_level_;
54  SAudioFrame() : SMediaFrame(), codec_(0), ts_(0), vad_(0), internalFlags_(0) {}
55 };
56 
57 using SharedSAudioFrame = std::shared_ptr<SAudioFrame>;
58 
59 struct SAudioPacket {
60  enum AUDIO_PACKET_TYPE {
61  AUDIO_PACKET_REXFERRED = 0x1,
62  AUDIO_PACKET_FROM_VOS = 0x2,
63  AUDIO_PACKET_FROM_P2P = 0x4,
64  };
65  int8_t vad_;
66  uint8_t codec_;
67  uint8_t internalFlags_;
68  uint16_t seq_;
69  uint16_t ssrc_;
70  uint16_t payloadLength_;
71  uint16_t latestFrameSeq_;
72  std::list<SharedSAudioFrame> frames_;
73  SAudioPacket()
74  : vad_(0),
75  codec_(0),
76  internalFlags_(0),
77  seq_(0),
78  ssrc_(0),
79  payloadLength_(0),
80  latestFrameSeq_(0) {}
81 };
82 
83 struct rtc_packet_t {
84  enum INTERNAL_FLAG_TYPE {
85  RTC_FLAG_REXFERRED = 0x1,
86  RTC_FLAG_FROM_VOS = 0x2,
87  RTC_FLAG_FROM_P2P = 0x4,
88  RTC_FLAG_FROM_BROADCAST = 0x8,
89  VIDEO_FLAG_TIMESTAMP_SET = 0x10,
90  VIDEO_FLAG_CACHED = 0x20,
91  VIDEO_FLAG_VIDEO3 = 0x40,
92  };
93  uid_t uid;
94  uint32_t seq;
95  uint16_t payload_length; // should be the same as payload.length()
96  uint64_t sent_ts;
97  uint64_t recv_ts;
98  int link_id;
99  uint8_t internal_flags;
100  std::string payload;
101  rtc_packet_t()
102  : uid(0), seq(0), payload_length(0), sent_ts(0), recv_ts(0), link_id(-1), internal_flags(0) {}
103  virtual ~rtc_packet_t() {}
104 };
105 
107  bool quit;
108  bool rtcp;
109  bool need_reliable;
110  bool real_quit;
112  : quit(false), rtcp(false), need_reliable(false), real_quit(false) {}
113 };
114 
115 struct audio_packet_t : public rtc_packet_t {
116  uint32_t ts;
117  int8_t vad;
118  uint8_t codec;
119  int last_error; // error code set by last filter
120  uint32_t reqMs; // for calculating RTT only
121  audio_packet_t() : ts(0), vad(0), codec(0), last_error(0), reqMs(0) {}
122 };
123 
124 struct video_packet_t : public rtc_packet_t {
125  enum VIDEO_STREAM_TYPE {
126  VIDEO_STREAM_UNKNOWN = -1,
127  VIDEO_STREAM_HIGH = 0,
128  VIDEO_STREAM_LOW = 1,
129  VIDEO_STREAM_MEDIUM = 2,
130  VIDEO_STREAM_LIVE = 3,
131  VIDEO_STREAM_MIN = VIDEO_STREAM_HIGH,
132  VIDEO_STREAM_MAX = VIDEO_STREAM_LIVE,
133  };
134 
135  enum VIDEO_FLAG_TYPE {
136  // below is for video2 only, not used in video3
137  VIDEO_FLAG_KEY_FRAME = 0x80,
138  VIDEO_FLAG_FEC = 0x40,
139  VIDEO_FLAG_LIVE = 0x20,
140  VIDEO_FLAG_STD_CODEC = 0x8, // also for video3 to differentiate std stream and private stream
141  VIDEO_FLAG_B_FRAME = 0x10,
142  // below is for video3
143  VIDEO_FLAG_HARDWARE_ENCODE = 0x4,
144  };
145 
146  enum VIDEO_FRAME_TYPE {
147  KEY_FRAME = 0,
148  DELTA_FRAME = 1,
149  B_FRAME = 2,
150  };
151 
152  // TODO(Bob): This should be removed and use public API definitions.
153  enum VIDEO_CODEC_TYPE {
154  VIDEO_CODEC_VP8 = 1, // std VP8
155  VIDEO_CODEC_H264 = 2, // std H264
156  VIDEO_CODEC_EVP = 3, // VP8 with BCM
157  VIDEO_CODEC_E264 = 4, // H264 with BCM
158  };
159 
160  enum VIDEO_EXTRA_FLAG_TYPE {
161  // marks if the |req_ms| field of PVideoRexferRes_v4 is set
162  VIDEO_EXTRA_FLAG_TIMESTAMP_SET = 0x1,
163  };
164 
165  enum EXTENSION_VERSION {
166  EXTENSION_VERSION_0 = 0,
167  EXTENSION_VERSION_1 = 1,
168  EXTENSION_VERSION_2 = 2,
169  };
170 
171  struct Extension {
172  bool has_extension_ = false;
173  uint16_t tag_ = EXTENSION_VERSION_0;
174  std::vector<uint32_t> content_;
175  };
176 
177  uint32_t frameSeq;
178  uint8_t frameType;
179  uint8_t streamType;
180  uint16_t packets;
181  uint16_t subseq;
182  uint8_t fecPkgNum;
183  uint8_t codec;
184  uint8_t flags;
185  uint8_t protocolVersion;
186  uint32_t reqMs; // for calculating RTT only
187  uint32_t reserve1;
188  Extension extension;
189  int64_t transport_seq; // for transport-cc
190  int8_t cc_type;
191 
193  : frameSeq(0),
194  frameType(0),
195  streamType(0),
196  packets(0),
197  subseq(0),
198  fecPkgNum(0),
199  codec(0),
200  flags(0),
201  protocolVersion(0),
202  reqMs(0),
203  reserve1(0),
204  transport_seq(-1),
205  cc_type(0) {}
206 
207  union video3_flags {
208  struct {
209  uint8_t stream_type : 4;
210  uint8_t frame_type : 4;
211  };
212  uint8_t video_type;
213  };
214 
215  void fromVideType(uint8_t f) {
216  video3_flags t;
217  t.video_type = f;
218  streamType = t.stream_type;
219  frameType = t.frame_type;
220  }
221 
222  uint8_t toVideoType() const {
223  video3_flags t;
224  t.stream_type = streamType;
225  t.frame_type = frameType;
226  return t.video_type;
227  }
228 
229  bool hasReserveBit(uint16_t bit) { return (reserve1 & (1 << bit)) == (1 << bit); }
230 };
231 
233  rtc::uid_t uid = 0;
234  bool from_vos = false;
235  std::string payload;
236 };
237 
239  rtc::uid_t uid;
240  std::string payload;
241 };
242 
244  rtc::uid_t uid;
245  int type;
246  std::string user_id;
247  std::string payload;
248 };
249 
250 enum VideoStreamType {
251  MASTER_VIDEO_STREAM = 0,
252  LOW_BITRATE_VIDEO_STREAM = 1,
253  MEDIUM_BITRATE_VIDEO_STREAM = 2,
254  LIVE_VIDEO_STREAM = 3,
255 };
256 
257 
259 {
260 public:
261  virtual ~IVideoListener() {}
263  {
264  int width;
265  int height;
266  unsigned int prevFrameNumber;
267  int sentBytes;
268  int sentFrames;
269  int sentQP;
270  int sentPkgNumber;
271 
272  void reset() {
273  sentBytes = 0;
274  sentFrames = 0;
275  sentQP = 0;
276  sentPkgNumber = 0;
277  }
278 
280  prevFrameNumber = 0;
281  width = 640;
282  height = 360;
283  reset();
284  }
285 
286  };
288  {
289 
290  // local stat
291  LocalVideoStreamStat highStream;
292  LocalVideoStreamStat lowStream;
293  int sentRtt;
294  int sentLoss;
295  int sentTargetBitRate;
296  int sentTargetFrameRate;
297  int sentBitrateExcludeFec;
298  unsigned int encodeTimeMs;
299  unsigned int minEncodeTimeMs;
300  unsigned int maxEncodeTimeMs;
301  int captureWidth;
302  int captureHeight;
303  int captureFrames;
304  int encoderRecvFrames;
305  int encodedFrames;
306  int encodedKeyFrames;
307  unsigned int totalEncodedFrames;
308  int encodeFailFrames;
309  int encoderSkipFrames;
310  int renderedFrames;
311  int duration;
312  int fecLevel;
313  int estimateBandwidth;
314  unsigned int maxFrameOutInterval;
315  int uplinkFreezeCount;
316  int uplinkFreezeTime;
317  int cameraOpenStatus;
318  int captureType;
319  int renderType;
320  uint64_t render_buffer_size;
321  int beautyCostTime;
322  // compressed states
323  /*
324  bit 31~28: version of this structure. value 1
325  bit 27: video engine exists. 0: not exist; 1: exit
326  bit 26: texture encode. 0: yuv encode; 1: texture encode (android only)
327  bit 25: app rendering. 0: sdk rendering; 1: app rendering
328  // bit 25: sender disabled. 0: sender enabled; 1: sender disabled (duplicate with SDK Mute Status)
329  bit 24~21: encoder codec index.
330  // bit 24: hardware encode. 0: software encode; 1: hardware encode (duplicate with Video Hardware Encode)
331  bit 20~19: fec type.
332  bit 18: has_intra_request (no periodic key frame) parameter setting.
333  bit 17~13: reserved.
334  bit 12~0: camera id.
335  */
336  unsigned int bitFieldStates;
337 
338  void reset() {
339  highStream.reset();
340  lowStream.reset();
341  sentRtt = 0;
342  sentLoss = 0;
343  sentTargetBitRate = 0;
344  sentTargetFrameRate = 0;
345  sentBitrateExcludeFec = 0;
346  encodeTimeMs = 0;
347  minEncodeTimeMs = 0;
348  maxEncodeTimeMs = 0;
349  captureWidth = 0;
350  captureHeight = 0;
351  captureFrames = 0;
352  encoderRecvFrames = 0;
353  encodedFrames = 0;
354  encodeFailFrames = 0;
355  encoderSkipFrames = 0;
356  renderedFrames = 0;
357  duration = 0;
358  fecLevel = 0;
359  estimateBandwidth = 0;
360  maxFrameOutInterval = 0;
361  cameraOpenStatus = -1;
362  captureType = 0;
363  renderType = -1;
364  render_buffer_size = 0;
365  beautyCostTime = 0;
366  }
367  };
368 
370  {
371  // remote stat
372  int uid;
373  int delay;
374  int renderedFrames;
375  int receivedBytes;
376  int recvPkgNumber;
377  int width;
378  int height;
379  int lossAfterFec;
380  int decodeFailedFrames;
381  int decodeRejectedFrames;
382  int streamType;
383  int decodedFrames;
384  int rendererRecvFrames;
385  int decodeTimeMs;
386  int duration;
387  int decodedQP; // Add the QP reportor of decoder
388  unsigned int maxRenderInterval;
389  unsigned int lastRenderMs;
390  unsigned int minFrameNumber;
391  unsigned int maxFrameNumber;
392  unsigned int freezeCnt;
393  unsigned int freezeCnt300;
394  unsigned int freezeCnt500;
395  int freezeTimeMs;
396  int freezeTimeMs300;
397  int freezeTimeMs500;
398  bool isHardwareDecoding;
399  int decoderInFrames;
400  int renderType;
401  bool muted;
402  uint64_t render_buffer_size;
403  int decodeBgDroppedFrames;
404  bool isSuperResolutionEnabled;
405 
406  void reset() {
407  uid = 0;
408  delay = 0;
409  width = 0;
410  height = 0;
411  renderedFrames = 0;
412  receivedBytes = 0;
413  lossAfterFec = 0;
414  decodeFailedFrames = 0;
415  //streamType = 0;
416  decodedFrames = 0;
417  rendererRecvFrames = 0;
418  decodeRejectedFrames = 0;
419  decodeTimeMs = 0;
420  recvPkgNumber = 0;
421  duration = 0;
422  maxRenderInterval = 0;
423  decodedQP = 0;
424  minFrameNumber = 0xFFFFFFFF;
425  maxFrameNumber = 0;
426  freezeCnt300 = 0;
427  freezeTimeMs300 = 0;
428  freezeCnt500 = 0;
429  freezeTimeMs500 = 0;
430  freezeCnt = 0;
431  freezeTimeMs = 0;
432  isHardwareDecoding = false;
433  decoderInFrames = 0;
434  renderType = -1;
435  render_buffer_size = 0;
436  decodeBgDroppedFrames = 0;
437  isSuperResolutionEnabled = false;
438  }
439  };
440 
441  // #define VIDEO_ENGINE_FLAG_KEY_FRAME 0x00000080
442  // #define VIDEO_ENGINE_FLAG_FEC 0x00000040
443  // #define VIDEO_ENGINE_FLAG_LIVE 0x00000020
444  // #define VIDEO_ENGINE_FLAG_B_FRAME 0x00000010
445  #define VIDEO_ENGINE_FLAG_SCALABLE_DELTA 0x00000080
446  #define VIDEO_ENGINE_FLAG_NASA 0x00000040
447  #define VIDEO_ENGINE_FLAG_HAS_PISE 0x00000020
448  #define VIDEO_ENGINE_FLAG_HAS_INTRA_REQUEST 0x00000010
449  #define VIDEO_ENGINE_FLAG_STD_CODEC 0x00000008
450  #define VIDEO_ENGINE_FLAG_HARDWARE_ENCODE 0x00000004
451  #define VIDEO_ENGINE_FLAG_NEW_AVSYNC_TIMESTAMP 0x00000002
452  #define VIDEO_ENGINE_FLAG_MULTI_STREAM 0x00000001
453 
454  enum VideoFrameType {
455  KEY_FRAME = 0,
456  DELTA_FRAME = 1,
457  B_FRAME = 2,
458  PISE_FRAME = 3,
459  SCALABLE_DELTA_FRAME = 4,
460  };
461  enum VideoCodecType {
462  VIDEO_CODEC_VP8 = 1,
463  VIDEO_CODEC_H264 = 2,
464  VIDEO_CODEC_EVP = 3,
465  VIDEO_CODEC_E264 = 4,
466  };
467  enum VideoFeedbackPacketType {
468  VIDEO_FEEDBACK_INTRA_REQUEST = 1,
469  VIDEO_FEEDBACK_DISABLE_INTRA_FEC = 2,
470  VIDEO_FEEDBACK_INTRA_REQUEST_QUICK = 3,
471  VIDEO_FEEDBACK_DECODE_FB = 4,
472  VIDEO_FEEDBACK_BANDWIDTH_ESTIMATION = 5,
473  VIDEO_FEEDBACK_REQUEST_MULTISTREAM = 6,
474  VIDEO_FEEDBACK_AUT = 7,
475  };
476  struct PacketInfo
477  {
478  const void* packet;
479  unsigned short packetLen;
480  unsigned int frame_num;
481  uint16_t frame_pkg_num;
482  uint16_t pkg_seq_in_frame;
483  unsigned char fec_num;
484  unsigned char fec_method;
485  int codec_type;
486  VideoFrameType frame_type;
487  VideoStreamType stream_type;
488  unsigned int flag;
489  int version;
490  unsigned short reserve1;
491  };
492  struct AutStreamData {
493  uint16_t bandwidth;
494  uint8_t recv_stream_type;
495  uint8_t expected_stream_type;
496  };
498  uint16_t rtt;
499  uint16_t lost_ratio;
500  uint16_t bwe;
501  uint16_t padding;
502  uint16_t total_sent;
503  uint16_t queueing_time;
504  uint16_t active_uid_counts;
505  uint16_t allocated;
506  uint16_t mtu;
507  uint16_t jitter95;
508  std::list<AutStreamData> stream_info;
509  };
510 
511  virtual int sendVideoPacket(const PacketInfo& info) = 0;
512  virtual int sendVideoRtcpPacket(unsigned int uid, const void* packet, unsigned short packetLen, bool isToVos) = 0;
513 
514  // new interface for feedback message
515  virtual int sendVideoRtcpFeedbackPacket(const VideoFeedbackPacketType type, unsigned int toUid, const void* packet, int packetSize) = 0;
516 
517  virtual void onRemoteFirstFrameDrawed(int viewIndex, unsigned int uid, int width, int height) = 0;
518  virtual void onLocalFirstFrameDrawed(int width, int height) = 0;
519  virtual void onRemoteFirstFrameDecoded(unsigned int uid, int width, int height) = 0;
520  virtual void onRemoteVideoInterrupted(unsigned int uid, unsigned int elapse_time) = 0;
521  virtual void onVideoStat(const LocalVideoStat& localStat, RemoteVideoStat* remoteStat, int remoteCount) = 0;
522  virtual void onVideoProfile(unsigned int width, unsigned int height, unsigned int fps, unsigned int bitrate) = 0;
523  virtual void switchVideoStream(unsigned int uid, VideoStreamType stream) = 0;
524  virtual void onBandWidthLevelChanged(int level) = 0;
525  virtual void onCameraFocusAreaChanged(int x, int y, int width, int height) = 0;
526  virtual void onCameraExposureAreaChanged(int x, int y, int width, int height) = 0;
527  virtual void onVideoViewSizeChanged(int userID, int newWidth, int newHeight) {
528  (void)userID;
529  (void)newWidth;
530  (void)newHeight;
531  }
532 
533  virtual void onVideoFrameFrozen(unsigned int uid, bool frozen) = 0;
534 
535  virtual void onAppSetVideoStartBitRate(int value) = 0;
536 
537  virtual int onEncodeVideoSEI(char** info, int *len) = 0;
538  virtual void onVideoSizeChanged(unsigned int uid, int newWidth, int newHeight, int newRotation) {
539  (void)uid;
540  (void)newWidth;
541  (void)newHeight;
542  (void)newRotation;
543  }
544 
545  virtual void onSendVideoPaced(bool status) = 0;
546  virtual void onBWELevel(int level) = 0;
547  virtual void onVideoRexferStatus(bool status, int target_bitrate) = 0;
548  virtual void onStartCaptureSuccess() = 0;
549  // add for sei
550  virtual int onSendSEI(char **info, int *len, long long timeStampMs, int streamType, bool isDualStream) = 0;
551  virtual int onReceiveSEI(char *info, int len, unsigned int uid, long long timeStampMs) = 0;
552  virtual void OnVideoStreamBitrateRangeChanged(VideoStreamType type,
553  uint32_t max_kbps, uint32_t min_kbps) = 0;
554 };
555 
556 } // namespace rtc
557 } // namespace agora
agora::rtc::IVideoListener::PacketInfo
Definition: packet_i.h:477
agora::rtc::IVideoListener::AutStreamData
Definition: packet_i.h:492
agora::rtc::IVideoListener::LocalVideoStat
Definition: packet_i.h:288
agora::rtc::video_packet_t::Extension
Definition: packet_i.h:171
agora::rtc::IVideoListener::LocalVideoStreamStat
Definition: packet_i.h:263
agora::rtc::video_packet_t
Definition: packet_i.h:124
agora::rtc::video_rtcp_packet_t
Definition: packet_i.h:232
agora::rtc::SMediaFrame
Definition: packet_i.h:36
agora::rtc::SAudioFrame
Definition: packet_i.h:48
agora::rtc::broadcast_packet_t
Definition: packet_i.h:106
agora::rtc::audio_packet_t
Definition: packet_i.h:115
agora::rtc::IVideoListener::RemoteVideoStat
Definition: packet_i.h:370
agora::rtc::video_report_packet_t
Definition: packet_i.h:243
agora::rtc::video_custom_ctrl_broadcast_packet_t
Definition: packet_i.h:238
agora::rtc::rtc_packet_t
Definition: packet_i.h:83
agora::rtc::SAudioPacket
Definition: packet_i.h:59
agora::rtc::video_packet_t::video3_flags
Definition: packet_i.h:207
agora::rtc::IVideoListener
Definition: packet_i.h:259
agora::rtc::IVideoListener::AutFeedbackData
Definition: packet_i.h:497