From a46ee7234c28609f13bca5c23e31e2aba2c310e9 Mon Sep 17 00:00:00 2001 From: Jganenokk Date: Thu, 6 Aug 2026 09:58:39 +0700 Subject: [PATCH] =?UTF-8?q?feat:=20=D1=81=D0=B5=D1=80=D0=B2=D0=B5=D1=80?= =?UTF-8?q?=D0=BD=D0=B0=D1=8F=20=D1=82=D0=BE=D0=BF=D0=BE=D0=BB=D0=BE=D0=B3?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B5!!!!!!!!!=20=D1=83=D0=B1=D0=B5=D0=B9=D1=82?= =?UTF-8?q?=D0=B5=20=D0=BC=D0=B5=D0=BD=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/backend/api.dart | 84 +++++ lib/backend/modules/calls.dart | 5 +- lib/core/calls/call_controller.dart | 10 + lib/core/calls/call_session.dart | 381 +++++++++++++++++--- lib/core/calls/sfu_data_channel.dart | 346 ++++++++++++++++++ lib/core/calls/ws2_signaling.dart | 35 +- lib/core/storage/spoofing_service.dart | 1 + lib/frontend/screens/calls/call_screen.dart | 49 ++- lib/frontend/widgets/glossy_pill.dart | 8 +- 9 files changed, 847 insertions(+), 72 deletions(-) create mode 100644 lib/core/calls/sfu_data_channel.dart diff --git a/lib/backend/api.dart b/lib/backend/api.dart index d8ec476..cd1432b 100644 --- a/lib/backend/api.dart +++ b/lib/backend/api.dart @@ -51,9 +51,13 @@ class Api { int? _callsSeed; String? _deviceId; + String? _callsDevice; + String? _callsOsVersion; int? get callsSeed => _callsSeed; String? get deviceId => _deviceId; + String? get callsDevice => _callsDevice; + String? get callsOsVersion => _callsOsVersion; /// Сырой доступ к сессии ядра — для медиа-загрузок (data-plane). KolibriSession? get session => _session; @@ -387,6 +391,10 @@ class Api { String instanceId = await DeviceIdentity.instanceId(); int clientSessionId = DeviceIdentity.clientSessionId; + String? androidManufacturer; + String? androidModel; + int? androidSdkInt; + if (Platform.isLinux) { final linuxInfo = await deviceInfo.linuxInfo; osVersion = linuxInfo.name; @@ -398,15 +406,20 @@ class Api { final androidInfo = await deviceInfo.androidInfo; osVersion = 'Android ${androidInfo.version.release}'; deviceName = '${androidInfo.manufacturer} ${androidInfo.model}'; + androidManufacturer = androidInfo.manufacturer; + androidModel = androidInfo.model; + androidSdkInt = androidInfo.version.sdkInt; } else if (Platform.isWindows) { final windowsInfo = await deviceInfo.windowsInfo; osVersion = windowsInfo.productName; } + String? spoofUserAgent; final spoofed = await SpoofingService.getSpoofedSessionData( scope: spoofScope, ); if (spoofed != null) { + spoofUserAgent = spoofed['user_agent'] as String?; final sDeviceType = spoofed['device_type'] as String?; if (sDeviceType != null && sDeviceType != 'IOS') deviceType = sDeviceType; final sDeviceName = spoofed['device_name'] as String?; @@ -448,6 +461,19 @@ class Api { if (sClientSession is int) clientSessionId = sClientSession; } + _callsDevice = _resolveCallsDevice( + spoofed: spoofed != null, + deviceName: deviceName, + spoofUserAgent: spoofUserAgent, + manufacturer: androidManufacturer, + model: androidModel, + ); + _callsOsVersion = _resolveCallsOsVersion( + spoofed: spoofed != null, + osVersion: osVersion, + sdkInt: androidSdkInt, + ); + _userAgent = { 'deviceType': deviceType, 'appVersion': appVersion, @@ -491,6 +517,64 @@ class Api { ); } + static String? _resolveCallsDevice({ + required bool spoofed, + required String deviceName, + String? spoofUserAgent, + String? manufacturer, + String? model, + }) { + if (!spoofed && + manufacturer != null && + manufacturer.isNotEmpty && + model != null && + model.isNotEmpty) { + return '$manufacturer/$model'; + } + final parts = deviceName.trim().split(RegExp(r'\s+')) + ..removeWhere((p) => p.isEmpty); + if (parts.isEmpty) return null; + final fallbackModel = parts.length > 1 + ? parts.sublist(1).join(' ') + : parts.first; + return '${parts.first}/' + '${_modelFromUserAgent(spoofUserAgent) ?? fallbackModel}'; + } + + static String? _modelFromUserAgent(String? userAgent) { + if (userAgent == null || userAgent.isEmpty) return null; + final match = RegExp(r'Android\s+[\d.]+;\s*([^;)]+)').firstMatch(userAgent); + final model = match + ?.group(1) + ?.replaceFirst(RegExp(r'\s+Build/.*$'), '') + .trim(); + return model == null || model.isEmpty ? null : model; + } + + static String _resolveCallsOsVersion({ + required bool spoofed, + required String osVersion, + int? sdkInt, + }) { + if (!spoofed && sdkInt != null && sdkInt > 0) return '$sdkInt'; + final release = RegExp( + r'^Android\s+(\d+)', + ).firstMatch(osVersion.trim())?.group(1); + return '${_androidSdkForRelease(int.tryParse(release ?? ''))}'; + } + + static int _androidSdkForRelease(int? release) => switch (release) { + null => 34, + <= 9 => 28, + 10 => 29, + 11 => 30, + 12 => 31, + 13 => 33, + 14 => 34, + 15 => 35, + _ => 36, + }; + static Future _buildProxyUrl() async { final p = await ProxyConfig.load(); if (!p.isEnabled) return null; diff --git a/lib/backend/modules/calls.dart b/lib/backend/modules/calls.dart index d663410..ade2ecd 100644 --- a/lib/backend/modules/calls.dart +++ b/lib/backend/modules/calls.dart @@ -2,6 +2,7 @@ import 'dart:convert'; import 'contacts.dart'; import '../api.dart'; +import '../../core/calls/ws2_signaling.dart'; import '../../core/protocol/opcode_map.dart'; import '../../core/utils/ids.dart'; import '../../core/utils/logger.dart'; @@ -178,13 +179,13 @@ class CallsModule { String _internalParams() => jsonEncode({ 'platform': 'ANDROID', - 'sdkVersion': '0.1.16.4', + 'sdkVersion': '0.2.1.3', 'clientAppKey': 'CGPGAGLGDIHBABABA', 'deviceId': _api.deviceId ?? '', 'protocolVersion': 5, 'onlyAdminCanRecord': false, 'isWaitForAdminEnabled': false, - 'hexCapability': '3c03f', + 'hexCapability': Ws2Config.defaultCapabilities, }); Future resolveCallLink(String url) async { diff --git a/lib/core/calls/call_controller.dart b/lib/core/calls/call_controller.dart index 7fc9284..0075abc 100644 --- a/lib/core/calls/call_controller.dart +++ b/lib/core/calls/call_controller.dart @@ -150,6 +150,8 @@ class CallController { final config = Ws2Config.fromEndpoint( out.endpoint, userId: out.callsUserId, + device: _api?.callsDevice, + osVersion: _api?.callsOsVersion, ); final session = CallSession(ws2Config: config, role: CallRole.caller); _bind(session); @@ -167,6 +169,8 @@ class CallController { final config = Ws2Config.fromEndpoint( out.endpoint, userId: out.callsUserId, + device: _api?.callsDevice, + osVersion: _api?.callsOsVersion, ); final session = CallSession( ws2Config: config, @@ -188,6 +192,8 @@ class CallController { final config = Ws2Config.fromEndpoint( params.endpoint, userId: params.callsUserId, + device: _api?.callsDevice, + osVersion: _api?.callsOsVersion, ); final session = CallSession( ws2Config: config, @@ -206,6 +212,8 @@ class CallController { final config = Ws2Config.fromVcp( call.params, conversationId: call.conversationId, + device: _api?.callsDevice, + osVersion: _api?.callsOsVersion, ); final session = CallSession( ws2Config: config, @@ -225,6 +233,8 @@ class CallController { final config = Ws2Config.fromVcp( call.params, conversationId: call.conversationId, + device: _api?.callsDevice, + osVersion: _api?.callsOsVersion, ); final signaling = Ws2Signaling(config); try { diff --git a/lib/core/calls/call_session.dart b/lib/core/calls/call_session.dart index fb7d7e7..fe6faf5 100644 --- a/lib/core/calls/call_session.dart +++ b/lib/core/calls/call_session.dart @@ -11,6 +11,7 @@ import 'call_admin.dart'; import 'call_bridge.dart'; import 'call_info.dart'; import 'conversation_params.dart'; +import 'sfu_data_channel.dart'; import 'ws2_signaling.dart'; enum CallRole { caller, callee, joiner } @@ -102,7 +103,7 @@ class CallSession { RTCRtpSender? _videoSender; RTCRtpSender? _screenSender; - Completer? _gatherReady; + Completer? _gatherDone; bool _gotConnection = false; bool _reconnecting = false; @@ -123,6 +124,27 @@ class CallSession { RTCDataChannel? _probeChannel; bool _peerIsKomet = false; + final List _sfuChannels = []; + SfuCommandChannel? _sfuCommands; + StreamSubscription>? _sfuSlotSub; + StreamSubscription>? _sfuLevelSub; + final Map _slotParticipant = {}; + Timer? _layoutDebounce; + Timer? _videoStatsTimer; + List _lastLayout = const []; + bool _layoutSent = false; + + static const int _maxVideoSlots = 10; + static const int _sfuSpeakLevel = 50; + static const Duration _levelTtl = Duration(seconds: 6); + final Map _levelState = {}; + + static const List _sfuChannelLabels = [ + 'producerCommand', + 'producerNotification', + ]; + + static const bool _kometProbeEnabled = false; static const String _probeQuestion = 'AreYouKomet?'; static const String _probeAnswer = 'YesImKomet😎'; @@ -206,6 +228,7 @@ class CallSession { void _notifyInfo() { if (!_info.isClosed) _info.add(null); + if (_topology == 'SERVER') _scheduleDisplayLayout(); } Future start() async { @@ -304,6 +327,7 @@ class CallSession { await _probeChannel?.close(); } catch (_) {} _probeChannel = null; + await _closeSfuChannels(); try { await _pc?.close(); @@ -339,7 +363,7 @@ class CallSession { Future _sampleLevels() async { final pc = _pc; - if (pc == null || _ended) return; + if (pc == null || _ended || _topology == 'SERVER') return; if (!_mediaConnected || _current != CallSessionState.active) return; var local = 0.0; @@ -456,7 +480,7 @@ class CallSession { void _onWs2Error(Map msg) { final err = msg['error']; - logger.w('[call] ws2 error: $err'); + logger.w('[call] ws2 error: $err raw=$msg'); if (err == 'conversation-ended') _end(); } @@ -569,20 +593,17 @@ class CallSession { if (externalId != null) p.externalId = externalId; if (state != null) p.state = state; if (mediaSettings is Map) { - final a = mediaSettings['isAudioEnabled']; - final v = mediaSettings['isVideoEnabled']; - final s = mediaSettings['isScreenSharingEnabled']; - if (a is bool) p.audioEnabled = a; - if (v is bool) p.videoEnabled = v; - if (s is bool) p.screenSharing = s; + p.audioEnabled = mediaSettings['isAudioEnabled'] == true; + p.videoEnabled = mediaSettings['isVideoEnabled'] == true; + p.screenSharing = mediaSettings['isScreenSharingEnabled'] == true; } if (muteStates is Map) { final a = muteStates['AUDIO']; final v = muteStates['VIDEO']; final s = muteStates['SCREEN_SHARING']; - if (a is String) p.audioEnabled = a == 'UNMUTE'; - if (v is String) p.videoEnabled = v == 'UNMUTE'; - if (s is String) p.screenSharing = s == 'UNMUTE'; + if (a is String && a != 'UNMUTE') p.audioEnabled = false; + if (v is String && v != 'UNMUTE') p.videoEnabled = false; + if (s is String && s != 'UNMUTE') p.screenSharing = false; } if (handRaised != null) p.handRaised = handRaised; if (roles is List) { @@ -606,12 +627,16 @@ class CallSession { void _onParticipantMedia(Map msg) { final id = _participantIdFrom(msg['participantId']); if (id == null) return; - _upsertParticipant( + final p = _upsertParticipant( id, externalId: _externalId(msg['externalId']), mediaSettings: msg['mediaSettings'], muteStates: msg['muteStates'], ); + logger.i( + '[call] media $id video=${p.videoEnabled} audio=${p.audioEnabled} ' + 'screen=${p.screenSharing} raw=${msg['mediaSettings']}', + ); _maybeAdoptPeer(id, msg); _notifyInfo(); } @@ -706,8 +731,8 @@ class CallSession { logger.i('[call] connection role=$role peer=$_peerId topology=$_topology'); if (_topology == 'SERVER') { - await _setupSfu(); await accept(activate: role != CallRole.caller); + await _setupSfu(); return; } @@ -744,8 +769,17 @@ class CallSession { 'audioJitterBufferMaxPackets': 200, }); pc.onIceCandidate = _onLocalCandidate; + pc.onIceGatheringState = (s) { + logger.i('[call] ice gathering $s'); + if (s != RTCIceGatheringState.RTCIceGatheringStateComplete) return; + final done = _gatherDone; + if (done != null && !done.isCompleted) done.complete(); + }; pc.onTrack = (event) => unawaited(_onRemoteTrack(event)); - pc.onDataChannel = (channel) => _bindProbeChannel(channel, ask: false); + pc.onDataChannel = (channel) { + if (!_kometProbeEnabled) return; + _bindProbeChannel(channel, ask: false); + }; pc.onIceConnectionState = (s) { logger.i('[call] ice $s'); if (s != RTCIceConnectionState.RTCIceConnectionStateFailed) return; @@ -797,8 +831,180 @@ class CallSession { } } + Future _openSfuChannels(RTCPeerConnection pc) async { + await _closeSfuChannels(); + final commands = SfuCommandChannel(); + _sfuCommands = commands; + _sfuSlotSub = commands.slotUpdates.listen(_onSfuSlots); + _sfuLevelSub = commands.audioLevels.listen(_onSfuLevels); + for (final label in _sfuChannelLabels) { + try { + final channel = await pc.createDataChannel( + label, + RTCDataChannelInit() + ..ordered = true + ..maxRetransmitTime = 10000000, + ); + channel.onDataChannelState = (state) { + logger.i('[call][sfu] data channel $label $state'); + if (state == RTCDataChannelState.RTCDataChannelOpen) { + _scheduleDisplayLayout(); + } + }; + commands.bind(channel); + _sfuChannels.add(channel); + } catch (e) { + logger.w('[call][sfu] data channel $label failed: $e'); + } + } + } + + void _onSfuLevels(Map levels) { + final now = DateTime.now(); + levels.forEach((key, level) { + final id = _participantIdFrom(key.split(':').first); + if (id != null) _levelState[id] = (level: level, at: now); + }); + _levelState.removeWhere((_, v) => now.difference(v.at) > _levelTtl); + + final loud = _levelState.entries + .where((e) => e.value.level >= _sfuSpeakLevel) + .map((e) => e.key) + .toSet(); + logger.i('[call][sfu] levels: $levels speaking=$loud'); + if (loud.length == _speaking.length && loud.containsAll(_speaking)) return; + _speaking = loud; + _notifyInfo(); + } + + void _onSfuSlots(Map slots) { + if (slots.isEmpty) return; + _slotParticipant.clear(); + slots.forEach((key, slot) { + if (slot < 0) return; + final id = _participantIdFrom(key.split(':').first); + if (id != null) _slotParticipant[slot] = id; + }); + unawaited(_rebindSlotTracks()); + } + + Future _rebindSlotTracks() async { + await _clearParticipantStreams(); + await _collectReceivers(); + _notifyInfo(); + } + + void _scheduleDisplayLayout() { + _layoutDebounce?.cancel(); + _layoutDebounce = Timer( + const Duration(milliseconds: 300), + () => unawaited(_publishDisplayLayout()), + ); + } + + Future _publishDisplayLayout({bool force = false}) async { + final commands = _sfuCommands; + if (commands == null || _topology != 'SERVER' || _ended) return; + final items = []; + for (final p in _participants.values) { + if (p.isSelf || items.length >= _maxVideoSlots) continue; + if (!p.videoEnabled && !p.screenSharing) continue; + items.add( + SfuLayoutItem( + trackKey: 'u${p.id}:${p.screenSharing ? 'sSCREEN' : 'sCAMERA'}', + ), + ); + } + final keys = items.map((i) => i.trackKey).toList(growable: false); + if (!force && + _layoutSent && + keys.length == _lastLayout.length && + keys.every(_lastLayout.contains)) { + return; + } + if (!await commands.sendDisplayLayout(items)) return; + _lastLayout = keys; + _layoutSent = true; + } + + Set _videoSlotMids(String sdp) { + final mids = {}; + String? kind; + String? mid; + var recvOnly = false; + + void flush() { + final id = mid; + if (kind == 'video' && recvOnly && id != null) mids.add(id); + } + + for (var line in sdp.split('\n')) { + line = line.trim(); + if (line.startsWith('m=')) { + flush(); + kind = line.substring(2).split(' ').first; + mid = null; + recvOnly = false; + } else if (line.startsWith('a=mid:')) { + mid = line.substring(6); + } else if (line == 'a=recvonly') { + recvOnly = true; + } + } + flush(); + return mids; + } + + Future _prepareVideoSlot(RTCPeerConnection pc, String offerSdp) async { + final mids = _videoSlotMids(offerSdp); + if (mids.isEmpty) return; + for (final transceiver in await pc.getTransceivers()) { + final mid = transceiver.mid; + if (!mids.contains(mid)) continue; + final tracks = + _cameraStream?.getVideoTracks() ?? const []; + if (tracks.isNotEmpty) { + try { + await transceiver.sender.replaceTrack(tracks.first); + } catch (e) { + logger.w('[call][sfu] video slot $mid replaceTrack failed: $e'); + } + } + try { + await transceiver.setDirection(TransceiverDirection.SendOnly); + } catch (e) { + logger.w('[call][sfu] video slot $mid setDirection failed: $e'); + continue; + } + _videoSender = transceiver.sender; + logger.i('[call][sfu] video slot mid=$mid -> sendonly'); + return; + } + } + + Future _closeSfuChannels() async { + _layoutDebounce?.cancel(); + _layoutDebounce = null; + await _sfuSlotSub?.cancel(); + _sfuSlotSub = null; + await _sfuLevelSub?.cancel(); + _sfuLevelSub = null; + await _sfuCommands?.dispose(); + _sfuCommands = null; + _slotParticipant.clear(); + _lastLayout = const []; + _layoutSent = false; + final channels = List.from(_sfuChannels); + _sfuChannels.clear(); + for (final channel in channels) { + try { + await channel.close(); + } catch (_) {} + } + } + Future _setupKometProbe(RTCPeerConnection pc) async { - if (_topology == 'SERVER') return; + if (!_kometProbeEnabled || _topology == 'SERVER') return; try { final channel = await pc.createDataChannel( 'komet', @@ -897,6 +1103,7 @@ class CallSession { Future _setupSfu() async { if (_pc != null) { + await _closeSfuChannels(); await _pc!.close(); _pc = null; _probeChannel = null; @@ -915,6 +1122,7 @@ class CallSession { _pc = pc; await _addLocalMedia(pc); await _republishVideo(pc); + await _openSfuChannels(pc); logger.i( '[call][sfu] allocate-consumer camera=$_localVideo screen=$_localScreen', ); @@ -927,6 +1135,7 @@ class CallSession { } Future _rebuildSfuPc() async { + await _closeSfuChannels(); try { await _pc?.close(); } catch (_) {} @@ -951,6 +1160,7 @@ class CallSession { _pc = pc; await _addLocalMedia(pc); await _republishVideo(pc); + await _openSfuChannels(pc); } Future _republishVideo(RTCPeerConnection pc) async { @@ -1020,10 +1230,12 @@ class CallSession { '(${_candidateTypes(sdp)}), ${_sdpSummary(sdp)}, ice=${_iceServerUrls()}', ); logger.i('[call][sfu] producer m-lines: ${_mLineDetails(sdp)}'); + logger.i('[call][sfu] producer video codecs: ${_videoCodecs(sdp)}'); await pc.setRemoteDescription(RTCSessionDescription(sdp, type)); _remoteDescSet = true; await _flushCandidates(); await _addRemoteCandidatesFromSdp(pc, sdp); + await _prepareVideoSlot(pc, sdp); final answer = await pc.createAnswer({}); if (_pc != pc) return; @@ -1033,7 +1245,7 @@ class CallSession { return; } - await _awaitReflexiveCandidates(pc); + await _awaitIceGathering(pc); if (_pc != pc) { logger.w('[call][sfu] peer connection replaced while gathering'); return; @@ -1054,14 +1266,18 @@ class CallSession { 'gathering=${pc.iceGatheringState}', ); logger.i('[call][sfu] answer m-lines: ${_mLineDetails(answerSdp)}'); + logger.i('[call][sfu] answer video codecs: ${_videoCodecs(answerSdp)}'); + logger.i( + '[call][sfu] video feedback: offer=[${_videoFeedback(sdp)}] ' + 'answer=[${_videoFeedback(answerSdp)}]', + ); await _logSenders(); try { - final localSsrcs = _extractSsrcs(answerSdp); - logger.i('[call][sfu] accept-producer ssrcs=$localSsrcs'); + logger.i('[call][sfu] accept-producer ssrcs=$ssrcs'); final reply = await _signaling?.acceptProducer( description: _labelLocalTracks(answerSdp), - ssrcs: localSsrcs, + ssrcs: ssrcs, sessionId: _sfuSessionId, ); logger.i('[call][sfu] accept-producer reply: $reply'); @@ -1072,9 +1288,18 @@ class CallSession { Timer(const Duration(seconds: 5), () { if (_pc == pc && !_ended) unawaited(_dumpIceStats(pc)); }); + _videoStatsTimer?.cancel(); + _videoStatsTimer = Timer.periodic(const Duration(seconds: 5), (t) { + if (_pc != pc || _ended) { + t.cancel(); + return; + } + unawaited(_dumpVideoStats(pc)); + }); if (_accepted) await _sendMediaSettings(); unawaited(_collectReceivers()); + unawaited(_publishDisplayLayout(force: true)); } int _countCandidates(String sdp) => @@ -1119,6 +1344,41 @@ class CallSession { ); } + Future _dumpVideoStats(RTCPeerConnection pc) async { + try { + final rows = []; + for (final r in await pc.getStats()) { + if (r.type != 'inbound-rtp') continue; + final v = r.values; + if (v['kind'] != 'video' && v['mediaType'] != 'video') continue; + rows.add( + '[ssrc=${v['ssrc']} bytes=${v['bytesReceived']} ' + 'packets=${v['packetsReceived']} decoded=${v['framesDecoded']} ' + '${v['frameWidth']}x${v['frameHeight']}]', + ); + } + var transportBytes = 0; + var audioBytes = 0; + for (final r in await pc.getStats()) { + final v = r.values; + if (r.type == 'transport') { + final b = v['bytesReceived']; + if (b is num) transportBytes += b.toInt(); + } else if (r.type == 'inbound-rtp' && + (v['kind'] == 'audio' || v['mediaType'] == 'audio')) { + final b = v['bytesReceived']; + if (b is num) audioBytes += b.toInt(); + } + } + logger.i( + '[call][sfu] inbound video: ${rows.join(' ')} ' + '| transport=$transportBytes audio=$audioBytes', + ); + } catch (e) { + logger.w('[call][sfu] video stats failed: $e'); + } + } + Future _dumpIceStats(RTCPeerConnection pc) async { try { final reports = await pc.getStats(); @@ -1181,6 +1441,38 @@ class CallSession { 'setup=$setup$lite'; } + String _videoFeedback(String sdp) { + final fb = {}; + var inVideo = false; + for (var line in sdp.split('\n')) { + line = line.trim(); + if (line.startsWith('m=')) { + inVideo = line.startsWith('m=video'); + } else if (inVideo && line.startsWith('a=rtcp-fb:')) { + final idx = line.indexOf(' '); + if (idx > 0) fb.add(line.substring(idx + 1)); + } else if (inVideo && line.startsWith('a=extmap:')) { + if (line.contains('transport-wide-cc')) fb.add('extmap:transport-cc'); + } + } + return fb.isEmpty ? 'нет' : fb.join(', '); + } + + String _videoCodecs(String sdp) { + final codecs = {}; + var inVideo = false; + for (var line in sdp.split('\n')) { + line = line.trim(); + if (line.startsWith('m=')) { + inVideo = line.startsWith('m=video'); + } else if (inVideo && line.startsWith('a=rtpmap:')) { + final m = RegExp(r'^a=rtpmap:\d+ ([^/]+)/').firstMatch(line); + if (m != null) codecs.add(m.group(1)!); + } + } + return codecs.isEmpty ? 'нет' : codecs.join(','); + } + int _mLines(String sdp) => RegExp(r'^m=', multiLine: true).allMatches(sdp).length; @@ -1298,6 +1590,10 @@ class CallSession { int? _participantFromTrackId(String? trackId) { if (trackId == null) return null; + final slot = RegExp(r'^video-pat-(\d+)$').firstMatch(trackId); + if (slot != null) { + return _slotParticipant[int.parse(slot.group(1)!)]; + } for (final prefix in const ['video-', 'audio-']) { if (trackId.length > prefix.length && trackId.startsWith(prefix)) { final parsed = _participantIdFrom(trackId.substring(prefix.length)); @@ -1505,41 +1801,31 @@ class CallSession { } } - static bool _isReflexive(String? line) => - line != null && - (line.contains(' typ srflx') || line.contains(' typ relay')); - - Future _awaitReflexiveCandidates( + Future _awaitIceGathering( RTCPeerConnection pc, { - Duration timeout = const Duration(seconds: 4), + Duration timeout = const Duration(seconds: 5), }) async { if (pc.iceGatheringState == RTCIceGatheringState.RTCIceGatheringStateComplete) { return; } + final done = Completer(); + _gatherDone = done; try { - final current = await pc.getLocalDescription(); - if (_isReflexive(current?.sdp)) return; - } catch (_) {} - - final ready = Completer(); - _gatherReady = ready; - try { - await ready.future.timeout(timeout); - logger.i('[call][sfu] reflexive candidate gathered'); + await done.future.timeout(timeout); + logger.i('[call][sfu] relay candidate gathered'); } catch (_) { - logger.w('[call][sfu] no reflexive candidate within $timeout'); + logger.w('[call][sfu] no relay candidate within $timeout'); } finally { - _gatherReady = null; + _gatherDone = null; } } void _onLocalCandidate(RTCIceCandidate candidate) { - final pending = _gatherReady; - if (pending != null && - !pending.isCompleted && - _isReflexive(candidate.candidate)) { - pending.complete(); + final line = candidate.candidate; + if (line != null && line.contains(' typ relay')) { + final done = _gatherDone; + if (done != null && !done.isCompleted) done.complete(); } if (_topology == 'SERVER') return; final peerId = _peerId; @@ -1732,9 +2018,14 @@ class CallSession { } Future _clearParticipantStreams() async { - final streams = _participantStreams.values.toList(growable: false); + final entries = Map.from(_participantStreams); _participantStreams.clear(); - for (final stream in streams) { + for (final id in entries.keys) { + if (!_participantStreamUpdates.isClosed) { + _participantStreamUpdates.add(id); + } + } + for (final stream in entries.values) { try { await stream.dispose(); } catch (_) {} @@ -1779,10 +2070,12 @@ class CallSession { Future _dispose() async { _levelTimer?.cancel(); + _videoStatsTimer?.cancel(); try { await _probeChannel?.close(); } catch (_) {} _probeChannel = null; + await _closeSfuChannels(); for (final track in _localStream?.getTracks() ?? []) { await track.stop(); } diff --git a/lib/core/calls/sfu_data_channel.dart b/lib/core/calls/sfu_data_channel.dart new file mode 100644 index 0000000..48ccb5f --- /dev/null +++ b/lib/core/calls/sfu_data_channel.dart @@ -0,0 +1,346 @@ +import 'dart:async'; +import 'dart:convert'; +import 'dart:typed_data'; + +import 'package:flutter_webrtc/flutter_webrtc.dart'; + +import '../utils/logger.dart'; + +class MsgpackWriter { + final BytesBuilder _out = BytesBuilder(); + + Uint8List takeBytes() => _out.takeBytes(); + + void raw(int byte) => _out.addByte(byte); + + void nil() => raw(0xC0); + + void boolean(bool value) => raw(value ? 0xC3 : 0xC2); + + void integer(int value) { + if (value >= 0) { + if (value < 0x80) return raw(value); + if (value <= 0xFF) { + raw(0xCC); + return raw(value); + } + if (value <= 0xFFFF) { + raw(0xCD); + return _uint(value, 2); + } + if (value <= 0xFFFFFFFF) { + raw(0xCE); + return _uint(value, 4); + } + raw(0xCF); + return _uint(value, 8); + } + if (value >= -32) return raw(0xE0 | (value + 32)); + if (value >= -128) { + raw(0xD0); + return _uint(value & 0xFF, 1); + } + if (value >= -32768) { + raw(0xD1); + return _uint(value & 0xFFFF, 2); + } + if (value >= -2147483648) { + raw(0xD2); + return _uint(value & 0xFFFFFFFF, 4); + } + raw(0xD3); + _uint(value, 8); + } + + void string(String value) { + final bytes = utf8.encode(value); + final length = bytes.length; + if (length < 32) { + raw(0xA0 | length); + } else if (length <= 0xFF) { + raw(0xD9); + raw(length); + } else if (length <= 0xFFFF) { + raw(0xDA); + _uint(length, 2); + } else { + raw(0xDB); + _uint(length, 4); + } + _out.add(bytes); + } + + void arrayHeader(int length) { + if (length < 16) return raw(0x90 | length); + if (length <= 0xFFFF) { + raw(0xDC); + return _uint(length, 2); + } + raw(0xDD); + _uint(length, 4); + } + + void _uint(int value, int bytes) { + for (var shift = (bytes - 1) * 8; shift >= 0; shift -= 8) { + raw((value >> shift) & 0xFF); + } + } +} + +class MsgpackReader { + final Uint8List _data; + int _pos = 0; + + MsgpackReader(this._data); + + bool get exhausted => _pos >= _data.length; + + bool get nextIsString { + final b = _data[_pos]; + return (b & 0xE0) == 0xA0 || b == 0xD9 || b == 0xDA || b == 0xDB; + } + + int readInt() { + final b = _data[_pos++]; + if (b < 0x80) return b; + if (b >= 0xE0) return b - 256; + switch (b) { + case 0xCC: + return _uint(1); + case 0xCD: + return _uint(2); + case 0xCE: + return _uint(4); + case 0xCF: + return _uint(8); + case 0xD0: + final v = _uint(1); + return v >= 0x80 ? v - 0x100 : v; + case 0xD1: + final v = _uint(2); + return v >= 0x8000 ? v - 0x10000 : v; + case 0xD2: + final v = _uint(4); + return v >= 0x80000000 ? v - 0x100000000 : v; + case 0xD3: + return _uint(8); + } + throw FormatException('не целое: 0x${b.toRadixString(16)}'); + } + + String readString() { + final b = _data[_pos++]; + int length; + if ((b & 0xE0) == 0xA0) { + length = b & 0x1F; + } else if (b == 0xD9) { + length = _uint(1); + } else if (b == 0xDA) { + length = _uint(2); + } else if (b == 0xDB) { + length = _uint(4); + } else { + throw FormatException('не строка: 0x${b.toRadixString(16)}'); + } + final value = utf8.decode(_data.sublist(_pos, _pos + length)); + _pos += length; + return value; + } + + int readMapHeader() { + final b = _data[_pos++]; + if ((b & 0xF0) == 0x80) return b & 0x0F; + if (b == 0xDE) return _uint(2); + if (b == 0xDF) return _uint(4); + throw FormatException('не map: 0x${b.toRadixString(16)}'); + } + + int readArrayHeader() { + final b = _data[_pos++]; + if ((b & 0xF0) == 0x90) return b & 0x0F; + if (b == 0xDC) return _uint(2); + if (b == 0xDD) return _uint(4); + throw FormatException('не array: 0x${b.toRadixString(16)}'); + } + + int _uint(int bytes) { + var value = 0; + for (var i = 0; i < bytes; i++) { + value = (value << 8) | _data[_pos++]; + } + return value; + } +} + +class SfuLayoutItem { + final String trackKey; + final int width; + final int height; + + const SfuLayoutItem({ + required this.trackKey, + this.width = 640, + this.height = 360, + }); +} + +class SfuCommandChannel { + static const int _commandDisplayLayout = 0; + static const int _fitMode = 0; + + static const int _notifyAliases = 1; + static const int _notifySlots = 2; + static const int _notifyAudioLevels = 6; + + RTCDataChannel? _command; + int _sequence = 1; + + final Map _aliases = {}; + final _slots = StreamController>.broadcast(); + final _levels = StreamController>.broadcast(); + + Stream> get slotUpdates => _slots.stream; + Stream> get audioLevels => _levels.stream; + + void bind(RTCDataChannel channel) { + if (channel.label == 'producerCommand') { + _command = channel; + channel.onMessage = _onCommandReply; + return; + } + if (channel.label != 'producerNotification') return; + channel.onMessage = _onNotification; + } + + bool get ready => _command?.state == RTCDataChannelState.RTCDataChannelOpen; + + Future sendDisplayLayout( + List items, { + bool snapshot = true, + }) async { + final channel = _command; + if (channel == null) return false; + if (channel.state != RTCDataChannelState.RTCDataChannelOpen) { + logger.w('[call][sfu] producerCommand не открыт, слои не отправлены'); + return false; + } + + final writer = MsgpackWriter() + ..integer(_commandDisplayLayout) + ..integer(0) + ..integer(_sequence++) + ..boolean(snapshot); + + if (items.isEmpty) { + writer.nil(); + } else { + writer.arrayHeader(items.length * 2); + for (final item in items) { + writer + ..string(item.trackKey) + ..integer(0) + ..nil() + ..integer(item.width) + ..integer(item.height) + ..integer(_fitMode); + } + } + writer.nil(); + + final payload = writer.takeBytes(); + try { + await channel.send(RTCDataChannelMessage.fromBinary(payload)); + logger.i( + '[call][sfu] update-display-layout: ' + '${items.map((i) => i.trackKey).join(', ')} ' + 'raw=${payload.map((b) => b.toRadixString(16).padLeft(2, '0')).join(' ')}', + ); + return true; + } catch (e) { + logger.w('[call][sfu] update-display-layout failed: $e'); + return false; + } + } + + void _onCommandReply(RTCDataChannelMessage message) { + if (!message.isBinary) return; + final bytes = message.binary; + final head = bytes.length > 32 ? bytes.sublist(0, 32) : bytes; + final hex = head.map((b) => b.toRadixString(16).padLeft(2, '0')).join(' '); + try { + final reader = MsgpackReader(bytes); + final type = reader.readInt(); + final version = reader.readInt(); + final error = reader.readInt(); + if (error != 0) { + logger.w( + '[call][sfu] command reply type=$type version=$version ' + 'ERROR=$error raw=$hex', + ); + return; + } + logger.i('[call][sfu] command reply type=$type ok raw=$hex'); + } catch (e) { + logger.w('[call][sfu] command reply parse failed: $e raw=$hex'); + } + } + + int _dumped = 0; + + void _onNotification(RTCDataChannelMessage message) { + if (!message.isBinary) return; + if (_dumped < 12) { + _dumped++; + final bytes = message.binary; + final head = bytes.length > 64 ? bytes.sublist(0, 64) : bytes; + logger.i( + '[call][sfu] notify raw len=${bytes.length} ' + '${head.map((b) => b.toRadixString(16).padLeft(2, '0')).join(' ')}', + ); + } + final bytes = message.binary; + if (bytes.isEmpty) return; + final type = bytes[0]; + final reader = MsgpackReader(Uint8List.sublistView(bytes, 1)); + try { + switch (type) { + case _notifyAliases: + final count = reader.readMapHeader(); + for (var i = 0; i < count; i++) { + final key = reader.readString(); + _aliases[reader.readInt()] = key; + } + break; + case _notifySlots: + final count = reader.readArrayHeader(); + final slots = {}; + for (var i = 0; i < count; i++) { + final key = _aliases[reader.readInt()]; + if (key != null) slots[key] = i; + } + logger.i('[call][sfu] slots: $slots'); + if (!_slots.isClosed) _slots.add(slots); + break; + case _notifyAudioLevels: + final count = reader.readMapHeader(); + final levels = {}; + for (var i = 0; i < count; i++) { + final key = _aliases[reader.readInt()]; + final level = reader.readInt(); + if (key != null) levels[key] = level; + } + if (!_levels.isClosed) _levels.add(levels); + break; + } + } catch (e) { + logger.w('[call][sfu] notify type=$type parse failed: $e'); + } + } + + Future dispose() async { + _command = null; + _aliases.clear(); + if (!_slots.isClosed) await _slots.close(); + if (!_levels.isClosed) await _levels.close(); + } +} diff --git a/lib/core/calls/ws2_signaling.dart b/lib/core/calls/ws2_signaling.dart index 1916a86..34ac957 100644 --- a/lib/core/calls/ws2_signaling.dart +++ b/lib/core/calls/ws2_signaling.dart @@ -20,8 +20,10 @@ class Ws2Config { const Ws2Config({required this.uri, required this.userId}); - static const defaultCapabilities = '3c03f'; - static const _appVersion = 'sdk-0.1.16.4'; + static const defaultCapabilities = '3c02f'; + static const _appVersion = 'sdk-0.2.1.3'; + static const defaultDevice = 'Android/Unknown'; + static const defaultOsVersion = '34'; /// Входящий звонок: из распакованных параметров [ConversationParams]. /// `userId` — часть после `:` в [ConversationParams.turnUser]. @@ -29,23 +31,22 @@ class Ws2Config { ConversationParams params, { required String conversationId, String capabilities = defaultCapabilities, - String device = 'Komet', - String osVersion = '36', + String? device, + String? osVersion, }) { final userId = int.tryParse((params.turnUser ?? '').split(':').last) ?? 0; final uri = Uri.parse(params.wsEndpoint).replace( queryParameters: { 'userId': '$userId', - 'entityType': 'USER', - 'conversationId': conversationId, 'token': params.token, + 'conversationId': conversationId, 'version': '5', 'capabilities': capabilities, - 'device': device, + 'device': device ?? defaultDevice, 'platform': 'ANDROID', 'clientType': 'ONE_ME', 'appVersion': _appVersion, - 'osVersion': osVersion, + 'osVersion': osVersion ?? defaultOsVersion, }, ); return Ws2Config(uri: uri, userId: userId); @@ -57,19 +58,20 @@ class Ws2Config { String endpoint, { required int userId, String capabilities = defaultCapabilities, - String device = 'Komet', + String? device, + String? osVersion, }) { final base = Uri.parse(endpoint); final uri = base.replace( queryParameters: { ...base.queryParameters, - 'platform': 'ANDROID', 'version': '5', 'capabilities': capabilities, + 'device': device ?? defaultDevice, + 'platform': 'ANDROID', 'clientType': 'ONE_ME', 'appVersion': _appVersion, - 'device': device, - 'tgt': 'start', + 'osVersion': osVersion ?? defaultOsVersion, }, ); return Ws2Config(uri: uri, userId: userId); @@ -283,16 +285,7 @@ class Ws2Signaling { 'singleSession': true, 'unifiedPlan': true, 'fastScreenShare': true, - 'producerScreenDataChannelVersion': 1, - 'consumerScreenDataChannelVersion': 1, - 'animojiDataChannelVersion': 2, - 'animojiBackendRender': true, - 'asrDataChannelVersion': 1, - 'consumerFastScreenShare': true, 'consumerFastScreenShareQualityOnDemand': true, - 'audioShare': true, - 'simulcast': true, - 'simulcastNativeOrder': true, 'red': true, 'videoTracksCount': 10, 'csrcAccessible': true, diff --git a/lib/core/storage/spoofing_service.dart b/lib/core/storage/spoofing_service.dart index 9f6b587..5162a63 100644 --- a/lib/core/storage/spoofing_service.dart +++ b/lib/core/storage/spoofing_service.dart @@ -144,6 +144,7 @@ class SpoofingService { 'instance_id': profile.instanceId, 'client_session_id': profile.clientSessionId, 'push_device_type': profile.pushDeviceType, + 'user_agent': profile.userAgent, }; } diff --git a/lib/frontend/screens/calls/call_screen.dart b/lib/frontend/screens/calls/call_screen.dart index dd06efd..2b05243 100644 --- a/lib/frontend/screens/calls/call_screen.dart +++ b/lib/frontend/screens/calls/call_screen.dart @@ -74,6 +74,8 @@ class _CallScreenState extends State with TickerProviderStateMixin { late final AnimationController _videoController; final RTCVideoRenderer _remoteRenderer = RTCVideoRenderer(); final RTCVideoRenderer _localRenderer = RTCVideoRenderer(); + final Map _tileRenderers = {}; + StreamSubscription? _tileStreamSub; bool _rendererReady = false; bool _localRendererReady = false; bool _videoAttached = false; @@ -89,6 +91,38 @@ class _CallScreenState extends State with TickerProviderStateMixin { bool get _isGroup => widget.isGroup || (_session?.participantCount ?? 0) > 2; + void _onTileStream(int id) { + final stream = _session?.streamOf(id); + final existing = _tileRenderers[id]; + if (existing != null) { + existing.srcObject = stream; + if (mounted) setState(() {}); + return; + } + if (stream == null) return; + unawaited(_createTileRenderer(id, stream)); + } + + Future _createTileRenderer(int id, MediaStream stream) async { + final renderer = RTCVideoRenderer(); + await renderer.initialize(); + if (!mounted) { + await renderer.dispose(); + return; + } + renderer.srcObject = stream; + _tileRenderers[id] = renderer; + setState(() {}); + } + + RTCVideoRenderer? _tileRenderer(CallParticipant p) { + if (p.isSelf) return null; + final own = _tileRenderers[p.id]; + final src = own?.srcObject; + if (src != null && src.getVideoTracks().isNotEmpty) return own; + return _tileVideoReady ? _remoteRenderer : null; + } + bool get _tileVideoReady { if (_session?.topology == 'SERVER') return false; final others = (_session?.participants ?? const []) @@ -215,6 +249,7 @@ class _CallScreenState extends State with TickerProviderStateMixin { setState(() {}); }); _remoteStreamSub = session.remoteStreamStream.listen(_attachStream); + _tileStreamSub = session.participantStreamUpdates.listen(_onTileStream); _kometSub = session.peerKometDetected.listen((_) => _showKometBadge()); _chatSub = session.chatMessages.listen(_onChatMessage); if (session.peerIsKomet) { @@ -383,6 +418,12 @@ class _CallScreenState extends State with TickerProviderStateMixin { _kometSub?.cancel(); _chatSub?.cancel(); _remoteStreamSub?.cancel(); + _tileStreamSub?.cancel(); + for (final renderer in _tileRenderers.values) { + renderer.srcObject = null; + renderer.dispose(); + } + _tileRenderers.clear(); _dotsController.dispose(); _videoController.dispose(); if (_rendererReady) _remoteRenderer.srcObject = null; @@ -622,8 +663,9 @@ class _CallScreenState extends State with TickerProviderStateMixin { final url = p.isSelf ? _avatarUrl : info?.avatar; final muted = p.isSelf ? _isMuted : !p.audioEnabled; final speaking = !muted && _session?.isSpeaking(p.id) == true; + final renderer = _tileRenderer(p); final showVideo = - !p.isSelf && (p.videoEnabled || p.screenSharing) && _tileVideoReady; + !p.isSelf && (p.videoEnabled || p.screenSharing) && renderer != null; return GlossyPill( color: cs.surfaceContainerHigh, @@ -634,7 +676,7 @@ class _CallScreenState extends State with TickerProviderStateMixin { : null, padding: EdgeInsets.all(showVideo ? 0 : 12), child: showVideo - ? _videoTile(cs, name, muted, p.handRaised, p.screenSharing) + ? _videoTile(cs, renderer, name, muted, p.handRaised, p.screenSharing) : _avatarTile(cs, name, url, muted, p.handRaised, p.screenSharing), ); } @@ -719,6 +761,7 @@ class _CallScreenState extends State with TickerProviderStateMixin { Widget _videoTile( ColorScheme cs, + RTCVideoRenderer renderer, String name, bool muted, bool hand, @@ -730,7 +773,7 @@ class _CallScreenState extends State with TickerProviderStateMixin { fit: StackFit.expand, children: [ RTCVideoView( - _remoteRenderer, + renderer, objectFit: RTCVideoViewObjectFit.RTCVideoViewObjectFitCover, ), Positioned( diff --git a/lib/frontend/widgets/glossy_pill.dart b/lib/frontend/widgets/glossy_pill.dart index b3f5a8d..9a6dc17 100644 --- a/lib/frontend/widgets/glossy_pill.dart +++ b/lib/frontend/widgets/glossy_pill.dart @@ -141,7 +141,9 @@ class GlossyPill extends StatelessWidget { child: DecoratedBox( decoration: BoxDecoration( borderRadius: borderRadius, - border: GlossyDecor.rimBorder(base), + border: borderSide != null + ? Border.fromBorderSide(borderSide!) + : GlossyDecor.rimBorder(base), boxShadow: [GlossyDecor.dropShadow(base, depth)], ), child: LiquidGlassSurface( @@ -204,7 +206,9 @@ class GlossyPill extends StatelessWidget { borderRadius: borderRadius, color: gradient ? null : base, gradient: gradient ? GlossyDecor.fillGradient(base) : null, - border: GlossyDecor.rimBorder(base), + border: borderSide != null + ? Border.fromBorderSide(borderSide!) + : GlossyDecor.rimBorder(base), boxShadow: [GlossyDecor.dropShadow(base, depth)], ), child: ClipRRect(