Add verified server rollout and live ghost-mode checks

tool/deploy_server_fixes.py

@@ -0,0 +1,102 @@
+"""Deploy the complete tested IKAR API build, preserving other containers."""
+from pathlib import Path
+import hashlib
+import json
+import os
+import shutil
+import sqlite3
+import subprocess
+import sys
+import tarfile
+import tempfile
+import time
+import urllib.request
+
+root = Path('/mnt/data/ikar_server')
+release = root / 'releases/20261007-komet'
+current = root / 'current'
+previous = current.resolve(strict=True)
+expected_sha = sys.argv[1].lower()
+if not os.path.ismount('/mnt/data') or previous != root / 'releases/20261007-video' or release.exists():
+    raise RuntimeError('Unexpected release or external disk state')
+
+def snapshot():
+    containers = subprocess.check_output(['docker', 'ps', '-q'], text=True).split()
+    return {item['Name']: {'id': item['Id'], 'startedAt': item['State']['StartedAt']}
+            for item in json.loads(subprocess.check_output(['docker', 'inspect', *containers], text=True))
+            if item['Name'] != '/ikar_server'}
+
+protected_before = snapshot()
+print('Creating database and attachment backup', flush=True)
+backup = Path(subprocess.check_output(['python3', str(previous / 'backup.py')], text=True).strip())
+with tempfile.TemporaryDirectory() as temp:
+    target = Path(temp) / 'ikar.db'
+    with tarfile.open(backup) as archive:
+        target.write_bytes(archive.extractfile('data/ikar.db').read())
+    with sqlite3.connect(target) as db:
+        if db.execute('PRAGMA integrity_check').fetchone()[0] != 'ok':
+            raise RuntimeError('Backup failed integrity verification')
+
+shutil.copytree(previous, release, ignore=shutil.ignore_patterns('publish', 'release.tar.gz'))
+with tarfile.open(root / 'server-fixes.tar.gz') as archive:
+    archive.extractall(release, filter='data')
+actual_sha = hashlib.sha256((release / 'publish/Ikar.Server.dll').read_bytes()).hexdigest()
+if actual_sha != expected_sha:
+    raise RuntimeError('Uploaded server differs from verified local build')
+compose = release / 'compose.yaml'
+text = compose.read_text()
+old_image = 'image: ikar_server:20261007-video'
+if text.count(old_image) != 1:
+    raise RuntimeError('Unexpected API image in compose')
+compose.write_text(text.replace(old_image, 'image: ikar_server:20261007-komet', 1))
+print('Building IKAR API image', flush=True)
+with (root / 'build-komet.log').open('w') as log:
+    subprocess.run(['docker', 'build', '-f', 'Dockerfile.release', '-t', 'ikar_server:20261007-komet', '.'],
+                   cwd=release, stdout=log, stderr=subprocess.STDOUT, check=True)
+
+def update(path):
+    subprocess.run(['docker', 'compose', '--project-directory', str(path), '--env-file', str(root / 'shared/turn.env'),
+                    'up', '-d', '--no-deps', 'ikar_server'], check=True)
+
+try:
+    print('Updating API container', flush=True)
+    update(release)
+    health = None
+    for attempt in range(30):
+        try:
+            with urllib.request.urlopen('http://127.0.0.1:5099/health', timeout=3) as response:
+                health = json.load(response)
+            if health.get('status') == 'ok':
+                break
+        except Exception:
+            pass
+        time.sleep(1)
+    if health is None or health.get('status') != 'ok':
+        raise RuntimeError('New API failed health verification')
+    running_sha = subprocess.check_output(['docker', 'exec', 'ikar_server', 'sha256sum', '/app/Ikar.Server.dll'], text=True).split()[0]
+    if running_sha != expected_sha:
+        raise RuntimeError('Running server differs from verified build')
+    with sqlite3.connect(f'file:{root}/shared/data/ikar.db?mode=ro', uri=True) as db:
+        integrity = db.execute('PRAGMA integrity_check').fetchone()[0]
+        columns = {row[1] for row in db.execute('PRAGMA table_info(Users)')}
+    if integrity != 'ok' or 'PublicLastSeenUnixMs' not in columns:
+        raise RuntimeError('Database or presence schema verification failed')
+    subprocess.run(['docker', 'exec', 'ikar_server', 'ffmpeg', '-version'], stdout=subprocess.DEVNULL, check=True)
+    pending = root / 'current.next-komet'
+    pending.symlink_to(release, target_is_directory=True)
+    os.replace(pending, current)
+except Exception:
+    print('Rolling back API to previous release', flush=True)
+    update(previous)
+    raise
+
+report = {
+    'passed': True, 'release': str(release), 'previousRelease': str(previous),
+    'backup': str(backup), 'backupIntegrity': 'ok', 'health': health,
+    'databaseIntegrity': integrity, 'publicPresenceColumn': True,
+    'serverSha256': running_sha, 'ffmpegAvailable': True,
+    'protectedServicesUnchanged': snapshot() == protected_before,
+    'protectedServices': protected_before,
+}
+(root / 'deployment-komet.json').write_text(json.dumps(report, indent=2))
+print(json.dumps(report), flush=True)

tool/live_ikar_smoke.dart

@@ -19,8 +19,9 @@ class Device {
     Uri base,
     String node,
     String phone,
-    String name,
-  ) async {
+    String name, {
+    bool interactive = true,
+  }) async {
     final process = await Process.start(node, ['tool/smoke_key.cjs']);
     final lines = StreamIterator(
       process.stdout.transform(utf8.decoder).transform(const LineSplitter()),
@@ -45,7 +46,8 @@ class Device {
       device.transport.token = registered['token'];
       device.refresh = registered['refreshToken'];
       device.id = registered['accountId'] as int;
-      await device.ok(19, {});
+      device.transport.interactive = interactive;
+      await device.ok(19, {'interactive': interactive});
       await device.transport.connectEvents();
       return device;
     } catch (_) {
@@ -121,7 +123,12 @@ Future<void> main(List<String> args) async {
       'chatId': chatId,
       'count': 40,
     });
-    check(forbidden['cmd'] == 3, 'Nonmember could read chat');
+    final outsiderMessages = (forbidden['payload'] as Map?)?['messages'];
+    check(
+      forbidden['cmd'] == 3 ||
+          (outsiderMessages is List && outsiderMessages.isEmpty),
+      'Nonmember could read chat: ${forbidden['payload']}',
+    );
     checks.add('persistent history and nonmember isolation');
     final retryCid = DateTime.now().microsecondsSinceEpoch;
     final richPayload = {

tool/verify_ghost_mode.dart

@@ -0,0 +1,191 @@
+import 'dart:convert';
+import 'dart:io';
+import '../lib/backend/ikar_transport.dart';
+import '../lib/core/transport/ikar_network.dart';
+import 'live_ikar_smoke.dart' as live;
+
+Future<void> main() async {
+  final peers = <live.Device>[];
+  IkarTransport? visibleConnection;
+  final base = Uri.parse('https://ikar.kusoft.xyz');
+  final checks = <String>[];
+  final client = IkarNetwork.client();
+  try {
+    final healthResponse = await (await client.getUrl(
+      base.resolve('/health'),
+    )).close();
+    final health = jsonDecode(await utf8.decodeStream(healthResponse)) as Map;
+    live.check(
+      healthResponse.statusCode == 200 && health['status'] == 'ok',
+      'Public TLS health failed',
+    );
+    checks.add('public HTTPS health with certificate validation');
+    final stamp = DateTime.now().microsecondsSinceEpoch.toString().substring(6);
+    peers.add(
+      await live.Device.create(
+        base,
+        'node',
+        '+999${stamp}0',
+        'Ghost verification',
+        interactive: false,
+      ),
+    );
+    peers.add(
+      await live.Device.create(
+        base,
+        'node',
+        '+999${stamp}1',
+        'Presence observer',
+      ),
+    );
+    final hidden = peers[0], observer = peers[1];
+    final chatId = hidden.id ^ observer.id;
+    await hidden.ok(64, {
+      'chatId': chatId,
+      'message': {'text': 'Ghost initial message'},
+    });
+
+    Future<Map> presence(int status) async {
+      for (var attempt = 0; attempt < 30; attempt++) {
+        final result = await observer.ok(35, {
+          'contactIds': [hidden.id],
+        });
+        final entry = result['presence']['${hidden.id}'] as Map;
+        if (entry['status'] == status) return entry;
+        await Future<void>.delayed(const Duration(milliseconds: 100));
+      }
+      throw StateError('Presence failed to reach $status');
+    }
+
+    final initial = await presence(0);
+    live.check(
+      initial['seen'] == 0,
+      'Invisible first connection exposed activity',
+    );
+    checks.add(
+      'invisible login and SignalR connection expose neither online nor last seen',
+    );
+    await hidden.ok(1, {'interactive': true});
+    final active = await presence(1);
+    live.check(
+      (active['seen'] as num) > 0,
+      'Visible ping failed to publish activity',
+    );
+    await Future<void>.delayed(const Duration(milliseconds: 1100));
+    await hidden.ok(1, {'interactive': false});
+    final inactive = await presence(0);
+    live.check(
+      inactive['seen'] == active['seen'],
+      'Ghost ping changed last seen',
+    );
+    checks.add('switching to ghost freezes the previous public timestamp');
+    hidden.transport.interactive = false;
+    await hidden.transport.connectEvents();
+    await hidden.ok(1, {'interactive': false});
+    live.check(
+      (await presence(0))['seen'] == active['seen'],
+      'Reconnect exposed activity',
+    );
+    checks.add('invisible event reconnection retains hidden presence');
+    final delivered = hidden.transport.events
+        .firstWhere(
+          (event) =>
+              event['opcode'] == 128 &&
+              event['payload']?['message']?['text'] == 'Delivery to ghost',
+        )
+        .timeout(const Duration(seconds: 15));
+    final sent = await observer.ok(64, {
+      'chatId': chatId,
+      'message': {'text': 'Delivery to ghost'},
+    });
+    await delivered;
+    final history = await hidden.ok(49, {'chatId': chatId});
+    live.check(
+      (history['messages'] as List).any(
+        (message) => message['id'] == sent['message']['id'],
+      ),
+      'Ghost history missed delivered message',
+    );
+    await hidden.ok(64, {
+      'chatId': chatId,
+      'message': {'text': 'Reply from ghost'},
+    });
+    live.check(
+      (await presence(0))['seen'] == active['seen'],
+      'Ghost messaging exposed security session activity',
+    );
+    checks.add(
+      'invisible client receives messages, reads history and replies without exposing activity',
+    );
+    final incoming = hidden.transport.events
+        .firstWhere((event) => event['opcode'] == 137)
+        .timeout(const Duration(seconds: 15));
+    final call = await observer.ok(78, {
+      'calleeIds': [hidden.id],
+      'isVideo': false,
+      'conversationId': 'ikar-ghost-$stamp',
+    });
+    final invitation = await incoming;
+    live.check(
+      invitation['payload']['callerId'] == observer.id,
+      'Ghost call invitation has wrong caller',
+    );
+    final endpoint =
+        jsonDecode(call['internalCallerParams'])['endpoint'] as String;
+    final callSocket = await IkarNetwork.connectWebSocket(endpoint);
+    final callSubscription = callSocket.listen((_) {});
+    try {
+      callSocket.add(
+        jsonEncode({'command': 'hangup', 'sequence': 1, 'reason': 'FINISHED'}),
+      );
+      await Future<void>.delayed(const Duration(milliseconds: 200));
+    } finally {
+      await callSocket.close();
+      await callSubscription.cancel();
+    }
+    live.check(
+      (await presence(0))['seen'] == active['seen'],
+      'Incoming call exposed ghost activity',
+    );
+    checks.add(
+      'invisible client receives a call invitation without exposing activity',
+    );
+    visibleConnection = IkarTransport(base)
+      ..token = hidden.transport.token
+      ..interactive = true;
+    await visibleConnection.connectEvents();
+    await presence(1);
+    await visibleConnection.close();
+    visibleConnection = null;
+    await presence(0);
+    checks.add(
+      'disconnecting a visible connection preserves the invisible connection',
+    );
+    await hidden.ok(1, {'interactive': true});
+    final restored = await presence(1);
+    live.check(
+      (restored['seen'] as num) > (active['seen'] as num),
+      'Visible activity did not resume',
+    );
+    checks.add('disabling ghost resumes online and public last seen');
+    final report = {
+      'passed': true,
+      'checks': checks,
+      'timestamp': DateTime.now().toUtc().toIso8601String(),
+    };
+    await File(
+      'artifacts/ghost-live-verification.json',
+    ).writeAsString(const JsonEncoder.withIndent('  ').convert(report));
+    stdout.writeln(jsonEncode(report));
+  } finally {
+    await visibleConnection?.close();
+    client.close(force: true);
+    for (final peer in peers.reversed) {
+      try {
+        await peer.ok(199, {});
+      } finally {
+        await peer.close();
+      }
+    }
+  }
+}