test: add webrtc stream benchmark

Author: DjDeveloperr

.github/workflows/ci.yml

@@ -242,6 +242,42 @@ jobs:
       - name: JS API simulator integration tests
         run: npm run test:integration:js-api
 
+  integration-webrtc:
+    name: WebRTC stream benchmark
+    runs-on: macos-15
+    timeout-minutes: 20
+    env:
+      DEVELOPER_DIR: /Applications/Xcode_16.4.app/Contents/Developer
+    needs:
+      - rust
+      - client
+      - packages
+      - build-artifacts
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Download integration artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: simdeck-integration-artifacts
+          path: .
+
+      - name: Make CLI executable
+        run: chmod +x build/simdeck build/simdeck-bin
+
+      - name: WebRTC software stream benchmark
+        run: npm run test:integration:webrtc
+        env:
+          SIMDECK_E2E_MIN_DECODED_FPS: "55"
+          SIMDECK_E2E_MIN_PRESENTED_FPS: "55"
+          SIMDECK_E2E_MIN_RECEIVED_FPS: "55"
+          SIMDECK_E2E_WEBRTC_MS: "20000"
+
   integration-android:
     name: Android emulator integration
     runs-on: ubuntu-latest

package.json

@@ -69,6 +69,7 @@
     "test:integration:cli:verbose": "SIMDECK_INTEGRATION_VERBOSE=1 SIMDECK_INTEGRATION_SHOW_SIMULATOR=1 node scripts/integration/cli.mjs",
     "test:integration:fixture": "node scripts/integration/prebuild-fixture.mjs",
     "test:integration:js-api": "node scripts/integration/js-api.mjs",
+    "test:integration:webrtc": "node scripts/integration/webrtc.mjs",
     "test:integration:js-api:verbose": "SIMDECK_INTEGRATION_SHOW_SIMULATOR=1 node scripts/integration/js-api.mjs",
     "test:integration:android": "node scripts/integration/android.mjs",
     "test:integration:android:verbose": "SIMDECK_INTEGRATION_VERBOSE=1 node scripts/integration/android.mjs",

packages/simdeck-test/dist/index.js

@@ -384,23 +384,24 @@ async function startIsolatedDaemon(cliPath, options) {
     const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), "simdeck-test-project-"));
     const metadataPath = path.join(os.tmpdir(), `simdeck-test-${process.pid}-${Date.now()}-${crypto.randomUUID()}.json`);
     const accessToken = crypto.randomBytes(32).toString("hex");
+    const packageRoot = options.projectRoot ?? process.cwd();
     const child = spawn(cliPath, [
-        "daemon",
+        "service",
         "run",
-        "--project-root",
-        projectRoot,
         "--metadata-path",
         metadataPath,
         "--port",
         String(port),
         "--bind",
         "127.0.0.1",
+        "--client-root",
+        path.join(packageRoot, "packages", "client", "dist"),
         "--access-token",
         accessToken,
         "--video-codec",
         options.videoCodec ?? "software",
     ], {
-        cwd: options.projectRoot,
+        cwd: projectRoot,
         stdio: ["ignore", "pipe", "pipe"],
     });
     const output = captureChildOutput(child);

packages/simdeck-test/src/index.ts

@@ -726,26 +726,27 @@ async function startIsolatedDaemon(
     `simdeck-test-${process.pid}-${Date.now()}-${crypto.randomUUID()}.json`,
   );
   const accessToken = crypto.randomBytes(32).toString("hex");
+  const packageRoot = options.projectRoot ?? process.cwd();
   const child = spawn(
     cliPath,
     [
-      "daemon",
+      "service",
       "run",
-      "--project-root",
-      projectRoot,
       "--metadata-path",
       metadataPath,
       "--port",
       String(port),
       "--bind",
       "127.0.0.1",
+      "--client-root",
+      path.join(packageRoot, "packages", "client", "dist"),
       "--access-token",
       accessToken,
       "--video-codec",
       options.videoCodec ?? "software",
     ],
     {
-      cwd: options.projectRoot,
+      cwd: projectRoot,
       stdio: ["ignore", "pipe", "pipe"],
     },
   );

scripts/e2e-webrtc-reliability.mjs

@@ -27,6 +27,11 @@ const maxPeerDisconnectedMs = Number(
   process.env.SIMDECK_E2E_MAX_PEER_DISCONNECTED_MS ?? 1000,
 );
 const maxDecoderDrops = Number(process.env.SIMDECK_E2E_MAX_DECODER_DROPS ?? 0);
+const minVideoWidth = Number(process.env.SIMDECK_E2E_MIN_VIDEO_WIDTH ?? 0);
+const minVideoHeight = Number(process.env.SIMDECK_E2E_MIN_VIDEO_HEIGHT ?? 0);
+const minDecodedFps = Number(process.env.SIMDECK_E2E_MIN_DECODED_FPS ?? 0);
+const minPresentedFps = Number(process.env.SIMDECK_E2E_MIN_PRESENTED_FPS ?? 0);
+const minReceivedFps = Number(process.env.SIMDECK_E2E_MIN_RECEIVED_FPS ?? 0);
 const visualSampleIntervalMs = Number(
   process.env.SIMDECK_E2E_VISUAL_SAMPLE_INTERVAL_MS ?? 5000,
 );
@@ -406,6 +411,13 @@ try {
     directStatsEnd.packetsReceived - directStatsStart.packetsReceived;
   const directPresentedDelta =
     directStatsEnd.totalVideoFrames - directStatsStart.totalVideoFrames;
+  const observedDurationSeconds = Math.max(
+    0.001,
+    (directStatsEnd.timestampMs - directStatsStart.timestampMs) / 1000,
+  );
+  const decodedFps = directDecodedDelta / observedDurationSeconds;
+  const presentedFps = directPresentedDelta / observedDurationSeconds;
+  const receivedFps = directPacketsDelta / observedDurationSeconds;
 
   if (decodedDelta <= 0 || receivedDelta <= 0) {
     failures.push(
@@ -429,6 +441,41 @@ try {
   if (reconnectDelta > 0) {
     failures.push(`peer reconnected ${reconnectDelta} times`);
   }
+  const finalVideoWidth = Math.max(
+    numeric(finalPage.width),
+    numeric(finalWebRtc.width),
+    numeric(directStatsEnd.videoWidth),
+  );
+  const finalVideoHeight = Math.max(
+    numeric(finalPage.height),
+    numeric(finalWebRtc.height),
+    numeric(directStatsEnd.videoHeight),
+  );
+  if (minVideoWidth > 0 && finalVideoWidth < minVideoWidth) {
+    failures.push(
+      `video width ${finalVideoWidth} did not meet minimum ${minVideoWidth}`,
+    );
+  }
+  if (minVideoHeight > 0 && finalVideoHeight < minVideoHeight) {
+    failures.push(
+      `video height ${finalVideoHeight} did not meet minimum ${minVideoHeight}`,
+    );
+  }
+  if (minDecodedFps > 0 && decodedFps < minDecodedFps) {
+    failures.push(
+      `decoded fps ${decodedFps.toFixed(2)} did not meet minimum ${minDecodedFps}`,
+    );
+  }
+  if (minPresentedFps > 0 && presentedFps < minPresentedFps) {
+    failures.push(
+      `presented fps ${presentedFps.toFixed(2)} did not meet minimum ${minPresentedFps}`,
+    );
+  }
+  if (minReceivedFps > 0 && receivedFps < minReceivedFps) {
+    failures.push(
+      `received packet fps ${receivedFps.toFixed(2)} did not meet minimum ${minReceivedFps}`,
+    );
+  }
   if (maxPeerDisconnectedObservedMs > maxPeerDisconnectedMs) {
     failures.push(
       `peer disconnected for ${maxPeerDisconnectedObservedMs}ms, exceeded ${maxPeerDisconnectedMs}ms`,
@@ -480,8 +527,14 @@ try {
     durationMs,
     finalPage,
     finalWebRtc,
+    finalVideoHeight,
+    finalVideoWidth,
     initialPage,
     initialWebRtc,
+    observedDurationSeconds,
+    decodedFps,
+    presentedFps,
+    receivedFps,
     maxObservedDecodeQueue,
     maxObservedFrameGapMs,
     maxPeerDisconnectedMs,
@@ -601,12 +654,15 @@ async function collectDirectWebRtcStats(cdp) {
     (async () => {
       const totals = {
         framesDecoded: 0,
-        framesDropped: 0,
-        jitter: 0,
-        packetsLost: 0,
-        packetsReceived: 0,
-        totalVideoFrames: 0,
-      };
+      framesDropped: 0,
+      jitter: 0,
+      packetsLost: 0,
+      packetsReceived: 0,
+      timestampMs: Date.now(),
+      totalVideoFrames: 0,
+      videoHeight: 0,
+      videoWidth: 0,
+    };
       for (const pc of window.__simdeckPeerConnections || []) {
         const reports = await pc.getStats();
         for (const report of reports.values()) {
@@ -622,6 +678,8 @@ async function collectDirectWebRtcStats(cdp) {
       for (const video of document.querySelectorAll("video.stream-video")) {
         const quality = video.getVideoPlaybackQuality?.();
         totals.totalVideoFrames += quality?.totalVideoFrames || 0;
+        totals.videoWidth = Math.max(totals.videoWidth, video.videoWidth || 0);
+        totals.videoHeight = Math.max(totals.videoHeight, video.videoHeight || 0);
       }
       return totals;
     })()

scripts/integration/fixture.mjs

@@ -140,7 +140,10 @@ function fixtureSource() {
 @interface FixtureViewController : UIViewController <UITextFieldDelegate>
 @property (nonatomic, strong) UILabel *statusLabel;
 @property (nonatomic, strong) UITextField *messageField;
+@property (nonatomic, strong) UIView *animationBar;
+@property (nonatomic, strong) CADisplayLink *displayLink;
 @property (nonatomic) NSInteger tapCount;
+@property (nonatomic) CFTimeInterval animationStartedAt;
 - (void)openFixtureURL:(NSURL *)url;
 @end
 
@@ -178,11 +181,18 @@ function fixtureSource() {
   [self.messageField addTarget:self action:@selector(messageChanged:) forControlEvents:UIControlEventEditingChanged];
   [self.messageField.widthAnchor constraintEqualToConstant:240.0].active = YES;
 
+  self.animationBar = [[UIView alloc] initWithFrame:CGRectZero];
+  self.animationBar.backgroundColor = UIColor.systemBlueColor;
+  self.animationBar.layer.cornerRadius = 6.0;
+  self.animationBar.accessibilityIdentifier = @"fixture.animation";
+  self.animationBar.translatesAutoresizingMaskIntoConstraints = NO;
+
   UIStackView *stack = [[UIStackView alloc] initWithArrangedSubviews:@[
     titleLabel,
     self.statusLabel,
     continueButton,
     self.messageField,
+    self.animationBar,
   ]];
   stack.axis = UILayoutConstraintAxisVertical;
   stack.alignment = UIStackViewAlignmentCenter;
@@ -195,6 +205,8 @@ function fixtureSource() {
     [stack.centerYAnchor constraintEqualToAnchor:self.view.centerYAnchor],
     [stack.leadingAnchor constraintGreaterThanOrEqualToAnchor:self.view.safeAreaLayoutGuide.leadingAnchor constant:24.0],
     [stack.trailingAnchor constraintLessThanOrEqualToAnchor:self.view.safeAreaLayoutGuide.trailingAnchor constant:-24.0],
+    [self.animationBar.widthAnchor constraintEqualToConstant:220.0],
+    [self.animationBar.heightAnchor constraintEqualToConstant:12.0],
   ]];
 }
 
@@ -207,8 +219,31 @@ function fixtureSource() {
   self.statusLabel.text = sender.text ?: @"";
 }
 
+- (void)startAnimation {
+  if (self.displayLink) {
+    return;
+  }
+  self.animationStartedAt = CACurrentMediaTime();
+  self.statusLabel.text = @"Animating";
+  self.displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(animationTick:)];
+  [self.displayLink addToRunLoop:NSRunLoop.mainRunLoop forMode:NSRunLoopCommonModes];
+}
+
+- (void)animationTick:(CADisplayLink *)displayLink {
+  CFTimeInterval elapsed = CACurrentMediaTime() - self.animationStartedAt;
+  CGFloat phase = (sin(elapsed * 4.0) + 1.0) * 0.5;
+  CGFloat x = -80.0 + phase * 160.0;
+  self.animationBar.transform = CGAffineTransformMakeTranslation(x, 0.0);
+  self.animationBar.backgroundColor = [UIColor colorWithHue:fmod(elapsed * 0.18, 1.0)
+                                                 saturation:0.85
+                                                 brightness:0.92
+                                                      alpha:1.0];
+}
+
 - (void)openFixtureURL:(NSURL *)url {
-  if ([url.host isEqualToString:@"focus-message"]) {
+  if ([url.host isEqualToString:@"animate"]) {
+    [self startAnimation];
+  } else if ([url.host isEqualToString:@"focus-message"]) {
     dispatch_async(dispatch_get_main_queue(), ^{
       [self.messageField becomeFirstResponder];
       dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(500 * NSEC_PER_MSEC)), dispatch_get_main_queue(), ^{