Using Surface from swap chains to render immersive frames instead of framebuffer textures.

Author: daoshengmu

app/src/common/shared/org/mozilla/vrbrowser/VRBrowserActivity.java

@@ -949,6 +949,17 @@ public void dismiss() {
         });
     }
 
+    public void setExternalVRSurface(int aIndex, Surface aSurface) {
+        if (aSurface == null) {
+            Log.e(LOGTAG, "aSurface is null in setExternalVRSurface...");
+        }
+        GeckoVRManager.insertExternalSurface(aIndex, aSurface);
+    }
+
+    public void releaseExternalVRSurfaces() {
+        GeckoVRManager.releaseExternalSurfaces();
+    }
+
     void createOffscreenDisplay() {
         int[] ids = new int[1];
         GLES20.glGenTextures(1, ids, 0);

app/src/common/shared/org/mozilla/vrbrowser/browser/engine/SessionUtils.java

@@ -10,6 +10,7 @@
 import org.mozilla.gecko.GeckoProfile;
 import org.mozilla.vrbrowser.BuildConfig;
 import org.mozilla.vrbrowser.browser.SettingsStore;
+import org.mozilla.vrbrowser.utils.DeviceType;
 
 import java.io.File;
 import java.io.FileNotFoundException;
@@ -55,6 +56,12 @@ public static void vrPrefsWorkAround(Context aContext, Bundle aExtras) {
                 out.write("pref(\"media.autoplay.enabled.ask-permission\", false);\n".getBytes());
                 out.write("pref(\"media.autoplay.default\", 0);\n".getBytes());
             }
+            // In Oculus platform, we can render WebGL immersive frames info AndroidSurface.
+            if (DeviceType.isOculusBuild()) {
+                out.write("pref(\"webgl.enable-externalvr-surface\", true);\n".getBytes());
+            } else {
+                out.write("pref(\"webgl.enable-externalvr-surface\", false);\n".getBytes());
+            }
         } catch (FileNotFoundException e) {
             Log.e(LOGTAG, "Unable to create file: '" + prefFileName + "' got exception: " + e.toString());
         } catch (IOException e) {

app/src/main/cpp/BrowserWorld.cpp

@@ -1177,6 +1177,7 @@ void
 BrowserWorld::DrawWorld() {
   m.externalVR->SetCompositorEnabled(true);
   m.device->SetRenderMode(device::RenderMode::StandAlone);
+  m.device->EnableExternalSurfaceRender(false);
   if (m.fadeAnimation) {
     m.fadeAnimation->UpdateAnimation();
   }
@@ -1244,23 +1245,29 @@ BrowserWorld::DrawImmersive() {
   m.device->StartFrame();
   VRB_GL_CHECK(glDepthMask(GL_FALSE));
   m.externalVR->PushFramePoses(m.device->GetHeadTransform(), m.controllers->GetControllers(), m.context->GetTimestamp());
+  mozilla::gfx::VRLayerTextureType surfaceType;
   int32_t surfaceHandle, textureWidth, textureHeight = 0;
   device::EyeRect leftEye, rightEye;
   bool aDiscardFrame = !m.externalVR->WaitFrameResult();
-  m.externalVR->GetFrameResult(surfaceHandle, textureWidth, textureHeight, leftEye, rightEye);
+  m.externalVR->GetFrameResult(surfaceType, surfaceHandle, textureWidth, textureHeight, leftEye, rightEye);
   ExternalVR::VRState state = m.externalVR->GetVRState();
   if (state == ExternalVR::VRState::Rendering) {
     if (!aDiscardFrame) {
       if (textureWidth > 0 && textureHeight > 0) {
         m.device->SetImmersiveSize((uint32_t) textureWidth/2, (uint32_t) textureHeight);
       }
-      m.blitter->StartFrame(surfaceHandle, leftEye, rightEye);
-      m.device->BindEye(device::Eye::Left);
-      m.blitter->Draw(device::Eye::Left);
+      m.device->EnableExternalSurfaceRender(surfaceType ==
+        mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface);
+      // In Oculus platform, we can render WebGL immersive frames info AndroidSurface.
+      if (surfaceType != mozilla::gfx::VRLayerTextureType::LayerTextureType_ExternalVRSurface) {
+        m.blitter->StartFrame(surfaceHandle, leftEye, rightEye);
+        m.device->BindEye(device::Eye::Left);
+        m.blitter->Draw(device::Eye::Left);
 #if !defined(VRBROWSER_NO_VR_API)
-      m.device->BindEye(device::Eye::Right);
-      m.blitter->Draw(device::Eye::Right);
+        m.device->BindEye(device::Eye::Right);
+        m.blitter->Draw(device::Eye::Right);
 #endif // !defined(VRBROWSER_NO_VR_API)
+      }
     }
     m.device->EndFrame(aDiscardFrame);
     m.blitter->EndFrame();

app/src/main/cpp/DeviceDelegate.h

@@ -37,6 +37,7 @@ class ImmersiveDisplay {
                               const double aRightDegrees,
                               const double aTopDegrees,
                               const double aBottomDegrees) = 0;
+  virtual void SetExternalSurfaceInfo(int32_t aExternalSurfLen, int32_t aExternalSurfId) = 0;
   virtual void SetEyeOffset(const device::Eye aEye, const float aX, const float aY, const float aZ) = 0;
   virtual void SetEyeResolution(const int32_t aWidth, const int32_t aHeight) = 0;
   virtual void SetSittingToStandingTransform(const vrb::Matrix& aTransform) = 0;
@@ -67,6 +68,7 @@ class DeviceDelegate {
   virtual void StartFrame() = 0;
   virtual void BindEye(const device::Eye aWhich) = 0;
   virtual void EndFrame(bool aDiscard = false) = 0;
+  virtual void EnableExternalSurfaceRender(bool aEnable) {}
   virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
                                          VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; }
   virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; }

app/src/main/cpp/ExternalVR.cpp

@@ -259,6 +259,12 @@ ExternalVR::SetCapabilityFlags(const device::CapabilityFlags aFlags) {
   m.system.sensorState.flags = m.system.displayState.capabilityFlags;
 }
 
+void
+ExternalVR::SetExternalSurfaceInfo(int32_t aExternalSurfLen, int32_t aExternalSurfId) {
+  m.system.displayState.externalSurfaceLength = aExternalSurfLen;
+  m.system.displayState.externalSurfaceId = aExternalSurfId;
+}
+
 void
 ExternalVR::SetFieldOfView(const device::Eye aEye, const double aLeftDegrees,
                            const double aRightDegrees,
@@ -476,10 +482,10 @@ ExternalVR::CompleteEnumeration()
   m.system.enumerationCompleted = true;
 }
 
-
 void
-ExternalVR::GetFrameResult(int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight,
+ExternalVR::GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType, int32_t& aSurfaceHandle, int32_t& aTextureWidth, int32_t& aTextureHeight,
     device::EyeRect& aLeftEye, device::EyeRect& aRightEye) const {
+  aSurfaceType = m.browser.layerState[0].layer_stereo_immersive.textureType;
   aSurfaceHandle = (int32_t)m.browser.layerState[0].layer_stereo_immersive.textureHandle;
   mozilla::gfx::VRLayerEyeRect& left = m.browser.layerState[0].layer_stereo_immersive.leftEyeRect;
   mozilla::gfx::VRLayerEyeRect& right = m.browser.layerState[0].layer_stereo_immersive.rightEyeRect;

app/src/main/cpp/ExternalVR.h

@@ -10,6 +10,7 @@
 #include "Controller.h"
 #include "DeviceDelegate.h"
 #include "Device.h"
+#include "moz_external_vr.h"
 #include <memory>
 #include <string>
 #include <vector>
@@ -38,6 +39,7 @@ class ExternalVR : public ImmersiveDisplay {
   // DeviceDisplay interface
   void SetDeviceName(const std::string& aName) override;
   void SetCapabilityFlags(const device::CapabilityFlags aFlags) override;
+  void SetExternalSurfaceInfo(int32_t aExternalSurfLen, int32_t aExternalSurfId) override;
   void SetFieldOfView(const device::Eye aEye, const double aLeftDegrees,
                       const double aRightDegrees,
                       const double aTopDegrees,
@@ -54,7 +56,8 @@ class ExternalVR : public ImmersiveDisplay {
   VRState GetVRState() const;
   void PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<Controller>& aControllers, const double aTimestamp);
   bool WaitFrameResult();
-  void GetFrameResult(int32_t& aSurfaceHandle,
+  void GetFrameResult(mozilla::gfx::VRLayerTextureType& aSurfaceType,
+                      int32_t& aSurfaceHandle,
                       int32_t& aTextureWidth,
                       int32_t& aTextureHeight,
                       device::EyeRect& aLeftEye,

app/src/main/cpp/VRBrowser.cpp

@@ -52,6 +52,10 @@ const char* kHaltActivity = "haltActivity";
 const char* kHaltActivitySignature = "(I)V";
 const char* kHandlePoorPerformance = "handlePoorPerformance";
 const char* kHandlePoorPerformanceSignature = "()V";
+const char* kSetExternalVRSurface = "setExternalVRSurface";
+const char* kSetExternalVRSurfaceSignature = "(ILandroid/view/Surface;)V";
+const char* kReleaseExternalSurfaces = "releaseExternalVRSurfaces";
+const char* kReleaseExternalSurfacesSignature = "()V";
 
 JNIEnv* sEnv = nullptr;
 jclass sBrowserClass = nullptr;
@@ -77,6 +81,8 @@ jmethodID sAreLayersEnabled = nullptr;
 jmethodID sSetDeviceType = nullptr;
 jmethodID sHaltActivity = nullptr;
 jmethodID sHandlePoorPerformance = nullptr;
+jmethodID sSetExternalVRSurface = nullptr;
+jmethodID sReleaseExternalSurfaces = nullptr;
 }
 
 namespace crow {
@@ -117,6 +123,8 @@ VRBrowser::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
   sSetDeviceType = FindJNIMethodID(sEnv, sBrowserClass, kSetDeviceType, kSetDeviceTypeSignature);
   sHaltActivity = FindJNIMethodID(sEnv, sBrowserClass, kHaltActivity, kHaltActivitySignature);
   sHandlePoorPerformance = FindJNIMethodID(sEnv, sBrowserClass, kHandlePoorPerformance, kHandlePoorPerformanceSignature);
+  sSetExternalVRSurface = FindJNIMethodID(sEnv, sBrowserClass, kSetExternalVRSurface, kSetExternalVRSurfaceSignature);
+  sReleaseExternalSurfaces = FindJNIMethodID(sEnv, sBrowserClass, kReleaseExternalSurfaces, kReleaseExternalSurfacesSignature);
 }
 
 void
@@ -151,6 +159,7 @@ VRBrowser::ShutdownJava() {
   sAreLayersEnabled = nullptr;
   sSetDeviceType = nullptr;
   sHaltActivity = nullptr;
+  sSetExternalVRSurface = nullptr;
   sEnv = nullptr;
 }
 
@@ -339,4 +348,17 @@ VRBrowser::HandlePoorPerformance() {
   CheckJNIException(sEnv, __FUNCTION__);
 }
 
+void
+VRBrowser::SetExternalVRSurface(jint aIndex, jobject aSurface) {
+  if (!ValidateMethodID(sEnv, sActivity, sSetExternalVRSurface, __FUNCTION__)) { return; }
+  sEnv->CallVoidMethod(sActivity, sSetExternalVRSurface, aIndex, aSurface);
+  CheckJNIException(sEnv, __FUNCTION__);
+}
+
+void
+VRBrowser::ReleaseExternalVRSurfaces() {
+  if (!ValidateMethodID(sEnv, sActivity, sReleaseExternalSurfaces, __FUNCTION__)) { return; }
+  sEnv->CallVoidMethod(sActivity, sReleaseExternalSurfaces);
+  CheckJNIException(sEnv, __FUNCTION__);
+}
 } // namespace crow

app/src/main/cpp/VRBrowser.h

@@ -39,6 +39,10 @@ bool AreLayersEnabled();
 void SetDeviceType(const jint aType);
 void HaltActivity(const jint aReason);
 void HandlePoorPerformance();
+void SetExternalVRSurfaceLength(jint aLen);
+void SetExternalVRSurfaceId(jint aId);
+void SetExternalVRSurface(jint aIndex, jobject aSurface);
+void ReleaseExternalVRSurfaces();
 } // namespace VRBrowser;
 
 } // namespace crow

app/src/main/cpp/moz_external_vr.h

@@ -40,7 +40,7 @@ enum class GamepadCapabilityFlags : uint16_t;
 #endif  //  MOZILLA_INTERNAL_API
 namespace gfx {
 
-static const int32_t kVRExternalVersion = 9;
+static const int32_t kVRExternalVersion = 10;
 
 // We assign VR presentations to groups with a bitmask.
 // Currently, we will only display either content or chrome.
@@ -308,6 +308,11 @@ struct VRDisplayState {
   uint64_t lastSubmittedFrameId;
   bool lastSubmittedFrameSuccessful;
   uint32_t presentingGeneration;
+#if defined(__ANDROID__)
+  // external VR surface
+  uint32_t externalSurfaceLength;
+  uint32_t externalSurfaceId;
+#endif
   // Telemetry
   bool reportsDroppedFrames;
   uint64_t droppedFrameCount;
@@ -364,7 +369,8 @@ enum class VRLayerTextureType : uint16_t {
   LayerTextureType_None = 0,
   LayerTextureType_D3D10SurfaceDescriptor = 1,
   LayerTextureType_MacIOSurface = 2,
-  LayerTextureType_GeckoSurfaceTexture = 3
+  LayerTextureType_GeckoSurfaceTexture = 3,
+  LayerTextureType_ExternalVRSurface = 4
 };
 
 struct VRLayer_2D_Content {

app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp

@@ -617,6 +617,8 @@ const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f);
 const vrb::Vector kAverageOculusHeight(0.0f, 1.65f, 0.0f);
 
 struct DeviceDelegateOculusVR::State {
+  const static uint32_t EXTERNAL_SURFACE_BUFFER_SIZE = 2;
+
   struct ControllerState {
     const int32_t index;
     const ElbowModel::HandEnum hand;
@@ -645,6 +647,7 @@ struct DeviceDelegateOculusVR::State {
   ovrJava java = {};
   ovrMobile* ovr = nullptr;
   OculusEyeSwapChainPtr eyeSwapChains[VRAPI_EYE_COUNT];
+  ovrTextureSwapChain* eyeSurfaceSwapChain[EXTERNAL_SURFACE_BUFFER_SIZE];
   OculusLayerCubePtr cubeLayer;
   OculusLayerEquirectPtr equirectLayer;
   std::vector<OculusLayerPtr> uiLayers;
@@ -707,6 +710,9 @@ struct DeviceDelegateOculusVR::State {
       exit(status);
       return;
     }
+    for (int i = 0; i < EXTERNAL_SURFACE_BUFFER_SIZE; ++i) {
+      eyeSurfaceSwapChain[i] = nullptr;
+    }
     initialized = true;
     SetRenderSize(device::RenderMode::StandAlone);
 
@@ -1128,6 +1134,19 @@ DeviceDelegateOculusVR::SetRenderMode(const device::RenderMode aMode) {
     m.eyeSwapChains[i]->Init(render, m.renderMode, m.renderWidth, m.renderHeight);
   }
 
+  if (aMode == device::RenderMode::Immersive) {
+    uint32_t width, height;
+    m.GetImmersiveRenderSize(width, height);
+    for (int i = 0; i < DeviceDelegateOculusVR::State::EXTERNAL_SURFACE_BUFFER_SIZE; ++i) {
+      if (!m.eyeSurfaceSwapChain[i]) {
+        m.eyeSurfaceSwapChain[i] = vrapi_CreateAndroidSurfaceSwapChain(width, height);
+        auto surfaceOut = vrapi_GetTextureSwapChainAndroidSurface(m.eyeSurfaceSwapChain[i]);
+        surfaceOut = m.java.Env->NewGlobalRef(surfaceOut);
+        VRBrowser::SetExternalVRSurface(i, surfaceOut);
+      }
+    }
+  }
+
   m.UpdateTrackingMode();
   m.UpdateFoveatedLevel();
   m.UpdateDisplayRefreshRate();
@@ -1340,8 +1359,6 @@ DeviceDelegateOculusVR::StartFrame() {
   ovrMatrix4f matrix = vrapi_GetTransformFromPose(&m.predictedTracking.HeadPose.Pose);
   vrb::Matrix head = vrb::Matrix::FromRowMajor(matrix.M[0]);
 
-
-
   if (m.renderMode == device::RenderMode::StandAlone) {
     head.TranslateInPlace(kAverageHeight);
   }
@@ -1359,6 +1376,7 @@ DeviceDelegateOculusVR::StartFrame() {
       caps |= device::PositionEmulated;
     }
     m.immersiveDisplay->SetCapabilityFlags(caps);
+    m.immersiveDisplay->SetExternalSurfaceInfo(State::EXTERNAL_SURFACE_BUFFER_SIZE, externalSurfaceId);
   }
 
   int lastReorientCount = m.reorientCount;
@@ -1458,13 +1476,33 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   projection.HeadPose = m.predictedTracking.HeadPose;
   projection.Header.SrcBlend = VRAPI_FRAME_LAYER_BLEND_ONE;
   projection.Header.DstBlend = VRAPI_FRAME_LAYER_BLEND_ONE_MINUS_SRC_ALPHA;
-  for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) {
-    const auto &eyeSwapChain = m.eyeSwapChains[i];
-    const int swapChainIndex = m.frameIndex % eyeSwapChain->swapChainLength;
-    // Set up OVR layer textures
-    projection.Textures[i].ColorSwapChain = eyeSwapChain->ovrSwapChain;
-    projection.Textures[i].SwapChainIndex = swapChainIndex;
-    projection.Textures[i].TexCoordsFromTanAngles = ovrMatrix4f_TanAngleMatrixFromProjection(&projectionMatrix);
+
+  if (!enableExternalSurfaceRender) {
+    for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) {
+      const auto &eyeSwapChain = m.eyeSwapChains[i];
+      const int swapChainIndex = m.frameIndex % eyeSwapChain->swapChainLength;
+      // Set up OVR layer textures
+      projection.Textures[i].ColorSwapChain = eyeSwapChain->ovrSwapChain;
+      projection.Textures[i].SwapChainIndex = swapChainIndex;
+      projection.Textures[i].TexCoordsFromTanAngles = ovrMatrix4f_TanAngleMatrixFromProjection(&projectionMatrix);
+    }
+  } else {
+    const int swapChainIndex = externalSurfaceId;
+    ovrMatrix4f proj(projectionMatrix);
+    // Flip texCoord in vertical when using WebGL frame textures.
+    proj.M[1][1] *= -1;
+    proj = ovrMatrix4f_TanAngleMatrixFromProjection(&proj);
+
+    for (int i = 0; i < VRAPI_FRAME_LAYER_EYE_MAX; ++i) {
+      const auto eyeSwapChain = m.eyeSurfaceSwapChain[swapChainIndex];
+      // Set up OVR layer textures
+      projection.Textures[i].ColorSwapChain = eyeSwapChain;
+      projection.Textures[i].SwapChainIndex = 0;
+      projection.Textures[i].TexCoordsFromTanAngles = proj;
+    }
+
+    // Switch to the next surface.
+    externalSurfaceId = (++externalSurfaceId) % State::EXTERNAL_SURFACE_BUFFER_SIZE;
   }
   layers[layerCount++] = &projection.Header;
 
@@ -1494,6 +1532,11 @@ DeviceDelegateOculusVR::EndFrame(const bool aDiscard) {
   vrapi_SubmitFrame2(m.ovr, &frameDesc);
 }
 
+void
+DeviceDelegateOculusVR::EnableExternalSurfaceRender(bool aEnable) {
+  enableExternalSurfaceRender = aEnable;
+}
+
 VRLayerQuadPtr
 DeviceDelegateOculusVR::CreateLayerQuad(int32_t aWidth, int32_t aHeight,
                                         VRLayerSurface::SurfaceType aSurfaceType) {
@@ -1671,6 +1714,13 @@ DeviceDelegateOculusVR::LeaveVR() {
   for (int i = 0; i < VRAPI_EYE_COUNT; ++i) {
     m.eyeSwapChains[i]->Destroy();
   }
+  for (int i = 0; i < DeviceDelegateOculusVR::State::EXTERNAL_SURFACE_BUFFER_SIZE; ++i) {
+    if (m.eyeSurfaceSwapChain[i]) {
+      vrapi_DestroyTextureSwapChain(m.eyeSurfaceSwapChain[i]);
+      m.eyeSurfaceSwapChain[i] = nullptr;
+      VRBrowser::ReleaseExternalVRSurfaces();
+    }
+  }
   if (m.cubeLayer) {
     m.cubeLayer->Destroy();
   }
@@ -1698,7 +1748,8 @@ DeviceDelegateOculusVR::ExitApp() {
   return true;
 }
 
-DeviceDelegateOculusVR::DeviceDelegateOculusVR(State &aState) : m(aState) {}
+DeviceDelegateOculusVR::DeviceDelegateOculusVR(State &aState) : m(aState),
+  externalSurfaceId(0), enableExternalSurfaceRender(false) {}
 
 DeviceDelegateOculusVR::~DeviceDelegateOculusVR() { m.Shutdown(); }