Modal transparency / ReactNativeIsland in 3rd party components

[austinhallock]

Just upgraded to 0.76.3 and am very close to having our RNW paper app fully migrated over :)

For context, I'm working on a browser built for creators/livestreamers. Most of the UI is RNW, but the web viewport itself is a Chromium HWND that I need to overlay certain UI elements on top of.

My old implementation in video below won't work for obvious reasons (RN HWND is below Chromium HWND), so I need to render popovers into a separate HWND.

2024-12-17.12-13-08.mp4

The new <Modal /> seems great for this (renders on top of Chromium HWND), but not quite there for my use-case. The main limitations right now being I don't seem to be able to set a custom position, and I didn't see an option to specify what happens when the "backdrop" is clicked or hovered into (ie being able to close the modal).

The easiest workaround would be to have <Modal /> support transparency and the ability to cover the entire screen - ie the transparency and presentationStyle props from https://reactnative.dev/docs/modal. Then I could handle positioning and backdrop behavior on my own.

---

In the meantime (trying to get this out as soon as possible) I've been trying to build my own implementation of a Modal/Popover, but haven't been able to get that to fully work. Will keep trying since this is my last blocker. There was a recent commit that I thought would let me do this, but I haven't had luck yet.

I also plan on using a similar implementation (3rd party component ReactNativeIsland) to support multiple app windows within the same React Native instance.

Here's the rough code I'm trying out for the popover

#include "pch.h"

#include "Popover.h"

#include "../../codegen/react/components/CustomModule/Popover.g.h"

#include <winrt/Microsoft.ReactNative.Composition.h>
#include <winrt/Microsoft.UI.Composition.h>
#include <winrt/Microsoft.UI.Content.h>
#include <winrt/Microsoft.UI.Windowing.h>
#include <winrt/Microsoft.ReactNative.h>
#include <windows.h>

#pragma comment(lib, "gdi32.lib")

namespace winrt::CustomModule {

LRESULT CALLBACK ModalBoxWndProc(HWND hwnd, UINT message, WPARAM wparam, LPARAM lparam) noexcept;

constexpr PCWSTR c_modalWindowClassName = L"TEST_MODAL";
constexpr auto CompHostProperty = L"CompHost";
constexpr int MODAL_MIN_WIDTH = 600;
constexpr int MODAL_MIN_HEIGHT = 600;

struct Popover : public winrt::implements<Popover, winrt::IInspectable>, Codegen::BasePopover<Popover> {
  Popover() {}

  void Initialize(const winrt::Microsoft::ReactNative::ComponentView &sender) noexcept override {
    auto view = sender.as<winrt::Microsoft::ReactNative::Composition::ViewComponentView>();
    CreateModalWindow(view);
  }

  winrt::Microsoft::UI::Composition::Visual VisualToMountChildrenInto(
      const winrt::Microsoft::ReactNative::ComponentView& view) noexcept {
      return m_reactNativeIsland.RootVisual(); // .Parent();
  }

private:
  void RegisterWndClass() noexcept {
    WNDCLASSEX wcex = {};
    wcex.cbSize = sizeof(wcex);
    wcex.style = CS_HREDRAW | CS_VREDRAW;
    wcex.lpfnWndProc = ModalBoxWndProc;
    wcex.cbClsExtra = DLGWINDOWEXTRA;
    wcex.cbWndExtra = sizeof(winrt::impl::abi<winrt::Microsoft::ReactNative::ICompositionHwndHost>::type *);
    wcex.hInstance = GetModuleHandle(NULL);
    wcex.hCursor = LoadCursor(nullptr, IDC_ARROW);
    wcex.hbrBackground = nullptr;
    wcex.lpszClassName = c_modalWindowClassName;

    if (RegisterClassEx(&wcex) == 0 && GetLastError() != ERROR_CLASS_ALREADY_EXISTS) {
      throw winrt::hresult_error(HRESULT_FROM_WIN32(GetLastError()), 
          L"Failed to register window class");
    }
  }

  void CreateModalWindow(const winrt::Microsoft::ReactNative::Composition::ViewComponentView &view) {
      RegisterWndClass();

      HWND parentHwnd = GetAncestor(GetActiveWindow(), GA_ROOT);
      if (!parentHwnd) {
          throw winrt::hresult_error(E_FAIL, L"Failed to get parent window handle");
      }

      HINSTANCE hInstance = GetModuleHandle(NULL);
      winrt::com_ptr<::IUnknown> spunk;
      
      RECT parentRect;
      GetWindowRect(parentHwnd, &parentRect);

      HWND hwnd = CreateWindow(
          c_modalWindowClassName,
          L"Modal Window",
          WS_POPUP | WS_EX_LAYERED,
          parentRect.left + 500,
          parentRect.top + 500,
          MODAL_MIN_WIDTH,
          MODAL_MIN_HEIGHT,
          parentHwnd,
          nullptr,
          hInstance,
          spunk.get()
      );

      if (!hwnd) {
          throw winrt::hresult_error(E_FAIL, L"Failed to create window");
      }
      
      winrt::Microsoft::UI::Content::DesktopChildSiteBridge bridge = winrt::Microsoft::UI::Content::DesktopChildSiteBridge::Create(
          view.Compositor(),
          winrt::Microsoft::UI::GetWindowIdFromWindow(hwnd)
      );

      m_reactNativeIsland = winrt::Microsoft::ReactNative::ReactNativeIsland(
          view.Compositor(),
          view.ReactContext(),
          view
      );

      bridge.Connect(m_reactNativeIsland.Island());
      bridge.ResizePolicy(winrt::Microsoft::UI::Content::ContentSizePolicy::ResizeContentToParentWindow);
      bridge.Show();
      
      winrt::Microsoft::ReactNative::LayoutConstraints constraints;
      constraints.LayoutDirection = winrt::Microsoft::ReactNative::LayoutDirection::Undefined;

      RECT rc;
      GetClientRect(parentHwnd, &rc);
      float scaleFactor = GetDpiForWindow(hwnd) / static_cast<float>(USER_DEFAULT_SCREEN_DPI);
      constraints.MaximumSize = {(rc.right - rc.left) * scaleFactor, (rc.bottom - rc.top) * scaleFactor};
      constraints.MinimumSize = {MODAL_MIN_WIDTH * scaleFactor, MODAL_MIN_HEIGHT * scaleFactor};
      m_reactNativeIsland.Arrange(constraints, {0, 0});
      bridge.ResizePolicy(winrt::Microsoft::UI::Content::ContentSizePolicy::ResizeContentToParentWindow);
      spunk.detach();
      
      ShowWindow(hwnd, SW_SHOW);
      UpdateWindow(hwnd);
  }

  winrt::Microsoft::ReactNative::ReactNativeIsland m_reactNativeIsland{nullptr};
};

void RegisterPopoverNativeComponent(
    winrt::Microsoft::ReactNative::IReactPackageBuilder const &packageBuilder) noexcept {
    Codegen::RegisterPopoverNativeComponent<Popover>(packageBuilder, [](winrt::Microsoft::ReactNative::Composition::IReactCompositionViewComponentBuilder const &compBuilder) noexcept {
        compBuilder.SetVisualToMountChildrenIntoHandler([](const winrt::Microsoft::ReactNative::ComponentView& view) noexcept {
            auto userData = view.UserData().as<Popover>();
            return userData->VisualToMountChildrenInto(view);
        });
    });
}

LRESULT CALLBACK ModalBoxWndProc(HWND hwnd, UINT message, WPARAM wparam, LPARAM lparam) noexcept {
  auto data = reinterpret_cast<::IUnknown*>(GetProp(hwnd, CompHostProperty));
  winrt::Microsoft::ReactNative::CompositionHwndHost host{nullptr};

  if (data) {
    winrt::check_hresult(data->QueryInterface(
        winrt::guid_of<winrt::Microsoft::ReactNative::CompositionHwndHost>(),
        winrt::put_abi(host)));
    auto result = static_cast<LRESULT>(host.TranslateMessage(message, wparam, lparam));
    if (result) {
      return result;
    }
  }

  switch (message) {
    case WM_NCCREATE: {
      auto createStruct = reinterpret_cast<CREATESTRUCT*>(lparam);
      data = static_cast<::IUnknown*>(createStruct->lpCreateParams);
      SetProp(hwnd, CompHostProperty, data);
      break;
    }
    case WM_CLOSE: {
      ::ShowWindow(hwnd, SW_HIDE);
      return 0;
    }
    case WM_DESTROY: {
      ::ShowWindow(hwnd, SW_HIDE);
      if (data) {
        data->Release();
      }
      SetProp(hwnd, CompHostProperty, nullptr);
      break;
    }
  }

  return DefWindowProc(hwnd, message, wparam, lparam);
}

} // namespace winrt::CustomModule

I can get it to render in the new HWND if I use ReactNativeIsland(view.Compositor()) (without passing in context and view). However, it's not interactive and still crashes if I dynamically add a new nested child on the tsx side (maybe because I have to use RootVisual since InternalRootVisual isn't exposed to 3rd party components)

With all 3 params to ReactNativeIsland, it'll always crash, with a non-obvious (to me) error/stack - it traces back to Start(), so something in the core loop

If y'all have an example of a 3rd party component that renders React elements into a separate HWND, or some guidance let me know <3

call_stack.txt