diff --git a/main.tex b/main.tex index a90880a..51f1c5d 100644 --- a/main.tex +++ b/main.tex @@ -1,478 +1,478 @@ \documentclass[aps,prx,a4paper,notitlepage,reprint,superscriptaddress]{revtex4-1} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% PACKAGE CONFIG. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \usepackage{amsmath,amssymb,amsfonts} % standard AMS packages \usepackage{mathrsfs} % use \ma۳thscr{} for script letters in math \usepackage{mathtools} % for proper typesetting of := and =: \usepackage{color} \usepackage{chemformula} \usepackage{siunitx} \usepackage{graphicx,float} \usepackage[colorlinks, linkcolor=red, citecolor=blue, urlcolor=red]{hyperref} \usepackage{cleveref} \usepackage{soul} \usepackage{changes} \interfootnotelinepenalty=10000 % prevents footnotes from splitting across pages %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% CUSTOM MACROS %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % macros for physics objects \newcommand{\ket}[1]{\vert{#1}\rangle} \newcommand{\bra}[1]{\langle{#1}\vert} \newcommand{\op}[1]{\hat{#1}} \newcommand{\tr}{\mathrm{Tr}} \newcommand{\avg}[1]{\left\langle{#1}\right\rangle} % macros for math objects \newcommand{\re}{\mathrm{Re}\,} \newcommand{\im}{\mathrm{Im}\,} \newcommand{\abs}[1]{\left\vert{#1}\right\vert} \newcommand{\symtext}[2]{\ensuremath{\stackrel{{#2}}{{#1}}}} % specific macros for this document \renewcommand{\t}[1]{\mathrm{#1}} \newcommand{\SiN}{Si$_3$N$_4\,$} \newcommand{\fnss}{\mathcal{F}} \newcommand{\mbrsize}[1]{#1\,mm$\times$#1\,mm} % useful reference macros \newcommand{\figref}[1]{Fig.~\ref{#1}} \renewcommand{\eqref}[1]{Eq.~\ref{#1}} \newcommand{\secref}[1]{Sec.~\ref{#1}} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% BEGIN DOCUMENT %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \begin{document} \title{Optomechanical thermal intermodulation noise} \author{S. A. Fedorov} \thanks{These authors contributed equally} \email{sergey.fedorov@epfl.ch} \affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} \author{A. Beccari} \thanks{These authors contributed equally} \affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} \author{A. Arabmoheghi} \affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} %\author{\\M. J. Bereyhi} %\affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} \author{D. J. Wilson} \affiliation{College of Optical Sciences, University of Arizona, Tucson, Arizona 85721, USA} \author{N. J. Engelsen} \email{nils.engelsen@epfl.ch} \affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} \author{T. J. Kippenberg} \email{tobias.kippenberg@epfl.ch} \affiliation{Institute of Physics (IPHYS), {\'E}cole Polytechnique F{\'e}d{\'e}rale de Lausanne, 1015 Lausanne, Switzerland} \begin{abstract} -Thermal fluctuations limit the sensitivity of precision measurements ranging from laser interferometer gravitational wave observatories to optical atomic clocks. In optical cavities, thermal fluctuations of length or refractive index create frequency noise. Here we report a previously unobserved broadband noise process, thermal intermodulation noise, originating from the transduction of frequency fluctuations by the nonlinearity in the cavity-laser detuning response. -We study thermal intermodulation noise due to the Brownian motion of a thin $\ch{Si3N4}$ membrane resonator in an optomechanical cavity at room temperature, and show that it creates intensity noise for resonant laser excitation. We study the laser detuning dependence of the noise and demonstrate that it scales with the quartic power of the ratio of the optomechanical coupling rate to the cavity linewidth; both of which are in excellent agreement with our developed theoretical model. The noise process is particularly relevant to quantum optomechanics. We utilize a phononic crystal membrane with a low mass, soft-clamped defect mode and operate in a regime where quantum fluctuations of radiation pressure are expected to dominate (i.e. a nominal quantum cooperativity exceeding unity). However, we find that the thermal intermodulation noise exceeds the vacuum fluctuations by orders of magnitude, even within the bandgap, thereby preventing the observation of pondermotive squeezing. +Thermal fluctuations limit the sensitivity of precision measurements ranging from laser interferometer gravitational wave observatories to optical atomic clocks. In optical cavities, thermal fluctuations of length or refractive index create frequency noise. Here we study a previously unreported broadband noise process, thermal intermodulation noise, originating from the transduction of frequency fluctuations by the nonlinearity in the cavity-laser detuning response. +We study thermal intermodulation noise due to the Brownian motion of a thin $\ch{Si3N4}$ membrane resonator in an optomechanical cavity at room temperature, and show that it creates intensity noise for resonant laser excitation. We study the laser detuning dependence of the noise and demonstrate that it scales with the quartic power of the ratio of the optomechanical coupling rate to the cavity linewidth; both of which are in excellent agreement with our developed theoretical model. The noise process is particularly relevant to quantum optomechanics. We utilize a phononic crystal membrane with a low mass, soft-clamped defect mode and operate in a regime where quantum fluctuations of radiation pressure are expected to dominate (i.e. a nominal quantum cooperativity exceeding unity). However, we find that the thermal intermodulation noise exceeds the vacuum fluctuations by orders of magnitude, even within the bandgap, thereby preventing the observation of ponderomotive squeezing. The described noise process is broadly relevant to cavity-based measurements, and is especially pronounced when thermally induced frequency fluctuations are comparable to the optical linewidth. \end{abstract} \date{\today} \maketitle %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% BEGIN MATTER %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Introduction} -Optical cavities are an enabling technology for precision metrology, with applications including gravitational wave detection \cite{ligo_collaboration_observation_gw_2016}, ultrastable lasers \cite{sterr_ultrastable_2009}, cavity QED \cite{ye_quantum_2008} to cavity optomechanics \cite{aspelmeyer_cavity_2014}. The ultimate limits in the frequency stability of such cavities are imposed by thermal noise. At finite temperature, the optical resonance frequencies exhibits frequency fluctuations which originate from Brownian motion of mirror surfaces and the thermorefractive and thermoelastic fluctuations of the mirror substrates \cite{braginsky_thermorefractive_2000,gorodetsky_thermal_noise_compensation_2008}. Historically, understanding and minimizing such thermal noises has played a key role in design of laser interferometer gravitational wave detectors, where it motivated changes in the choice of mirror substrate \cite{braginsky_thermodynamical_1999}. Moreover, thermal noises have been combated by the development of crystalline mirror coatings \cite{cole_tenfold_2013} and cryogenic operation \cite{robinson_crystalline_2019}. Fundamental thermodynamical noises are particularly prominent in optical micro-cavities, which due to their lower mode volume exhibit larger fluctuations \cite{gorodetsky_fundamental_2004,anetsberger_near-field_2009}. In the regime where the cavity response is linear in the cavity-laser detuning, these thermal fluctuations manifest as excess phase noise in an optical field resonant with the cavity. +Optical cavities are an enabling technology for precision metrology, with applications including gravitational wave detection \cite{ligo_collaboration_observation_gw_2016}, ultrastable lasers \cite{sterr_ultrastable_2009}, cavity QED \cite{ye_quantum_2008} to cavity optomechanics \cite{aspelmeyer_cavity_2014}. The ultimate limits in the frequency stability of such cavities are imposed by thermal noise. At finite temperature, the optical resonance frequencies exhibits frequency fluctuations which originate from Brownian motion of mirror surfaces and the thermorefractive and thermoelastic fluctuations of the mirror substrates \cite{braginsky_thermorefractive_2000,gorodetsky_thermal_noise_compensation_2008}. Historically, understanding and minimizing these thermal noises has played a key role in the design of laser interferometer gravitational wave detectors, where it motivated changes in the choice of mirror substrate \cite{braginsky_thermodynamical_1999}. Moreover, thermal noises have been combated by the development of crystalline mirror coatings \cite{cole_tenfold_2013} and cryogenic operation \cite{robinson_crystalline_2019}. Fundamental thermodynamical noises are particularly prominent in optical micro-cavities, which due to their lower mode volume exhibit larger fluctuations \cite{gorodetsky_fundamental_2004,anetsberger_near-field_2009}. In the regime where the cavity response is linear in the cavity-laser detuning, these thermal fluctuations manifest as excess phase noise in an optical field resonant with the cavity. However, the inherent nonlinearity of the cavity response with respect to the cavity-laser detuning can create noise in the amplitude quadrature of a resonant laser field. This effect is known as \emph{intermodulation noise}, as it mixes different Fourier components of the frequency noise \cite{riehle_frequency_2005}. When frequency locking using modulation sidebands, such technical intermodulation noise in the error signal is known to limit the stability of frequency standards \cite{audoin_intermodulation_1991} and cavity-stabilized lasers \cite{ferguson_laser-noise-induced_1990,bahoura_ultimate_2003}. -Here we report and study \emph{optomechanical thermal intermodulation noise} (TIN) which has a fundamental thermodynamic origin. Specifically, we show that this noise arises in optomechanical systems, caused by thermal motion of multiple mechanical modes coupled to the cavity field, and presents a key obstacle in room temperature quantum optomechanics experiments. Our platform is the membrane-in-the-middle (MIM) system, which is considered promising for accessing quantum optomechanical experiments at room temperature \cite{thompson_strong_2008,wilson_cavity_2009}. Recent advances in high-stress \SiN membrane resonators hosting high-$Q$ and low mass soft-clamped modes \cite{tsaturyan_ultracoherent_2017,reetz_analysis_2019} have made it theoretically possible to reach the radiation pressure quantum backaction dominated regime at room temperature using microwatt-level optical input powers. +Here we report and study \emph{optomechanical thermal intermodulation noise} (TIN) which has a fundamental thermodynamic origin. Specifically, we show that this noise arises in optomechanical systems, caused by thermal motion of multiple mechanical modes coupled to the cavity field, and presents a key obstacle in room temperature quantum optomechanics experiments. Our platform is the membrane-in-the-middle (MIM) system, which is considered promising for conducting quantum optomechanical experiments at room temperature \cite{thompson_strong_2008,wilson_cavity_2009}. Recent advances in high-stress \SiN membrane resonators hosting high-$Q$ and low mass soft-clamped modes \cite{tsaturyan_ultracoherent_2017,reetz_analysis_2019} have made it theoretically possible to reach the radiation pressure quantum backaction dominated regime at room temperature using microwatt-level optical input powers. Yet, concomitant with this approach is a dense spectrum of membrane modes coupled to the optical field, which produce a high level of TIN and prevent access to the quantum backaction dominated regime. The optical transduction nonlinearity which creates thermal intermodulation noise was reported in optomechanical systems previously \cite{brawley_nonlinear_2016,leijssen_nonlinear_2017}. To the lowest order in the displacement divided by dynamic range, it manifests as the measurement of mechanical displacement squared. Such measurements have enticing applications in quantum optomechanics: they can be used for the observation of phononic jumps \cite{gangat_phonon_2011}, phononic shot noise \cite{clerk_quantum_2010}, and the creation of mechanical squeezed states \cite{nunnenkamp_cooling_and_squeezing_2010} if the effects of linear measurement backaction are kept small \cite{martin_measurement_2007,brawley_nonlinear_2016}. Experiments that demonstrated quadratic optomechanical position measurements using position-squared coupling to the cavity frequency \cite{paraiso_position-squared_2015} remain deep in the classical regime due to small coupling rates. Optical cavity transduction can produce large, effective quadratic nonlinearity \cite{brawley_nonlinear_2016}, but it is inevitably accompanied by linear quantum backaction. Our work shows that a new noise source arises as a further consequence of nonlinear cavity transduction: optomechanical thermal intermodulation noise. \section{Theory of intermodulation noise} \label{sec:genTheor} We begin by presenting the theory of thermal intermodulation noise with the assumption that the cavity frequency fluctuations are slow compared to the optical decay rate. We concentrate on the lowest-order, i.e. quadratic, nonlinearity of the cavity detuning transduction. We consider (as in our experimental setup) an optical cavity with two ports, which is driven by a laser coupled to port one. The output from port two is directly detected on a photodiode. In the classical regime, i.e. neglecting vacuum fluctuations, the complex amplitude of the intracavity optical field, $a$, and the output field $s_\t{out,2}$ can be found from the input-output relations \begin{align} &\frac{da(t)}{dt}=\left(i\Delta(t)-\frac{\kappa}{2}\right)a(t)+\sqrt{\kappa_1}\, s_\t{in,1},\\ & s_\t{out,2}(t)=-\sqrt{\kappa_2}a(t),\label{eq:oi2} \end{align} where $s_\t{in,1}$ is the constant coherent drive amplitude, $\Delta(t)=\omega_L-\omega_c(t)$ is the laser detuning from the cavity resonance, modulated by the cavity frequency noise, and $\kappa_{1,2}$ are the external coupling rates of ports one and two and $\kappa=\kappa_1+\kappa_2$. In the fast cavity limit, when the optical field adiabatically follows $\Delta(t)$, the intracavity field is found as \begin{equation}\label{eq:aFull} a(t)=2\sqrt{\frac{\eta_1}{\kappa}} L(\nu(t))\, s_\t{in,1}, \end{equation} where we introduced for brevity the normalized detuning $\nu=2\Delta/\kappa$, the cavity decay ratios $\eta_{1,2}=\kappa_{1,2}/\kappa$ and Lorentzian susceptibility \begin{equation} L(\nu)=\frac{1}{1-i\nu}. \end{equation} Expanding $L$ in \eqref{eq:aFull} over small detuning fluctuations $\delta\nu$ around the mean value $\nu_0$ up to second order we find the intracavity field as \begin{equation}\label{eq:aSq} a=2\sqrt{\frac{\eta_1}{\kappa}}L(\nu_0)(1+iL(\nu_0)\delta\nu -L(\nu_0)^2\delta\nu^2) s_\t{in,1}. \end{equation} According to \eqref{eq:aSq}, the intracavity field is modulated by the cavity frequency excursion, $\delta\nu$, and the frequency excursions squared, $\delta\nu^2$. If $\delta\nu(t)$ is a stationary Gaussian noise process, like typical thermal noises, the linear and quadratic contributions are uncorrelated (despite clearly not being independent). This is due to the fact that odd-order correlations vanish for Gaussian noise, \begin{equation} \langle\delta\nu(t)^2\delta\nu(t+\tau)\rangle=0, \end{equation} where $\langle ...\rangle$ is the time average, for an arbitrary time delay $\tau$. Next, we consider the photodetected signal, which, up to a conversion factor, equals the intensity of the output light and is found to be \begin{multline}\label{eq:detectedNoise} I(t)=|s_\t{out,2}(t)|^2\propto \\ |L(\nu_0)|^2 \left(1-\frac{2\nu_0}{1+\nu_0^2}\delta\nu(t)+\frac{3\nu_0^2-1}{(1+\nu_0^2)^2}\delta\nu(t)^2\right). \end{multline} \noindent Notice that $\delta\nu(t)$ and $\delta\nu(t)^2$ can be distinguished by their detuning dependence. The linearly transduced fluctuations vanish on resonance ($\nu=0$), where $\partial L/\partial \nu=0$. Similarly, when $\partial^2 L/\partial\nu^2=0$, the quadratic frequency fluctuations vanish, and thus also the thermal intermodulation noise. We will denote these ``magic'' detunings by $\nu_0$, which is given by \begin{equation} \nu_0=\pm 1/\sqrt{3} \end{equation} \noindent In the following experiments, we will make measurements at $\nu=\nu_0$ to independently characterize the spectra of $\delta\nu(t)$ and $\delta\nu(t)^2$. The total spectrum \cite{specta_notations} of the detected signal, $I(t)$, is an incoherent sum of the linear term, \begin{equation}\label{eq:Snu} S_{\nu\nu}[\omega]=\int_{-\infty}^{\infty}\langle \delta\nu(t) \delta\nu(t+\tau) \rangle e^{i\omega \tau}d\tau, \end{equation} and the quadratic term, which for Gaussian noise can be found using Wick's theorem \cite{gardiner_handbook_1985} \begin{equation} \langle \delta\nu(t)^2 \delta\nu(t+\tau)^2 \rangle=\langle\delta\nu(t)^2 \rangle^2+2\langle \delta\nu(t) \delta\nu(t+\tau) \rangle^2, \end{equation} as \begin{multline}\label{eq:Snu2} S_{\nu\nu}^{(2)}[\omega]=\int_{-\infty}^{\infty}\langle \delta\nu(t)^2 \delta\nu(t+\tau)^2 \rangle e^{i\omega \tau}d\tau=\\ 2\pi\langle \delta\nu^2\rangle^2 \delta[\omega]+2\times\frac{1}{2\pi} \int_{-\infty}^{\infty} S_{\nu\nu}[\omega']S_{\nu\nu}[\omega-\omega']d\omega', \end{multline} where $\delta[\omega]$ is the Dirac delta function. \begin{figure}[t] \centering \includegraphics[width=\columnwidth]{fig_intro} \caption{Physical mechanism of optomechanical thermal intermodulation noise. \small{a) Transduction of the oscillator's motion to the phase (upper panel) and amplitude (lower panel) quadratures of resonant intracavity light. b) Spectra of linear (upper panel) and quadratic (lower panel) position fluctuations of a multimode resonator, which can lead to the emergence of a wideband noise process. c) Experimental setup in which TIN is studied consisting of a membrane-in-the-middle optomechanical system. PM: Phase modulator. AM: Amplitude modulator. ESA: Electronic spectrum analyzer.}} \label{fig:intro} \end{figure} \begin{figure*}[t] \centering \includegraphics[width=\textwidth]{fig_rect_mbr} \caption{Observation of optomechanical thermal intermodulation noise. \small{a), b) and c) show measurements for a membrane-in-the middle cavity with 2 mm square membrane. a) Cavity reflection signal as the laser is scanned over two resonances, with low (left) and high (right) optomechanical coupling. b) Dependence of resonant RIN, averaged over $0.6-1.6$ MHz, on the input power. Parameters: $\kappa/2\pi=9.9$ MHz, $g_0/2\pi=84$ Hz for the fundamental mode. The interval of $\pm$ one standard deviation around the mean is shaded gray. c) Dependence of the average RIN in a $0.6-1.6$ MHz band on $g_0/\kappa$. d) Noise from a MIM cavity with the laser detuned from resonance (top) and on resonance (bottom) using a 1 mm square membrane, with $\kappa/2\pi=26.6$ MHz and $g_0/2\pi=330$ Hz for the fundamental mode.\hl{It would help to indicate the elevated noisefloor in the figure, which is our wideband TIN noise source. Second: what are the detuning values shown in d? The 'magic' value is not shown it appears.} }} \label{fig:rectMbr} \end{figure*} % Wavelengths: % b), the the power dependence, is taken at \lambda=837.7 nm, % d), the example transmission noises with 1 mm membrane, are taken at \lambda=838.2 nm, \section{Thermal intermodulation noise} In an optomechanical cavity, the dominant source of cavity frequency fluctuations is the Brownian motion of mechanical modes coupled to the cavity, \begin{equation}\label{eq:dnuOpt} \delta\nu(t)= 2\frac{G}{\kappa} x(t), \end{equation} where $G=-\partial \omega_c/\partial x$ is the linear optomechanical coupling constant, and $x$ is the total resonator displacement, i.e. the sum of independent contributions $x_n$ of different mechanical modes (the effect of the finite cavity mode waist is treated in the SI). The spectrum of the Brownian frequency noise is then found to be \begin{equation}\label{eq:mbrFreqFluct} S_{\nu\nu}[\omega]=\left(\frac{2G}{\kappa}\right)^{2}\sum_n S_{xx,n}[\omega], \end{equation} where $S_{xx,n}[\omega]$ are the displacement spectra of individual mechanical modes (see SI for more details). The thermomechanical frequency noise given by \eqref{eq:mbrFreqFluct} produces TIN which contains peaks at sums and differences of mechanical resonance frequencies and a broadband background due to off-resonant components of thermal noise, as illustrated in \figref{fig:intro}b. The magnitude of the intermodulation noise is related to the quadratic spectrum of the total mechanical displacement, $S_{xx}^{(2)}$, as \begin{equation}\label{eq:intNoiseSxx2} S_{\nu\nu}^{(2)}=(2G/\kappa)^{4}S_{xx}^{(2)}. \end{equation} A reservation needs to be made: the theory presented in \secref{sec:genTheor} is only strictly applicable to an optomechanical cavity when the input power is sufficiently low, such that the driving of mechanical motion by radiation pressure fluctuations created by the intermodulation noise is negligible; otherwise the fluctuations of $x(t)$ and $\delta \nu(t)$ may deviate from purely Gaussian and correlations exist between $\delta\nu(t)$ and $\delta\nu(t)^2$. On a practical level, this reservation has minor significance for our experiment. Also, the presence of linear dynamical backaction of radiation pressure does not change the results of \secref{sec:genTheor} but does modify $S_{xx}$. Thermal intermodulation noise can preclude the observation of linear quantum correlations, which are induced by the vacuum fluctuations of radiation pressure between the quadratures of light and manifest as ponderomotive squeezing \cite{purdy_strong_2013,safavi-naeini_squeezed_2013}, Raman sideband asymmetry \cite{sudhir_appearance_2017} and the cancellation of shot noise in force measurements \cite{kampel_improving_2017,sudhir_quantum_2017}. The observation of quantum correlations typically requires selecting a mechanical mode with high $Q$, a spectral neighbourhood free from other modes, and a high optomechanical coupling rate. If TIN is taken into account, the following condition also needs to be satisfied: \begin{equation}\label{eq:qbaCond} C_q\left(\frac{g_0}{\kappa}\right)^2 \Gamma_m n_\t{th}\frac{S^{(2)}_{xx}[\omega]}{x_\t{zpf}^4}\ll 1. \end{equation} Here $C_q=4 g_0^2\bar{n}_c/(\kappa \Gamma_m n_\t{th})\gtrsim 1$ is the quantum cooperativity, $g_0=G\sqrt{\hbar /2 m_\t{eff} \Omega_m}$, $\bar{n}_c$ is the intracavity photon number. The selected mechanical mode is characterized by the resonance frequency $\Omega_m$, the damping rate $\Gamma_m$, the effective mass $m_\t{eff}$ and the thermal phonon occupancy $n_\t{th}=\hbar\Omega_m/(k_B T)$. From the condition given by \eqref{eq:qbaCond}, we can learn that simply increasing the quantum cooperativity is not necessarily a successful strategy when limited by intermodulation noise. One can immediately observe that by reducing the mechanical dissipation and $g_0/\kappa$, one can keep the quantum cooperativity constant while lowering the intermodulation noise. The engineering of mode spectrum to reduce $S^{(2)}_{xx}$ at the desired frequency might also be a fruitful approach. The nonlinearity of the cavity-laser detuning response, producing TIN, modulates the optical field proportional to $x^2$ in a way analogous to, but not equivalent to, quadratic optomechanical coupling, $\partial^2 \omega_c/\partial x^2$. It was noticed that the cavity transduction commonly results in effective quadratic coupling which is orders of magnitude stronger than the highest experimentally reported $\partial^2 \omega_c/\partial x^2$ (in terms of the optical signal proportional to $x^2$ \cite{brawley_nonlinear_2016}). In the Supplementary Information, it is shown that the same is true in the MIM system. Here, the quadratic signal originating from nonlinear transduction, which creates the intermodulation noise, is larger than the signals due to nonlinear optomechanical coupling, $\partial^2 \omega_c/\partial x^2$, by a factor of $r \mathcal{F}$, where $r$ is the membrane reflectivity and $\mathcal{F}$ is the optical finesse. \section{Experimental observation of thermal intermodulation noise} TIN has a number of manifestations that are qualitatively different from other thermal noises in optical cavities. Namely, TIN is present in the \emph{amplitude quadrature} of an optical field coupled to a cavity on resonance, and its magnitude depends very sensitively on the ratio of RMS cavity frequency fluctuations to the linewidth. In this section we present the observation of broadband classical intensity noise in the optical field resonant with membrane-in-the-middle optomechanical cavity at room temperature, and verify that this noise is due to the intermodulation of Brownian motion of membrane modes. Our experimental setup, shown in \figref{fig:intro}c, comprises a membrane-in-the-middle cavity, consisting of two high-reflectivity mirrors and a chip which high-stress stoichiometric \SiN membrane sandwiched directly between them. The MIM cavity is situated in a vacuum chamber at room temperature and probed in transmission. See \secref{sec:expDetails} for more details. \subsection{Intermodulation noise in a cavity with a uniform membrane} \label{sec:rectMbr} -We first characterize the TIN in cavities with 20 nm-thick uniform square membranes of different sizes. The optomechanical cooperativity was kept low during in order to eliminate dynamical backaction of the light, which was achieved by increasing the vacuum pressure and keeping the mechanical modes gas damped to $Q\sim 10^3$. +We first characterize the TIN in cavities with 20 nm-thick uniform square membranes of different sizes. The optomechanical cooperativity was kept low in order to eliminate dynamical backaction of the light, which was achieved by increasing the vacuum pressure and keeping the mechanical modes gas damped to $Q\sim 10^3$. -The reflection signals of two resonances of a MIM cavity with a \mbrsize{2} membrane are presented in \figref{fig:rectMbr}a. The resonances have similar optical linewidths of about 15 MHz but different by a factor of ten optomechanical couplings. The resonance with high coupling ($g_0/2\pi=150$ Hz) shows clear signatures of thermal noise. For this resonance the total RMS thermal frequency fluctuations are expected to be around 2 MHz, which is still well below the cavity linewidth, $\kappa/2\pi=16$ MHz. +The reflection signals of two resonances of a MIM cavity with a \mbrsize{2} membrane are presented in \figref{fig:rectMbr}a. The resonances have similar optical linewidths (about 15 MHz) but their optomechanical coupling is different by a factor of ten. The resonance with high coupling ($g_0/2\pi=150$ Hz) shows clear signatures of thermal noise. For this resonance the total RMS thermal frequency fluctuations are expected to be around 2 MHz, which is still well below the cavity linewidth, $\kappa/2\pi=16$ MHz. The thermal fluctuations imprinted on the optical field by the cavity are significant even when the laser is resonant with the cavity. Typical spectra of the detected noise are shown in \figref{fig:rectMbr}d for a cavity with a different, \mbrsize{1}, square membrane. When the laser is locked detuned from the cavity resonance (close to the ``magic" detuning, $\nu_0\approx -1/\sqrt{3}$), the transmission signal is dominated by the Brownian motion of membrane modes linearly transduced by the cavity (shown in the top of \figref{fig:rectMbr}d). The magnitude of thermomechanical noise is gradually reduced at high frequencies due to the averaging of membrane mode profiles \cite{zhao_wilson_suppression_2012,wilson_thesis_2012} over the cavity waist, until it meets shot noise at around 15 MHz (verified by the optical power dependence, see SI). When the laser is locked on resonance, the output light also contains a large amount of thermal noise---at an input power of 5 $\mu$W the classical RIN exceeds the shot noise level by about 25 dB at MHz frequencies (see the bottom of \figref{fig:rectMbr}d). Again, at high frequency the noise level approaches shot noise. An unambiguous proof of the intermodulation origin of the resonant intensity noise is obtained by examining the scaling of the noise level with $G/\kappa$. In thermal equilibrium the spectral density of frequency fluctuations, $\delta\nu(t)$, created by a particular membrane is proportional to $(G/\kappa)^2$, and therefore the spectral density of intermodulation noise is expected to be proportional to $(G/\kappa)^4$. We confirm this scaling by measuring the resonant intensity noise for different optical resonances of the same cavity with a \mbrsize{2} membrane and present in \figref{fig:rectMbr}b the noise magnitude, averaged over the frequency band from 0.6 to 1.6 MHz, as a function of $g_0/\kappa$, where $g_0$ is the optomechanical coupling of the fundamental mode. By performing a sweep of the input laser power on one of the resonances of the same cavity we show (see \figref{fig:rectMbr}b) that the resonant intensity noise level is power-independent and therefore the noise is not related to radiation pressure effects. \begin{figure}[t] \centering \includegraphics[width=\columnwidth]{fig_theor} -\caption{\hl{Short figure captions should be added to guide the reader.}\small{Detuning fluctuation (top row) and relative intensity noise (bottom row) spectra produced by the modes of a 20-nm, \mbrsize{0.3}, rectangular, Si$_3$N$_4$ membrane. Red shows experimental data and blue is the theoretical prediction. }} +\caption{\small{Comparison of theoretical and experimental frequency and intensity noises. Detuning fluctuation (top row) and relative intensity noise (bottom row) spectra produced by the modes of a 20-nm, \mbrsize{0.3}, rectangular, Si$_3$N$_4$ membrane. Red shows experimental data and blue is the theoretical prediction. }} \label{fig:theor} \end{figure} The TIN observed in our experiments agrees well with our model. By calculating the spectrum of total membrane fluctuations according to \eqref{eq:mbrFreqFluct} and applying the convolution formula from \eqref{eq:Snu2} (see SI for full details), we can accurately reproduce the observed noise. In \figref{fig:theor}, we compare the measured detuning and intensity noise spectra with the theoretical model. Here, we assume that the damping rate of all the membrane modes are identical as the experiment is operated in the gas-damping-dominated regime. While this model is not detailed enough to reproduce all the noise features, it accurately reproduces the overall magnitude and the broadband envelope of the intermodulation noise observed in the experiment. A comparison of the linear and quadratic displacements of rectangular membranes of different sizes is made in the SI, where we observe that the overall magnitude of the noise increases with increasing membrane size, owing to the increased mode density of larger structures. \begin{figure}[t] \centering \includegraphics[width=\columnwidth]{fig_soft_clamped} \caption{\small{Microscope images of PnC membranes (top) and ringdowns of their soft-clamped, localized modes (bottom). a) 3.6\,mm$\times$3.3\,mm$\times$40\,nm, with a localized mode at 853 kHz, b) 2\,mm$\times$2\,mm$\times$20\,nm membrane with a localized mode at 1.46 MHz.}} \label{fig:softClamped} \end{figure} \begin{figure*}[t] \centering \includegraphics[width=\textwidth]{fig_detuning_sweep} \caption{\small{Observation and detuning dependence of thermal optomechanical intermodulation noise in a phononic bandgap membrane. a) Blue---protocurrent noise spectrum detected with the cavity-laser detuning set to $2\Delta/\kappa\approx-0.3$, red---shot noise level. The shaded region shows the noise averaging band for the plot in b. The inset shows an optical cavity mode (imaged at $\lambda\approx780$ nm) overlapping with the PnC membrane defect. b) The variation of the relative intensity noise at bandgap frequencies with cavity-laser detuning. Red dots are experimental measurements, blue line---fit \eqref{eq:Siidelta}, orange line --- cavity phase noise inferred from the fit, shaded blue region---independently calibrated cavity noise, with uncertainty from the selection of the averaging band (see SI).}} \label{fig:detuningSweep} \end{figure*} -We would like to address two potential confounding effects: \emph{laser frequency noise} and \emph{dissipative coupling}. Laser frequency fluctuations contribute to detuning fluctuations in the same way as cavity frequency fluctuations. However, the noise of the Ti:Sa laser used in our experiments was much lower \hl{Very qualitative statement. Can you be quantitative?} than the thermomechanical frequency noise of the short MIM cavities, and was therefore neglected. Additionally, we did not observe any significant effect of the laser lock performance on the magnitude of TIN, which indicates that the up-conversion of detuning noise from low frequencies($<10$ kHz), where the laser noise is largest, contributes negligibly to the TIN in our cavities. As dissipative coupling leads to the modulation of optical linewidth by mechanical position, it could also potentially explain intensity noise in a resonant optical field. Although dissipative coupling is generally present in MIM cavities \cite{wilson_thesis_2012}, the magnitude of this noise would expected to be orders of magnitude below that measured in our experiments (see SI for more details). Moreover, dissipative coupling cannot explain the observed scaling of resonant RIN ($\propto (G/\kappa)^4$) and the absence of correlation between the RIN level and the excess optical loss added by the membrane. +We would like to address two potential confounding effects: \emph{laser frequency noise} and \emph{dissipative coupling}. Laser frequency fluctuations contribute to detuning fluctuations in the same way as cavity frequency fluctuations. However, the noise of the Ti:Sa laser used in our experiments was much lower than the thermomechanical frequency noise of the short MIM cavities, and was therefore neglected (see \hl{add reference to relevant appendix here}). Additionally, we did not observe any significant effect of the laser lock performance on the magnitude of TIN, which indicates that the up-conversion of detuning noise from low frequencies($<10$ kHz), where the laser noise is largest, contributes negligibly to the TIN in our cavities. As dissipative coupling leads to the modulation of optical linewidth by mechanical position, it could also potentially explain intensity noise in a resonant optical field. Although dissipative coupling is generally present in MIM cavities \cite{wilson_thesis_2012}, the magnitude of this noise would expected to be orders of magnitude below that measured in our experiments (see SI for more details). Moreover, dissipative coupling cannot explain the observed scaling of resonant RIN ($\propto (G/\kappa)^4$) and the absence of correlation between the RIN level and the excess optical loss added by the membrane. \subsection{Thermal intermodulation noise caused by a soft-clamped phononic crystal membrane} % Comment: Raw numbers for the input and output powers, uncorrected for the transmission of the vacuum chamber: % 137 uW input -> 15 uW output (resonance with kappa/2 pi = 34 MHz, at \lambda = 849.8 nm) % 30 uW input -> 8.3 uW output (resonance with kappa/2 pi = 25 MHz, at \lambda = 840.8 nm) Localized (``soft-clamped") defect modes in stressed phononic crystal (PnC) resonators can have quality factors in excess of $10^8$ at room temperature due to enhanced dissipation dilution \cite{tsaturyan_ultracoherent_2017,ghadimi_strain_2017}. Owing to their high $Q$ and low effective mass, which result in low thermal force noise, $S_\t{FF,th}= 2k_B T m_\t{eff}\Gamma_m$ \cite{saulson_thermal_1990}, these modes are promising for quantum optomechanics experiments \cite{rossi_measurement-based_2018}. In \figref{fig:softClamped}a and b we present \SiN PnC membranes with soft-clamped modes optimized for low effective mass and high $Q$. The phononic crystals are formed by the hexagonal pattern of circular holes introduced in Ref.~\cite{tsaturyan_ultracoherent_2017}, which creates a bandgap for flexural modes. The phononic crystal is terminated to the silicon frame at half the hole radii in order to prevent mode localization at the membrane edges---such modes have low $Q$ and can have frequencies within the phononic bandgap, contaminating the spectrum. \figref{fig:softClamped}a shows a microscope image of a resonator with a trampoline defect, featuring $m_\t{eff}=3.8$ ng and $Q=1.65\times 10^8$ at $0.853$ MHz, corresponding to a thermal force noise $S_\t{FF,th}=13$ aN/$\sqrt{\text{Hz}}$. Another resonator, shown in \figref{fig:softClamped}b, is a \mbrsize{2} phononic crystal membrane with a defect engineered to create a single mode localized in the middle of the phononic bandgap. The displayed sample has $Q=7.4\times 10^7$ at $1.46$ MHz and $m_\t{eff}=1.1$ ng, corresponding to $S_\t{FF,th}=34$ aN/$\sqrt{\text{Hz}}$. The phononic bandgap spectrally isolates soft-clamped modes from the thermomechanical noise created by the rest of the membrane spectrum. Nevertheless, when a PnC membrane is incorporated in a MIM cavity the entire multitude of membrane modes contributes to the TIN \emph{even within bandgap frequencies}, as TIN is produced by a nonlinear process. The in-bandgap excess noise in a MIM cavity at room temperature is dominated by TIN at all detunings except for the immediate vicinity of the ``magic" detuning $\nu_0=-1/\sqrt{3}$. Around $\nu_0=-1/\sqrt{3}$ the cavity noise, characterized by measuring the noise of an empty cavity (see SI), is the dominant excess noise. The measurements were conducted using a 2mm square PnC membrane with the patterning shown in \figref{fig:softClamped}b, but made of 40 nm-thick \SiN. The membrane has a single soft-clamped mode with $Q=4.1\times 10^7$ at $1.55$ MHz. The quality factor was characterized immediately before inserting the membrane in the cavity assembly. The measurements presented in this section were made using the same setup described in \secref{sec:rectMbr} and shown in \figref{fig:intro}c, the only difference being that the vacuum pressure was kept below $5\times 10^{-7}$ mBar in order to eliminate gas damping. \figref{fig:detuningSweep}a shows the spectrum of light transmitted through a resonance of membrane-in-the-middle cavity with $g_0/2\pi=0.9$ kHz for the soft-clamped mode, $\kappa/2\pi=34$ MHz (estimated roundtrip excess optical loss is 300 ppm) and $C_0=2.5$. The input power in the measurement was 100 $\mathrm{\mu W}$ after correcting for spatial mode matching, which corresponds to a nominal $C_Q\sim1$. The shot noise level was calibrated in a separate measurement by directing an independent laser beam on the detector. The noise at bandgap frequencies is dominated by TIN, which exceeds the shot noise by four orders of magnitude. The spectrum also shows a dispersive feature in the middle of the bandgap, which is a signature of classical correlations due to the intracavity TIN exciting the localized mechanical mode. We next present in \figref{fig:detuningSweep}b the dependence of the bandgap noise level on the laser detuning, measured on a different optical resonance of the same MIM cavity and at lower input power. In this measurement $g_0/2\pi=360$ Hz for the localized mode, $\kappa/2\pi=24.8$ MHz (estimated 150 ppm excess loss per roundtrip) and the input power was 30 $\mu$W. The bandgap noise was averaged over a 35 kHz band indicated in \figref{fig:detuningSweep}a. With the laser drive detuned from the resonance, both linear, $\delta\nu(t)$, and quadratic, $\delta\nu(t)^2$, cavity frequency fluctuations contribute to the detected signal, as described in \eqref{eq:detectedNoise}. $S_{\nu\nu}$ is dominated by cavity noise at bandgap frequencies, while $S_{\nu\nu}^{(2)}$ consists of the intermodulation products of all the membrane modes. Moreover, radiation pressure cooling of membrane modes must be taken into account (see \secref{sec:detDep} for full details of the model). As can be seen from \figref{fig:detuningSweep}b, our model reproduces the observed variation of output noise with detuning very well. While at small detunings from resonance, TIN is the dominant contribution to the overall noise, around the ``magic" detuning, TIN is suppressed and the output noise is limited by the cavity noise. Notice that the intensity of the detected light in our measurement is proportional to the intensity of the intracavity field. Therefore, the suppresion of TIN in the output necessarily implies the suppression of the corresponding radiation pressure noise, which can lead to classical heating of the mechanical oscillator and thereby limit the true quantum cooperativity. \section{Conclusions and outlook} To summarize, we have presented the observation and characterization of a previously unreported broadband thermal noise in optical cavities, TIN, which originates from the quadratic transduction of thermal cavity frequency noise. Although produced by the cavity frequency noise, TIN is not correlated with it (neglecting radiation pressure effects) and therefore in many ways behaves as an independent noise. The key qualitative feature of TIN is that it creates intensity fluctuations in an optical field resonant with the cavity. The TIN magnitude grows quadratically in the ratio of RMS thermal frequency fluctuations by the optical linewidth, and therefore it strongly affects high-finesse optical cavities with large frequency fluctuations, such as optomechanical membrane-in-the-middle cavities at room temperature. Thermal intermodulation noise in optomechanical experiments can be avoided by using cavities with low finesse (equivalently, low $g_0/\kappa$), and by coupling them to mechanical resonators with lower total thermal fluctuations, i.e. which have fewer mechanical modes, higher frequency, and higher $Q$ for all modes. The latter consideration could make the fundamental modes of mechanical resonators (e.g. low-mass trampolines \cite{reinhardt_ultralow-noise_2016}) seem preferable compared to high-$Q$ but high-order PnC defect soft-clamped modes. In this context, a newly proposed method of exploiting self-similar structures as mechanical resonators with soft-clamped fundamental modes \cite{fedorov_fractal-like_2020} could potentially be fruitful for overcoming TIN. Another way of reducing the TIN is laser cooling of mechanical motion, either by dynamical backaction of a red-detuned beam or by active feedback. In this case, however, all mechanical modes that contribute to the total cavity noise must be efficiently cooled, which could be technically challenging. The raw measurement data, analysis scripts and membrane designs are available in \cite{zenodo_repos}. \section{Acknowledgements} The authors thank Ryan Schilling for fabrication advice. All samples were fabricated and grown in the Center of MicroNanoTechnology (CMi) at EPFL. This work was supported by the Swiss National Science Foundation under grant no. 182103 and the Defense Advanced Research Projects Agency (DARPA), Defense Sciences Office (DSO), under contract no. D19AP00016 (QUORT). A.B. acknowledges support from the European Union's Horizon 2020 research and innovation program under the Marie Sklodowska-Curie grant agreement no. 722923 (OMT). N.J.E. acknowledges support from the Swiss National Science Foundation under grant no. 185870 (Ambizione). % SUPPLEMENTARY %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %\clearpage \appendix \section{Experimental details} \label{sec:expDetails} Our experimental setup, as shown in \figref{fig:intro}c, comprises a membrane-in-the-middle cavity, consisting of two high-reflectivity, dielectric mirrors with 100 ppm transmission and a 200 $\mu$m-thick silicon chip which is sandwiched directly between the mirrors and hosts a suspended high-stress stoichiometric \SiN membrane. The total length of the cavity is around 350 $\mu$m, the cavity waist is $35$ $\mu$m. The MIM cavity is situated in a vacuum chamber at room temperature and probed using a Ti:Sa or a tunable external cavity diode laser at a wavelength around 840 nm, close to the maximum reflectivity wavelength of the mirrors. The Ti:Sa laser was used in all the thermal noise measurements, whereas the diode laser was only used for the characterization of optical linewidths. The measurement signal was generated by direct detection of the light transmitted through the cavity on an avalanche photodiode. The reflected light, separated using a circulator, was used for Pound-Drever-Hall (PDH) locking of the Ti:Sa frequency. The one-sided spectra \cite{specta_notations} of signals were detected in transmission and calibrated either as relative intensity noise (RIN) or as effective cavity detuning fluctuations, $S_\Delta$, with the help of calibration tones applied to the amplitude or phase quadratures of the laser, respectively. Optomechanical vacuum coupling rates, $g_0$, were measured using frequency noise calibration as described in Ref. \cite{gorodetksy_determination_2010}. The characterization of TIN in \secref{sec:rectMbr} was performed using 20 nm-thick, square membranes with different side lengths as mechanical resonators. The insertion of a membrane into the cavity resulted in excess loss for most of the optical resonances. Nevertheless, for some resonances, the optical quality factors were reduced by only 10\% (a typical variation of optical loss with wavelength is shown in \figref{fig:extRectMbr}b). The optomechanical cooperativity was kept low during the noise measurements to eliminate dynamical backaction of the light (damping or amplification of mechanical motion). For this purpose the residual pressure in the vacuum chamber was kept high, $0.22\pm 0.03$ mBar, such that the quality factors of the fundamental modes of the membranes were limited by gas damping to $Q\sim 10^3$. For taking the data in \figref{fig:detuningSweep}, the detuning of the laser from the cavity resonance was controlled by and inferred from the locking offset. For detunings greater than $2\Delta/\kappa\approx 0.5$, where the PDH error flips sign, side of the line locking was used instead of PDH. \section{Membrane fabrication} Patterned and unpatterned membrane samples are fabricated on the same \SI{100}{\milli \meter} wafer. Stoichiometric, high stress \ch{Si_3N_4} is grown by low pressure chemical vapor deposition (LPCVD) on both sides of a \SI{200}{\micro \meter}-thick silicon wafer. The initial deposition stress is estimated a posteriori from the observation of membrane resonant frequencies, and varies in the range \SI{900}{}-\SI{1100}{MPa}, changing slightly with deposition run. The fabrication process relies on bulk wet etching of silicon in \ch{KOH} through the whole wafer thickness, to create openings for optical access to the membrane samples \cite{tsaturyan_ultracoherent_2017,reinhardt_ultralow-noise_2016,gartner2018integrated}. The extremely high selectivity of \ch{Si_3N_4} to \ch{Si} during \ch{KOH} etching allows the use of the backside nitride layer as a mask, to define the outline of the membranes on the frontside. Initially, the frontside nitride (\ch{Si_3N_4}) layer is patterned with h-line photolithography and \ch{CHF_3}/\ch{SF6}-based reactive ion etching (RIE) (steps 2-3 of figure \ref{fig:processflow}). The photoresist film is then stripped with a sequence of hot N-Methyl-2-pyrrolidone (NMP) and \ch{O_2} plasma; this procedure is carefully repeated after each etching step. The frontside nitride layer is then protected by spinning a thick layer of negative-tone photoresist (MicroChemicals AZ\textregistered 15nXT), prior to flipping the wafer and beginning the patterning of membrane windows on the backside nitride layer (steps 4-5). We noticed a reduction in the occurrence of local defects and increased overall membrane yield when the unreleased membranes on the frontside were protected from contact with hot plates, spin-coaters and plasma etcher chucks. The backside layer is then patterned with membrane windows, in a completely analogous way. The exposure step requires a wafer thickness-dependent rescaling of membrane windows, to account for the slope of slow-etching planes in \ch{KOH}, and careful alignment with frontside features. After stripping the photoresist, the wafer is installed in a PTFE holder for the first wet etching step in \ch{KOH} at $\approx \SI{75}{\celsius}$ (step 6). The holder clamps the wafer along its rim, sealing off the wafer frontside with a rubber O-ring, while exposing the backside to chemical etching by \ch{KOH}. This procedure is necessary to ensure that PnC membranes are suspended correctly: we noticed that releasing PnC samples by etching from both sides of the wafer produced a large number of defects in the phononic crystal, probably due to the particular dynamics of undercut and stress relaxation in the film. The wafer is etched until \SI{30}{}-\SI{40}{\micro\meter} of silicon remains, leaving the samples robust during the subsequent fabrication steps. The wafer is then removed from the KOH bath and the PTFE holder, rinsed and cleaned in concentrated \ch{HCl} at room temperature for 2 hours \cite{nielsen2004particle}. Subsequently, the wafer is coated with thick, protective photoresist and diced into $\SI{8.75}{mm}\times\SI{8.75}{mm}$ chips, and the remainder of the process is carried on chip-wise. The chips are cleaned again with hot solvents and \ch{O_2} plasma, and the membrane release is completed by exposing the chips to \ch{KOH} from both sides (step 7). The temperature of the solution is lowered ($\approx \SI{55}{}-\SI{60}{\celsius}$), to mitigate the perturbation of fragile samples by buoyant \ch{N_2} bubbles, a byproduct of the etching reaction. After the undercut is complete, the samples are carefully rinsed, cleaned in \ch{HCl}, transferred to an ethanol bath and gently dried in a critical point dryer (CPD). \begin{figure}[t] \includegraphics[width=\columnwidth]{fig_process_flow.pdf} \centering \caption{\small{Main steps of the fabrication process. Magenta - \ch{Si_3N_4}; gray - Si; green - photoresist.}} \label{fig:processflow} \end{figure} \section{Quadratic mechanical displacement transduction by the optical cavity versus quadratic optomechanical coupling} Nonlinear cavity transduction can produce signals quadratic in mechanic displacement which are orders of magnitude stronger than previously experimentally demonstrated quadratic coupling arising from $\partial^2 \omega_c/\partial x^2$ terms \cite{Brawley_2016}. Below we derive the classical dynamics of the optical field in an optomechanical cavity taking into account terms that are quadratic in displacement. We show that in a membrane-in-the-middle cavity, the quadratic signals originating from nonlinear transduction are $r\mathcal{F}$ larger than the signals due to the nonlinear optomechanical coupling, $\partial^2 \omega_c/\partial x^2$. The fluctuations of $\nu$ due to the mechanical displacement are given by \begin{equation} \delta\nu(t)\approx 2\frac{G}{\kappa} x(t)+\frac{G_2}{\kappa} x(t)^2, \end{equation} where $G=-\partial \omega_c/\partial x$ and $G_2=-\partial^2 \omega_c/\partial^2 x$ are the linear and quadratic optomechanical coupling, respectively, and the total displacement $x$ consists of partial contributions of different modes $x_n$ \begin{equation} x(t) = \sum_n x_n(t). \end{equation} For a resonant laser probe we can find the intracavity field as \begin{multline}\label{eq:aResDisp} a(t)\approx 2\sqrt{\frac{\eta_1}{\kappa}}(1 -i\nu(t) -\nu(t)^2) s_\t{in,1}=\\ 2\sqrt{\frac{\eta_1}{\kappa}}\left(1 -2i\frac{G}{\kappa} x(t) -\left(\left(2\frac{G}{\kappa} \right)^2+i\frac{G_2}{\kappa}\right) x(t)^2\right) s_\t{in,1}. \end{multline} It is instructive to compare the magnitudes of the two contributions to the prefactor of $x(t)^2$. The typical value for $G$ (assuming the membrane is not very close to one of the mirrors) is \begin{equation} G\sim 2r \frac{\omega_c}{l_c}, \end{equation} while the typical value for $G_2$ is \cite{thompson_strong_2008} \begin{equation} G_2\sim 4 \frac{r \omega_c^2}{c \, l_c}, \end{equation} where $c$ is the speed of light, $r$ is the membrane reflectivity and $l_c$ is the cavity length. The ratio of the two contributions is evaluated as \begin{equation} \left. \left(2\frac{G}{\kappa} \right)^2\right/\left(\frac{G_2}{\kappa}\right)\sim\mathcal{F}r. \end{equation} As the cavity finesse $\mathcal{F}$ is typically large, on on the order of $10^3$ to $10^5$, and the membrane reflectivity $r$ is between $0.1$ and $0.5$, we conclude that linear optomechanical coupling needs to extremely well suppressed in order for the quadratic coupling $G_2$ to contribute. \section{Dissipative coupling} In an optomechanical membrane-in-the-middle cavity dissipative coupling, $\partial \kappa/\partial x$, exists in addition to the dispersive coupling, $\partial \omega_c/\partial x$. Dissipative coupling modulates the optical decay rate, both external coupling and intrinsic loss, and can potentially produce intensity noise in a resonantly locked probe laser. However, for the parameters of our experiment the dissipative coupling is negligible. The noise due to dissipative coupling can be upper-bound as follows. The cavity linewidth cannot change by more than $\kappa$ as the membrane is translated by $\lambda$ inside the cavity, and therefore the dissipative coupling rate is limited by \begin{equation} \frac{\partial \kappa}{\partial x}\lesssim \frac{\kappa}{\lambda}= \frac{1}{\mathcal{F}}\frac{\omega_c}{2l_c}\sim \frac{G}{\mathcal{F}}, \end{equation} where in the last transition it was assumed that the membrane reflectivity is not very much smaller than one. Resonant intracavity field modulated by dissipative coupling only is given by \begin{equation} a(t)\approx 2\sqrt{\frac{\eta_1}{\kappa}}\left(1 - \frac{1}{2\kappa}\frac{\partial \kappa}{\partial x} x(t)\right) s_\t{in,1}. \end{equation} Comparing to \eqref{eq:aResDisp}, we find that the noise produced by dissipative coupling is negligible compared to the intermodulation noise if \begin{equation} \frac{G}{\kappa}x \gg \frac{1}{\mathcal{F}}. \end{equation} In all the experiments presented in this work this condition is satisfied, $Gx/\kappa$ ranges from \textcolor{red}{0.1 to 0.01 [verify exact numbers]}, whereas $1/\mathcal{F}$ is always less than $10^{-4}$. \section{Details of TIN calculations} \noindent For an optomechanical system, the cavity resonance frequency shifts due to a displacement field $\vec{u}(\vec r)$ is, to the first order in $\vec{u}(\vec r)$, given by \cite{Lai_perturbation_1990, Gorodetksy_vacuum_2010} \begin{equation} \frac{\Delta\omega_c}{\omega_c} = \frac{1}{2}\frac{\int |\vec E{(\vec r)}|^2\nabla\epsilon(\vec r)\cdot \vec{u}(\vec r)}{\int |\vec E{(\vec r)}|^2\epsilon(\vec r)} \end{equation} where $\epsilon(\vec r)$ is the dielectric constant. For membrane in the middle systems the gradient of the dielectric constant is constant over the displacement field region. Therefore, the resonance frequency shift and correspondingly the linear optomechanical coupling constant ($G$) is only proportional to the average of the displacement field over the cavity mode shape at the position of the membrane. Hence, the optomechanical coupling constant for different flexural modes of a membrane can be written in the form $G_n = \eta_n\cdot G$, where $n$ denotes the mode number, $G$ is a constant and $\eta_n$ are the overlap factors, proportional to the average of the mode shape on the cavity mode. For a membrane (in the x-y plane) with flexural modes $\{u_n(x,y)\}$, by choosing $G$ to be equal to the coupling constant for the fundamental mode ($G_1$) the overlap factors have the form \begin{equation} \eta_n = \frac{\int u_n(x,y)I(x,y)\,dxdy}{\int u_1(x,y)I(x,y)\,dxdy}, \end{equation} where $u_1$ denotes the fundamental mode and $I(x,y) = |\vec E{(x,y)}|^2$ is the cavity mode shape. For the TEM$_{00}$ mode of a Fabry-Perot cavity the transverse mode profile is given by \begin{equation} I(x,y) = \sqrt{\frac{2}{\pi w^2}} e^{-2((x-x_0)^2+(y-y_0)^2)/w^2} \end{equation} where $w$ is the waist of the mode at the position of the membrane and $x_0$ and $y_0$ are the position coordinates of the beam center on the membrane. We normalize the coupling constants to the value for the fundamental mode since in the actual experiment we calibrate the coupling constant using the fundamental mode. With $x_n(t)$ the amplitude of the $n_{th}$ mode, the total fluctuations of the cavity normalized detuning are given by \begin{equation} \delta\nu(t) = \frac{2G}{\kappa} \sum_n \eta_n x_n(t). \end{equation} Comparing to Eq.. (12) of the main text total displacement, $x(t)$ is defined as \begin{equation} x(t) = \sum_n \eta_n x_n(t). \end{equation} The spectrum of the linear fluctuations of $x$, $S_{xx}^{(1)}[\omega]$, is a linear combination of thermal spectra of each mode, $S_{xx,n}^{(1)}[\omega]$, due to the fact that the Brownian motion of different modes are statistically independent. $S_{xx,n}^{(1)}[\omega]$ is also given by the fluctuation-dissipation theorem. \begin{equation}\label{S_xx} S_{xx}^{(1)}[\omega] = \sum_n \eta_n^2 S_{xx,n}^{(1)}[\omega] = \sum_n \eta_n^2 \frac{2kT}{\omega}\Im{\chi_n[\omega]} \end{equation} where $\chi_n[\omega]$ is the susceptibility of mode $n$. The quadratic fluctuations of $x$ can be calculated using a relation similar to Eq. (11) of the main text \begin{equation}\label{S_xx2} S_{xx}^{(2)}[\omega] = 2\pi\langle x^2\rangle \delta[\omega] + \frac{1}{\pi}\int_{-\infty}^{\infty}S_{xx}^{(1)}[\omega']S_{xx}^{(1)}[\omega-\omega']d\omega' \end{equation} Having computed the linear and quadratic spectra of displacement fluctuations, the linear and quadratic frequency fluctuations are also calculated accordingly using the Eq. (13) and (14) of the main text, and finally the total photocurrent spectrum can be calculated from Eq. (18). For a thin high stress sqaure membrane with side L, the flexural modes are given by sine waves as \begin{equation} u_{nm} = \frac{2}{L}\sin{(\frac{2\pi x}{L})}\sin{(\frac{2\pi y}{L})}, \end{equation} with the mode frequencies are $\Omega_{nm} = \frac{\pi c}{L}\sqrt{n^2 + m^2}$ where $L$ is the side of the square and $c=\sqrt{\frac{\sigma}{\rho}}$ is the speed of the acoustic wave in a film with density of $\rho$ and stress $\sigma$. The effective mass for all modes is equal to $M/4$, a quarter of the total mass of the membrane. In the data presented in Fig. (3) of the main text, the membrane is under a rather high pressure, such that the damping process is dominated by viscous damping which implies a constant damping rate given by $\Gamma_{nm} = \Omega_{nm}/Q_{nm}$. Piecing it all together, the susceptibility of the mode ${nm}$ is given by \begin{equation} \chi_{nm}[\omega] = \frac{1}{M/4}\frac{1}{\Omega_{nm}^2 - \omega^2 - i\Gamma_{nm}\omega}. \end{equation} Finally, we can analytically calculate the linear spectrum using Eq. (\ref{S_xx}) and then find the TIN by numerically computing the convolution integral in Eq. (\ref{S_xx2}). %%%%%%%%%%%%%%% \section{The model of detuning dependence of total output light noise for MIM cavity with PnC membrane} \label{sec:detDep} As shown in the main manuscript text, the intensity of light, $I(t)$, and therefore the photodiode signal, is related to the linear ($\delta\nu(t)$) and quadratic ($\delta\nu(t)^2$) fluctuations of the cavity frequency as \begin{multline} I(t)=|s_\t{out,2}(t)|^2\propto\\ |L(\nu_0)|^2 \left(1-\frac{2\nu_0}{1+\nu_0^2}\delta\nu(t)+\frac{3\nu_0^2-1}{(1+\nu_0^2)^2}\delta\nu(t)^2\right), \end{multline} where $\nu_0=2\Delta_0/\kappa$ is normalized detuning. The spectrum of intensity fluctuations of the output light is given by, \begin{equation} S_{II}[\omega]\propto \frac{4 \nu_0^2}{(1+\nu_0^2)^2} S_{\nu\nu}[\omega]+\frac{(3\nu_0^2-1)^2}{(1+\nu_0^2)^4} S^{(2)}_{\nu\nu}[\omega]. \end{equation} In an optomechanical cavity operated at high input power $S_{\nu\nu}$ and $S^{(2)}_{\nu\nu}$ in general are detuning-dependent because of the dynamic backaction of light, most importantly because of the laser cooling/amplification of mechanical motion. In order to find the dependence of $S_{II}$ on $\Delta$ some specific assumptions need to be made about the operating regime and the frequency of interest. Considering the case of data in Fig.~5b of the main text, here the noise level is estimated at the bandgap frequency and therefore only the mirror noise is expected to contribute to $S_{\nu\nu}$. The mechanical modes of the mirrors are relatively weakly coupled to the intracavity light and therefore the dynamical backaction for them can be neglected, resulting in detuning-independent $S_{\nu\nu}$. The intermodulation noise contribution, on the contrary, is significantly affected by laser cooling. It is natural to suggest (and it is advocated for by the very good agreement of our conclusions with experimental data) that TIN at bandgap frequencies is dominated by the mixing products of resonant and off-resonant parts of the membrane thermomechanical spectrum. Dynamical backaction reduces the mechanical spectral density on resonance $\propto 1/\Gamma_\t{DBA}$, where $\Gamma_\t{DBA}$ is the optical damping rate and $\Gamma_\t{DBA}\gg \Gamma_m$ is assumed, and it does not affect the off-resonant spectral density. In the unresolved-sideband regime, which is typically well fulfilled in our measurements, the optical damping rate is given by \begin{equation} %\Gamma_\t{DBA}=-8\Omega_m\left(\frac{2g_0}{\kappa}\right)^2 \frac{\nu_0}{(1+\nu_0^2)^2}n_c \Gamma_\t{DBA}= -32\frac{\Omega_m}{\kappa}\left(\frac{2g_0}{\kappa}\right)^2\frac{\nu_0}{(1+\nu_0^2)^3} \eta_1|s_\t{in,1}|^2, \end{equation} and under our assumptions the spectral density of quadratic frequency fluctuations at PnC bandgap frequencies follows the detuning dependence of $1/\Gamma_\t{DBA}$, \begin{equation} S^{(2)}_{\nu\nu}\propto \frac{(1+\nu_0^2)^3}{|\nu_0|}, \end{equation} for $\nu_0<0$. Motivated by this consideration, the experimental data in Fig.~5b is fitted with the model \begin{equation} S_{II}\propto \frac{4 \nu_0^2}{(1+\nu_0^2)^2} C_1+\frac{1}{|\nu_0|}\frac{(3\nu_0^2-1)^2}{1+\nu_0^2} C_2, \end{equation} where $C_1$ and $C_2$ are free parameters. It was found that the model very well reproduces the observed variation of output noise with detuning and the value of $C_1$ found from the fit is indeed consistent with independently measured mirror noise, as shown in \figref{fig:mirrorNoise}. \section{Extended data} \textcolor{red}{[This section is a draft]} \textcolor{red}{[Add a calibrated measurement of the amplitude noise of Ti:Sa laser]} \begin{figure*}[t] \includegraphics[width=\textwidth]{fig_power_sweep.pdf} \centering \caption{\small{a) Spectra of resonant relative intensity noise for a 2mm$\times$2mm square unpatterned membrane (resonance wavelength 837.7 nm, $g_0/2\pi=84$ Hz, $\kappa/2\pi=9.9$ MHz) at different input powers. The inset shows the same plot zoomed in at low frequencies. The RIN levels plotted in Fig. 2 of main manuscript are averaged over the frequency range shaded gray. b) The reproduction of the average RIN from Fig. 2 of main manuscript.}} \label{fig:powerSweep} \end{figure*} The spectra of resonant RIN taken at different powers (shown in \figref{fig:powerSweep}a) show that the transmission signal is shot noise-limited at the frequency $\gtrsim 15$ MHz and therefore validates the shot noise estimate in Fig 2 of the main text. \begin{figure*}[t] \includegraphics[width=\textwidth]{fig_ext_rect_mbr.pdf} \centering \caption{\small{a) Low frequency zoom-in of the data in Fig. 2 of the main text. b) Green points---measured linewidths of different optical resonances of MIM cavity with a 2mm$\times$2mm$\times$20nm unpatterned membrane, the dashed line is a guide to eye. Orange line---linewidth of an empty cavity with the same length.}} \label{fig:extRectMbr} \end{figure*} \begin{figure*}[t] \includegraphics[width=\textwidth]{fig_mirror_noise.pdf} \centering \caption{\small{a) Spectrum of detuning fluctuations due to the mirror noise. b) Mirror noise overlapped with a trace from detuning sweep presented in Fig. 5 of the main text corresponding to $2\Delta/\kappa=-0.51$.}} \label{fig:mirrorNoise} \end{figure*} \bibliography{references} \end{document}