notes.tex

\documentclass{article}
\usepackage[margin=0.7in]{geometry}
\usepackage[parfill]{parskip}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amssymb,amsfonts,amsthm,bm}
\usepackage{graphicx}
\usepackage{titlesec}
\usepackage[table]{xcolor}
\usepackage[colorlinks=true, linkcolor=dblue, citecolor=dred]{hyperref} 
\usepackage[nameinlink]{cleveref}
\usepackage{natbib}
\usepackage{braket}
\usepackage{url}
\usepackage{lmodern}

\definecolor{dred}{rgb}{0.6,0,0}
\definecolor{dpurple}{HTML}{A020F0}
\definecolor{dblue}{rgb}{0,0,0.6}
\Crefname{equation}{Equation}{Equations}
\Crefname{figure}{Figure}{Figures}
\creflabelformat{equation}{#2#1#3}
\crefrangelabelformat{equation}{#3#1#4-#5#2#6}

\renewcommand\b\bm

\begin{document}

\begin{equation}
    C_{ii} = var(m_i) = f (1-f)
\end{equation}

\begin{align}
    dim &= \frac{(\sum_i \lambda_i)^2}{\sum_i \lambda_i^2}\\
    &= \frac{tr(\b{C})^2}{tr(\b{C} \b{C})}\\
    &= \frac{tr(\b{C})^2}{\sum_{ij}C_{ij}^2}\\
    &= \frac{(M f (1-f))^2}{M f^2 (1-f)^2 + \sum_{i \neq j} C_{ij}^2}\\
    &\approx \frac{M^2 f^2 (1-f)^2}{M f^2 (1-f)^2 + M (M-1) \braket{C_{i \neq j}^2}}\\
    &= \frac{M}{1 + (M-1)f^{-2}(1-f)^{-2}\braket{C_{i \neq j}^2}}.
\end{align}

Define $\b{\Sigma} = \b{J}^T \b{J}$ and $\b{z}_{ij} = [z_i; z_j] \sim \mathcal{N}(\b{z}, \b{\mu} = 0, \b{\Sigma}[i;j, i;j] ) $.
\begin{align}
    C_{ij} &= \braket{m_i m_j} - f^2\\
    &= p(z_i > \theta_i \cup  z_j > \theta_j) - f^2\\
    &= \int_{z_i = \theta_i} \left [ \int_{z_j = \theta_j} p(z_j| z_i) dz_j \right ] p(z_i) dz_i.
\end{align}

Define $S_{ij} = s_{ij}^2 = \braket{(\b{j}_i^T \b{x})^T (\b{j}_j^T \b{x})}$ as the output correlation before the nonlinearity.
Can we make a convexity argument about $\braket{C_{ij}^2}$ and $\braket{S_{ij}^2}$ and apply Jensen's inequality?
This would be useful since we can analytically compute $\braket{S_{ij}^2}$:
\begin{align}
    \braket{S_{ij}^2} &= \int S^2 p(\rho|\b{j}_1, \b{j}_2) p(\b{j}_1) p(\b{j}_2) dS d\b{j}_1 d\b{j}_2\\
    &= \int (\b{j}_1^T \b{j}_2)^2 p(\b{j}_1) p(\b{j}_2) d\b{j}_1 d\b{j}_2\\
    &= \int (\b{j}_1^T \b{j}_2)^2 p(\b{j}_1|K_1) p(\b{j}_2|K_2) d\b{j}_1 d\b{j}_2 dK_1 dK_2\\
    &= \int S(O; K_1, K_2)^2 p(O|K_1, K_2) p(K_1) p(K_2) dO dK_1 dK_2\\
    &= \int S(O; K_1, K_2)^2 Hypergeom(N, K_1, K_2, O) p(K_1) p(K_2) dO dK_1 dK_2,
\end{align}
where 
\begin{align}
    S(O; K_1, K_2) &= O j_+^2 + (K_1 + K_2 - 2O) j_- j_+ + (N - K_1 - K_2 + O) j_-^2\\
    &= O(j_+^2 + j_-^2 - 2 j_+ j_-) + (K_1 + K_2)(j_+ j_- - j_-^2)+Nj_-^2\\
    &= O (j_+ - j_-)^2 + (K_1 + K_2)((1- \mu_j)(-\mu_j) - (-\mu_j)^2) + N \mu_j^2\\
    &= O(1 - \mu_j + \mu_j) - \mu_j (K_1 + K_2) + \mu_j^2 N
    &:= O + \beta
\end{align}
with $j_+ = 1 - \mu_j$ and $j_- = 0 - \mu_j$ in the presence of inhibition.
Now
\begin{equation}
    S(O; K_1, K_2)^2 = O^2 + 2O \beta + \beta^2.
\end{equation}
We now note that
\begin{equation}
    \int O Hypergeom(N, K_1, K_2, O) dO = K_1 K_2 / N
\end{equation}
and
\begin{equation}
    \int O^2 Hypergeom(N, K_1, K_2, O) dO = 
    N^{-2} \left ( (K_1 K_2)^2 + \frac{K_1 K_2 (N-K_1) (N-K_2)}{N-1} \right ) := \gamma.
\end{equation}
This leads to
\begin{align}
    \braket{S_{ij}^2}
    &= \int (\gamma + 2\beta K_1 K_2/N + \beta^2 ) p(K_1) p(K_2) dK_1 dK_2.
\end{align}
We now treat these terms one at a time, remembering that $\beta = N \mu_j^2 - \mu_j (K_1 + K_2)$, noting that $\mu_j = \frac{\braket{K}}{N} := \frac{\mu_k}{N}$, and denoting $v_k := \braket{K^2}$.
\begin{align}
    I_1 &:= \int \gamma p(K_1) p(K_2) dK_1 dK_2\\
    &= \braket{K^2}^2/N^2 + (N-1)^{-1} (\braket{K}^2 + \braket{K^2}^2/N^2 - 2 \braket{K} \braket{K^2} / N)\\
    &= \frac{\braket{K^2}^2}{N (N-1)} + \frac{\braket{K}^2}{N-1} - 2 \frac{\braket{K} \braket{K^2}}{N (N-1)}\\
    &= \frac{v_k^2}{N (N-1)} + \frac{\mu_k^2}{N-1} - 2 \frac{\mu_k v_k}{N (N-1)}
\end{align}

\begin{align}
    I_2 &:= \int (2\beta K_1 K_2/N) p(K_1) p(K_2) dK_1 dK_2\\
    &= 2/N \int ((-\mu_j (K_1+K_2) + N \mu_j^2) K_1 K_2) p(K_1) p(K_2) dK_1 dK_2\\
    &=2/N \left ( N \mu_j^2 \braket{K}^2 - 2 \mu_j \braket{K}\braket{K^2} \right )\\
    &= 2 \mu_j^2 \braket{K}^2 - 4 \frac{\mu_j \braket{K}\braket{K^2}}{N}\\
    &= 2 \frac{\mu_k^4}{N^2} - 4 \frac{\mu_k^2 v_k}{N^2}\\
    &= 2 \frac{\mu_k^2}{N^2}(\mu_k^2 - 2 v_k) 
\end{align}

\begin{align}
    I_3 &:= \int \beta^2 p(K_1) p(K_2) dK_1 dK_2\\
    &= \int (N \mu_j^2 - \mu_j (K_1+K_2))^2 p(K_1) p(K_2) dK_1 dK_2\\
    &= N^2 \mu_j^4 - 4 N \mu_j^3 \braket{K} + 2 \mu_j^2 (\braket{K}^2 + \braket{K^2})\\
    &= \frac{\mu_k^4}{N^2} - 4 \frac{\mu_k^4}{N^2} + 2 \frac{\mu_k^4 + \mu_k^2 v_k}{N^2}\\
    &= \frac{\mu_k^2}{N^2}(2 v_k - \mu_k^2)
\end{align}

This gives us
\begin{align}
    \braket{S_{ij}^2} &= I_1 + I_2 + I_3\\
    &= \frac{v_k^2}{N (N-1)} + \frac{\mu_k^2}{N-1} - 2 \frac{\mu_k v_k}{N (N-1)} + 2 \frac{\mu_k^2}{N^2}(\mu_k^2 - 2 v_k) + \frac{\mu_k^2}{N^2}(2 v_k - \mu_k^2)\\
    &= \frac{v_k^2}{N (N-1)} + \frac{\mu_k^2}{N-1} - 2 \frac{\mu_k v_k}{N (N-1)} + \frac{\mu_k^4}{N^2} - 2 \frac{\mu_k^2}{N^2} v_k.
\end{align}

We can take the derivative of this result to find the optimum:
\begin{align}
    \frac{\partial \braket{S_{ij}^2}}{\partial v_k} &= 
    2 \frac{v_k}{N (N-1)} - 2 \frac{\mu_k}{N (N-1)} - 2 \frac{\mu_k^2}{N^2}\\
    &\propto v_k - \mu_k - \mu_k^2 + \frac{\mu_k^2}{N}.
\end{align}
We see that this is zero, and the correlation thus optimized, when the variance is
\begin{equation}
    v_k - \mu_k^2 = \mu_k ( 1 - \frac{\mu_k}{N}).
\end{equation}

We can also note that
\begin{align}
    \braket{S_{ii}} &= \braket{\b{j}_i^T \b{j}_i}\\
    &= \braket{K j_+^2 + (N-K) j_-^2}\\
    &= \braket{K (1 + \mu_j^2 - 2 \mu_j)  + N \mu_j^2 - K \mu_j^2}\\
    &= \braket{K - 2 K \mu_k/N + \mu_k^2/N}\\
    &= \mu_k - 2 \mu_k^2/N + \mu_k^2 / N\\
    &= \mu_k (1 - \mu_k / N)
\end{align}
and
\begin{align}
    \braket{S_{ii}^2} &= \braket{(K(1 - 2 \mu_k/N) + \mu_k^2/N)^2}\\
    &= \braket{K^2(1 + 4 \mu_k^2 / N^2 - 4 \mu_k/N) + \mu_k^4/N^2 + 2 K (\mu_k^2/N - 2 \mu_k^3/N^2)}\\
    &= v_k(1 + 4 \mu_k^2 / N^2 - 4 \mu_k/N) + \mu_k^4/N^2 + 2 \mu_k^3/N - 4 \mu_k^4/N^2\\
    &= v_k(1 + 4 \mu_k^2 / N^2 - 4 \mu_k/N) + 2 \mu_k^3/N - 3 \mu_k^4/N^2.
\end{align}

This allows us to compute the input dimensionality
\begin{align}
    dim &= \frac{M \braket{S_{ii}}^2}{\braket{S_{ii}^2} + (M-1) \braket{S_{ij}^2}}.
\end{align}

Since $\braket{S_{ii}}^2$ does not depend on $v_k$, the dimensionality is minimized when the denominator is maximized, which happens when
\begin{equation}
    v_k - \mu_k^2 = \mu_k ( 1 - \frac{\mu_k}{N}) - N(N-1)/(2M-2) (1 + 4 \mu_k^2 / N^2 - 4 \mu_k/N).
\end{equation}

\end{document}