% -----------------------------------------------------------------------------
% -*-TeX-*- -*-Hard-*- Smart Wrapping
% -----------------------------------------------------------------------------
\def\baselinestretch{1}

\chapter{On Invariant Subspaces of Essentially Self--Adjoint Operators}

\def\baselinestretch{1.66}

%%% ---------------------------------------------------------------------------

An application of the main result of the previous chapter to the algebra
generated by an essentially self--adjoint operator $A$ yields the existence
of nonzero vectors $x,y\in\h$ such that $\tau(p)=\seq{p(A)x,y}$ is a positive
functional on the space of all polynomials on the essential spectrum of $A$.
This result immediately implies the existence of real invariant subspaces for
essentially self--adjoint operators acting on a complex Hilbert space.
Elementary convex analysis techniques, applied to the space of certain vector
states, yield the existence of invariant subspaces for essentially
self--adjoint operators acting on an infinite--dimensional real Hilbert
space.

%%% ---------------------------------------------------------------------------
\goodbreak
\section{Introduction}

The existence of invariant subspaces for compact perturbations of
self--adjoint operators appears to be one of the most difficult questions in
the theory of invariant subspaces~\cite{Lom92}. The positive results about
the existence of the invariant subspaces for the Schatten--class
perturbations of self--adjoint operators, acting on a complex Hilbert space,
date back to the late 1950's. For the facts concerning such operators see
Chapter~6 in~\cite{RR73}, where a brief history of the problem, together with
the references to the related topics is given. The proofs of those results
are based on the concept of the separation of spectra. However, Ljubi\v{c}
and Macaev~\cite{LM65} showed that there is no general spectral theory by
constructing an example of an operator $A$ such that
$\sigma(A|\cal{M})=[0,1]$ whenever $\cal{M}$ is a nonzero invariant subspace
for $A$. This suggests that different techniques might be needed to establish
the existence of invariant subspaces for essentially self--adjoint operators.

\smallskip

The fact that the right--hand side of the inequality~(\ref{e:ESSCON}) depends
only on the essential norm of the real part of the operator $A$, suggests
that Proposition~\ref{p:ESS} might have applications to the invariant
subspace problem for compact perturbations of self--adjoint operators. In
this chapter we apply Proposition~\ref{p:ESS} in order to construct positive
functionals $\tau(p)=\seq{p(A)x,y}$ on the space of all polynomials
restricted to the essential spectrum of $A$. Finally, in the case when the
underlying Hilbert space is real, the existence of invariant subspaces for
$A$ is established after solving an extreme problem concerning certain convex
subspaces of vector states.

\begin{defn}
Suppose $\h$ is a real or complex Hilbert space. An operator $A\in\BH$ is
called \emph{essentially self--adjoint}, if $\pi(A)$ is a self--adjoint
element in the Calkin algebra $\BH/\KH$, where $\pi\colon\BH\To\BH/\KH$ is
the quotient mapping.
\end{defn}

\begin{rem}
Clearly, by definition of the Calkin algebra, $A$ is essentially
self--adjoint if and only if $A=S+K$, where $S\in\BH$ is self--adjoint and
$K$ is a compact operator. Hence, saying that $A$ is essentially
self--adjoint, is the same as saying that $A$ is a compact perturbation of a
self--adjoint operator. Note, however, that this is {\em{}false} if we
replace self--adjoint operators by normal ones.
\end{rem}

%%% ---------------------------------------------------------------------------
\goodbreak
\section{On Real Invariant Subspaces}

Recently V.I.~Lomonosov~\cite{Lom92} proved that every essentially
self--adjoint operator acting on a complex Hilbert space has a nontrivial
closed \emph{real} invariant subspace. We give an alternative proof, based on
Proposition~\ref{p:ESS}, and thus introduce the idea that will be later
generalized in order to yield the existence of proper invariant subspaces for
essentially self--adjoint operators acting on a real Hilbert space.

Recall that a \emph{real subspace} of a complex Hilbert space $\h$ is a
subset that is closed under addition and multiplication by the \emph{real}
scalars. A real subspace $\cal{M}\subset\h$ is invariant for an operator
$A\in\BH$ if and only if $\cal{M}$ is invariant under all operators in the
\emph{real} algebra generated by $A$, i.e. the algebra of all real
polynomials in $A$.

\begin{prop}\label{p:RIS}
Suppose $\h$ is an infinite--dimensional complex Hilbert space and let $\A$
be a convex set of commuting essentially self--adjoint operators. Then the
set of non--cyclic vectors for $\A$ is dense in $\h$.
\end{prop}

\begin{proof}
Suppose not; then there exists a unit vector $f_0$ and a positive number
$r\in(0,1)$ such that all vectors in the set
\[  \s=\set{f\in\h\,\,|\,\,\,\norm{f_0-f}\leq\tfrac{r}{\sqrt{1-r^2}}}, \]
are cyclic for $\A$. In particular, for every vector $g\in\h$ and
$\norm{g}\leq1$, there exists an operator $A\in\A$ such that
\[ \RE\seq{A\left(f_0+\tfrac{r}{\sqrt{1-r^2}}g\right),
          -i\left(f_0-\tfrac{\sqrt{1-r^2}}{r}g\right)} > 0, \]
or equivalently,
\[ \RE\seq{ i A\left(f_0+\tfrac{r}{\sqrt{1-r^2}}g\right),
                     f_0-\tfrac{\sqrt{1-r^2}}{r}g} > 0. \]
Since $A$ is an essentially self--adjoint operator, it follows that
\[ \essnorm{\IM{A}}=\essnorm{\RE(i{A})}=0, \]
and consequently the convex set $i\A=\set{i{A}\,\,|\,\,\,A\in\A}$, satisfies
the hypothesis of Proposition~\ref{p:ESS}. Therefore, there exists an element
$A_0\in\A$ ($A_0\neq{z}I$), with an eigenvector $f_1\in\s$. Since the
operators in $\A$ commute, $f_1$ cannot be a cyclic vector for $\A$,
contradicting the assumption that all vectors in $\s$ are cyclic for $\A$.
\end{proof}

\begin{cor}[V.I.\,Lomonosov, 1992]
Every essentially self--adjoint operator on an infinite--dimensional complex
Hilbert space has a nontrivial closed real invariant subspace.
\end{cor}

\begin{proof}
The commutative algebra $\A_{\Real}$ of all real polynomials in $A$ consists
of essentially self--adjoint operators whenever $A$ is essentially
self--adjoint. By Proposition~\ref{p:RIS} the set of non--cyclic vectors for
$\A_{\Real}$ is dense in $\h$. Since for every nonzero vector $f\in\h$ the
closure of the orbit $\A_{\Real}f=\set{T{f}\,\,|\,\,\,T\in\A_{\Real}}$ is a
real invariant subspace for $A$, it follows that $A$ has a nontrivial closed
real invariant subspace.
\end{proof}

\begin{rem}
If $A$ is a self--adjoint operator acting on a complex Hilbert space $\h$,
then for every vector $f\in\h$ and every real polynomial $p$ we have:
\begin{equation} \label{e:I0}
  \IM\seq{p(A){f},f}=0.
\end{equation}
The condition (\ref{e:I0}) in fact characterizes self--adjoint operators on a
complex Hilbert space~\cite[p.~103]{KR83}. Roughly speaking,
Proposition~\ref{p:RIS} and its corollary establish a similar fact for
essentially self--adjoint operators acting on a complex Hilbert space.
\end{rem}

%%% ---------------------------------------------------------------------------
\goodbreak
\section{The Space of Vector States}

In the previous section we applied our machinery only to the imaginary part
of an essentially self--adjoint operator $A$. An application to the real part
yields the existence of ``vector states'' on the space of all polynomials
restricted to the essential spectrum of $A$. Before proceeding, we make the
following conventions that hold through the rest of this chapter:

\bigskip

As usual, let $\h$ be an infinite--dimensional real or complex Hilbert space.
The underlying field of real or complex numbers (respectively) is denoted by
$\Field$. Suppose $A\in\BH$ is a fixed essentially self--adjoint operator
without non--trivial closed invariant subspaces and let $E$ denote its
essential spectrum. Furthermore, we may assume that $\essnorm{A}\leq1$, and
consequently, $E\subset[-1,1]$. Let $\A\subset\BH$ be an algebra generated by
$A$, i.e. $\A$ is the algebra of all polynomials $p(A)$ with the coefficients
in the underlying field $\Field$.

\medskip

\noindent The algebra of all polynomials with the coefficients in $\Field$,
equipped with the norm
\[ \norm{p}_\infty=\max_{t\in{E}} \abs{p(t)}, \]
is denoted by $\Poly$.

\smallskip

\begin{defn}
Let $\EssD\subset\h$ be the set of all nonzero vectors $x\in\h$ for which
there exists a nonzero vector $y\in\h$ satisfying the following inequality
for every polynomial $p\in\Poly$
\begin{equation} \label{e:PF1}
  \RE\seq{p(A)x,y} \leq \norm{\RE p}_\infty\seq{x,y}.
\end{equation}
\end{defn}

\medskip

\goodbreak

\begin{lem}\label{l:PF1}
The set $\EssD$ is dense in $\h$.
\end{lem}

\begin{proof}
Since the operator $A$ has no invariant subspaces the condition of
Proposition~\ref{p:ESS} is never satisfied for the algebra $\A$. More
precisely, for every unit vector $f_0\in\h$ and any positive number
$r\in(0,1)$ there exists a vector $g\perp{}f_0$ such that for every
polynomial $p\in\Poly$
\[ \RE\seq{p(A)\left(f_0+\tfrac{r}{\sqrt{1-r^2}}g\right),
                  f_0-\tfrac{\sqrt{1-r^2}}{r}g} \leq
   \essnorm{\RE{}p(A)}(1 - \norm{g}^2). \]
Clearly, for every polynomial $p\in\Poly$ we have
\[ \essnorm{\RE p(A)} = \essnorm{(\RE p)(A)} = \norm{\RE p}_\infty. \]
The vectors
\[ x=f_0+\tfrac{r}{\sqrt{1-r^2}}g
   \quad \text{ and } \quad
   y=f_0-\tfrac{\sqrt{1-r^2}}{r}g \]
satisfy the inequality (\ref{e:PF1}). Letting $r\to0$, and replacing the
vector $x$ by $\lambda{x}$, where $\lambda>0$, implies the required density
of $\EssD$.
\end{proof}

\begin{lem} \label{l:PF2}
For fixed vectors $x,y\in\h$ define a linear functional
$\tau\colon\Poly\To\Field$
\[ \tau(p) = \seq{p(A)x,y}. \]
Then $\tau$ is a bounded positive functional on the space $\Poly$ if and only
if the following inequality is satisfied for every polynomial $p\in\Poly$:
\[ \RE\seq{p(A)x,y} \leq \norm{\RE p}_\infty\seq{x,y}. \]
\end{lem}

\begin{proof}
Suppose that $\tau$ is a positive functional on $\Poly$. Then $\RE
\seq{p(A)x,y} = \seq{(\RE p)(A)x,y}$. Since $\norm{\RE p}_\infty - \RE p$ is
a positive polynomial on $E$, we have
\[ \tau(\norm{\RE p}_\infty - \RE p) =
   \seq{(\norm{\RE p}_\infty - \RE p)(A)x, y} \geq 0, \]
or equivalently,
\[ \RE\seq{p(A)x,y} \leq \norm{\RE p}_\infty\seq{x,y}. \]

Conversely, suppose $\tau$ is not a bounded positive functional on $\Poly$.
Then either there exists a real polynomial $p$ such that
$\IM\seq{p(A)x,y}\neq0$, or $\seq{p(A)x,y}<0$ for some positive polynomial
$p\in\Poly$. After replacing $p$ by $\pm i p$ it is easy to see that
$\IM\seq{p(A)x,y}\neq0$ contradicts (\ref{e:PF1}). Similarly, for a positive
polynomial $p$ we have
\[ \norm{\norm{p}_\infty-p}_\infty \leq \norm{p}_\infty. \]
Therefore $\seq{p(A)x,y}<0$ and $\seq{x,y}\geq0$ imply
\[ \seq{ \big(\norm{p}_\infty-p(A)\big)x,y} > \norm{p}_\infty\seq{x,y}
   \geq  \norm{\norm{p}_\infty-p}_\infty\seq{x,y}, \]
contradicting (\ref{e:PF1}). Finally, in the case when $\seq{x,y}<0$ the
inequality (\ref{e:PF1}) fails for the polynomial $p\equiv-1$.
\end{proof}

\begin{defn}
The set of all bounded positive linear functionals on $\Poly$ is denoted by
$\cal T$. For each vector $x\in\h$ define the set
\[ \cal{T}_x = \set{y\in\h\,\,|\,\,\,
               \tau(p) = \seq{p(A)x,y} \in \cal{T}}. \]
\end{defn}

\begin{lem} \label{l:PF3}
For every vector $x\in\h$, $\cal{T}_x$ is a closed convex subset of $\h$.
\end{lem}

\begin{proof}
Convexity of the set $\cal T_x$ is obvious. It remains to prove that the
complement of $\cal T_x$ is an open subset of $\h$. If $y\not\in\cal{T}_x$
then there exists a positive polynomial $p\in\Poly$ such that $\seq{p(A)x, y}
\not\geq 0$. In that case there exists a weak neighborhood $\cal W$ of $y$
such that $\seq{p(A)x,z}\not\geq0$ for every $z\in\cal W$. Consequently, the
complement of the set $\cal T_x$ is a (weakly) open subset of $\h$.
\end{proof}

\begin{defn}
A positive functional $\tau\in\cal{T}$ is called a \emph{state} if
$\norm{\tau}=1$, or equivalently $\tau(1)=1$. The space of all states on
$\Poly$ is denoted by $\cal{T}'$. Similarly, for every vector $x\in\h$ the
set $\cal{T}'_x$ is defined by
\[ \cal{T}'_x = \set{y\in\h\,\,|\,\,\, \tau(p) =
   \seq{p(A)x,y} \in \cal{T}'}. \]
\end{defn}

\begin{rem}
From Lemma~\ref{l:PF1} and Lemma~\ref{l:PF2} it follows that the set $\EssD$
of all vectors $x\in\h$ for which the set $\cal{T}_x$ contains a nonzero
vector is dense in $\h$. If $x$ and $y$ are nonzero vectors and
$y\in\cal{T}_x$ then $\seq{x,y}\geq0$. However, since a positive functional
always attains its norm on the identity function, the equality $\seq{x,y}=0$
implies that $\tau(p)=\seq{p(A)x,y}=0$ for every polynomial $p\in\Poly$,
contradicting the fact that the operator $A$ has no invariant subspaces.
Therefore, the set $\cal{T}'_x$ is nonempty for every vector $x$ in a dense
set $\EssD\subset\h$. In fact, for every vector $x\in\EssD$ the set
$\cal{T}'_x$ is the intersection of the cone $\cal{T}_x$ and the hyperplane
$\cal{M}_x=\set{y\in\h\,\,|\,\,\,\seq{y,x}=1}$. Note also, that for nonzero
vectors $x\in\EssD$ and $y\in\cal{T}_x$, we have:
$\seq{x,y}^{-1}y\in\cal{T}'_x$.
\end{rem}

\medskip

By Lemma~\ref{l:PF3} the set $\cal{T}'_x$ is a weakly closed convex subset of
$\h$. We show that the set $\cal{T}'_x$ has no extreme points.

\smallskip

\begin{lem} \label{l:EXTR}
For every vector $x\in\h$ the set $\cal{T}'_x$ has no extreme points.
\end{lem}

\begin{proof}
Suppose $y_0$ is an extreme point in $\cal{T}'_x$. By definition of the set
$\cal{T}'_x$, the functional $\tau'(p)=\seq{p(A)x,y_0}$ is a state on
$\Poly$. Hence,
\[ \omega(p)=\tau((1-t)p(t))=\seq{p(A)x, (1-A^*)y_0} \]
is a positive functional on $\Poly$. Consequently,
\[ y_1 = \seq{(1-A)x, y_0}^{-1}(1-A^*)y_0 \in \cal{T}'_x. \]
Similarly,
\[ y_2 = \seq{(1+A)x, y_0}^{-1}(1+A^*)y_0 \in \cal{T}'_x. \]
From
\[ y_0 = \frac{\seq{(1-A)x, y_0}}{2} y_1 +
         \frac{\seq{(1+A)x, y_0}}{2} y_2, \]
we conclude that $y_0=y_1=y_2$. Therefore, $(1-A^*)y_0=\seq{(1-A)x,y}y_0$
implies that $y_0$ is an eigenvector for $A^*$, contradicting the
nonexistence of invariant subspaces for the operator $A$.
\end{proof}

\begin{cor} \label{c:UNBOUND}
For every vector $x\in\h$ the set $\cal{T}'_x$ is either empty or unbounded.
\end{cor}

\begin{proof}
By the Krein--Milman Theorem the set $\cal{T}'_x$ cannot be weakly compact
due to the lack of extreme points.
\end{proof}

\medskip

Although the set $\cal{T}'_x$ is unbounded for every vector $x\in\EssD$, the
following lemma shows that it contains no line segments of infinite length.
In particular, $\cal{T}'_x$ is a \emph{proper} subset of the hyperplane
\[ \cal{M}_x=\set{y\in\h\,\,|\,\,\,\seq{y,x}=1}. \]

\medskip

\begin{lem} \label{l:FL}
Every line segment in $\cal{T}'_x$ has a finite length.
\end{lem}

\begin{proof}
Suppose the set $\cal{T}'_x$ contains a line segment of infinite length. Then
there exists a vector $y\in\cal{T}'_x$, and a unit vector $u\perp{x}$ such
that $y+\lambda u\in\cal{T}'_x$ for every $\lambda\geq0$. For every power
$k=0,1,\ldots$, and every vector $z\in\cal{T}'_x$, we have: $\abs{\seq{A^k
x,z}}\leq1$. Applying this inequality to a vector $y+\lambda u$ and letting
$\lambda\to\infty$ implies that $\seq{A^k x,u}=0$, contradicting the fact
that $x$ is a cyclic vector for $A$.
\end{proof}

%%% ---------------------------------------------------------------------------
\goodbreak
\section{Invariant Subspaces on a Real Hilbert Space}

In this section we use vector states in order to establish the existence of
invariant subspaces for essentially self--adjoint operators acting on an
infinite--dimensional real Hilbert space. The invariant subspace problem for
essentially self--adjoint operator will be translated into an extreme problem
and the solution will be obtained upon differentiating certain functions at
their extreme. Once again we will employ the differentiability of the Hilbert
norm. We start with the following lemma.

\begin{lem}\label{l:DIFF}
Suppose $x$ and $y$ are any vectors in $\h$ such that $\RE\seq{x,y}=1$. Fix a
nonzero operator $T\in\BH$ and let $a=(\norm{T}\norm{x}\norm{y})^{-1}$. Then
for every vector $z\in\h$ the function $\psi(\lambda)\colon(-a,a)\To\RPlus$,
defined by
\[ \psi(\lambda) = \norm{
   \big(\RE\seq{(1+\lambda T)y, x}\big)^{-1} (1+\lambda T)y - z}^2 \]
is differentiable on $(-a,a)$. Furthermore, if $\psi'$ denotes the derivative
of $\psi$ then
\[ \psi'(0) = 2\RE\seq{T y, y - z - (\norm{y}^2-\RE\seq{y,z}) x}. \]
\end{lem}

\begin{proof}
Since for $\lambda\in(-a,a)$ we have $\RE\seq{(1+\lambda T)y, x}>0$, it
follows that the function $\psi$ is well defined on $(-a,a)$. In order to
compute its derivative $\psi'(0)$ first apply the polar identity to $\psi$
and then use the product and chain rules for differentiation. A
straightforward calculation yields the required formula.
\end{proof}

\begin{defn}
For every vector $x\in\EssD$, define $P_x\colon\h\To\cal{T}'_x$ to be the
projection to the set $\cal{T}'_x$, \,\,i.e. for every $z\in\h$
\[ \norm{P_x z - z} = \inf_{y\in\cal{T}'_x} \norm{y-z}. \]
\end{defn}

\begin{rem}
Since for $x\in\EssD$ the set $\cal{T}'_x$ is nonempty, closed, and convex it
follows that the projection $P_x$ is well defined on the whole space $\h$.
\end{rem}

\medskip

\begin{lem}\label{l:REIS}
If $x\in\EssD$ then for every vector $z\in\h$ and every power $k=0,1,\ldots$,
the following condition is satisfied:
\[ \RE\seq{A^k\big((\norm{P_x z}^2 - \RE\seq{P_x z,z})x + (I-P_x)z\big),
   P_x z} = 0.  \]
\end{lem}

\begin{proof}
Let $T=A^{*k}$, and fix a vector $y\in\cal{T}'_x$. The function
$\Phi(\lambda)\colon(-1,1)\To\h$ is defined by
\[ \Phi(\lambda) = \seq{(1+\lambda T)y, x}^{-1} (1+\lambda T)y. \]
The same argument as in the proof of Lemma~\ref{l:EXTR} shows that $\Phi$ is
well defined and $\Phi(\lambda)\in\cal{T}'_x$ for every $\lambda\in(-1,1)$.

Choose any vector $z\in\h$ and consider the function
$\psi(\lambda)\colon(-1,1)\To\RPlus$, defined by
\[ \psi(\lambda)=\norm{\Phi(\lambda) - z}^2. \]

By Lemma~\ref{l:DIFF} the function $\psi$ is differentiable, and
\[ \psi'(0) = 2\RE\seq{T y, y - z - (\norm{y}^2-\RE\seq{y,z}) x}. \]

By definition of the projection $P_x$ the function $\psi$ attains its global
minimum at the point $\lambda=0$ whenever $y=P_x z$. Consequently,
$\psi'(0)=0$ for $y=P_x z$, which completes the proof.
\end{proof}

\medskip

A remarkable fact is that Lemma~\ref{l:REIS} holds on a real or complex
infinite--dimensional Hilbert space. It is now easy to establish the
existence of proper invariant subspaces for essentially self--adjoint
operators acting on a real Hilbert space.

\begin{thm}\label{t:ISR}
Every essentially self--adjoint operator acting on a real
infinite--dimensional Hilbert space $\h$ has a nontrivial closed invariant
subspace.
\end{thm}

\begin{proof}
Suppose $A$ is an essentially self--adjoint operator acting on a real
infinite--dimensional Hilbert space $\h$. We may assume that
$\essnorm{A}\leq1$. If the operator $A$ has no nontrivial invariant subspaces
then we can apply Lemma~\ref{l:REIS} and Lemma~\ref{l:FL}. We will show that
this contradicts the non--existence of invariant subspaces for $A$.

On a real Hilbert space Lemma~\ref{l:REIS} implies that for every
$k=0,1,\ldots$:
\begin{equation*}
  \begin{split}
     \RE\seq{A^k\big((\norm{P_x z}^2 - \RE\seq{P_x z,z})x + (I-P_x)z\big),
     P_x z} & = \\
        \seq{A^k\big((\norm{P_x z}^2 - \RE\seq{P_x z,z})x + (I-P_x)z\big),
     P_x z} & = \, 0.
  \end{split}
\end{equation*}

Since $P_x z\neq0$ it follows that
\begin{equation*}
   y_z = \big(\norm{P_x z}^2 - \RE\seq{P_x z,z}\big)x + (I-P_x)z
\end{equation*}
is a non--cyclic vector for $\A$ whenever $x\in\EssD$. The proof is therefore
completed if we show that $y_z\neq0$ for a suitable choice of the vector
$z\in\h$.

Recall that the set $\cal{T}'_x$ lies in the hyperplane
$\cal{M}_x=\set{y\in\h\,\,|\,\,\,\seq{y,x}=1}$. By definition of the
projection $P_x$, the vector $y_z=0$ for $z\in\cal{M}_x$ if and only if
$z\in\cal{T}'_x$. Lemma~\ref{l:FL} implies that $\cal{T}'_x$ is a proper
subset of the hyperplane $\cal{M}_x$ and thus completes the proof.
\end{proof}

\smallskip

\begin{rem}
Theorem~\ref{t:ISR} yields the existence of invariant subspaces for an
essentially self--adjoint operator $A$ acting on a complex Hilbert space,
whenever the operator $A$ has a matrix representation with real coefficients.
Although considerable efforts have been made to reduce the general complex
case to the real one, so far all such attempts have been unsuccessful.
\end{rem}

\bigskip
\goodbreak

\emph{We suggest that further research in this direction is likely going to
reveal additional properties of essentially self--adjoint operators and thus
contribute to our understanding of how such operators act on the underlying
Hilbert space in terms of invariant subspaces.}

% -----------------------------------------------------------------------------
