Scale, rotate and transform ellipse in TikZ picture

You are right when you say that the second transformation does not match its description, yet strictly speaking that's already the case for the first one. (Note: in a previous version I was claiming that the transformations are nonlinear, which they are not, but the transformations I implemented were those following your prescription. They coincide with what follows, except that the following is simpler, of course.)

enter image description here

\documentclass[11pt]{article}
\usepackage[top=0.7in,bottom=0.7in,left=0.5in,right=0.5in]{geometry}
\usepackage{tikz} 
\usepackage{mathtools}
% \usepackage[active,tightpage]{preview}
% \PreviewEnvironment{tikzpicture}
% \setlength\PreviewBorder{5pt}%
\usetikzlibrary{calc}
\newcommand{\MyConst}{-0.8}




\newcommand{\addphasespace}[2]{
\begin{scope}[xshift=#1]
\draw (-1.5,-1.5) rectangle (1.5,1.5);
\draw (-1.5,0) -- (1.5,0);
\draw (0,-1.5) -- (0,1.5);
\node[inner sep=1pt,font=\large] at (0,-1.75) {$x$};
\node[inner sep=1pt,font=\large] at (1.75,0) {$y$};
\node[inner sep=1pt,font=\large] at (0,1.9) {#2};
\end{scope}
}

\newcommand{\drawellipse}[4][]{
\begin{scope}[#4]
\fill[black!20,yshift=0cm] (0,0) circle (#2 and #3);
\draw[yshift=0cm,red] (-#2,0) -- (#2,0);
\foreach \X in {0,0.1,...,#2}
{
\pgfmathsetmacro{\Y}{#3*sin(acos(\X/#2))}
\draw[yshift=0cm,blue] (\X,-\Y) -- (\X,\Y);
\draw[yshift=0cm,blue] (-\X,-\Y) -- (-\X,\Y);
}
\draw[yshift=0cm] (0,0) circle (#2 and #3);
\end{scope}
}

\newcommand{\trafoone}[1]{
\pgftransformcm{1}{#1}{0}{1}{\pgfpoint{0pt}{0pt}}
}
\newcommand{\trafotwo}[1]{
\pgftransformcm{1}{0}{#1}{1}{\pgfpoint{0pt}{0pt}}
}
\newcommand{\trafooneaftertwo}[2]{
\pgftransformcm{1}{#1}{#2}{1+#1*#2}{\pgfpoint{0pt}{0pt}}
}
\newcommand{\trafotwoafterone}[2]{
\pgftransformcm{1+#1*#2}{#2}{#1}{1}{\pgfpoint{0pt}{0pt}}
}

%%%%%%%%%%%%%%%

\begin{document}


You are looking at transformations of the type
\[
\begin{pmatrix} x\\ y\end{pmatrix}\xmapsto{~f_1~}
\begin{pmatrix} x\\ y+c_1\, x\end{pmatrix}
\quad\text{and}\quad
\begin{pmatrix} x\\ y\end{pmatrix}\xmapsto{~f_2~}
\begin{pmatrix} x+c_2\,y\\ y\end{pmatrix}\;,
\]
which can be written in matrix form as
\[
\begin{pmatrix} x\\ y\end{pmatrix}\xmapsto{~f_1~}
\underbrace{\begin{pmatrix} 1 & 0\\ c_1 & 1\end{pmatrix}}_{=A_1}\cdot
\begin{pmatrix} x\\ y\end{pmatrix}
\quad\text{and}\quad
\begin{pmatrix} x\\ y\end{pmatrix}\xmapsto{~f_2~}
\underbrace{\begin{pmatrix} 1 & c_2\\ 0 & 1\end{pmatrix}}_{=A_2}\cdot
\begin{pmatrix} x\\ y\end{pmatrix}\;.
\]
These transformations do not commute,
\[
 A_1\cdot A_2=
 \begin{pmatrix} 1 & c_2\\ c_1 & 1+c_1\,c_2\end{pmatrix}
 \ne
 \begin{pmatrix} 1+c_1\,c_2 & c_2\\ c_1 & 1\end{pmatrix}=A_2\cdot A_1\;.
\]
You can implement these transformations with \verb|\pgftransformcm|, which is
equivalent to \texttt{[cm=\dots]} in a scope. This version comes with four commands
\[ \verb|\trafoone{#1}|\;,\quad\verb|\trafotwo{#1}|\;,\quad
\verb|\trafooneaftertwo{#1}{#2}|\quad\text{and}\quad \verb|\trafotwoafterone{#1}{#2}|\;,
\] 
which correspond to the transformation matrices
\[A_1\;,\quad A_2\;,\quad A_1\cdot A_2 \quad\text{and}\quad
A_2\cdot A_1\;, %@DavidCarlisle @barbarabeeton please don't kill me for the spaces ;-)
\]
respectively. See Figure \ref{fig:trafos} for some examples, which are chosen to resemble
your screenshot.
\begin{figure}
% @egreg please don't kill me for using \centerline ;-)
\centering
\begin{tikzpicture}
\draw[help lines] (-3,-3) grid (15,3);
% area ellipse  = pi*0.8*0.2 = 0.5026
%Plot1
\addphasespace{0cm}{1}
\drawellipse{0.8}{0.2}{}
%Plot2
\addphasespace{4cm}{2}
\begin{scope}[xshift=4cm]
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\trafoone{\MyConst}
\drawellipse{0.8}{0.2}{}
\end{scope}
\end{scope}
%%Plot3
\addphasespace{8cm}{3}
\begin{scope}[xshift=8cm]
\trafoone{\MyConst}
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\trafotwoafterone{\MyConst}{-\MyConst}
\drawellipse{0.8}{0.2}{};
\end{scope}
\end{scope}
%%Plot4
\addphasespace{12cm}{4}
\begin{scope}[xshift=12cm]
\trafotwo{\MyConst}
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\trafooneaftertwo{-\MyConst}{\MyConst}
\drawellipse{0.8}{0.2}{};
\end{scope}
\end{scope}
\end{tikzpicture}
\caption{$f_1$, $f_2$, $f_2\circ f_1$ and $f_1\circ f_2$.}
\label{fig:trafos}
\end{figure}
\end{document}

And an animation, like in @J Leon V.'s answer.

\documentclass[tikz,border=3.14]{standalone}


\newcommand{\addphasespace}[2]{
\begin{scope}[xshift=#1]
\draw (-1.5,-1.5) rectangle (1.5,1.5);
\draw (-1.5,0) -- (1.5,0);
\draw (0,-1.5) -- (0,1.5);
\node[inner sep=1pt,font=\large] at (0,-1.75) {$x$};
\node[inner sep=1pt,font=\large] at (1.75,0) {$y$};
\node[inner sep=1pt,font=\large] at (0,1.9) {#2};
\end{scope}
}

\newcommand{\drawellipse}[4][]{
\begin{scope}[#4]
\fill[black!20,yshift=0cm] (0,0) circle (#2 and #3);
\draw[yshift=0cm,red] (-#2,0) -- (#2,0);
\foreach \X in {0,0.1,...,#2}
{
\pgfmathsetmacro{\Y}{#3*sin(acos(\X/#2))}
\draw[yshift=0cm,blue] (\X,-\Y) -- (\X,\Y);
\draw[yshift=0cm,blue] (-\X,-\Y) -- (-\X,\Y);
}
\draw[yshift=0cm] (0,0) circle (#2 and #3);
\end{scope}
}

%%%%%%%%%%%%%%%
\begin{document}
\foreach \MyConst [count=\Z] in {0,0.1,...,2}
{\begin{tikzpicture}
\draw[help lines] (-3,-3) grid (15,3);
% area ellipse  = pi*0.8*0.2 = 0.5026
%Plot1
\addphasespace{0cm}{1}
\drawellipse{0.8}{0.2}{}
%Plot2
\addphasespace{4cm}{2}
\begin{scope}[xshift=4cm]
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\pgftransformcm{1}{\MyConst}{0}{1}{\pgfpoint{0pt}{0pt}}
\drawellipse{0.8}{0.2}{}
\coordinate (l1) at (-0.8,0);
\coordinate (r1) at (0.8,0);
\end{scope}
\ifnum\Z=1
\else
\draw[-latex] (-0.8,0) -- (l1);
\draw[-latex] (0.8,0) -- (r1);
\fi
\end{scope}
%%Plot3
\addphasespace{8cm}{3}
\begin{scope}[xshift=8cm]
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\pgftransformcm{1}{0}{\MyConst}{1}{\pgfpoint{0pt}{0pt}}
\drawellipse{0.8}{0.2}{};
\coordinate (l2) at (-0.8,0);
\coordinate (r2) at (0.8,0);
\end{scope}
\end{scope}
%%Plot4
\addphasespace{12cm}{4}
\begin{scope}[xshift=12cm]
\drawellipse{0.8}{0.2}{opacity = 0.3}
\begin{scope}
\pgftransformcm{1}{\MyConst}{\MyConst}{{1+\MyConst*\MyConst}}{\pgfpoint{0pt}{0pt}}
\drawellipse{0.8}{0.2}{};
\coordinate (l3) at (-0.8,0);
\coordinate (r3) at (0.8,0);
\end{scope}
\ifnum\Z=1
\else
\draw[-latex] (-0.8,0) -- (l3);
\draw[-latex] (0.8,0) -- (r3);
\fi
\end{scope}
\end{tikzpicture}}
\end{document}

enter image description here

As you can see, the transformations do preserve the area, as they should (since they have determinant 1).


Although it is not very clear to me, one way to change and scale according to the arrows that you place, is using the option /tikz/cm={<a>,<b>,<c>,<d>,<coordinate>}, the first figure simply rotating the shape without changing the area, the following are scaling according to the arrows, let me know if it is what you are looking for, you can experiment with the code.

UPDATE: A modification to visualize an arbitrary variation in the area, which simulates that it conserves the constant area.

%Plot1
\addphasespace{0cm}{0.8}{0.2}{rotate=360/20*\X}{1}
%Plot2
\draw[->] (4.8,0) -- ++(0,-0.85);
\draw[->] (3.2,0) -- ++(0,0.85);
\addphasespace{4cm}{0.8}{0.2}{opacity = 0.3}{2}
\addphasespace{4cm}{0.8}{0.2}{cm={1,-1*\X/20,0.5*\X/20,1-0.5*\X/20,(0,0)}}{2}
%Plot3
\draw[->] (7.2,0.8) -- ++(0.85,0);
\draw[->] (8.8,-0.8) -- ++(-0.85,0);
\addphasespace{8cm}{0.8}{0.2}{opacity = 0.3,cm={1,-1,0.5,0.5,(0,0)}}{3}
\addphasespace{8cm}{0.8}{0.2}{cm={1-1*\X/20,-1,0.5+0.5*\X/20,0.5-0.5*\X/20,(0,0)}}{3}
%Plot4
\draw[->]  (12.82,0.8) -- ++(0,-0.85);
\draw[->]  (11.18,-0.8) -- ++(0,0.85);
\addphasespace{12cm}{0.8}{0.2}{opacity = 0.3,cm={1,1,-0.5,0.5,(0,0)}}{4}
\addphasespace{12cm}{0.8}{0.2}{cm={1,1-1*\X/20,-0.5+0.5*\X/20,0.5+0.5*\X/20,(0,0)}}{4}

UPDATE RESULT: enter image description here

RESULT: enter image description here

MWE:

% arara: pdflatex: {synctex: yes, action: nonstopmode}
% arara: animate: {density: 150, delay: 8, other: -background white -alpha remove}
% arara: showanimate

\documentclass[tikz,border=1pt]{standalone}
\usepackage{tikz} 
\usetikzlibrary{calc}
\begin{document}
    \foreach \X in {1,...,20}
    {\begin{tikzpicture}
        \newcommand{\addphasespace}[5]{
        \begin{scope}[xshift=#1]
        \draw (-1.5,-1.5) rectangle (1.5,1.5);
        \draw (-1.5,0) -- (1.5,0);
        \draw (0,-1.5) -- (0,1.5);
        \node[inner sep=1pt,font=\large] at (0,-1.75) {$x$};
        \node[inner sep=1pt,font=\large] at (1.75,0) {$y$};
        \node[inner sep=1pt,font=\large] at (0,1.9) {#5};
        \begin{scope}[#4]
        \fill[black!20] (0,0) circle (#2 and #3);
        \draw[red] (-#2,0) -- (#2,0);
        \draw[blue] (0,-#3) -- (0,#3);
        \draw[blue] (0.1,-0.195) -- (0.1,0.195);
        \draw[blue] (-0.1,-0.195) -- (-0.1,0.195);
        \draw[blue] (0.2,-0.19) -- (0.2,0.19);
        \draw[blue] (-0.2,-0.19) -- (-0.2,0.19);
        \draw[blue] (0.3,-0.18) -- (0.3,0.18);
        \draw[blue] (-0.3,-0.18) -- (-0.3,0.18);
        \draw[blue] (0.4,-0.17) -- (0.4,0.17);
        \draw[blue] (-0.4,-0.17) -- (-0.4,0.17);
        \draw[blue] (0.5,-0.158) -- (0.5,0.158);
        \draw[blue] (-0.5,-0.158) -- (-0.5,0.158);
        \draw[blue] (0.6,-0.135) -- (0.6,0.135);
        \draw[blue] (-0.6,-0.135) -- (-0.6,0.135);
        \draw[blue] (0.7,-0.1) -- (0.7,0.1);
        \draw[blue] (-0.7,-0.1) -- (-0.7,0.1);
        \draw(0,0) circle (#2 and #3);
        \end{scope}
        \end{scope}
    }
    %%%%%%%%%%%%%%%
    \draw[help lines] (-3,-3) grid (15,3);
    % area ellipse  = pi*0.8*0.2 = 0.5026
    %Plot1
    \addphasespace{0cm}{0.8}{0.2}{rotate=360/20*\X}{1}
    %Plot2
    %Plot2
    \draw[->] (4.8,0) -- ++(0,-0.85);
    \draw[->] (3.2,0) -- ++(0,0.85);
    \addphasespace{4cm}{0.8}{0.2}{opacity = 0.3}{2}
    \addphasespace{4cm}{0.8}{0.2}{cm={1,-1*\X/20,1*\X/20,1,(0,0)}}{2}


    %%Plot3
    \draw[->] (7.2,0.8) -- ++(0.85,0);
    \draw[->] (8.8,-0.8) -- ++(-0.85,0);
    \addphasespace{8cm}{0.8}{0.2}{opacity = 0.3,cm={1,-1,1,1,(0,0)}}{3}
    \addphasespace{8cm}{0.8}{0.2}{cm={1-1*\X/20,-1,1,1-1*\X/20,(0,0)}}{3}


    %%Plot4
    \draw[->]  (12.82,0.8) -- ++(0,-0.85);
    \draw[->]  (11.18,-0.8) -- ++(0,0.85);
    \addphasespace{12cm}{0.8}{0.2}{opacity = 0.3,cm={1,1,-1,1,(0,0)}}{4}
    \addphasespace{12cm}{0.8}{0.2}{cm={1,1-1*\X/20,-1+\X/20,1,(0,0)}}{4}
    \end{tikzpicture}}
\end{document}

PSD: For animation I use imagemagic, that converts pdf files to gif documents.