Cayley's Matrix Notation

My proposal:

\documentclass{article}
\usepackage{amsmath,amssymb,graphicx}

\DeclareMathSymbol{A}{\mathalpha}{operators}{`A}
\DeclareMathSymbol{B}{\mathalpha}{operators}{`B}
\DeclareMathSymbol{C}{\mathalpha}{operators}{`C}
\DeclareMathSymbol{D}{\mathalpha}{operators}{`D}
\DeclareMathSymbol{E}{\mathalpha}{operators}{`E}
\DeclareMathSymbol{F}{\mathalpha}{operators}{`F}
\DeclareMathSymbol{G}{\mathalpha}{operators}{`G}
\DeclareMathSymbol{H}{\mathalpha}{operators}{`H}
\DeclareMathSymbol{I}{\mathalpha}{operators}{`I}
\DeclareMathSymbol{J}{\mathalpha}{operators}{`J}
\DeclareMathSymbol{K}{\mathalpha}{operators}{`K}
\DeclareMathSymbol{L}{\mathalpha}{operators}{`L}
\DeclareMathSymbol{M}{\mathalpha}{operators}{`M}
\DeclareMathSymbol{N}{\mathalpha}{operators}{`N}
\DeclareMathSymbol{O}{\mathalpha}{operators}{`O}
\DeclareMathSymbol{P}{\mathalpha}{operators}{`P}
\DeclareMathSymbol{Q}{\mathalpha}{operators}{`Q}
\DeclareMathSymbol{R}{\mathalpha}{operators}{`R}
\DeclareMathSymbol{S}{\mathalpha}{operators}{`S}
\DeclareMathSymbol{T}{\mathalpha}{operators}{`T}
\DeclareMathSymbol{U}{\mathalpha}{operators}{`U}
\DeclareMathSymbol{V}{\mathalpha}{operators}{`V}
\DeclareMathSymbol{W}{\mathalpha}{operators}{`W}
\DeclareMathSymbol{X}{\mathalpha}{operators}{`X}
\DeclareMathSymbol{Y}{\mathalpha}{operators}{`Y}
\DeclareMathSymbol{Z}{\mathalpha}{operators}{`Z}


\newsavebox{\cayleybox}
\newlength{\cayleyheight}
\newenvironment{cayleymatrix}[1]
 {\begin{lrbox}{\cayleybox}
  $\begin{array}{@{\enspace}l*{\numexpr#1-1}{@{{,}\enspace}l}@{\enspace}}}
 {\end{array}$\end{lrbox}%
  \setlength{\cayleyheight}{\dimexpr\ht\cayleybox+\dp\cayleybox}%
  \kern-0.8pt
  \vtop to \cayleyheight{\hbox{\vphantom{$\between$}}\leaders\vrule\vfill}%
  \raisebox{\dimexpr\ht\strutbox-\ht\cayleybox}{\usebox\cayleybox}%
  \vtop to \cayleyheight{\hbox{\vphantom{$\between$}}\leaders\vrule\vfill}%
  \kern-0.8pt
 }
\newcommand{\by}{\kern-0.4pt{\between}\kern-0.4pt}

\thinmuskip=6mu
\medmuskip=0mu
\thickmuskip=0mu

\begin{document}

\[ % page 17
(X,Y,Z)=
(\begin{cayleymatrix}{3}
a   & b   & c \\
a'  & b'  & c' \\
a'' & b'' & c''
\end{cayleymatrix}
\by x,y,z)
\]

\[ % page 32
(\begin{cayleymatrix}{2} a & c \\ b & d \end{cayleymatrix} \by
\begin{cayleymatrix}{2} a & b \\ c & d \end{cayleymatrix} \by
\begin{cayleymatrix}{2} a & c \\ b & d \end{cayleymatrix} \by
\begin{cayleymatrix}{2} a & b \\ c & d \end{cayleymatrix} )
=
\bigl(
  (\begin{cayleymatrix}{2} a & c \\ b & d \end{cayleymatrix}\by
  \begin{cayleymatrix}{2} a & b \\ c & d \end{cayleymatrix})
\bigr)^2
\]

\end{document}

Extending Bernard's idea slightly:

...but not quite up to the standards of egreg's nice solution:)

\documentclass{article}
\usepackage{array}
\newcounter{cayley}
\newcolumntype{L}{>{\stepcounter{cayley}%
\ifnum\value{cayley}=1\raisebox{-0.4ex}{(}\kern0.5ex\else\vline\fi}c@{}}
  \newcolumntype{R}{@{}c<{\space\ifnum\value{cayley}=1\kern0.5ex\raisebox{-0.4ex}{)}\else\vline\fi}}
\newenvironment{cayley}[1][3]{\setcounter{cayley}{0}\array{L*{#1}{c}R}}{\endarray}

\begin{document}

  \[
     \begin{cayley}
       &a  & b  & c  &\\
       &a' & b' & c' &\\
       &a''& b''& c''&\\
     \end{cayley}
     \qquad
     \begin{cayley}[4]
       &a  & b  & c  & d  &\\
       &a' & b' & c' & d' &\\
       &a''& b''& c''& d''&\\
     \end{cayley}
  \]

\end{document}

A TikZ proposal. Similarly to Andrew's nice answer you can focus on typing the matrix.

\documentclass{article}
\usepackage{tikz}
\makeatletter% https://tex.stackexchange.com/a/85531/121799
\long\def\ifnodedefined#1#2#3{%
    \@ifundefined{pgf@sh@ns@#1}{#3}{#2}%
}
\makeatother
\newcounter{CM}
\usetikzlibrary{matrix}
\newcommand{\CayleyMatrix}[2][]{\stepcounter{CM}%
\begin{tikzpicture}[baseline=(mat-\number\value{CM}-1-1.base),inner sep=2pt,#1]
  \matrix[matrix of math nodes,ampersand replacement=\&] (mat-\number\value{CM})
 { #2
  };
 \ifnodedefined{mat-\number\value{CM}-2-1}{%
 \draw[semithick] (mat-\number\value{CM}.south west) -- (mat-\number\value{CM}.south west|-mat-\number\value{CM}-1-1.south) to[out=135,in=-135]
 (mat-\number\value{CM}.south west|-mat-\number\value{CM}-1-1.north)
 (mat-\number\value{CM}.south east) -- (mat-\number\value{CM}.south east|-mat-\number\value{CM}-1-1.south) to[out=45,in=-45]
 (mat-\number\value{CM}.south east|-mat-\number\value{CM}-1-1.north) ;}{
 \draw[semithick] (mat-\number\value{CM}.south west|-mat-\number\value{CM}-1-1.south) to[out=135,in=-135]
 (mat-\number\value{CM}.south west|-mat-\number\value{CM}-1-1.north)
 (mat-\number\value{CM}.south east|-mat-\number\value{CM}-1-1.south) to[out=45,in=-45]
 (mat-\number\value{CM}.south east|-mat-\number\value{CM}-1-1.north) ;
 }
\end{tikzpicture}}
\begin{document}
\[\CayleyMatrix{
X   \&Y   \& Z \\
}=\CayleyMatrix{
a   \&b   \& c \\
  a'  \&b' \& c' \\
  a'' \& b'' \& c'' \\
}\!\!\CayleyMatrix{
x   \&y   \& z \\
}\]
\end{document}