\bordermatrix with brackets [ ] instead of parentheses ( )

An almost direct modification of \bordermatrix:

\documentclass{article}
\makeatletter
\def\bbordermatrix#1{\begingroup \m@th
  \@tempdima 4.75\p@
  \setbox\z@\vbox{%
    \def\cr{\crcr\noalign{\kern2\p@\global\let\cr\endline}}%
    \ialign{$##$\hfil\kern2\p@\kern\@tempdima&\thinspace\hfil$##$\hfil
      &&\quad\hfil$##$\hfil\crcr
      \omit\strut\hfil\crcr\noalign{\kern-\baselineskip}%
      #1\crcr\omit\strut\cr}}%
  \setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{$\kern\wd\@ne\kern-\@tempdima\left[\kern-\wd\@ne
    \global\setbox\@ne\vbox{\box\@ne\kern2\p@}%
    \vcenter{\kern-\ht\@ne\unvbox\z@\kern-\baselineskip}\,\right]$}%
  \null\;\vbox{\kern\ht\@ne\box\tw@}\endgroup}
\makeatother

\begin{document}
$
M = \bbordermatrix{ & x & y \cr
              A & 1 & 0 \cr
              B & 0 & 1 \cr}
$
\end{document}

I acted on \left( and \right), but also changed the value given to \@tempdima, smaller because brackets are slimmer than parentheses.

If you don't want such a messy code, just do

\usepackage{etoolbox}
\let\bbordermatrix\bordermatrix
\patchcmd{\bbordermatrix}{8.75}{4.75}{}{}
\patchcmd{\bbordermatrix}{\left(}{\left[}{}{}
\patchcmd{\bbordermatrix}{\right)}{\right]}{}{}

that does exactly the same changes.

A slightly different version where the border entries are set in \scriptstyle.

\documentclass{article}
\makeatletter
\def\bbordermatrix#1{\begingroup \m@th
  \global\let\perhaps@scriptstyle\scriptstyle
  \@tempdima 4.75\p@
  \setbox\z@\vbox{%
    \def\cr{%
      \crcr
      \noalign{%
        \kern2\p@
        \global\let\cr\endline
        \global\let\perhaps@scriptstyle\relax
      }%
    }%
    \ialign{$\make@scriptstyle{##}$\hfil\kern2\p@\kern\@tempdima
      &\thinspace\hfil$\perhaps@scriptstyle##$\hfil
      &&\quad\hfil$\perhaps@scriptstyle##$\hfil\crcr
      \omit\strut\hfil\crcr
      \noalign{\kern-\baselineskip}%
      #1\crcr\omit\strut\cr}}%
  \setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
  \setbox\tw@\hbox{$\kern\wd\@ne\kern-\@tempdima\left[\kern-\wd\@ne
    \global\setbox\@ne\vbox{\box\@ne\kern2\p@}%
    \vcenter{\kern-\ht\@ne\unvbox\z@\kern-\baselineskip}\,\right]$}%
  \null\;\vbox{\kern\ht\@ne\box\tw@}\endgroup}
\def\make@scriptstyle#1{\vcenter{\hbox{$\scriptstyle#1$}}}
\makeatother

\begin{document}
$
M = \bbordermatrix{
   & x & y \cr
 A & 1 & 0 \cr
 B & 0 & 1 \cr
}
$
\end{document}

Following Herbert Voss' excellent mathmode document, you can redefine the bordermatrix command to take an optional argument that determines the delimeters

Note that Herbert also details that the amsmath package offers the same or better macros/environments

\documentclass{article}
\makeatletter
\newif\if@borderstar
\def\bordermatrix{\@ifnextchar*{%
\@borderstartrue\@bordermatrix@i}{\@borderstarfalse\@bordermatrix@i*}%
}
\def\@bordermatrix@i*{\@ifnextchar[{\@bordermatrix@ii}{\@bordermatrix@ii[()]}}
\def\@bordermatrix@ii[#1]#2{%
\begingroup
\m@th\@tempdima8.75\p@\setbox\z@\vbox{%
\def\cr{\crcr\noalign{\kern 2\p@\global\let\cr\endline }}%
\ialign {$##$\hfil\kern 2\p@\kern\@tempdima & \thinspace %
\hfil $##$\hfil && \quad\hfil $##$\hfil\crcr\omit\strut %
\hfil\crcr\noalign{\kern -\baselineskip}#2\crcr\omit %
\strut\cr}}%
\setbox\tw@\vbox{\unvcopy\z@\global\setbox\@ne\lastbox}%
\setbox\tw@\hbox{\unhbox\@ne\unskip\global\setbox\@ne\lastbox}%
\setbox\tw@\hbox{%
$\kern\wd\@ne\kern -\@tempdima\left\@firstoftwo#1%
\if@borderstar\kern2pt\else\kern -\wd\@ne\fi%
\global\setbox\@ne\vbox{\box\@ne\if@borderstar\else\kern 2\p@\fi}%
\vcenter{\if@borderstar\else\kern -\ht\@ne\fi%
\unvbox\z@\kern-\if@borderstar2\fi\baselineskip}%
\if@borderstar\kern-2\@tempdima\kern2\p@\else\,\fi\right\@secondoftwo#1 $%
}\null \;\vbox{\kern\ht\@ne\box\tw@}%
\endgroup
}
\makeatother


\begin{document}
$
M = \bordermatrix[{[]}]{%
            & x & y \cr
          A & 1 & 0 \cr
          B & 0 & 1 }
$

$
M = \bordermatrix[\{\}]{%
            & x & y \cr
          A & 1 & 0 \cr
          B & 0 & 1 }
$
\end{document}