Cells with text are slighlty raised in TikZ

add anchor=north in the definition of nodes

as below

[mat/.style={matrix of nodes,
            nodes={draw, minimum size=10mm, minimum width=15mm, fill=gray!10,   anchor=north, },

the output is

also changed the options slightly for the arrow

\draw [-{Latex[length=3mm]}]  (array-2-5.south)--(array-5-8.north) node [near end, above, sloped] (TextNode) {\&}; 

the midway replaced with near end now no requirement of hspace

MWE

\documentclass[10pt,a4paper]{article}
\usepackage[utf8]{inputenc}
\usepackage[T1]{fontenc}
\usepackage{tikz, }
\usetikzlibrary{matrix, arrows.meta}
\begin{document}
    \begin{tikzpicture}
        [mat/.style={matrix of nodes,
            nodes={draw, minimum size=10mm, minimum width=15mm, fill=gray!10,   anchor=north, },
            column sep=-\pgflinewidth, 
            row sep=0.5mm, 
            nodes in empty cells, 
            row 1 column 1/.style={nodes={draw=none}},
            row 2 column 1/.style={nodes={draw=none, fill=none}},
            row 2 column 2/.style={nodes={draw=none, fill=none}},
            row 2 column 8/.style={nodes={draw=none, fill=none}},
            row 2 column 9/.style={nodes={draw=none, fill=none}},
            row 5 column 4/.style={nodes={draw=none, fill=none}},
            row 5 column 5/.style={nodes={draw=none, fill=none}},
            row 5 column 6/.style={nodes={draw=none, fill=none}},
            row 1/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}}, 
            row 3/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}}, 
            row 4/.style={nodes={font=\ttfamily, draw=none, fill=none, minimum size=5mm}}, 
        }]
        
        \matrix[mat] (array) 
        { 
            &   &   &   &   dhaka   &   &   &   &       \\
            &   &   &   &   19      &   &   &   &       \\
            &   &   & 20021 &   20022 & 20023 & &   &       \\
            &   sylhet  &   &   &           &   &   &   khulna  &       \\
            & 19    &   &   &           &   &   &   20022 &     \\
        }; 
        
        
        \draw [-{Latex[length=3mm]}]  (array-2-5.south)--(array-5-8.north) node [near end, above, sloped] (TextNode) {\&}; 
        \draw [-{Latex[length=3mm]}]  (array-2-5.south)--(array-5-2.north); 
        
    \end{tikzpicture}
\end{document}