Scale of braces of cases environment in tabular

With use of makegapedcells defined in the makecell package:

\documentclass[12pt]{article}
\usepackage[a4paper, margin= .1cm, 
            bindingoffset=0cm]{geometry}
\usepackage{mathtools}
\usepackage{makecell, multirow}

\newcommand{\ROC}{\text{ROC}}

\begin{document}
    \[
\setcellgapes{3pt}
\makegapedcells
\begin{array}{@{}c|c|c@{}}
F(z)    &   \ROC(f)         &   f[.]    \\  
    \hline
\multirow{4.4}{*}{$\dfrac{z}{z-p}$} 
        &   |z| > |p|       &   f[k] = \begin{cases}
                            \hphantom{-}0, \quad    &  k<0   \\
                            \hphantom{-}p^k, \quad  &   k\ge 0
                                    \end{cases} \\
    \cline{2-3}
        &   0 < |z| < |p|   &   f[k] = \begin{cases}
                            -p^k, \quad         &   k<0 \\\
                            \hphantom{-}0, \quad &   k\ge 0
                                        \end{cases} \\
    \hline
\multirow{4.4}{*}{$\begin{aligned}
    & \dfrac{Az}{z-p} + \dfrac{\overline{A}z}{z-p}  \\
    & = 2z\dfrac{z\text{Re}(A)-\text{Re}(A\overline{p})}
                {z^2 - 2z\text{Re}(p)+|p|^2}
                \end{aligned}$} 
        &   |z| > |p|       &   f[k] = \begin{cases}
                                \hphantom{-}0, & k<0    \\
                                \hphantom{-}2|A||p|^k\cos(\Omega k+\varphi), &  k\ge 0
                                        \end{cases}     \\
    \cline{2-3}
    &   0<|z|<|p|           &   f[k] = \begin{cases}
                            -2|A||p|^k\cos(\Omega k+\varphi),   & k<0\\
                            \hphantom{-} 0,                     &k\ge 0   
                                \end{cases}
\end{array}
    \]
\end{document}

Note: with use of the array inside of math environment instead of the tabular, you can omit all $ in table except in \multirow cells.

One possibility is to use booktabs package and replace every \hline by \midrule and \cline{2-3} with \cmidrule{2-3} as follows:

\documentclass[12pt]{article}

\usepackage[a4paper, top = 0.1cm, left = 0.1cm, right = 0.1cm, bottom = 0.1cm, bindingoffset=0cm]{geometry}
\usepackage[utf8]{inputenc}
\usepackage{mathtools}
\usepackage{multirow}
\usepackage{booktabs}

\newcommand{\ROC}{\text{ROC}}

\begin{document}

\begin{tabular}{@{}c|c|c@{}}
    $F(z)$ & $\ROC(f)$ & $f[.]$\\\midrule
    \multirow{2}{*}{$\dfrac{z}{z-p}$} & $|z| > |p|$ & $f[k] = \begin{cases}\hphantom{-}0, \quad &k<0\\\hphantom{-}p^k, \quad &k\ge 0\end{cases}$\\\cmidrule{2-3}
    & $0 < |z| < |p|$ & $f[k] =  \begin{cases}-p^k, \quad &k<0\\\hphantom{-}0, \quad &k\ge 0\end{cases}$\\\midrule 
    \multirow{2}{*}{\shortstack{$\dfrac{Az}{z-p} + \dfrac{\overline{A}z}{z-p}$\\$=2z\dfrac{z\text{Re}(A)-\text{Re}(A\overline{p})}{z^2 - 2z\text{Re}(p)+|p|^2}$}} & $|z| > |p|$ & $f[k] = \begin{cases}\hphantom{-}0, & k<0\\\hphantom{-}2|A||p|^k\cos(\Omega k+\varphi), &k\ge 0   \end{cases}$\\\cmidrule{2-3}
    & $0<|z|<|p|$ & $f[k] = \begin{cases}-2|A||p|^k\cos(\Omega k+\varphi), & k<0\\\hphantom{-} 0, &k\ge 0   \end{cases}$
\end{tabular}

This gives the following output:

I would get rid of all vertical and almost all horizontal rules, left-align the cell contents, use array instead of tabular, double the value of \arraycolsep, and use the \addlinespace macro to create whitespace-type ("negative space") between the rows. I'd also clean up and streamline the LaTeX code; e.g., replace \overline with \bar, use an aligned environment rather than a \shortstack directive, and set up \ROC and \Re as math operators. (Many thanks to @egreg for suggesting the latter changes.)

\documentclass[12pt]{article}

\usepackage[a4paper, margin=5mm, bindingoffset=0cm]{geometry}
% \usepackage[utf8]{inputenc} % is the default
\usepackage[T1]{fontenc}
\usepackage{mathtools} % for '\DeclarePairedDelimiter' macro
\DeclarePairedDelimiter\abs\lvert\rvert
\usepackage{booktabs}

\DeclareMathOperator{\ROC}{ROC}
\renewcommand\Re{\operatorname{Re}}

\begin{document}

\[
\setlength\arraycolsep{10pt} % default: 5pt
\begin{array}{@{} lll @{}}
    F(z) & \ROC(f) & f[k] \\
    \midrule %\hline
    \dfrac{z}{z-p}
    & \abs{z} > \abs{p} 
    & \begin{cases}
             \hphantom{-}0   &\text{if $k<0$}\\
             \hphantom{-}p^k &\text{if $k\ge0$}
             \end{cases} \\
    \addlinespace %\cline{2-3}
    & 0 < \abs{z} < \abs{p} 
    & \begin{cases}
          -p^k          &\text{if $k<0$}\\
          \hphantom{-}0 &\text{if $k\ge0$}
      \end{cases} \\
    \addlinespace[2\defaultaddspace] % \hline 
     \smash[b]{%
         \begin{aligned}[t]
            &\frac{Az}{z-p} + \frac{\bar{A}z}{z-p} \\
            &=2z\frac{z\Re(A)-\Re(A\bar{p})}{z^2 - 2z\Re(p)+\abs{p}^2}
         \end{aligned}}
    & \abs{z} > \abs{p} 
    & \begin{cases}
             \hphantom{-}0  & \text{if $k<0$}\\
             \hphantom{-}2\abs{A}\abs{p}^k\cos(\Omega k+\varphi) &\text{if $k\ge0$}   
      \end{cases} \\
    \addlinespace % \cline{2-3}
    & 0<\abs{z}<\abs{p} 
    & \begin{cases}
             -2\abs{A}\abs{p}^k\cos(\Omega k+\varphi) & \text{if $k<0$}\\
             \hphantom{-} 0 &\text{if $k\ge0$}   
      \end{cases}
\end{array}
\]

\end{document}