How can I draw the circle is inscribed in triangle in 3D?

You provide some wonderful code! You only need to draw the circle in the plane of the triangle.

\documentclass[12pt, border = 1mm,tikz]{standalone}
\usepackage{tikz-3dplot}
\usetikzlibrary{3dtools}
\def\pgfmathsetmacroFPU#1#2{\begingroup% 
\pgfkeys{/pgf/fpu,/pgf/fpu/output format=fixed}% 
\pgfmathsetmacro{#1}{#2}% 
\pgfmathsmuggle#1\endgroup}% 
\begin{document}
    \tdplotsetmaincoords{70}{123}
    \begin{tikzpicture}[tdplot_main_coords,scale=1,tdplot_main_coords,
    declare function={xA = 8; yA=5;zA=5; xB=1;yB=2;zB=0; xC=5;yC=-5;zC=0 );%
    }]
 \path
   (xA,yA,zA) coordinate (A) 
   (xB,yB,zB) coordinate (B) 
   (xC,yC,zC) coordinate (C) ;
 \pgfmathsetmacro{\AB}{{sqrt((xB- xA)*(xB- xA) +(yB- yA)*(yB- yA) + (zB- zA)*(zB- zA))} };
 \pgfmathsetmacro{\AC}{{sqrt((xC- xA)*(xC- xA) +(yC- yA)*(yC- yA) + (zC- zA)*(zC- zA))} };
 \pgfmathsetmacro{\BC}{{sqrt((xC- xB)*(xC- xB) +(yC- yB)*(yC- yB) + (zC- zB)*(zC- zB))} };
 \pgfmathsetmacro{\P}{{(\AB + \BC + \AC)/2} };
 \pgfmathsetmacro{\myarea}{{sqrt(\P*(\P - \AB) *(\P - \AC)*(\P - \BC) )} };
 \pgfmathsetmacro{\r}{{\myarea/\P} };  
 \coordinate (I) at ({(xA*\BC + xB*\AC + xC*\AB)/(\AB + \BC + \AC)},{(yA*\BC + yB*\AC + yC*\AB)/(\AB + \BC + \AC)},{(zA*\BC + zB*\AC + zC*\AB)/(\AB + \BC + \AC)});
 \foreach \p in {A,B,C,I}
 \draw[fill=black] (\p) circle (1.5 pt);
 \foreach \p/\g in {A/-90,B/-90,C/-90,I/0}
 \path (\p)+(\g:3mm) node{$\p$};
 \draw[blue] (A) -- (B) -- (C) -- cycle; 
 \path[overlay] [3d coordinate={(mya)=(C)-(B)},
    3d coordinate={(myb)=(C)-(A)},3d coordinate={(myc)=(B)-(A)}];
 \pgfmathsetmacroFPU{\tmpa}{1/sqrt(TD("(mya)o(mya)"))}   
 \pgfmathsetmacroFPU{\tmpb}{1/sqrt(TD("(myb)o(myb)"))}   
 \pgfmathsetmacroFPU{\tmpc}{1/sqrt(TD("(myc)o(myc)"))}   
 \path[overlay] [3d coordinate={(mya)=\tmpa*(mya)},
   3d coordinate={(myb)=\tmpb*(myb)},3d coordinate={(myc)=\tmpc*(myc)},
    3d coordinate={(myn)=(myc)x(mya)},
    3d coordinate={(mym)=(myn)x(B)-(A)}]; 
 \pgfmathsetmacroFPU{\tmpm}{1/sqrt(TD("(mym)o(mym)"))}   
 \path[overlay] [3d coordinate={(mym)=\tmpm*(mym)}]; 
 \begin{scope}[x={(myc)},y={(mym)}]
 \draw[red,dashed] (I) circle[radius= \r];
 \end{scope}
\end{tikzpicture}
\end{document}

You can choose the coordinates (more or less) as you wish

\documentclass[12pt, border = 1mm,tikz]{standalone}
\usepackage{tikz-3dplot}
\usetikzlibrary{3dtools}
\def\pgfmathsetmacroFPU#1#2{\begingroup% 
\pgfkeys{/pgf/fpu,/pgf/fpu/output format=fixed}% 
\pgfmathsetmacro{#1}{#2}% 
\pgfmathsmuggle#1\endgroup}% 
\begin{document}
\tdplotsetmaincoords{70}{123}
\foreach \X in {-6,-5.8,...,6}
{\begin{tikzpicture}[tdplot_main_coords,scale=1,tdplot_main_coords,
    declare function={xA = 8; yA=5;zA=\X; xB=1;yB=2;zB=0; xC=5;yC=-5;zC=0 );%
    }]
 \draw[tdplot_screen_coords] (-8,-11) rectangle (3,3);
 \path
   (xA,yA,zA) coordinate (A) 
   (xB,yB,zB) coordinate (B) 
   (xC,yC,zC) coordinate (C) ;
 \pgfmathsetmacro{\AB}{{sqrt((xB- xA)*(xB- xA) +(yB- yA)*(yB- yA) + (zB- zA)*(zB- zA))} };
 \pgfmathsetmacro{\AC}{{sqrt((xC- xA)*(xC- xA) +(yC- yA)*(yC- yA) + (zC- zA)*(zC- zA))} };
 \pgfmathsetmacro{\BC}{{sqrt((xC- xB)*(xC- xB) +(yC- yB)*(yC- yB) + (zC- zB)*(zC- zB))} };
 \pgfmathsetmacro{\P}{{(\AB + \BC + \AC)/2} };
 \pgfmathsetmacro{\myarea}{{sqrt(\P*(\P - \AB) *(\P - \AC)*(\P - \BC) )} };
 \pgfmathsetmacro{\r}{{\myarea/\P} };  
 \coordinate (I) at ({(xA*\BC + xB*\AC + xC*\AB)/(\AB + \BC + \AC)},{(yA*\BC + yB*\AC + yC*\AB)/(\AB + \BC + \AC)},{(zA*\BC + zB*\AC + zC*\AB)/(\AB + \BC + \AC)});
 \foreach \p in {A,B,C,I}
 \draw[fill=black] (\p) circle (1.5 pt);
 \foreach \p/\g in {A/-90,B/-90,C/-90,I/0}
 \path (\p)+(\g:3mm) node{$\p$};
 \draw[blue] (A) -- (B) -- (C) -- cycle; 
 \path[overlay] [3d coordinate={(mya)=(C)-(B)},
    3d coordinate={(myb)=(C)-(A)},3d coordinate={(myc)=(B)-(A)}];
 \pgfmathsetmacroFPU{\tmpa}{1/sqrt(TD("(mya)o(mya)"))}   
 \pgfmathsetmacroFPU{\tmpb}{1/sqrt(TD("(myb)o(myb)"))}   
 \pgfmathsetmacroFPU{\tmpc}{1/sqrt(TD("(myc)o(myc)"))}   
 \path[overlay] [3d coordinate={(mya)=\tmpa*(mya)},
   3d coordinate={(myb)=\tmpb*(myb)},3d coordinate={(myc)=\tmpc*(myc)},
    3d coordinate={(myn)=(myc)x(mya)},
    3d coordinate={(mym)=(myn)x(B)-(A)}]; 
 \pgfmathsetmacroFPU{\tmpm}{1/sqrt(TD("(mym)o(mym)"))}   
 \path[overlay] [3d coordinate={(mym)=\tmpm*(mym)}]; 
 \begin{scope}[x={(myc)},y={(mym)}]
 \draw[red,dashed] (I) circle[radius= \r];
 \end{scope}
\end{tikzpicture}}
\end{document}

Note that I normalized various vectors to avoid dimension too large errors.