Turn off caption prefix WITHOUT using caption package

Some surgery on tufte-common.def in order to accommodate for a \varcaption command which has the syntax

\varcaption[<offset>]{<caption text>}

(the optional argument is similar to the second optional argument for \caption).

\documentclass{tufte-book}
\usepackage{regexpatch}

\usepackage{lipsum} % just for the example

\makeatletter
\newcommand{\varcaption}[2][0pt]{%
  \gsetlength{\@tufte@caption@vertical@offset}{-#1}%
  \gdef\@tufte@stored@varcaption{#2}%
}
\xpatchcmd{\end@tufte@float}
  {\par\vspace}
  {\ifthenelse{\NOT\equal{\@tufte@stored@varcaption}{}}{\@tufte@varcaption{\@tufte@stored@varcaption}}{}\par\vspace}
  {}{}
\xapptocmd{\end@tufte@float}{\gdef\@tufte@stored@varcaption{}}{}{}
\gdef\@tufte@stored@varcaption{} % initialize
\newcommand\@tufte@varcaption[1]{%
  \begingroup
  \@parboxrestore 
  \if@minipage\@setminipage\fi
  \@tufte@caption@font
  \@tufte@caption@justification
  \noindent #1\par
  \endgroup
}
\makeatother

\begin{document}

\listoffigures

\chapter{Title}

\lipsum[2]

\begin{figure}
Something
\caption{A numbered caption}
\end{figure}

\lipsum[3]

\begin{figure}
Something
\varcaption{An unnumbered caption}
\end{figure}

\lipsum[4]

\end{document}

You'll see that the second caption is not listed in the List of Figures.

A version without regexpatch; copy in your document preamble everything from \makeatletter to \makeatother (included).

\documentclass{tufte-book}

\usepackage{lipsum} % just for the example

\makeatletter
\newcommand{\varcaption}[2][0pt]{%
  \gsetlength{\@tufte@caption@vertical@offset}{-#1}%
  \gdef\@tufte@stored@varcaption{#2}%
}
\gdef\@tufte@stored@varcaption{} % initialize
\newcommand\@tufte@varcaption[1]{%
  \begingroup
  \@parboxrestore 
  \if@minipage\@setminipage\fi
  \@tufte@caption@font
  \@tufte@caption@justification
  \noindent #1\par
  \endgroup
}
\def\end@tufte@float{%
      \par\hbox{}\vspace{-\baselineskip}\ifthenelse{\prevdepth>0}{\vspace{-\prevdepth}}{}% align baselines of boxes
      \end{minipage}%
    \end{lrbox}%
    % build the caption box
    \begin{lrbox}{\@tufte@caption@box}%
      \begin{minipage}[\floatalignment]{\marginparwidth}\hbox{}%
        \ifthenelse{\NOT\equal{\@tufte@stored@caption}{}}{\@tufte@orig@caption[\@tufte@stored@shortcaption]{\@tufte@stored@caption}}{}%
        \ifthenelse{\NOT\equal{\@tufte@stored@label}{}}{\@tufte@orig@label{\@tufte@stored@label}}{}%
        \ifthenelse{\NOT\equal{\@tufte@stored@varcaption}{}}{\@tufte@varcaption{\@tufte@stored@varcaption}}{}
        \par\vspace{-\prevdepth}%% TODO: DOUBLE-CHECK FOR SAFETY
      \end{minipage}%
    \end{lrbox}%
    % now typeset the stored boxes
    \begin{fullwidth}%
      \begin{minipage}[\floatalignment]{\linewidth}%
        \ifthenelse{\boolean{@tufte@float@star}}%
          {\@tufte@float@fullwidth[\@tufte@caption@vertical@offset]{\@tufte@figure@box}{\@tufte@caption@box}}%
          {\@tufte@float@textwidth[\@tufte@caption@vertical@offset]{\@tufte@figure@box}{\@tufte@caption@box}}%
      \end{minipage}%
    \end{fullwidth}%
    \@tufte@orig@endfloat% end original LaTeX float environment
    % output debug info
    \ifthenelse{\boolean{@tufte@debug}}{%
      \typeout{^^J^^J----------- Tufte-LaTeX float information ----------}%
      \ifthenelse{\equal{\@tufte@stored@label}{}}%
        {\typeout{Warning: Float unlabeled!}}%
        {\typeout{Float label: [\@tufte@stored@label]}}%
      \typeout{Page number: [\thepage]}%
      \def\MessageBreak{^^J}%
      \typeout{\@tufte@float@debug@info}%
      \ifthenelse{\boolean{@tufte@symmetric}}%
        {\typeout{Symmetric: [true]}}%
        {\typeout{Symmetric: [false]}}%
      \typeout{----------------------------------------------------^^J^^J}%
    }{}%
    % reset commands and temp boxes and captions
    \gdef\@tufte@float@debug@info{}%
    \let\caption\@tufte@orig@caption%
    \let\label\@tufte@orig@label%
    \begin{lrbox}{\@tufte@figure@box}\hbox{}\end{lrbox}%
    \begin{lrbox}{\@tufte@caption@box}\hbox{}\end{lrbox}%
    \gdef\@tufte@stored@shortcaption{}%
    \gdef\@tufte@stored@caption{}%
    \gdef\@tufte@stored@label{}%
    \gsetlength{\@tufte@caption@vertical@offset}{0pt}% reset caption offset
    \gdef\@tufte@stored@varcaption{}
  }
\makeatother

\begin{document}

\listoffigures

\chapter{Title}

\lipsum[2]

\begin{figure}
Something
\caption{A numbered caption}
\end{figure}

\lipsum[3]

\begin{figure}
Something
\varcaption{An unnumbered caption}
\end{figure}

\lipsum[4]

\end{document}

Support for marginfigure and margintable

With the following code that must be added to the previous one before the final \makeatother, you can use \varcaption also in marginfigure and margintable.

\renewenvironment{@tufte@margin@float}[2][-1.2ex]%
  {\FloatBarrier% process all floats before this point so the figure/table numbers stay in order.
  \begin{lrbox}{\@tufte@margin@floatbox}%
  \begin{minipage}{\marginparwidth}%
    \renewcommand\varcaption[2][]{\par\@tufte@varcaption{##2}}%
    \@tufte@caption@font%
    \def\@captype{#2}%
    \hbox{}\vspace*{#1}%
    \@tufte@caption@justification%
    \@tufte@margin@par%
    \noindent%
  }
  {\end{minipage}%
  \end{lrbox}%
  \marginpar{\usebox{\@tufte@margin@floatbox}}%
  }

In a comment, I told the OP how to add a figure in the text, without numbering. Here, based on a follow-up comment by the OP, I show how to get the unnumbered figure into the margin, similar to marginfigure of the tufte-book class.

\documentclass{tufte-book}
\usepackage{lipsum,graphicx}
\begin{document}
\listoffigures

\noindent\hrulefill

\lipsum[1]
\marginpar{
\includegraphics[width=.5in]{example-image-A}\par
\footnotesize This is my caption
}
\lipsum[2]
\begin{marginfigure}
\includegraphics[width=.5in]{example-image-A}\par
\caption{margin figure}
\end{marginfigure}
\lipsum[3]
\end{document}