Can I create a node list from some text entirely within lua?

(This is taken from the LuaTeX wiki and updated for LuaTeX shipped with tl2016/17/18/...?)

This look difficult, but it isn't. Well, it is, I don't admit it.

\documentclass{article}
\usepackage{luacode}
\usepackage{libertine}
\begin{document}
\begin{luacode*}
function newglue(parameters)
    local g = node.new("glue")
    local tmp_spec
    if node.has_field(g,"spec") then
        g.spec = node.new("glue_spec")
        tmp_spec = g.spec
    else
        tmp_spec = g
    end
    for k,v in pairs(parameters) do
        tmp_spec[k] = v
    end
    return g
end

function mknodes( text )
  local current_font = font.current()
  local font_parameters = font.getfont(current_font).parameters
  local n, head, last
  -- we should insert the paragraph indentation at the beginning
  head = newglue({width = 20 * 2^16})
  last = head

  for s in string.utfvalues( text ) do
    local char = unicode.utf8.char(s)
    if unicode.utf8.match(char,"%s") then
      -- its a space
      n = newglue({width = font_parameters.space,shrink  = font_parameters.space_shrink, stretch = font_parameters.space_stretch})
    else -- a glyph
      n = node.new("glyph")
      n.font = current_font
      n.subtype = 1
      n.char = s
      n.lang = tex.language
      n.uchyph = 1
      n.left = tex.lefthyphenmin
      n.right = tex.righthyphenmin
    end

    last.next = n
    last = n
  end

  -- now add the final parts: a penalty and the parfillskip glue
  local penalty = node.new("penalty")
  penalty.penalty = 10000

  local parfillskip = newglue({stretch = 2^16,stretch_order = 2})

  last.next = penalty
  penalty.next = parfillskip

  -- just to create the prev pointers for tex.linebreak
  node.slide(head)
  return head
end

local txt = "A wonderful serenity has taken possession of my entire soul, like these sweet mornings of spring which I enjoy with my whole heart. I am alone, and feel the charm of existence in this spot, which was created for the bliss of souls like mine."

tex.baselineskip = newglue({width = 14 * 2^16})

local head = mknodes(txt)
lang.hyphenate(head)
head = node.kerning(head)
head = node.ligaturing(head)

local vbox = tex.linebreak(head,{ hsize = tex.sp("3in")})
node.write(vbox)
\end{luacode*}
\end{document}

I create glyph nodes manually, add some glue and call the internal TeX functions for ligaturing, keming and linebreaking.

The result is a vbox which I write directly into TeX's current list, but you could do other things with it.

run with luatex. It creates a file xlist2.nodes:

  0    ->nil (hlist->nil)
8,6    ->  0 (whatsit->hlist)
  0    -> 37 (hlist->glyph)
 37"h" -> 37 (glyph->glyph)
 37"e" -> 37 (glyph->glyph)
 37"l" -> 37 (glyph->glyph)
 37"l" -> 37 (glyph->glyph)
 37"o" -> 10 (glyph->glue)
 10    -> 37 (glue->glyph)
 37"w" -> 11 (glyph->kern)
 11    -> 37 (kern->glyph)
 37"o" -> 37 (glyph->glyph)
 37"r" -> 37 (glyph->glyph)
 37"l" -> 37 (glyph->glyph)
 37"d" -> 12 (glyph->penalty)
 12    -> 10 (penalty->glue)
 10    -> 10 (glue->glue)
 10    ->nil (glue->nil)


\nopagenumbers
\begingroup
\catcode`\%=12
\directlua{local out=assert(io.open("xlist2.nodes","w"))
local n=node.types()
function printnode(head)
while head do
  if head.id==8 then out:write(head.id..","..head.subtype)
                else out:write(string.format("%3d",head.id))
  end
  if head.id==37 then
    out:write("\string\"",string.char(head.char),"\string\" ->")
  else out:write("\space\space\space\space->")
  end
  if head.next==nil then out:write("nil")
                    else out:write(string.format("%3d",head.next.id))
  end
  out:write(" ("..n[head.id].."->")
  if head.next==nil then out:write("nil)\string\n")
  else out:write(n[head.next.id]..")\string\n")
  end
  if head.id==0 or head.id==1 then printnode(head.head) end
  head=head.next
end
return true
end
callback.register("post_linebreak_filter",printnode,"printnode")}
\endgroup
hello world
\bye