How do I create a class object in Lua-C API 5.2?

Here's how I would satisfy both your criteria as well as j_schultz's

#include <lua.h>
#include <lauxlib.h>
#include <stdlib.h>
#include <stdio.h>

#define LUA_FOO "Foo"

typedef struct {
    int x;
} Foo;

static int foo_gc(lua_State *L) {
    printf("## __gc\n");
    Foo *foo = *(Foo**)luaL_checkudata(L, 1, LUA_FOO);
    free(foo);
    return 0;
}

static int foo_doSomething(lua_State *L) {
    printf("## doSomething\n");
    Foo *foo = *(Foo**)luaL_checkudata(L, 1, LUA_FOO);
    lua_pushinteger(L, foo->x);
    return 1;
}

static int foo_new(lua_State* L) {
    printf("## new\n");
    Foo *foo = malloc(sizeof(Foo));
    int i = 1 + lua_istable(L, 1);
    foo->x = !lua_isnoneornil(L, i) ? luaL_checkinteger(L, i) : 0;
    *(Foo**)lua_newuserdata(L, sizeof(Foo*)) = foo;
    luaL_setmetatable(L, LUA_FOO);
    return 1;
}

static int foo_index(lua_State *L) {
    printf("## index\n");
    int i = luaL_checkinteger(L, 2);
    lua_pushinteger(L, i);
    return 1;
}

int luaopen_foo(lua_State *L) {
    // instance functions
    static const luaL_Reg meta[] =
    {   { "__gc"        ,foo_gc          },
        { NULL          ,NULL            }  };
    static const luaL_Reg meth[] =
    {   { "doSomething" ,foo_doSomething },
        { NULL          ,NULL            }  };
    luaL_newmetatable(L, LUA_FOO);
    luaL_setfuncs    (L, meta, 0);
    luaL_newlib      (L, meth);
    lua_setfield     (L, -2, "__index");
    lua_pop          (L, 1);

    // static functions
    static const luaL_Reg static_meta[] =
    {   { "__index" ,foo_index },
        { "__call"  ,foo_new   },
        { NULL      ,NULL      }  };
    static const luaL_Reg static_meth[] =
    {   { "new"     ,foo_new   },
        { NULL      ,NULL      }  };
    luaL_newlib      (L, static_meth);
    luaL_newlib      (L, static_meta);
    lua_setmetatable (L, -2);
    return 1;
}

Lua code:

local Foo = require('foo')
local foo = Foo.new(12)
local bar = Foo(24)

print(Foo[13])
print(foo:doSomething())
print(bar:doSomething())

Lua output:

## new
## new
## index
13
## doSomething
12
## doSomething
24
## __gc
## __gc

I tried replying to your solution but apparently I don't have the reputation to do so yet, so here goes a separate answer.

Your solution is pretty nice, but it does not allow for something that I'd like to do: Have both "array-like" access to an object and still have functions on it. Have a look at this Lua code:

Foo = {}

mt = {
__index = function(table, key)
  print("Accessing array index ", tostring(key), "\n")
  return 42
end
}
setmetatable(Foo, mt)

Foo.bar = function()
  return 43
end

print(tostring(Foo[13]), "\n")
print(tostring(Foo.bar()), "\n")

--[[
Output:
Accessing array index 13
42
43
]]--

Registering a class using your solution does not seem to allow for this, as the __index entry is overwritten. It might not make sense to have both array access and function access on a class, but for the sake of simplicity (offering one C function for registering both types of classes) I'd like to use the same code everywhere. Does anyone have an idea how this restriction could be circumvented, so that I can create a class from C which has both a function Foo.bar() but also Foo[13]?

Ok, got it working. I had to add __index and __metatable to Foo's new metatable, as shown below:

void register_foo_class(lua_State* L) {
  int lib_id, meta_id;

  /* newclass = {} */
  lua_createtable(L, 0, 0);
  lib_id = lua_gettop(L);

  /* metatable = {} */
  luaL_newmetatable(L, "Foo");
  meta_id = lua_gettop(L);
  luaL_setfuncs(L, _meta, 0);

  /* metatable.__index = _methods */
  luaL_newlib(L, _methods);
  lua_setfield(L, meta_id, "__index");  

  /* metatable.__metatable = _meta */
  luaL_newlib(L, _meta);
  lua_setfield(L, meta_id, "__metatable");

  /* class.__metatable = metatable */
  lua_setmetatable(L, lib_id);

  /* _G["Foo"] = newclass */
  lua_setglobal(L, "Foo");
}