if not modules then modules = { } end modules ['toks-aux'] = {
    version   = 1.001,
    author    = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
    copyright = "PRAGMA ADE / ConTeXt Development Team",
    license   = "see context related readme files"
}

local type, tostring, next = type, tostring, next
local max = math.max
local formatters, gsub, char, gmatch = string.formatters, string.gsub, string.char, string.gmatch
local concat, swapped = table.concat, table.swapped

local allocate      = utilities.storage.allocate
local texintegerdef = tex.integerdef

do

 -- tex.magicconstants = { -- we use tex.constants for something else
 --     running  = -1073741824, -- null_flag
 --     maxdimen =  1073741823, -- max_dimen
 --  -- trueinch =     4736286, -- obsolete
 -- }

    local c = status.getconstants()
    local t = { }
    for k, v in next, c do
        t[gsub(k,"_","")] = v
    end

    tex.magicconstants = table.setmetatableindex(t,c)

end

do

    -- todo : locals from scanners

    local function flags(t)
        if type(t) == "string" then
            t = token.create(t)
        end
        local r = { }
        if t.frozen    then r[#r+1] = "frozen"    end
        if t.permanent then r[#r+1] = "permanent" end
        if t.immutable then r[#r+1] = "immutable" end
        if t.primitive then r[#r+1] = "primitive" end
        if t.mutable   then r[#r+1] = "mutable"   end
        if t.noaligned then r[#r+1] = "noaligned" end
        if t.instance  then r[#r+1] = "instance"  end
        if t.tolerant  then r[#r+1] = "tolerant"  end
        if t.protected then r[#r+1] = "protected" end
        if t.constant  then r[#r+1] = "constant"  end
        return r
    end

    tokens.flags= flags

    interfaces.implement {
        name      = "showluatokens",
        public    = true,
        protected = true,
        actions   = function()
            local f0 = formatters["%s: %s"]
            local nl = logs.newline
            local wr = logs.writer
            local t  = token.peeknext() -- local t = token.scannext() token.putback(t)
            local n  = ""
            local w  = ""
            local c  = t.cmdname
            local tl = nil
            if c == "left_brace" then
                w = "given token list"
                t = token.scantoks(false)
            elseif c == "register_toks" then
                token.scannext()
                w = "token register"
                n = t.csname or t.index
                t = tex.gettoks(n,true)
            elseif c == "internal_toks" then
                token.scannext()
                w = "internal token variable"
                n = t.csname or t.index
                t = tex.gettoks(n,true)
            elseif t.active then
                token.scannext()
                local r = flags(t)
                local m, l = token.getmeaning("\xEF\xBF\xBF"..t.csname,true,false,true)
                r[#r+1] = "active character"
                if type(m) == "table" then
                    t = m
                    tl = l
                else
                    t = { t }
                end
                w = concat(r, " ")
                n = t.csname
            else
                n = token.scancsname()
                local r = flags(t)
                local m, l = token.getmeaning(n,true,false,true) -- last argument: we want original links
                r[#r+1] = "control sequence"
                if type(m) == "table" then
                    t = m
                    tl = l
                else
                    t = { t }
                end
                w = concat(r, " ")
            end
            wr(f0(w,n))
            nl()
            if type(t) == "table" then
                local w1 = 4
                local w2 = 1
                local w3 = 3
                local w4 = 3
                for i=1,#t do
                    local ti = t[i]
                    local li = tl and tl[i]
                    w1 = max(w1,#tostring(li or ti.id))
                    w2 = max(w2,#tostring(ti.command))
                    w3 = max(w3,#tostring(ti.index))
                    w4 = max(w4,#ti.cmdname)
                end
                local f1 = formatters["%" .. w1 .. "i  %" .. w2 .. "i  %" .. w3 .. "i  %-" .. w4 .. "s  %s"]
                local f2 = formatters["%" .. w1 .. "i  %" .. w2 .. "i  %" .. w3 .. "i  %-" .. w4 .. "s"]
                local f3 = formatters["%" .. w1 .. "i  %" .. w2 .. "i  %" .. w3 .. "i  %-" .. w4 .. "s  %C"]
                for i=1,#t do
                    local ti = t[i]
                    local li = tl and tl[i]
                    local cs = ti.csname
                    local id = li or ti.id
                    local ix = ti.index
                    local cd = ti.command
                    local cn = ti.cmdname
                    if cn == "prefix" and not cs then
                        cs = "always enforced"
                    end
                    cn = gsub(cn,"_"," ")
                    if cs then
                        wr(f1(id,cd,ix,cn,cs))
                    elseif cn == "letter" or cn == "other char" then
                        wr(f3(id,cd,ix,cn,ix))
                    elseif cn == "match" then
                        -- needs checking for additions
                        local s
                            if ix == 32 then s = "optional spacer"     -- space
                        elseif ix == 42 then s = "skip spaces"         -- *
                        elseif ix == 43 then s = "keep braces"         -- +
                        elseif ix == 45 then s = "thrash"              -- -
                        elseif ix == 47 then s = "prune"               -- /
                        elseif ix == 58 then s = "continue"            -- :
                        elseif ix == 59 then s = "quit"                -- ;
                        elseif ix == 61 then s = "mandate braces"      -- =
                        elseif ix == 94 then s = "keep spaces"         -- ^
                        elseif ix == 95 then s = "keep mandate braces" -- _
                        else                 s = "argument " .. char(ix)
                        end
                        wr(f1(id,cd,ix,cn,s))
                    else
                        wr(f2(id,cd,ix,cn))
                        if cn == "end match" then
                            wr("--------------")
                        end
                    end
                end
                nl()
            end
        end
    }

end

-- For the moment here, will move to initex only (also see node-ini.lua); we need
-- to actually store these.

local context = context

local function getthem(getter,post)
    local codes = { }
    if getter then
        for k, v in next, getter() do
            codes[k] = gsub(v,"[_ ]","")
        end
        if post and post ~= "" then
            if environment.initex then
                for k, v in next, codes do
                    texintegerdef(v .. post,k,"immutable")
                end
            end
            interfaces.implement {
                name      = post .. "string",
                public    = true,
                arguments = "integer",
                actions   = function(i)
                    context(codes[i] or "unknown")
                end
            }
        end
    end
    return allocate(swapped(codes,codes))
end

-- For the moment we do it like this:

local function setthem(codes,post)
    for k, v in next, codes do
        texintegerdef(k .. post,v,"immutable")
    end
    return allocate(swapped(codes,codes))
end

tex.userglyphoptioncodes = setthem({
    textcheckitalic = 0x01000000,
    mathcheckitalic = 0x02000000,
},"glyphoptioncode")

tex.userdiscoptioncodes = setthem({
    textcheckitalic = 0x01000000,
},"discoptioncode")

-- Not all of these make sense at the tex end (can't be set).

tex.glyphoptioncodes         = getthem(tex.getglyphoptionvalues,"glyphoptioncode")
tex.discoptioncodes          = getthem(tex.getdiscoptionvalues,"discoptioncode")
tex.ruleoptioncodes          = getthem(tex.getruleoptionvalues,"ruleoptioncode")
tex.glueoptioncodes          = getthem(tex.getglueoptionvalues)
tex.mathoptioncodes          = getthem(tex.getmathoptionvalues)
tex.penaltyoptioncodes       = getthem(tex.getpenaltyoptionvalues)
tex.flagcodes                = getthem(tex.getflagvalues,"flagcode")
tex.frozenparcodes           = getthem(tex.getfrozenparvalues,"frozenparcode")
tex.groupcodes               = getthem(tex.getgroupvalues,"groupcode")
tex.hyphenationcodes         = getthem(tex.gethyphenationvalues,"hyphenationcode")
tex.mathcontrolcodes         = getthem(tex.getmathcontrolvalues,"mathcontrolcode")
---.mathflattencodes         = getthem(tex.getmathflattenvalues,"mathflattencode")
tex.noadoptioncodes          = getthem(tex.getnoadoptionvalues)
tex.normalizelinecodes       = getthem(tex.getnormalizelinevalues,"normalizecode") -- we keep the short name
tex.normalizeparcodes        = getthem(tex.getnormalizeparvalues,"normalizeparcode")
tex.automigrationcodes       = getthem(tex.getautomigrationvalues,"automigrationcode")
tex.autoparagraphcodes       = getthem(tex.getautoparagraphvalues)
tex.parcontextcodes          = getthem(tex.getparcontextvalues,"parcontextcode")
tex.partriggercodes          = getthem(tex.getpartriggervalues,"partriggercode")
tex.textcontrolcodes         = getthem(tex.gettextcontrolvalues)
---.fitnesscodes             = getthem(tex.getfitnessvalues)
---.parpassclasscodes        = getthem(tex.getparpassclassvalues,"parpassclass" )
tex.listanchorcodes          = getthem(tex.getlistanchorvalues,"listanchorcode")
tex.listsigncodes            = getthem(tex.getlistsignvalues,"listsigncode")
tex.listgeometrycodes        = getthem(tex.getlistgeometryvalues)
tex.classoptioncodes         = getthem(tex.getmathclassoptionvalues,"classoptioncode")
---.alignmentcontextcodes    = getthem(tex.getalignmentcontextvalues)
tex.specialmathclasscodes    = getthem(tex.getspecialmathclassvalues)
tex.directioncodes           = getthem(tex.getdirectionvalues)
tex.fillcodes                = getthem(tex.getfillvalues)
tex.mathparametercodes       = getthem(tex.getmathparametervalues)
tex.breakcodes               = getthem(tex.getbreakcontextvalues)
tex.buildcodes               = getthem(tex.getbuildcontextvalues)
tex.splitcodes               = getthem(tex.getvsplitcontextvalues)
tex.interactioncodes         = getthem(tex.getinteractionmodes,"modecode")
tex.charactertagcodes        = getthem(tex.getcharactertagvalues)
tex.protrusionboundarycodes  = getthem(tex.getprotrusionboundaryvalues)
tex.shapingpenaltiescodes    = getthem(tex.getshapingpenaltiesvalues)
tex.mathscriptordercodes     = getthem(tex.getmathscriptordervalues)
tex.mathscriptsmodecodes     = getthem(tex.getmathscriptsmodevalues,"scriptsmodecode")
tex.doublescriptoptioncodes  = getthem(tex.getdoublescriptoptionvalues,"doublescriptmodecode")
tex.charactercontrolcodes    = getthem(tex.getcharactercontrolvalues,"charactercontrolcode")
tex.specificationoptioncodes = getthem(tex.getspecificationoptionvalues,"specificationoptioncode")
tex.discpartcodes            = getthem(tex.getdiscpartvalues)
tex.glyphdisccodes           = getthem(tex.getglyphdiscvalues)
tex.runstatecodes            = getthem(tex.getrunstatevalues)
tex.mvloptioncodes           = getthem(tex.getmvloptionvalues,"mvloptioncode")
tex.balancecallbackcodes     = getthem(tex.getbalancecallbackvalues)
tex.badnessmodecodes         = getthem(tex.getbadnessmodevalues,"badnessmodecode")

local linebreakstatecodes    = getthem(tex.getlinebreakstatevalues)
tex.linebreakstatecodes      = linebreakstatecodes

linebreakstatecodes.text     = linebreakstatecodes.glyph | linebreakstatecodes.disc
linebreakstatecodes.content  = linebreakstatecodes.text  | linebreakstatecodes.math

function tex.stringtocodesbitmap(str,codes)
    local bitmap = 0
    if codes then
        if not str or str == "" then
            -- zero
        elseif str == "all" then
            for k in next, codes do
                if type(k) == "number" then
                    bitmap = bitmap | k
                end
            end
        else
            str = gsub(str,"[_ ]","")
            for s in gmatch(str,"[^%,]+") do
                local b = codes[s]
                if b then
                    bitmap = bitmap | b
                end
            end
        end
    end
    return bitmap
end

if environment.initex then
    for k, v in next, tex.getmathvariantvalues() do
        texintegerdef("math"..v.."stylepreset",tex.getmathvariantpresets(k),"immutable")
    end
end