minify/optlex.lua

--[[--------------------------------------------------------------------

  optlex.lua: does lexer-based optimizations

  This file is part of LuaSrcDiet.

  Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net>

  The COPYRIGHT file describes the conditions

  under which this software may be distributed.

  See the ChangeLog for more information.

----------------------------------------------------------------------]]

--[[--------------------------------------------------------------------

-- NOTES:

-- * For more lexer-based optimization ideas, see the TODO items or

--   look at technotes.txt.

-- * TODO: general string delimiter conversion optimizer

-- * TODO: (numbers) warn if overly significant digit

----------------------------------------------------------------------]]

local base = _G

local string = require "string"

local match = string.match

local sub = string.sub

local find = string.find

local rep = string.rep

local print

------------------------------------------------------------------------

-- variables and data structures

------------------------------------------------------------------------

-- error function, can override by setting own function into module

error = base.error

warn = {}                       -- table for warning flags

local stoks, sinfos, stoklns    -- source lists

local is_realtoken = {          -- significant (grammar) tokens

  TK_KEYWORD = true,

  TK_NAME = true,

  TK_NUMBER = true,

  TK_STRING = true,

  TK_LSTRING = true,

  TK_OP = true,

  TK_EOS = true,

}

local is_faketoken = {          -- whitespace (non-grammar) tokens

  TK_COMMENT = true,

  TK_LCOMMENT = true,

  TK_EOL = true,

  TK_SPACE = true,

}

local opt_details               -- for extra information

------------------------------------------------------------------------

-- true if current token is at the start of a line

-- * skips over deleted tokens via recursion

------------------------------------------------------------------------

local function atlinestart(i)

  local tok = stoks[i - 1]

  if i <= 1 or tok == "TK_EOL" then

    return true

  elseif tok == "" then

    return atlinestart(i - 1)

  end

  return false

end

------------------------------------------------------------------------

-- true if current token is at the end of a line

-- * skips over deleted tokens via recursion

------------------------------------------------------------------------

local function atlineend(i)

  local tok = stoks[i + 1]

  if i >= #stoks or tok == "TK_EOL" or tok == "TK_EOS" then

    return true

  elseif tok == "" then

    return atlineend(i + 1)

  end

  return false

end

------------------------------------------------------------------------

-- counts comment EOLs inside a long comment

-- * in order to keep line numbering, EOLs need to be reinserted

------------------------------------------------------------------------

local function commenteols(lcomment)

  local sep = #match(lcomment, "^%-%-%[=*%[")

  local z = sub(lcomment, sep + 1, -(sep - 1))  -- remove delims

  local i, c = 1, 0

  while true do

    local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i)

    if not p then break end     -- if no matches, done

    i = p + 1

    c = c + 1

    if #s > 0 and r ~= s then   -- skip CRLF or LFCR

      i = i + 1

    end

  end

  return c

end

------------------------------------------------------------------------

-- compares two tokens (i, j) and returns the whitespace required

-- * important! see technotes.txt for more information

-- * only two grammar/real tokens are being considered

-- * if "", no separation is needed

-- * if " ", then at least one whitespace (or EOL) is required

------------------------------------------------------------------------

local function checkpair(i, j)

  local match = match

  local t1, t2 = stoks[i], stoks[j]

  --------------------------------------------------------------------

  if t1 == "TK_STRING" or t1 == "TK_LSTRING" or

     t2 == "TK_STRING" or t2 == "TK_LSTRING" then

    return ""

  --------------------------------------------------------------------

  elseif t1 == "TK_OP" or t2 == "TK_OP" then

    if (t1 == "TK_OP" and (t2 == "TK_KEYWORD" or t2 == "TK_NAME")) or

       (t2 == "TK_OP" and (t1 == "TK_KEYWORD" or t1 == "TK_NAME")) then

      return ""

    end

    if t1 == "TK_OP" and t2 == "TK_OP" then

      -- for TK_OP/TK_OP pairs, see notes in technotes.txt

      local op, op2 = sinfos[i], sinfos[j]

      if (match(op, "^%.%.?$") and match(op2, "^%.")) or

         (match(op, "^[~=<>]$") and op2 == "=") or

         (op == "[" and (op2 == "[" or op2 == "=")) then

        return " "

      end

      return ""

    end

    -- "TK_OP" + "TK_NUMBER" case

    local op = sinfos[i]

    if t2 == "TK_OP" then op = sinfos[j] end

    if match(op, "^%.%.?%.?$") then

      return " "

    end

    return ""

  --------------------------------------------------------------------

  else-- "TK_KEYWORD" | "TK_NAME" | "TK_NUMBER" then

    return " "

  --------------------------------------------------------------------

  end

end

------------------------------------------------------------------------

-- repack tokens, removing deletions caused by optimization process

------------------------------------------------------------------------

local function repack_tokens()

  local dtoks, dinfos, dtoklns = {}, {}, {}

  local j = 1

  for i = 1, #stoks do

    local tok = stoks[i]

    if tok ~= "" then

      dtoks[j], dinfos[j], dtoklns[j] = tok, sinfos[i], stoklns[i]

      j = j + 1

    end

  end

  stoks, sinfos, stoklns = dtoks, dinfos, dtoklns

end

------------------------------------------------------------------------

-- number optimization

-- * optimization using string formatting functions is one way of doing

--   this, but here, we consider all cases and handle them separately

--   (possibly an idiotic approach...)

-- * scientific notation being generated is not in canonical form, this

--   may or may not be a bad thing, feedback welcome

-- * note: intermediate portions need to fit into a normal number range

-- * optimizations can be divided based on number patterns:

-- * hexadecimal:

--   (1) no need to remove leading zeros, just skip to (2)

--   (2) convert to integer if size equal or smaller

--       * change if equal size -> lose the 'x' to reduce entropy

--   (3) number is then processed as an integer

--   (4) note: does not make 0[xX] consistent

-- * integer:

--   (1) note: includes anything with trailing ".", ".0", ...

--   (2) remove useless fractional part, if present, e.g. 123.000

--   (3) remove leading zeros, e.g. 000123

--   (4) switch to scientific if shorter, e.g. 123000 -> 123e3

-- * with fraction:

--   (1) split into digits dot digits

--   (2) if no integer portion, take as zero (can omit later)

--   (3) handle degenerate .000 case, after which the fractional part

--       must be non-zero (if zero, it's matched as an integer)

--   (4) remove trailing zeros for fractional portion

--   (5) p.q where p > 0 and q > 0 cannot be shortened any more

--   (6) otherwise p == 0 and the form is .q, e.g. .000123

--   (7) if scientific shorter, convert, e.g. .000123 -> 123e-6

-- * scientific:

--   (1) split into (digits dot digits) [eE] ([+-] digits)

--   (2) if significand has ".", shift it out so it becomes an integer

--   (3) if significand is zero, just use zero

--   (4) remove leading zeros for significand

--   (5) shift out trailing zeros for significand

--   (6) examine exponent and determine which format is best:

--       integer, with fraction, scientific

------------------------------------------------------------------------

local function do_number(i)

  local before = sinfos[i]      -- 'before'

  local z = before              -- working representation

  local y                       -- 'after', if better

  --------------------------------------------------------------------

  if match(z, "^0[xX]") then            -- hexadecimal number

    local v = base.tostring(base.tonumber(z))

    if #v <= #z then

      z = v  -- change to integer, AND continue

    else

      return  -- no change; stick to hex

    end

  end

  --------------------------------------------------------------------

  if match(z, "^%d+%.?0*$") then        -- integer or has useless frac

    z = match(z, "^(%d+)%.?0*$")  -- int portion only

    if z + 0 > 0 then

      z = match(z, "^0*([1-9]%d*)$")  -- remove leading zeros

      local v = #match(z, "0*$")

      local nv = base.tostring(v)

      if v > #nv + 1 then  -- scientific is shorter

        z = sub(z, 1, #z - v).."e"..nv

      end

      y = z

    else

      y = "0"  -- basic zero

    end

  --------------------------------------------------------------------

  elseif not match(z, "[eE]") then      -- number with fraction part

    local p, q = match(z, "^(%d*)%.(%d+)$")  -- split

    if p == "" then p = 0 end  -- int part zero

    if q + 0 == 0 and p == 0 then

      y = "0"  -- degenerate .000 case

    else

      -- now, q > 0 holds and p is a number

      local v = #match(q, "0*$")  -- remove trailing zeros

      if v > 0 then

        q = sub(q, 1, #q - v)

      end

      -- if p > 0, nothing else we can do to simplify p.q case

      if p + 0 > 0 then

        y = p.."."..q

      else

        y = "."..q  -- tentative, e.g. .000123

        local v = #match(q, "^0*")  -- # leading spaces

        local w = #q - v            -- # significant digits

        local nv = base.tostring(#q)

        -- e.g. compare 123e-6 versus .000123

        if w + 2 + #nv < 1 + #q then

          y = sub(q, -w).."e-"..nv

        end

      end

    end

  --------------------------------------------------------------------

  else                                  -- scientific number

    local sig, ex = match(z, "^([^eE]+)[eE]([%+%-]?%d+)$")

    ex = base.tonumber(ex)

    -- if got ".", shift out fractional portion of significand

    local p, q = match(sig, "^(%d*)%.(%d*)$")

    if p then

      ex = ex - #q

      sig = p..q

    end

    if sig + 0 == 0 then

      y = "0"  -- basic zero

    else

      local v = #match(sig, "^0*")  -- remove leading zeros

      sig = sub(sig, v + 1)

      v = #match(sig, "0*$") -- shift out trailing zeros

      if v > 0 then

        sig = sub(sig, 1, #sig - v)

        ex = ex + v

      end

      -- examine exponent and determine which format is best

      local nex = base.tostring(ex)

      if ex == 0 then  -- it's just an integer

        y = sig

      elseif ex > 0 and (ex <= 1 + #nex) then  -- a number

        y = sig..rep("0", ex)

      elseif ex < 0 and (ex >= -#sig) then  -- fraction, e.g. .123

        v = #sig + ex

        y = sub(sig, 1, v).."."..sub(sig, v + 1)

      elseif ex < 0 and (#nex >= -ex - #sig) then

        -- e.g. compare 1234e-5 versus .01234

        -- gives: #sig + 1 + #nex >= 1 + (-ex - #sig) + #sig

        --     -> #nex >= -ex - #sig

        v = -ex - #sig

        y = "."..rep("0", v)..sig

      else  -- non-canonical scientific representation

        y = sig.."e"..ex

      end

    end--if sig

  end

  --------------------------------------------------------------------

  if y and y ~= sinfos[i] then

    if opt_details then

      print("<number> (line "..stoklns[i]..") "..sinfos[i].." -> "..y)

      opt_details = opt_details + 1

    end

    sinfos[i] = y

  end

end

------------------------------------------------------------------------

-- string optimization

-- * note: works on well-formed strings only!

-- * optimizations on characters can be summarized as follows:

--   \a\b\f\n\r\t\v -- no change

--   \\ -- no change

--   \"\' -- depends on delim, other can remove \

--   \[\] -- remove \

--   \<char> -- general escape, remove \

--   \<eol> -- normalize the EOL only

--   \ddd -- if \a\b\f\n\r\t\v, change to latter

--           if other < ascii 32, keep ddd but zap leading zeros

--           if >= ascii 32, translate it into the literal, then also

--                           do escapes for \\,\",\' cases

--   <other> -- no change

-- * switch delimiters if string becomes shorter

------------------------------------------------------------------------

local function do_string(I)

  local info = sinfos[I]

  local delim = sub(info, 1, 1)                 -- delimiter used

  local ndelim = (delim == "'") and '"' or "'"  -- opposite " <-> '

  local z = sub(info, 2, -2)                    -- actual string

  local i = 1

  local c_delim, c_ndelim = 0, 0                -- "/' counts

  --------------------------------------------------------------------

  while i <= #z do

    local c = sub(z, i, i)

    ----------------------------------------------------------------

    if c == "\\" then                   -- escaped stuff

      local j = i + 1

      local d = sub(z, j, j)

      local p = find("abfnrtv\\\n\r\"\'0123456789", d, 1, true)

      ------------------------------------------------------------

      if not p then                     -- \<char> -- remove \

        z = sub(z, 1, i - 1)..sub(z, j)

        i = i + 1

      ------------------------------------------------------------

      elseif p <= 8 then                -- \a\b\f\n\r\t\v\\

        i = i + 2                       -- no change

      ------------------------------------------------------------

      elseif p <= 10 then               -- \<eol> -- normalize EOL

        local eol = sub(z, j, j + 1)

        if eol == "\r\n" or eol == "\n\r" then

          z = sub(z, 1, i).."\n"..sub(z, j + 2)

        elseif p == 10 then  -- \r case

          z = sub(z, 1, i).."\n"..sub(z, j + 1)

        end

        i = i + 2

      ------------------------------------------------------------

      elseif p <= 12 then               -- \"\' -- remove \ for ndelim

        if d == delim then

          c_delim = c_delim + 1

          i = i + 2

        else

          c_ndelim = c_ndelim + 1

          z = sub(z, 1, i - 1)..sub(z, j)

          i = i + 1

        end

      ------------------------------------------------------------

      else                              -- \ddd -- various steps

        local s = match(z, "^(%d%d?%d?)", j)

        j = i + 1 + #s                  -- skip to location

        local cv = s + 0

        local cc = string.char(cv)

        local p = find("\a\b\f\n\r\t\v", cc, 1, true)

        if p then                       -- special escapes

          s = "\\"..sub("abfnrtv", p, p)

        elseif cv < 32 then             -- normalized \ddd

          s = "\\"..cv

        elseif cc == delim then         -- \<delim>

          s = "\\"..cc

          c_delim = c_delim + 1

        elseif cc == "\\" then          -- \\

          s = "\\\\"

        else                            -- literal character

          s = cc

          if cc == ndelim then

            c_ndelim = c_ndelim + 1

          end

        end

        z = sub(z, 1, i - 1)..s..sub(z, j)

        i = i + #s

      ------------------------------------------------------------

      end--if p

    ----------------------------------------------------------------

    else-- c ~= "\\"                    -- <other> -- no change

      i = i + 1

      if c == ndelim then  -- count ndelim, for switching delimiters

        c_ndelim = c_ndelim + 1

      end

    ----------------------------------------------------------------

    end--if c

  end--while

  --------------------------------------------------------------------

  -- switching delimiters, a long-winded derivation:

  -- (1) delim takes 2+2*c_delim bytes, ndelim takes c_ndelim bytes

  -- (2) delim becomes c_delim bytes, ndelim becomes 2+2*c_ndelim bytes

  -- simplifying the condition (1)>(2) --> c_delim > c_ndelim

  if c_delim > c_ndelim then

    i = 1

    while i <= #z do

      local p, q, r = find(z, "([\'\"])", i)

      if not p then break end

      if r == delim then                -- \<delim> -> <delim>

        z = sub(z, 1, p - 2)..sub(z, p)

        i = p

      else-- r == ndelim                -- <ndelim> -> \<ndelim>

        z = sub(z, 1, p - 1).."\\"..sub(z, p)

        i = p + 2

      end

    end--while

    delim = ndelim  -- actually change delimiters

  end

  --------------------------------------------------------------------

  z = delim..z..delim

  if z ~= sinfos[I] then

    if opt_details then

      print("<string> (line "..stoklns[I]..") "..sinfos[I].." -> "..z)

      opt_details = opt_details + 1

    end

    sinfos[I] = z

  end

end

------------------------------------------------------------------------

-- long string optimization

-- * note: warning flagged if trailing whitespace found, not trimmed

-- * remove first optional newline

-- * normalize embedded newlines

-- * reduce '=' separators in delimiters if possible

------------------------------------------------------------------------

local function do_lstring(I)

  local info = sinfos[I]

  local delim1 = match(info, "^%[=*%[")  -- cut out delimiters

  local sep = #delim1

  local delim2 = sub(info, -sep, -1)

  local z = sub(info, sep + 1, -(sep + 1))  -- lstring without delims

  local y = ""

  local i = 1

  --------------------------------------------------------------------

  while true do

    local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i)

    -- deal with a single line

    local ln

    if not p then

      ln = sub(z, i)

    elseif p >= i then

      ln = sub(z, i, p - 1)

    end

    if ln ~= "" then

      -- flag a warning if there are trailing spaces, won't optimize!

      if match(ln, "%s+$") then

        warn.lstring = "trailing whitespace in long string near line "..stoklns[I]

      end

      y = y..ln

    end

    if not p then  -- done if no more EOLs

      break

    end

    -- deal with line endings, normalize them

    i = p + 1

    if p then

      if #s > 0 and r ~= s then  -- skip CRLF or LFCR

        i = i + 1

      end

      -- skip first newline, which can be safely deleted

      if not(i == 1 and i == p) then

        y = y.."\n"

      end

    end

  end--while

  --------------------------------------------------------------------

  -- handle possible deletion of one or more '=' separators

  if sep >= 3 then

    local chk, okay = sep - 1

    -- loop to test ending delimiter with less of '=' down to zero

    while chk >= 2 do

      local delim = "%]"..rep("=", chk - 2).."%]"

      if not match(y, delim) then okay = chk end

      chk = chk - 1

    end

    if okay then  -- change delimiters

      sep = rep("=", okay - 2)

      delim1, delim2 = "["..sep.."[", "]"..sep.."]"

    end

  end

  --------------------------------------------------------------------

  sinfos[I] = delim1..y..delim2

end

------------------------------------------------------------------------

-- long comment optimization

-- * note: does not remove first optional newline

-- * trim trailing whitespace

-- * normalize embedded newlines

-- * reduce '=' separators in delimiters if possible

------------------------------------------------------------------------

local function do_lcomment(I)

  local info = sinfos[I]

  local delim1 = match(info, "^%-%-%[=*%[")  -- cut out delimiters

  local sep = #delim1

  local delim2 = sub(info, -sep, -1)

  local z = sub(info, sep + 1, -(sep - 1))  -- comment without delims

  local y = ""

  local i = 1

  --------------------------------------------------------------------

  while true do

    local p, q, r, s = find(z, "([\r\n])([\r\n]?)", i)

    -- deal with a single line, extract and check trailing whitespace

    local ln

    if not p then

      ln = sub(z, i)

    elseif p >= i then

      ln = sub(z, i, p - 1)

    end

    if ln ~= "" then

      -- trim trailing whitespace if non-empty line

      local ws = match(ln, "%s*$")

      if #ws > 0 then ln = sub(ln, 1, -(ws + 1)) end

      y = y..ln

    end

    if not p then  -- done if no more EOLs

      break

    end

    -- deal with line endings, normalize them

    i = p + 1

    if p then

      if #s > 0 and r ~= s then  -- skip CRLF or LFCR

        i = i + 1

      end

      y = y.."\n"

    end

  end--while

  --------------------------------------------------------------------

  -- handle possible deletion of one or more '=' separators

  sep = sep - 2

  if sep >= 3 then

    local chk, okay = sep - 1

    -- loop to test ending delimiter with less of '=' down to zero

    while chk >= 2 do

      local delim = "%]"..rep("=", chk - 2).."%]"

      if not match(y, delim) then okay = chk end

      chk = chk - 1

    end

    if okay then  -- change delimiters

      sep = rep("=", okay - 2)

      delim1, delim2 = "--["..sep.."[", "]"..sep.."]"

    end

  end

  --------------------------------------------------------------------

  sinfos[I] = delim1..y..delim2

end

------------------------------------------------------------------------

-- short comment optimization

-- * trim trailing whitespace

------------------------------------------------------------------------

local function do_comment(i)

  local info = sinfos[i]

  local ws = match(info, "%s*$")        -- just look from end of string

  if #ws > 0 then

    info = sub(info, 1, -(ws + 1))      -- trim trailing whitespace

  end

  sinfos[i] = info

end

------------------------------------------------------------------------

-- returns true if string found in long comment

-- * this is a feature to keep copyright or license texts

------------------------------------------------------------------------

local function keep_lcomment(opt_keep, info)

  if not opt_keep then return false end  -- option not set

  local delim1 = match(info, "^%-%-%[=*%[")  -- cut out delimiters

  local sep = #delim1

  local delim2 = sub(info, -sep, -1)

  local z = sub(info, sep + 1, -(sep - 1))  -- comment without delims

  if find(z, opt_keep, 1, true) then  -- try to match

    return true

  end

end

------------------------------------------------------------------------

-- main entry point

-- * currently, lexer processing has 2 passes

-- * processing is done on a line-oriented basis, which is easier to

--   grok due to the next point...

-- * since there are various options that can be enabled or disabled,

--   processing is a little messy or convoluted

------------------------------------------------------------------------

function optimize(option, toklist, semlist, toklnlist)

  --------------------------------------------------------------------

  -- set option flags

  --------------------------------------------------------------------

  local opt_comments = option["opt-comments"]

  local opt_whitespace = option["opt-whitespace"]

  local opt_emptylines = option["opt-emptylines"]

  local opt_eols = option["opt-eols"]

  local opt_strings = option["opt-strings"]

  local opt_numbers = option["opt-numbers"]

  local opt_keep = option.KEEP

  opt_details = option.DETAILS and 0  -- upvalues for details display

  print = print or base.print

  if opt_eols then  -- forced settings, otherwise won't work properly

    opt_comments = true

    opt_whitespace = true

    opt_emptylines = true

  end

  --------------------------------------------------------------------

  -- variable initialization

  --------------------------------------------------------------------

  stoks, sinfos, stoklns                -- set source lists

    = toklist, semlist, toklnlist

  local i = 1                           -- token position

  local tok, info                       -- current token

  local prev    -- position of last grammar token

                -- on same line (for TK_SPACE stuff)

  --------------------------------------------------------------------

  -- changes a token, info pair

  --------------------------------------------------------------------

  local function settoken(tok, info, I)

    I = I or i

    stoks[I] = tok or ""

    sinfos[I] = info or ""

  end

  --------------------------------------------------------------------

  -- processing loop (PASS 1)

  --------------------------------------------------------------------

  while true do

    tok, info = stoks[i], sinfos[i]

    ----------------------------------------------------------------

    local atstart = atlinestart(i)      -- set line begin flag

    if atstart then prev = nil end

    ----------------------------------------------------------------

    if tok == "TK_EOS" then             -- end of stream/pass

      break

    ----------------------------------------------------------------

    elseif tok == "TK_KEYWORD" or       -- keywords, identifiers,

           tok == "TK_NAME" or          -- operators

           tok == "TK_OP" then

      -- TK_KEYWORD and TK_OP can't be optimized without a big

      -- optimization framework; it would be more of an optimizing

      -- compiler, not a source code compressor

      -- TK_NAME that are locals needs parser to analyze/optimize

      prev = i

    ----------------------------------------------------------------

    elseif tok == "TK_NUMBER" then      -- numbers

      if opt_numbers then

        do_number(i)  -- optimize

      end

      prev = i

    ----------------------------------------------------------------

    elseif tok == "TK_STRING" or        -- strings, long strings

           tok == "TK_LSTRING" then

      if opt_strings then

        if tok == "TK_STRING" then

          do_string(i)  -- optimize

        else

          do_lstring(i)  -- optimize

        end

      end

      prev = i

    ----------------------------------------------------------------

    elseif tok == "TK_COMMENT" then     -- short comments

      if opt_comments then

        if i == 1 and sub(info, 1, 1) == "#" then

          -- keep shbang comment, trim whitespace

          do_comment(i)

        else

          -- safe to delete, as a TK_EOL (or TK_EOS) always follows

          settoken()  -- remove entirely

        end

      elseif opt_whitespace then        -- trim whitespace only

        do_comment(i)

      end

    ----------------------------------------------------------------

    elseif tok == "TK_LCOMMENT" then    -- long comments

      if keep_lcomment(opt_keep, info) then

        ------------------------------------------------------------

        -- if --keep, we keep a long comment if <msg> is found;

        -- this is a feature to keep copyright or license texts

        if opt_whitespace then          -- trim whitespace only

          do_lcomment(i)

        end

        prev = i

      elseif opt_comments then

        local eols = commenteols(info)

        ------------------------------------------------------------

        -- prepare opt_emptylines case first, if a disposable token

        -- follows, current one is safe to dump, else keep a space;

        -- it is implied that the operation is safe for '-', because

        -- current is a TK_LCOMMENT, and must be separate from a '-'

        if is_faketoken[stoks[i + 1]] then

          settoken()  -- remove entirely

          tok = ""

        else

          settoken("TK_SPACE", " ")

        end

        ------------------------------------------------------------

        -- if there are embedded EOLs to keep and opt_emptylines is

        -- disabled, then switch the token into one or more EOLs

        if not opt_emptylines and eols > 0 then

          settoken("TK_EOL", rep("\n", eols))

        end

        ------------------------------------------------------------

        -- if optimizing whitespaces, force reinterpretation of the

        -- token to give a chance for the space to be optimized away

        if opt_whitespace and tok ~= "" then

          i = i - 1  -- to reinterpret

        end

        ------------------------------------------------------------

      else                              -- disabled case

        if opt_whitespace then          -- trim whitespace only

          do_lcomment(i)

        end

        prev = i

      end

    ----------------------------------------------------------------

    elseif tok == "TK_EOL" then         -- line endings

      if atstart and opt_emptylines then

        settoken()  -- remove entirely

      elseif info == "\r\n" or info == "\n\r" then

        -- normalize the rest of the EOLs for CRLF/LFCR only

        -- (note that TK_LCOMMENT can change into several EOLs)

        settoken("TK_EOL", "\n")

      end

    ----------------------------------------------------------------

    elseif tok == "TK_SPACE" then       -- whitespace

      if opt_whitespace then

        if atstart or atlineend(i) then

          -- delete leading and trailing whitespace

          settoken()  -- remove entirely

        else

          ------------------------------------------------------------

          -- at this point, since leading whitespace have been removed,

          -- there should be a either a real token or a TK_LCOMMENT

          -- prior to hitting this whitespace; the TK_LCOMMENT case

          -- only happens if opt_comments is disabled; so prev ~= nil

          local ptok = stoks[prev]

          if ptok == "TK_LCOMMENT" then

            -- previous TK_LCOMMENT can abut with anything

            settoken()  -- remove entirely

          else

            -- prev must be a grammar token; consecutive TK_SPACE

            -- tokens is impossible when optimizing whitespace

            local ntok = stoks[i + 1]

            if is_faketoken[ntok] then

              -- handle special case where a '-' cannot abut with

              -- either a short comment or a long comment

              if (ntok == "TK_COMMENT" or ntok == "TK_LCOMMENT") and

                 ptok == "TK_OP" and sinfos[prev] == "-" then

                -- keep token

              else

                settoken()  -- remove entirely

              end

            else--is_realtoken

              -- check a pair of grammar tokens, if can abut, then

              -- delete space token entirely, otherwise keep one space

              local s = checkpair(prev, i + 1)

              if s == "" then

                settoken()  -- remove entirely

              else

                settoken("TK_SPACE", " ")

              end

            end

          end

          ------------------------------------------------------------

        end

      end

    ----------------------------------------------------------------

    else

      error("unidentified token encountered")

    end

    ----------------------------------------------------------------

    i = i + 1

  end--while

  repack_tokens()

  --------------------------------------------------------------------

  -- processing loop (PASS 2)

  --------------------------------------------------------------------

  if opt_eols then

    i = 1

    -- aggressive EOL removal only works with most non-grammar tokens

    -- optimized away because it is a rather simple scheme -- basically

    -- it just checks 'real' token pairs around EOLs

    if stoks[1] == "TK_COMMENT" then

      -- first comment still existing must be shbang, skip whole line

      i = 3

    end

    while true do

      tok, info = stoks[i], sinfos[i]

      --------------------------------------------------------------

      if tok == "TK_EOS" then           -- end of stream/pass

        break

      --------------------------------------------------------------

      elseif tok == "TK_EOL" then       -- consider each TK_EOL

        local t1, t2 = stoks[i - 1], stoks[i + 1]

        if is_realtoken[t1] and is_realtoken[t2] then  -- sanity check

          local s = checkpair(i - 1, i + 1)

          if s == "" then

            settoken()  -- remove entirely

          end

        end

      end--if tok

      --------------------------------------------------------------

      i = i + 1

    end--while

    repack_tokens()

  end

  --------------------------------------------------------------------

  if opt_details and opt_details > 0 then print() end -- spacing

  return stoks, sinfos, stoklns

end

return {optimize = optimize}