+++ /dev/null
-#!/usr/bin/env lua
---[[--------------------------------------------------------------------
-
- LuaSrcDiet
- Compresses Lua source code by removing unnecessary characters.
- For Lua 5.1.x source code.
-
- Copyright (c) 2008 Kein-Hong Man <khman@users.sf.net>
- The COPYRIGHT file describes the conditions
- under which this software may be distributed.
-
- See the ChangeLog for more information.
-
-----------------------------------------------------------------------]]
-
---[[--------------------------------------------------------------------
--- NOTES:
--- * Remember to update version and date information below (MSG_TITLE)
--- * TODO: to implement pcall() to properly handle lexer etc. errors
--- * TODO: verify token stream or double-check binary chunk?
--- * TODO: need some automatic testing for a semblance of sanity
--- * TODO: the plugin module is highly experimental and unstable
-----------------------------------------------------------------------]]
-
--- standard libraries, functions
-local string = string
-local math = math
-local table = table
-local require = require
-local print = print
-local sub = string.sub
-local gmatch = string.gmatch
-
--- support modules
-local llex = require "llex"
-local lparser = require "lparser"
-local optlex = require "optlex"
-local optparser = require "optparser"
-local plugin
-
---[[--------------------------------------------------------------------
--- messages and textual data
-----------------------------------------------------------------------]]
-
-local MSG_TITLE = [[
-LuaSrcDiet: Puts your Lua 5.1 source code on a diet
-Version 0.11.2 (20080608) Copyright (c) 2005-2008 Kein-Hong Man
-The COPYRIGHT file describes the conditions under which this
-software may be distributed.
-]]
-
-local MSG_USAGE = [[
-usage: LuaSrcDiet [options] [filenames]
-
-example:
- >LuaSrcDiet myscript.lua -o myscript_.lua
-
-options:
- -v, --version prints version information
- -h, --help prints usage information
- -o <file> specify file name to write output
- -s <suffix> suffix for output files (default '_')
- --keep <msg> keep block comment with <msg> inside
- --plugin <module> run <module> in plugin/ directory
- - stop handling arguments
-
- (optimization levels)
- --none all optimizations off (normalizes EOLs only)
- --basic lexer-based optimizations only
- --maximum maximize reduction of source
-
- (informational)
- --quiet process files quietly
- --read-only read file and print token stats only
- --dump-lexer dump raw tokens from lexer to stdout
- --dump-parser dump variable tracking tables from parser
- --details extra info (strings, numbers, locals)
-
-features (to disable, insert 'no' prefix like --noopt-comments):
-%s
-default settings:
-%s]]
-
-------------------------------------------------------------------------
--- optimization options, for ease of switching on and off
--- * positive to enable optimization, negative (no) to disable
--- * these options should follow --opt-* and --noopt-* style for now
-------------------------------------------------------------------------
-
-local OPTION = [[
---opt-comments,'remove comments and block comments'
---opt-whitespace,'remove whitespace excluding EOLs'
---opt-emptylines,'remove empty lines'
---opt-eols,'all above, plus remove unnecessary EOLs'
---opt-strings,'optimize strings and long strings'
---opt-numbers,'optimize numbers'
---opt-locals,'optimize local variable names'
---opt-entropy,'tries to reduce symbol entropy of locals'
-]]
-
--- preset configuration
-local DEFAULT_CONFIG = [[
- --opt-comments --opt-whitespace --opt-emptylines
- --opt-numbers --opt-locals
-]]
--- override configurations: MUST explicitly enable/disable everything
-local BASIC_CONFIG = [[
- --opt-comments --opt-whitespace --opt-emptylines
- --noopt-eols --noopt-strings --noopt-numbers
- --noopt-locals
-]]
-local MAXIMUM_CONFIG = [[
- --opt-comments --opt-whitespace --opt-emptylines
- --opt-eols --opt-strings --opt-numbers
- --opt-locals --opt-entropy
-]]
-local NONE_CONFIG = [[
- --noopt-comments --noopt-whitespace --noopt-emptylines
- --noopt-eols --noopt-strings --noopt-numbers
- --noopt-locals
-]]
-
-local DEFAULT_SUFFIX = "_" -- default suffix for file renaming
-local PLUGIN_SUFFIX = "plugin/" -- relative location of plugins
-
---[[--------------------------------------------------------------------
--- startup and initialize option list handling
-----------------------------------------------------------------------]]
-
--- simple error message handler; change to error if traceback wanted
-local function die(msg)
- print("LuaSrcDiet: "..msg); os.exit()
-end
---die = error--DEBUG
-
-if not string.match(_VERSION, "5.1", 1, 1) then -- sanity check
- die("requires Lua 5.1 to run")
-end
-
-------------------------------------------------------------------------
--- prepares text for list of optimizations, prepare lookup table
-------------------------------------------------------------------------
-
-local MSG_OPTIONS = ""
-do
- local WIDTH = 24
- local o = {}
- for op, desc in gmatch(OPTION, "%s*([^,]+),'([^']+)'") do
- local msg = " "..op
- msg = msg..string.rep(" ", WIDTH - #msg)..desc.."\n"
- MSG_OPTIONS = MSG_OPTIONS..msg
- o[op] = true
- o["--no"..sub(op, 3)] = true
- end
- OPTION = o -- replace OPTION with lookup table
-end
-
-MSG_USAGE = string.format(MSG_USAGE, MSG_OPTIONS, DEFAULT_CONFIG)
-
-------------------------------------------------------------------------
--- global variable initialization, option set handling
-------------------------------------------------------------------------
-
-local suffix = DEFAULT_SUFFIX -- file suffix
-local option = {} -- program options
-local stat_c, stat_l -- statistics tables
-
--- function to set option lookup table based on a text list of options
--- note: additional forced settings for --opt-eols is done in optlex.lua
-local function set_options(CONFIG)
- for op in gmatch(CONFIG, "(%-%-%S+)") do
- if sub(op, 3, 4) == "no" and -- handle negative options
- OPTION["--"..sub(op, 5)] then
- option[sub(op, 5)] = false
- else
- option[sub(op, 3)] = true
- end
- end
-end
-
---[[--------------------------------------------------------------------
--- support functions
-----------------------------------------------------------------------]]
-
--- list of token types, parser-significant types are up to TTYPE_GRAMMAR
--- while the rest are not used by parsers; arranged for stats display
-local TTYPES = {
- "TK_KEYWORD", "TK_NAME", "TK_NUMBER", -- grammar
- "TK_STRING", "TK_LSTRING", "TK_OP",
- "TK_EOS",
- "TK_COMMENT", "TK_LCOMMENT", -- non-grammar
- "TK_EOL", "TK_SPACE",
-}
-local TTYPE_GRAMMAR = 7
-
-local EOLTYPES = { -- EOL names for token dump
- ["\n"] = "LF", ["\r"] = "CR",
- ["\n\r"] = "LFCR", ["\r\n"] = "CRLF",
-}
-
-------------------------------------------------------------------------
--- read source code from file
-------------------------------------------------------------------------
-
-local function load_file(fname)
- local INF = io.open(fname, "rb")
- if not INF then die("cannot open \""..fname.."\" for reading") end
- local dat = INF:read("*a")
- if not dat then die("cannot read from \""..fname.."\"") end
- INF:close()
- return dat
-end
-
-------------------------------------------------------------------------
--- save source code to file
-------------------------------------------------------------------------
-
-local function save_file(fname, dat)
- local OUTF = io.open(fname, "wb")
- if not OUTF then die("cannot open \""..fname.."\" for writing") end
- local status = OUTF:write(dat)
- if not status then die("cannot write to \""..fname.."\"") end
- OUTF:close()
-end
-
-------------------------------------------------------------------------
--- functions to deal with statistics
-------------------------------------------------------------------------
-
--- initialize statistics table
-local function stat_init()
- stat_c, stat_l = {}, {}
- for i = 1, #TTYPES do
- local ttype = TTYPES[i]
- stat_c[ttype], stat_l[ttype] = 0, 0
- end
-end
-
--- add a token to statistics table
-local function stat_add(tok, seminfo)
- stat_c[tok] = stat_c[tok] + 1
- stat_l[tok] = stat_l[tok] + #seminfo
-end
-
--- do totals for statistics table, return average table
-local function stat_calc()
- local function avg(c, l) -- safe average function
- if c == 0 then return 0 end
- return l / c
- end
- local stat_a = {}
- local c, l = 0, 0
- for i = 1, TTYPE_GRAMMAR do -- total grammar tokens
- local ttype = TTYPES[i]
- c = c + stat_c[ttype]; l = l + stat_l[ttype]
- end
- stat_c.TOTAL_TOK, stat_l.TOTAL_TOK = c, l
- stat_a.TOTAL_TOK = avg(c, l)
- c, l = 0, 0
- for i = 1, #TTYPES do -- total all tokens
- local ttype = TTYPES[i]
- c = c + stat_c[ttype]; l = l + stat_l[ttype]
- stat_a[ttype] = avg(stat_c[ttype], stat_l[ttype])
- end
- stat_c.TOTAL_ALL, stat_l.TOTAL_ALL = c, l
- stat_a.TOTAL_ALL = avg(c, l)
- return stat_a
-end
-
---[[--------------------------------------------------------------------
--- main tasks
-----------------------------------------------------------------------]]
-
-------------------------------------------------------------------------
--- a simple token dumper, minimal translation of seminfo data
-------------------------------------------------------------------------
-
-local function dump_tokens(srcfl)
- --------------------------------------------------------------------
- -- load file and process source input into tokens
- --------------------------------------------------------------------
- local z = load_file(srcfl)
- llex.init(z)
- llex.llex()
- local toklist, seminfolist = llex.tok, llex.seminfo
- --------------------------------------------------------------------
- -- display output
- --------------------------------------------------------------------
- for i = 1, #toklist do
- local tok, seminfo = toklist[i], seminfolist[i]
- if tok == "TK_OP" and string.byte(seminfo) < 32 then
- seminfo = "(".. string.byte(seminfo)..")"
- elseif tok == "TK_EOL" then
- seminfo = EOLTYPES[seminfo]
- else
- seminfo = "'"..seminfo.."'"
- end
- print(tok.." "..seminfo)
- end--for
-end
-
-----------------------------------------------------------------------
--- parser dump; dump globalinfo and localinfo tables
-----------------------------------------------------------------------
-
-local function dump_parser(srcfl)
- local print = print
- --------------------------------------------------------------------
- -- load file and process source input into tokens
- --------------------------------------------------------------------
- local z = load_file(srcfl)
- llex.init(z)
- llex.llex()
- local toklist, seminfolist, toklnlist
- = llex.tok, llex.seminfo, llex.tokln
- --------------------------------------------------------------------
- -- do parser optimization here
- --------------------------------------------------------------------
- lparser.init(toklist, seminfolist, toklnlist)
- local globalinfo, localinfo = lparser.parser()
- --------------------------------------------------------------------
- -- display output
- --------------------------------------------------------------------
- local hl = string.rep("-", 72)
- print("*** Local/Global Variable Tracker Tables ***")
- print(hl.."\n GLOBALS\n"..hl)
- -- global tables have a list of xref numbers only
- for i = 1, #globalinfo do
- local obj = globalinfo[i]
- local msg = "("..i..") '"..obj.name.."' -> "
- local xref = obj.xref
- for j = 1, #xref do msg = msg..xref[j].." " end
- print(msg)
- end
- -- local tables have xref numbers and a few other special
- -- numbers that are specially named: decl (declaration xref),
- -- act (activation xref), rem (removal xref)
- print(hl.."\n LOCALS (decl=declared act=activated rem=removed)\n"..hl)
- for i = 1, #localinfo do
- local obj = localinfo[i]
- local msg = "("..i..") '"..obj.name.."' decl:"..obj.decl..
- " act:"..obj.act.." rem:"..obj.rem
- if obj.isself then
- msg = msg.." isself"
- end
- msg = msg.." -> "
- local xref = obj.xref
- for j = 1, #xref do msg = msg..xref[j].." " end
- print(msg)
- end
- print(hl.."\n")
-end
-
-------------------------------------------------------------------------
--- reads source file(s) and reports some statistics
-------------------------------------------------------------------------
-
-local function read_only(srcfl)
- local print = print
- --------------------------------------------------------------------
- -- load file and process source input into tokens
- --------------------------------------------------------------------
- local z = load_file(srcfl)
- llex.init(z)
- llex.llex()
- local toklist, seminfolist = llex.tok, llex.seminfo
- print(MSG_TITLE)
- print("Statistics for: "..srcfl.."\n")
- --------------------------------------------------------------------
- -- collect statistics
- --------------------------------------------------------------------
- stat_init()
- for i = 1, #toklist do
- local tok, seminfo = toklist[i], seminfolist[i]
- stat_add(tok, seminfo)
- end--for
- local stat_a = stat_calc()
- --------------------------------------------------------------------
- -- display output
- --------------------------------------------------------------------
- local fmt = string.format
- local function figures(tt)
- return stat_c[tt], stat_l[tt], stat_a[tt]
- end
- local tabf1, tabf2 = "%-16s%8s%8s%10s", "%-16s%8d%8d%10.2f"
- local hl = string.rep("-", 42)
- print(fmt(tabf1, "Lexical", "Input", "Input", "Input"))
- print(fmt(tabf1, "Elements", "Count", "Bytes", "Average"))
- print(hl)
- for i = 1, #TTYPES do
- local ttype = TTYPES[i]
- print(fmt(tabf2, ttype, figures(ttype)))
- if ttype == "TK_EOS" then print(hl) end
- end
- print(hl)
- print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
- print(hl)
- print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
- print(hl.."\n")
-end
-
-------------------------------------------------------------------------
--- process source file(s), write output and reports some statistics
-------------------------------------------------------------------------
-
-local function process_file(srcfl, destfl)
- local function print(...) -- handle quiet option
- if option.QUIET then return end
- _G.print(...)
- end
- if plugin and plugin.init then -- plugin init
- option.EXIT = false
- plugin.init(option, srcfl, destfl)
- if option.EXIT then return end
- end
- print(MSG_TITLE) -- title message
- --------------------------------------------------------------------
- -- load file and process source input into tokens
- --------------------------------------------------------------------
- local z = load_file(srcfl)
- if plugin and plugin.post_load then -- plugin post-load
- z = plugin.post_load(z) or z
- if option.EXIT then return end
- end
- llex.init(z)
- llex.llex()
- local toklist, seminfolist, toklnlist
- = llex.tok, llex.seminfo, llex.tokln
- if plugin and plugin.post_lex then -- plugin post-lex
- plugin.post_lex(toklist, seminfolist, toklnlist)
- if option.EXIT then return end
- end
- --------------------------------------------------------------------
- -- collect 'before' statistics
- --------------------------------------------------------------------
- stat_init()
- for i = 1, #toklist do
- local tok, seminfo = toklist[i], seminfolist[i]
- stat_add(tok, seminfo)
- end--for
- local stat1_a = stat_calc()
- local stat1_c, stat1_l = stat_c, stat_l
- --------------------------------------------------------------------
- -- do parser optimization here
- --------------------------------------------------------------------
- if option["opt-locals"] then
- optparser.print = print -- hack
- lparser.init(toklist, seminfolist, toklnlist)
- local globalinfo, localinfo = lparser.parser()
- if plugin and plugin.post_parse then -- plugin post-parse
- plugin.post_parse(globalinfo, localinfo)
- if option.EXIT then return end
- end
- optparser.optimize(option, toklist, seminfolist, globalinfo, localinfo)
- if plugin and plugin.post_optparse then -- plugin post-optparse
- plugin.post_optparse()
- if option.EXIT then return end
- end
- end
- --------------------------------------------------------------------
- -- do lexer optimization here, save output file
- --------------------------------------------------------------------
- optlex.print = print -- hack
- toklist, seminfolist, toklnlist
- = optlex.optimize(option, toklist, seminfolist, toklnlist)
- if plugin and plugin.post_optlex then -- plugin post-optlex
- plugin.post_optlex(toklist, seminfolist, toklnlist)
- if option.EXIT then return end
- end
- local dat = table.concat(seminfolist)
- -- depending on options selected, embedded EOLs in long strings and
- -- long comments may not have been translated to \n, tack a warning
- if string.find(dat, "\r\n", 1, 1) or
- string.find(dat, "\n\r", 1, 1) then
- optlex.warn.mixedeol = true
- end
- -- save optimized source stream to output file
- save_file(destfl, dat)
- --------------------------------------------------------------------
- -- collect 'after' statistics
- --------------------------------------------------------------------
- stat_init()
- for i = 1, #toklist do
- local tok, seminfo = toklist[i], seminfolist[i]
- stat_add(tok, seminfo)
- end--for
- local stat_a = stat_calc()
- --------------------------------------------------------------------
- -- display output
- --------------------------------------------------------------------
- print("Statistics for: "..srcfl.." -> "..destfl.."\n")
- local fmt = string.format
- local function figures(tt)
- return stat1_c[tt], stat1_l[tt], stat1_a[tt],
- stat_c[tt], stat_l[tt], stat_a[tt]
- end
- local tabf1, tabf2 = "%-16s%8s%8s%10s%8s%8s%10s",
- "%-16s%8d%8d%10.2f%8d%8d%10.2f"
- local hl = string.rep("-", 68)
- print("*** lexer-based optimizations summary ***\n"..hl)
- print(fmt(tabf1, "Lexical",
- "Input", "Input", "Input",
- "Output", "Output", "Output"))
- print(fmt(tabf1, "Elements",
- "Count", "Bytes", "Average",
- "Count", "Bytes", "Average"))
- print(hl)
- for i = 1, #TTYPES do
- local ttype = TTYPES[i]
- print(fmt(tabf2, ttype, figures(ttype)))
- if ttype == "TK_EOS" then print(hl) end
- end
- print(hl)
- print(fmt(tabf2, "Total Elements", figures("TOTAL_ALL")))
- print(hl)
- print(fmt(tabf2, "Total Tokens", figures("TOTAL_TOK")))
- print(hl)
- --------------------------------------------------------------------
- -- report warning flags from optimizing process
- --------------------------------------------------------------------
- if optlex.warn.lstring then
- print("* WARNING: "..optlex.warn.lstring)
- elseif optlex.warn.mixedeol then
- print("* WARNING: ".."output still contains some CRLF or LFCR line endings")
- end
- print()
-end
-
---[[--------------------------------------------------------------------
--- main functions
-----------------------------------------------------------------------]]
-
-local arg = {...} -- program arguments
-local fspec = {}
-set_options(DEFAULT_CONFIG) -- set to default options at beginning
-
-------------------------------------------------------------------------
--- per-file handling, ship off to tasks
-------------------------------------------------------------------------
-
-local function do_files(fspec)
- for _, srcfl in ipairs(fspec) do
- local destfl
- ------------------------------------------------------------------
- -- find and replace extension for filenames
- ------------------------------------------------------------------
- local extb, exte = string.find(srcfl, "%.[^%.%\\%/]*$")
- local basename, extension = srcfl, ""
- if extb and extb > 1 then
- basename = sub(srcfl, 1, extb - 1)
- extension = sub(srcfl, extb, exte)
- end
- destfl = basename..suffix..extension
- if #fspec == 1 and option.OUTPUT_FILE then
- destfl = option.OUTPUT_FILE
- end
- if srcfl == destfl then
- die("output filename identical to input filename")
- end
- ------------------------------------------------------------------
- -- perform requested operations
- ------------------------------------------------------------------
- if option.DUMP_LEXER then
- dump_tokens(srcfl)
- elseif option.DUMP_PARSER then
- dump_parser(srcfl)
- elseif option.READ_ONLY then
- read_only(srcfl)
- else
- process_file(srcfl, destfl)
- end
- end--for
-end
-
-------------------------------------------------------------------------
--- main function (entry point is after this definition)
-------------------------------------------------------------------------
-
-local function main()
- local argn, i = #arg, 1
- if argn == 0 then
- option.HELP = true
- end
- --------------------------------------------------------------------
- -- handle arguments
- --------------------------------------------------------------------
- while i <= argn do
- local o, p = arg[i], arg[i + 1]
- local dash = string.match(o, "^%-%-?")
- if dash == "-" then -- single-dash options
- if o == "-h" then
- option.HELP = true; break
- elseif o == "-v" then
- option.VERSION = true; break
- elseif o == "-s" then
- if not p then die("-s option needs suffix specification") end
- suffix = p
- i = i + 1
- elseif o == "-o" then
- if not p then die("-o option needs a file name") end
- option.OUTPUT_FILE = p
- i = i + 1
- elseif o == "-" then
- break -- ignore rest of args
- else
- die("unrecognized option "..o)
- end
- elseif dash == "--" then -- double-dash options
- if o == "--help" then
- option.HELP = true; break
- elseif o == "--version" then
- option.VERSION = true; break
- elseif o == "--keep" then
- if not p then die("--keep option needs a string to match for") end
- option.KEEP = p
- i = i + 1
- elseif o == "--plugin" then
- if not p then die("--plugin option needs a module name") end
- if option.PLUGIN then die("only one plugin can be specified") end
- option.PLUGIN = p
- plugin = require(PLUGIN_SUFFIX..p)
- i = i + 1
- elseif o == "--quiet" then
- option.QUIET = true
- elseif o == "--read-only" then
- option.READ_ONLY = true
- elseif o == "--basic" then
- set_options(BASIC_CONFIG)
- elseif o == "--maximum" then
- set_options(MAXIMUM_CONFIG)
- elseif o == "--none" then
- set_options(NONE_CONFIG)
- elseif o == "--dump-lexer" then
- option.DUMP_LEXER = true
- elseif o == "--dump-parser" then
- option.DUMP_PARSER = true
- elseif o == "--details" then
- option.DETAILS = true
- elseif OPTION[o] then -- lookup optimization options
- set_options(o)
- else
- die("unrecognized option "..o)
- end
- else
- fspec[#fspec + 1] = o -- potential filename
- end
- i = i + 1
- end--while
- if option.HELP then
- print(MSG_TITLE..MSG_USAGE); return true
- elseif option.VERSION then
- print(MSG_TITLE); return true
- end
- if #fspec > 0 then
- if #fspec > 1 and option.OUTPUT_FILE then
- die("with -o, only one source file can be specified")
- end
- do_files(fspec)
- return true
- else
- die("nothing to do!")
- end
-end
-
--- entry point -> main() -> do_files()
-if not main() then
- die("Please run with option -h or --help for usage information")
-end
-
--- end of script
projects / project / luci.git / blobdiff