modules/luci-base/luasrc/http/protocol.lua

   1 -- Copyright 2008 Freifunk Leipzig / Jo-Philipp Wich <jow@openwrt.org>
   2 -- Licensed to the public under the Apache License 2.0.
   3
   4 -- This class contains several functions useful for http message- and content
   5 -- decoding and to retrive form data from raw http messages.
   6 module("luci.http.protocol", package.seeall)
   7
   8 local ltn12 = require("luci.ltn12")
   9
  10 HTTP_MAX_CONTENT      = 1024*8          -- 8 kB maximum content size
  11
  12 -- the "+" sign to " " - and return the decoded string.
  13 function urldecode( str, no_plus )
  14
  15         local function __chrdec( hex )
  16                 return string.char( tonumber( hex, 16 ) )
  17         end
  18
  19         if type(str) == "string" then
  20                 if not no_plus then
  21                         str = str:gsub( "+", " " )
  22                 end
  23
  24                 str = str:gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
  25         end
  26
  27         return str
  28 end
  29
  30 -- from given url or string. Returns a table with urldecoded values.
  31 -- Simple parameters are stored as string values associated with the parameter
  32 -- name within the table. Parameters with multiple values are stored as array
  33 -- containing the corresponding values.
  34 function urldecode_params( url, tbl )
  35
  36         local params = tbl or { }
  37
  38         if url:find("?") then
  39                 url = url:gsub( "^.+%?([^?]+)", "%1" )
  40         end
  41
  42         for pair in url:gmatch( "[^&;]+" ) do
  43
  44                 -- find key and value
  45                 local key = urldecode( pair:match("^([^=]+)")     )
  46                 local val = urldecode( pair:match("^[^=]+=(.+)$") )
  47
  48                 -- store
  49                 if type(key) == "string" and key:len() > 0 then
  50                         if type(val) ~= "string" then val = "" end
  51
  52                         if not params[key] then
  53                                 params[key] = val
  54                         elseif type(params[key]) ~= "table" then
  55                                 params[key] = { params[key], val }
  56                         else
  57                                 table.insert( params[key], val )
  58                         end
  59                 end
  60         end
  61
  62         return params
  63 end
  64
  65 function urlencode( str )
  66
  67         local function __chrenc( chr )
  68                 return string.format(
  69                         "%%%02x", string.byte( chr )
  70                 )
  71         end
  72
  73         if type(str) == "string" then
  74                 str = str:gsub(
  75                         "([^a-zA-Z0-9$_%-%.%~])",
  76                         __chrenc
  77                 )
  78         end
  79
  80         return str
  81 end
  82
  83 -- separated by "&". Tables are encoded as parameters with multiple values by
  84 -- repeating the parameter name with each value.
  85 function urlencode_params( tbl )
  86         local enc = ""
  87
  88         for k, v in pairs(tbl) do
  89                 if type(v) == "table" then
  90                         for i, v2 in ipairs(v) do
  91                                 enc = enc .. ( #enc > 0 and "&" or "" ) ..
  92                                         urlencode(k) .. "=" .. urlencode(v2)
  93                         end
  94                 else
  95                         enc = enc .. ( #enc > 0 and "&" or "" ) ..
  96                                 urlencode(k) .. "=" .. urlencode(v)
  97                 end
  98         end
  99
 100         return enc
 101 end
 102
 103 -- (Internal function)
 104 -- Initialize given parameter and coerce string into table when the parameter
 105 -- already exists.
 106 local function __initval( tbl, key )
 107         if tbl[key] == nil then
 108                 tbl[key] = ""
 109         elseif type(tbl[key]) == "string" then
 110                 tbl[key] = { tbl[key], "" }
 111         else
 112                 table.insert( tbl[key], "" )
 113         end
 114 end
 115
 116 -- (Internal function)
 117 -- Initialize given file parameter.
 118 local function __initfileval( tbl, key, filename, fd )
 119         if tbl[key] == nil then
 120                 tbl[key] = { file=filename, fd=fd, name=key, "" }
 121         else
 122                 table.insert( tbl[key], "" )
 123         end
 124 end
 125
 126 -- (Internal function)
 127 -- Append given data to given parameter, either by extending the string value
 128 -- or by appending it to the last string in the parameter's value table.
 129 local function __appendval( tbl, key, chunk )
 130         if type(tbl[key]) == "table" then
 131                 tbl[key][#tbl[key]] = tbl[key][#tbl[key]] .. chunk
 132         else
 133                 tbl[key] = tbl[key] .. chunk
 134         end
 135 end
 136
 137 -- (Internal function)
 138 -- Finish the value of given parameter, either by transforming the string value
 139 -- or - in the case of multi value parameters - the last element in the
 140 -- associated values table.
 141 local function __finishval( tbl, key, handler )
 142         if handler then
 143                 if type(tbl[key]) == "table" then
 144                         tbl[key][#tbl[key]] = handler( tbl[key][#tbl[key]] )
 145                 else
 146                         tbl[key] = handler( tbl[key] )
 147                 end
 148         end
 149 end
 150
 151
 152 -- Table of our process states
 153 local process_states = { }
 154
 155 -- Extract "magic", the first line of a http message.
 156 -- Extracts the message type ("get", "post" or "response"), the requested uri
 157 -- or the status code if the line descripes a http response.
 158 process_states['magic'] = function( msg, chunk, err )
 159
 160         if chunk ~= nil then
 161                 -- ignore empty lines before request
 162                 if #chunk == 0 then
 163                         return true, nil
 164                 end
 165
 166                 -- Is it a request?
 167                 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
 168
 169                 -- Yup, it is
 170                 if method then
 171
 172                         msg.type           = "request"
 173                         msg.request_method = method:lower()
 174                         msg.request_uri    = uri
 175                         msg.http_version   = tonumber( http_ver )
 176                         msg.headers        = { }
 177
 178                         -- We're done, next state is header parsing
 179                         return true, function( chunk )
 180                                 return process_states['headers']( msg, chunk )
 181                         end
 182
 183                 -- Is it a response?
 184                 else
 185
 186                         local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
 187
 188                         -- Is a response
 189                         if code then
 190
 191                                 msg.type           = "response"
 192                                 msg.status_code    = code
 193                                 msg.status_message = message
 194                                 msg.http_version   = tonumber( http_ver )
 195                                 msg.headers        = { }
 196
 197                                 -- We're done, next state is header parsing
 198                                 return true, function( chunk )
 199                                         return process_states['headers']( msg, chunk )
 200                                 end
 201                         end
 202                 end
 203         end
 204
 205         -- Can't handle it
 206         return nil, "Invalid HTTP message magic"
 207 end
 208
 209
 210 -- Extract headers from given string.
 211 process_states['headers'] = function( msg, chunk )
 212
 213         if chunk ~= nil then
 214
 215                 -- Look for a valid header format
 216                 local hdr, val = chunk:match( "^([A-Za-z][A-Za-z0-9%-_]+): +(.+)$" )
 217
 218                 if type(hdr) == "string" and hdr:len() > 0 and
 219                    type(val) == "string" and val:len() > 0
 220                 then
 221                         msg.headers[hdr] = val
 222
 223                         -- Valid header line, proceed
 224                         return true, nil
 225
 226                 elseif #chunk == 0 then
 227                         -- Empty line, we won't accept data anymore
 228                         return false, nil
 229                 else
 230                         -- Junk data
 231                         return nil, "Invalid HTTP header received"
 232                 end
 233         else
 234                 return nil, "Unexpected EOF"
 235         end
 236 end
 237
 238
 239 -- data line by line with the trailing \r\n stripped of.
 240 function header_source( sock )
 241         return ltn12.source.simplify( function()
 242
 243                 local chunk, err, part = sock:receive("*l")
 244
 245                 -- Line too long
 246                 if chunk == nil then
 247                         if err ~= "timeout" then
 248                                 return nil, part
 249                                         and "Line exceeds maximum allowed length"
 250                                         or  "Unexpected EOF"
 251                         else
 252                                 return nil, err
 253                         end
 254
 255                 -- Line ok
 256                 elseif chunk ~= nil then
 257
 258                         -- Strip trailing CR
 259                         chunk = chunk:gsub("\r$","")
 260
 261                         return chunk, nil
 262                 end
 263         end )
 264 end
 265
 266 -- Content-Type. Stores all extracted data associated with its parameter name
 267 -- in the params table within the given message object. Multiple parameter
 268 -- values are stored as tables, ordinary ones as strings.
 269 -- If an optional file callback function is given then it is feeded with the
 270 -- file contents chunk by chunk and only the extracted file name is stored
 271 -- within the params table. The callback function will be called subsequently
 272 -- with three arguments:
 273 --  o Table containing decoded (name, file) and raw (headers) mime header data
 274 --  o String value containing a chunk of the file data
 275 --  o Boolean which indicates wheather the current chunk is the last one (eof)
 276 function mimedecode_message_body( src, msg, filecb )
 277
 278         if msg and msg.env.CONTENT_TYPE then
 279                 msg.mime_boundary = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)$")
 280         end
 281
 282         if not msg.mime_boundary then
 283                 return nil, "Invalid Content-Type found"
 284         end
 285
 286
 287         local tlen   = 0
 288         local inhdr  = false
 289         local field  = nil
 290         local store  = nil
 291         local lchunk = nil
 292
 293         local function parse_headers( chunk, field )
 294
 295                 local stat
 296                 repeat
 297                         chunk, stat = chunk:gsub(
 298                                 "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n",
 299                                 function(k,v)
 300                                         field.headers[k] = v
 301                                         return ""
 302                                 end
 303                         )
 304                 until stat == 0
 305
 306                 chunk, stat = chunk:gsub("^\r\n","")
 307
 308                 -- End of headers
 309                 if stat > 0 then
 310                         if field.headers["Content-Disposition"] then
 311                                 if field.headers["Content-Disposition"]:match("^form%-data; ") then
 312                                         field.name = field.headers["Content-Disposition"]:match('name="(.-)"')
 313                                         field.file = field.headers["Content-Disposition"]:match('filename="(.+)"$')
 314                                 end
 315                         end
 316
 317                         if not field.headers["Content-Type"] then
 318                                 field.headers["Content-Type"] = "text/plain"
 319                         end
 320
 321                         if field.name and field.file and filecb then
 322                                 __initval( msg.params, field.name )
 323                                 __appendval( msg.params, field.name, field.file )
 324
 325                                 store = filecb
 326                         elseif field.name and field.file then
 327                                 local nxf = require "nixio"
 328                                 local fd = nxf.mkstemp(field.name)
 329                                 __initfileval ( msg.params, field.name, field.file, fd )
 330                                 if fd then
 331                                         store = function(hdr, buf, eof)
 332                                                 fd:write(buf)
 333                                                 if (eof) then
 334                                                         fd:seek(0, "set")
 335                                                 end
 336                                         end
 337                                 else
 338                                         store = function( hdr, buf, eof )
 339                                                 __appendval( msg.params, field.name, buf )
 340                                         end
 341                                 end
 342                         elseif field.name then
 343                                 __initval( msg.params, field.name )
 344
 345                                 store = function( hdr, buf, eof )
 346                                         __appendval( msg.params, field.name, buf )
 347                                 end
 348                         else
 349                                 store = nil
 350                         end
 351
 352                         return chunk, true
 353                 end
 354
 355                 return chunk, false
 356         end
 357
 358         local function snk( chunk )
 359
 360                 tlen = tlen + ( chunk and #chunk or 0 )
 361
 362                 if msg.env.CONTENT_LENGTH and tlen > tonumber(msg.env.CONTENT_LENGTH) + 2 then
 363                         return nil, "Message body size exceeds Content-Length"
 364                 end
 365
 366                 if chunk and not lchunk then
 367                         lchunk = "\r\n" .. chunk
 368
 369                 elseif lchunk then
 370                         local data = lchunk .. ( chunk or "" )
 371                         local spos, epos, found
 372
 373                         repeat
 374                                 spos, epos = data:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
 375
 376                                 if not spos then
 377                                         spos, epos = data:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
 378                                 end
 379
 380
 381                                 if spos then
 382                                         local predata = data:sub( 1, spos - 1 )
 383
 384                                         if inhdr then
 385                                                 predata, eof = parse_headers( predata, field )
 386
 387                                                 if not eof then
 388                                                         return nil, "Invalid MIME section header"
 389                                                 elseif not field.name then
 390                                                         return nil, "Invalid Content-Disposition header"
 391                                                 end
 392                                         end
 393
 394                                         if store then
 395                                                 store( field, predata, true )
 396                                         end
 397
 398
 399                                         field = { headers = { } }
 400                                         found = found or true
 401
 402                                         data, eof = parse_headers( data:sub( epos + 1, #data ), field )
 403                                         inhdr = not eof
 404                                 end
 405                         until not spos
 406
 407                         if found then
 408                                 -- We found at least some boundary. Save
 409                                 -- the unparsed remaining data for the
 410                                 -- next chunk.
 411                                 lchunk, data = data, nil
 412                         else
 413                                 -- There was a complete chunk without a boundary. Parse it as headers or
 414                                 -- append it as data, depending on our current state.
 415                                 if inhdr then
 416                                         lchunk, eof = parse_headers( data, field )
 417                                         inhdr = not eof
 418                                 else
 419                                         -- We're inside data, so append the data. Note that we only append
 420                                         -- lchunk, not all of data, since there is a chance that chunk
 421                                         -- contains half a boundary. Assuming that each chunk is at least the
 422                                         -- boundary in size, this should prevent problems
 423                                         store( field, lchunk, false )
 424                                         lchunk, chunk = chunk, nil
 425                                 end
 426                         end
 427                 end
 428
 429                 return true
 430         end
 431
 432         return ltn12.pump.all( src, snk )
 433 end
 434
 435 -- Content-Type. Stores all extracted data associated with its parameter name
 436 -- in the params table within the given message object. Multiple parameter
 437 -- values are stored as tables, ordinary ones as strings.
 438 function urldecode_message_body( src, msg )
 439
 440         local tlen   = 0
 441         local lchunk = nil
 442
 443         local function snk( chunk )
 444
 445                 tlen = tlen + ( chunk and #chunk or 0 )
 446
 447                 if msg.env.CONTENT_LENGTH and tlen > tonumber(msg.env.CONTENT_LENGTH) + 2 then
 448                         return nil, "Message body size exceeds Content-Length"
 449                 elseif tlen > HTTP_MAX_CONTENT then
 450                         return nil, "Message body size exceeds maximum allowed length"
 451                 end
 452
 453                 if not lchunk and chunk then
 454                         lchunk = chunk
 455
 456                 elseif lchunk then
 457                         local data = lchunk .. ( chunk or "&" )
 458                         local spos, epos
 459
 460                         repeat
 461                                 spos, epos = data:find("^.-[;&]")
 462
 463                                 if spos then
 464                                         local pair = data:sub( spos, epos - 1 )
 465                                         local key  = pair:match("^(.-)=")
 466                                         local val  = pair:match("=([^%s]*)%s*$")
 467
 468                                         if key and #key > 0 then
 469                                                 __initval( msg.params, key )
 470                                                 __appendval( msg.params, key, val )
 471                                                 __finishval( msg.params, key, urldecode )
 472                                         end
 473
 474                                         data = data:sub( epos + 1, #data )
 475                                 end
 476                         until not spos
 477
 478                         lchunk = data
 479                 end
 480
 481                 return true
 482         end
 483
 484         return ltn12.pump.all( src, snk )
 485 end
 486
 487 -- version, message headers and resulting CGI environment variables from the
 488 -- given ltn12 source.
 489 function parse_message_header( src )
 490
 491         local ok   = true
 492         local msg  = { }
 493
 494         local sink = ltn12.sink.simplify(
 495                 function( chunk )
 496                         return process_states['magic']( msg, chunk )
 497                 end
 498         )
 499
 500         -- Pump input data...
 501         while ok do
 502
 503                 -- get data
 504                 ok, err = ltn12.pump.step( src, sink )
 505
 506                 -- error
 507                 if not ok and err then
 508                         return nil, err
 509
 510                 -- eof
 511                 elseif not ok then
 512
 513                         -- Process get parameters
 514                         if ( msg.request_method == "get" or msg.request_method == "post" ) and
 515                            msg.request_uri:match("?")
 516                         then
 517                                 msg.params = urldecode_params( msg.request_uri )
 518                         else
 519                                 msg.params = { }
 520                         end
 521
 522                         -- Populate common environment variables
 523                         msg.env = {
 524                                 CONTENT_LENGTH    = msg.headers['Content-Length'];
 525                                 CONTENT_TYPE      = msg.headers['Content-Type'] or msg.headers['Content-type'];
 526                                 REQUEST_METHOD    = msg.request_method:upper();
 527                                 REQUEST_URI       = msg.request_uri;
 528                                 SCRIPT_NAME       = msg.request_uri:gsub("?.+$","");
 529                                 SCRIPT_FILENAME   = "";         -- XXX implement me
 530                                 SERVER_PROTOCOL   = "HTTP/" .. string.format("%.1f", msg.http_version);
 531                                 QUERY_STRING      = msg.request_uri:match("?")
 532                                         and msg.request_uri:gsub("^.+?","") or ""
 533                         }
 534
 535                         -- Populate HTTP_* environment variables
 536                         for i, hdr in ipairs( {
 537                                 'Accept',
 538                                 'Accept-Charset',
 539                                 'Accept-Encoding',
 540                                 'Accept-Language',
 541                                 'Connection',
 542                                 'Cookie',
 543                                 'Host',
 544                                 'Referer',
 545                                 'User-Agent',
 546                         } ) do
 547                                 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
 548                                 local val = msg.headers[hdr]
 549
 550                                 msg.env[var] = val
 551                         end
 552                 end
 553         end
 554
 555         return msg
 556 end
 557
 558 -- This function will examine the Content-Type within the given message object
 559 -- to select the appropriate content decoder.
 560 -- Currently the application/x-www-urlencoded and application/form-data
 561 -- mime types are supported. If the encountered content encoding can't be
 562 -- handled then the whole message body will be stored unaltered as "content"
 563 -- property within the given message object.
 564 function parse_message_body( src, msg, filecb )
 565         -- Is it multipart/mime ?
 566         if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
 567            msg.env.CONTENT_TYPE:match("^multipart/form%-data")
 568         then
 569
 570                 return mimedecode_message_body( src, msg, filecb )
 571
 572         -- Is it application/x-www-form-urlencoded ?
 573         elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
 574                msg.env.CONTENT_TYPE:match("^application/x%-www%-form%-urlencoded")
 575         then
 576                 return urldecode_message_body( src, msg, filecb )
 577
 578
 579         -- Unhandled encoding
 580         -- If a file callback is given then feed it chunk by chunk, else
 581         -- store whole buffer in message.content
 582         else
 583
 584                 local sink
 585
 586                 -- If we have a file callback then feed it
 587                 if type(filecb) == "function" then
 588                         local meta = {
 589                                 name = "raw",
 590                                 encoding = msg.env.CONTENT_TYPE
 591                         }
 592                         sink = function( chunk )
 593                                 if chunk then
 594                                         return filecb(meta, chunk, false)
 595                                 else
 596                                         return filecb(meta, nil, true)
 597                                 end
 598                         end
 599                 -- ... else append to .content
 600                 else
 601                         msg.content = ""
 602                         msg.content_length = 0
 603
 604                         sink = function( chunk )
 605                                 if chunk then
 606                                         if ( msg.content_length + #chunk ) <= HTTP_MAX_CONTENT then
 607                                                 msg.content        = msg.content        .. chunk
 608                                                 msg.content_length = msg.content_length + #chunk
 609                                                 return true
 610                                         else
 611                                                 return nil, "POST data exceeds maximum allowed length"
 612                                         end
 613                                 end
 614                                 return true
 615                         end
 616                 end
 617
 618                 -- Pump data...
 619                 while true do
 620                         local ok, err = ltn12.pump.step( src, sink )
 621
 622                         if not ok and err then
 623                                 return nil, err
 624                         elseif not ok then -- eof
 625                                 return true
 626                         end
 627                 end
 628
 629                 return true
 630         end
 631 end
 632
 633 statusmsg = {
 634         [200] = "OK",
 635         [206] = "Partial Content",
 636         [301] = "Moved Permanently",
 637         [302] = "Found",
 638         [304] = "Not Modified",
 639         [400] = "Bad Request",
 640         [403] = "Forbidden",
 641         [404] = "Not Found",
 642         [405] = "Method Not Allowed",
 643         [408] = "Request Time-out",
 644         [411] = "Length Required",
 645         [412] = "Precondition Failed",
 646         [416] = "Requested range not satisfiable",
 647         [500] = "Internal Server Error",
 648         [503] = "Server Unavailable",
 649 }