libs/http/luasrc/http/protocol.lua

   1 --[[
   2
   3 HTTP protocol implementation for LuCI
   4 (c) 2008 Freifunk Leipzig / Jo-Philipp Wich <xm@leipzig.freifunk.net>
   5
   6 Licensed under the Apache License, Version 2.0 (the "License");
   7 you may not use this file except in compliance with the License.
   8 You may obtain a copy of the License at
   9
  10         http://www.apache.org/licenses/LICENSE-2.0
  11
  12 $Id$
  13
  14 ]]--
  15
  16 module("luci.http.protocol", package.seeall)
  17
  18 require("ltn12")
  19 require("luci.util")
  20
  21 HTTP_MAX_CONTENT      = 1024*4          -- 4 kB maximum content size
  22 HTTP_URLENC_MAXKEYLEN = 1024            -- maximum allowd size of urlencoded parameter names
  23
  24
  25 -- Decode an urlencoded string.
  26 -- Returns the decoded value.
  27 function urldecode( str )
  28
  29         local function __chrdec( hex )
  30                 return string.char( tonumber( hex, 16 ) )
  31         end
  32
  33         if type(str) == "string" then
  34                 str = str:gsub( "+", " " ):gsub( "%%([a-fA-F0-9][a-fA-F0-9])", __chrdec )
  35         end
  36
  37         return str
  38 end
  39
  40
  41 -- Extract and split urlencoded data pairs, separated bei either "&" or ";" from given url.
  42 -- Returns a table value with urldecoded values.
  43 function urldecode_params( url, tbl )
  44
  45         local params = tbl or { }
  46
  47         if url:find("?") then
  48                 url = url:gsub( "^.+%?([^?]+)", "%1" )
  49         end
  50
  51         for i, pair in ipairs(luci.util.split( url, "[&;]+", nil, true )) do
  52
  53                 -- find key and value
  54                 local key = urldecode( pair:match("^([^=]+)")     )
  55                 local val = urldecode( pair:match("^[^=]+=(.+)$") )
  56
  57                 -- store
  58                 if type(key) == "string" and key:len() > 0 then
  59                         if type(val) ~= "string" then val = "" end
  60
  61                         if not params[key] then
  62                                 params[key] = val
  63                         elseif type(params[key]) ~= "table" then
  64                                 params[key] = { params[key], val }
  65                         else
  66                                 table.insert( params[key], val )
  67                         end
  68                 end
  69         end
  70
  71         return params
  72 end
  73
  74
  75 -- Encode given string in urlencoded format.
  76 -- Returns the encoded string.
  77 function urlencode( str )
  78
  79         local function __chrenc( chr )
  80                 return string.format(
  81                         "%%%02x", string.byte( chr )
  82                 )
  83         end
  84
  85         if type(str) == "string" then
  86                 str = str:gsub(
  87                         "([^a-zA-Z0-9$_%-%.+!*'(),])",
  88                         __chrenc
  89                 )
  90         end
  91
  92         return str
  93 end
  94
  95
  96 -- Encode given table to urlencoded string.
  97 -- Returns the encoded string.
  98 function urlencode_params( tbl )
  99         local enc = ""
 100
 101         for k, v in pairs(tbl) do
 102                 enc = enc .. ( enc and "&" or "" ) ..
 103                         urlencode(k) .. "="  ..
 104                         urlencode(v)
 105         end
 106
 107         return enc
 108 end
 109
 110
 111 -- Table of our process states
 112 local process_states = { }
 113
 114 -- Extract "magic", the first line of a http message.
 115 -- Extracts the message type ("get", "post" or "response"), the requested uri
 116 -- or the status code if the line descripes a http response.
 117 process_states['magic'] = function( msg, chunk )
 118
 119         if chunk ~= nil then
 120
 121                 -- Is it a request?
 122                 local method, uri, http_ver = chunk:match("^([A-Z]+) ([^ ]+) HTTP/([01]%.[019])$")
 123
 124                 -- Yup, it is
 125                 if method then
 126
 127                         msg.type           = "request"
 128                         msg.request_method = method:lower()
 129                         msg.request_uri    = uri
 130                         msg.http_version   = http_ver
 131                         msg.headers        = { }
 132
 133                         -- We're done, next state is header parsing
 134                         return true, function( chunk )
 135                                 return process_states['headers']( msg, chunk )
 136                         end
 137
 138                 -- Is it a response?
 139                 else
 140
 141                         local http_ver, code, message = chunk:match("^HTTP/([01]%.[019]) ([0-9]+) ([^\r\n]+)$")
 142
 143                         -- Is a response
 144                         if code then
 145
 146                                 msg.type           = "response"
 147                                 msg.status_code    = code
 148                                 msg.status_message = message
 149                                 msg.http_version   = http_ver
 150                                 msg.headers        = { }
 151
 152                                 -- We're done, next state is header parsing
 153                                 return true, function( chunk )
 154                                         return process_states['headers']( msg, chunk )
 155                                 end
 156                         end
 157                 end
 158         end
 159
 160         -- Can't handle it
 161         return nil, "Invalid HTTP message magic"
 162 end
 163
 164
 165 -- Extract headers from given string.
 166 process_states['headers'] = function( msg, chunk )
 167
 168         if chunk ~= nil then
 169
 170                 -- Look for a valid header format
 171                 local hdr, val = chunk:match( "^([A-Z][A-Za-z0-9%-_]+): +(.+)$" )
 172
 173                 if type(hdr) == "string" and hdr:len() > 0 and
 174                    type(val) == "string" and val:len() > 0
 175                 then
 176                         msg.headers[hdr] = val
 177
 178                         -- Valid header line, proceed
 179                         return true, nil
 180
 181                 elseif #chunk == 0 then
 182                         -- Empty line, we won't accept data anymore
 183                         return false, nil
 184                 else
 185                         -- Junk data
 186                         return nil, "Invalid HTTP header received"
 187                 end
 188         else
 189                 return nil, "Unexpected EOF"
 190         end
 191 end
 192
 193
 194 -- Find first MIME boundary
 195 process_states['mime-init'] = function( msg, chunk, filecb )
 196
 197         if chunk ~= nil then
 198                 if #chunk >= #msg.mime_boundary + 2 then
 199                         local boundary = chunk:sub( 1, #msg.mime_boundary + 4 )
 200
 201                         if boundary == "--" .. msg.mime_boundary .. "\r\n" then
 202
 203                                 -- Store remaining data in buffer
 204                                 msg._mimebuffer = chunk:sub( #msg.mime_boundary + 5, #chunk )
 205
 206                                 -- Switch to header processing state
 207                                 return true, function( chunk )
 208                                         return process_states['mime-headers']( msg, chunk, filecb )
 209                                 end
 210                         else
 211                                 return nil, "Invalid MIME boundary"
 212                         end
 213                 else
 214                         return true
 215                 end
 216         else
 217                 return nil, "Unexpected EOF"
 218         end
 219 end
 220
 221
 222 -- Read MIME part headers
 223 process_states['mime-headers'] = function( msg, chunk, filecb )
 224
 225         if chunk ~= nil then
 226
 227                 -- Combine look-behind buffer with current chunk
 228                 chunk = msg._mimebuffer .. chunk
 229
 230                 if not msg._mimeheaders then
 231                         msg._mimeheaders = { }
 232                 end
 233
 234                 local function __storehdr( k, v )
 235                         msg._mimeheaders[k] = v
 236                         return ""
 237                 end
 238
 239                 -- Read all header lines
 240                 local ok, count = 1, 0
 241                 while ok > 0 do
 242                         chunk, ok = chunk:gsub( "^([A-Z][A-Za-z0-9%-_]+): +([^\r\n]+)\r\n", __storehdr )
 243                         count = count + ok
 244                 end
 245
 246                 -- Headers processed, check for empty line
 247                 chunk, ok = chunk:gsub( "^\r\n", "" )
 248
 249                 -- Store remaining buffer contents
 250                 msg._mimebuffer = chunk
 251
 252                 -- End of headers
 253                 if ok > 0 then
 254
 255                         -- When no Content-Type header is given assume text/plain
 256                         if not msg._mimeheaders['Content-Type'] then
 257                                 msg._mimeheaders['Content-Type'] = 'text/plain'
 258                         end
 259
 260                         -- Check Content-Disposition
 261                         if msg._mimeheaders['Content-Disposition'] then
 262                                 -- Check for "form-data" token
 263                                 if msg._mimeheaders['Content-Disposition']:match("^form%-data; ") then
 264                                         -- Check for field name, filename
 265                                         local field = msg._mimeheaders['Content-Disposition']:match('name="(.-)"')
 266                                         local file  = msg._mimeheaders['Content-Disposition']:match('filename="(.+)"$')
 267
 268                                         -- Is a file field and we have a callback
 269                                         if file and filecb then
 270                                                 msg.params[field] = file
 271                                                 msg._mimecallback = function(chunk,eof)
 272                                                         filecb( {
 273                                                                 name    = field;
 274                                                                 file    = file;
 275                                                                 headers = msg._mimeheaders
 276                                                         }, chunk, eof )
 277                                                 end
 278
 279                                         -- Treat as form field
 280                                         else
 281                                                 msg.params[field] = ""
 282                                                 msg._mimecallback = function(chunk,eof)
 283                                                         msg.params[field] = msg.params[field] .. chunk
 284                                                 end
 285                                         end
 286
 287                                         -- Header was valid, continue with mime-data
 288                                         return true, function( chunk )
 289                                                 return process_states['mime-data']( msg, chunk, filecb )
 290                                         end
 291                                 else
 292                                         -- Unknown Content-Disposition, abort
 293                                         return nil, "Unexpected Content-Disposition MIME section header"
 294                                 end
 295                         else
 296                                 -- Content-Disposition is required, abort without
 297                                 return nil, "Missing Content-Disposition MIME section header"
 298                         end
 299
 300                 -- We parsed no headers yet and buffer is almost empty
 301                 elseif count > 0 or #chunk < 128 then
 302                         -- Keep feeding me with chunks
 303                         return true, nil
 304                 end
 305
 306                 -- Buffer looks like garbage
 307                 return nil, "Malformed MIME section header"
 308         else
 309                 return nil, "Unexpected EOF"
 310         end
 311 end
 312
 313
 314 -- Read MIME part data
 315 process_states['mime-data'] = function( msg, chunk, filecb )
 316
 317         if chunk ~= nil then
 318
 319                 -- Combine look-behind buffer with current chunk
 320                 local buffer = msg._mimebuffer .. chunk
 321
 322                 -- Look for MIME boundary
 323                 local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "\r\n", 1, true )
 324
 325                 if spos then
 326                         -- Content data
 327                         msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
 328
 329                         -- Store remainder
 330                         msg._mimebuffer = buffer:sub( epos + 1, #buffer )
 331
 332                         -- Next state is mime-header processing
 333                         return true, function( chunk )
 334                                 return process_states['mime-headers']( msg, chunk, filecb )
 335                         end
 336                 else
 337                         -- Look for EOF?
 338                         local spos, epos = buffer:find( "\r\n--" .. msg.mime_boundary .. "--\r\n", 1, true )
 339
 340                         if spos then
 341                                 -- Content data
 342                                 msg._mimecallback( buffer:sub( 1, spos - 1 ), true )
 343
 344                                 -- We processed the final MIME boundary, cleanup
 345                                 msg._mimebuffer   = nil
 346                                 msg._mimeheaders  = nil
 347                                 msg._mimecallback = nil
 348
 349                                 -- We won't accept data anymore
 350                                 return false
 351                         else
 352                                 -- We're somewhere within a data section and our buffer is full
 353                                 if #buffer > #chunk then
 354                                         -- Flush buffered data
 355                                         msg._mimecallback( buffer:sub( 1, #buffer - #chunk ), false )
 356
 357                                         -- Store new data
 358                                         msg._mimebuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
 359
 360                                 -- Buffer is not full yet, append new data
 361                                 else
 362                                         msg._mimebuffer = buffer
 363                                 end
 364
 365                                 -- Keep feeding me
 366                                 return true
 367                         end
 368                 end
 369         else
 370                 return nil, "Unexpected EOF"
 371         end
 372 end
 373
 374
 375 -- Init urldecoding stream
 376 process_states['urldecode-init'] = function( msg, chunk, filecb )
 377
 378         if chunk ~= nil then
 379
 380                 -- Check for Content-Length
 381                 if msg.env.CONTENT_LENGTH then
 382                         msg.content_length = tonumber(msg.env.CONTENT_LENGTH)
 383
 384                         if msg.content_length <= HTTP_MAX_CONTENT then
 385                                 -- Initialize buffer
 386                                 msg._urldecbuffer = chunk
 387                                 msg._urldeclength = 0
 388
 389                                 -- Switch to urldecode-key state
 390                                 return true, function(chunk)
 391                                         return process_states['urldecode-key']( msg, chunk, filecb )
 392                                 end
 393                         else
 394                                 return nil, "Request exceeds maximum allowed size"
 395                         end
 396                 else
 397                         return nil, "Missing Content-Length header"
 398                 end
 399         else
 400                 return nil, "Unexpected EOF"
 401         end
 402 end
 403
 404
 405 -- Process urldecoding stream, read and validate parameter key
 406 process_states['urldecode-key'] = function( msg, chunk, filecb )
 407         if chunk ~= nil then
 408
 409                 -- Prevent oversized requests
 410                 if msg._urldeclength >= msg.content_length then
 411                         return nil, "Request exceeds maximum allowed size"
 412                 end
 413
 414                 -- Combine look-behind buffer with current chunk
 415                 local buffer = msg._urldecbuffer .. chunk
 416                 local spos, epos = buffer:find("=")
 417
 418                 -- Found param
 419                 if spos then
 420
 421                         -- Check that key doesn't exceed maximum allowed key length
 422                         if ( spos - 1 ) <= HTTP_URLENC_MAXKEYLEN then
 423                                 local key = urldecode( buffer:sub( 1, spos - 1 ) )
 424
 425                                 -- Prepare buffers
 426                                 msg.params[key]         = ""
 427                                 msg._urldeclength   = msg._urldeclength + epos
 428                                 msg._urldecbuffer   = buffer:sub( epos + 1, #buffer )
 429
 430                                 -- Use file callback or store values inside msg.params
 431                                 if filecb then
 432                                         msg._urldeccallback = function( chunk, eof )
 433                                                 filecb( field, chunk, eof )
 434                                         end
 435                                 else
 436                                         msg._urldeccallback = function( chunk, eof )
 437                                                 msg.params[key] = msg.params[key] .. chunk
 438
 439                                                 -- FIXME: Use a filter
 440                                                 if eof then
 441                                                         msg.params[key] = urldecode( msg.params[key] )
 442                                                 end
 443                                         end
 444                                 end
 445
 446                                 -- Proceed with urldecode-value state
 447                                 return true, function( chunk )
 448                                         return process_states['urldecode-value']( msg, chunk, filecb )
 449                                 end
 450                         else
 451                                 return nil, "POST parameter exceeds maximum allowed length"
 452                         end
 453                 else
 454                         return nil, "POST data exceeds maximum allowed length"
 455                 end
 456         else
 457                 return nil, "Unexpected EOF"
 458         end
 459 end
 460
 461
 462 -- Process urldecoding stream, read parameter value
 463 process_states['urldecode-value'] = function( msg, chunk, filecb )
 464
 465         if chunk ~= nil then
 466
 467                 -- Combine look-behind buffer with current chunk
 468                 local buffer = msg._urldecbuffer .. chunk
 469
 470                 -- Check for EOF
 471                 if #buffer == 0 then
 472                         -- Compare processed length
 473                         if msg._urldeclength == msg.content_length then
 474                                 -- Cleanup
 475                                 msg._urldeclength   = nil
 476                                 msg._urldecbuffer   = nil
 477                                 msg._urldeccallback = nil
 478
 479                                 -- We won't accept data anymore
 480                                 return false
 481                         else
 482                                 return nil, "Content-Length mismatch"
 483                         end
 484                 end
 485
 486                 -- Check for end of value
 487                 local spos, epos = buffer:find("[&;]")
 488                 if spos then
 489
 490                         -- Flush buffer, send eof
 491                         msg._urldeccallback( buffer:sub( 1, spos - 1 ), true )
 492                         msg._urldecbuffer = buffer:sub( epos + 1, #buffer )
 493                         msg._urldeclength = msg._urldeclength + epos
 494
 495                         -- Back to urldecode-key state
 496                         return true, function( chunk )
 497                                 return process_states['urldecode-key']( msg, chunk, filecb )
 498                         end
 499                 else
 500                         -- We're somewhere within a data section and our buffer is full
 501                         if #buffer > #chunk then
 502                                 -- Flush buffered data
 503                                 msg._urldeccallback( buffer:sub( 1, #buffer - #chunk ), false )
 504
 505                                 -- Store new data
 506                                 msg._urldeclength = msg._urldeclength + #buffer - #chunk
 507                                 msg._urldecbuffer = buffer:sub( #buffer - #chunk + 1, #buffer )
 508
 509                         -- Buffer is not full yet, append new data
 510                         else
 511                                 msg._urldecbuffer = buffer
 512                         end
 513
 514                         -- Keep feeding me
 515                         return true
 516                 end
 517         else
 518                 return nil, "Unexpected EOF"
 519         end
 520 end
 521
 522
 523 -- Decode MIME encoded data.
 524 function mimedecode_message_body( source, msg, filecb )
 525
 526         -- Find mime boundary
 527         if msg and msg.env.CONTENT_TYPE then
 528
 529                 local bound = msg.env.CONTENT_TYPE:match("^multipart/form%-data; boundary=(.+)")
 530
 531                 if bound then
 532                         msg.mime_boundary = bound
 533                 else
 534                         return nil, "No MIME boundary found or invalid content type given"
 535                 end
 536         end
 537
 538         -- Create an initial LTN12 sink
 539         -- The whole MIME parsing process is implemented as fancy sink, sinks replace themself
 540         -- depending on current processing state (init, header, data). Return the initial state.
 541         local sink = ltn12.sink.simplify(
 542                 function( chunk )
 543                         return process_states['mime-init']( msg, chunk, filecb )
 544                 end
 545         )
 546
 547         -- Create a throttling LTN12 source
 548         -- Frequent state switching in the mime parsing process leads to unwanted buffer aggregation.
 549         -- This source checks wheather there's still data in our internal read buffer and returns an
 550         -- empty string if there's already enough data in the processing queue. If the internal buffer
 551         -- runs empty we're calling the original source to get the next chunk of data.
 552         local tsrc = function()
 553
 554                 -- XXX: we schould propably keep the maximum buffer size in sync with
 555                 --      the blocksize of our original source... but doesn't really matter
 556                 if msg._mimebuffer ~= null and #msg._mimebuffer > 256 then
 557                         return ""
 558                 else
 559                         return source()
 560                 end
 561         end
 562
 563         -- Pump input data...
 564         while true do
 565                 -- get data
 566                 local ok, err = ltn12.pump.step( tsrc, sink )
 567
 568                 -- error
 569                 if not ok and err then
 570                         return nil, err
 571
 572                 -- eof
 573                 elseif not ok then
 574                         return true
 575                 end
 576         end
 577 end
 578
 579
 580 -- Decode urlencoded data.
 581 function urldecode_message_body( source, msg )
 582
 583         -- Create an initial LTN12 sink
 584         -- Return the initial state.
 585         local sink = ltn12.sink.simplify(
 586                 function( chunk )
 587                         return process_states['urldecode-init']( msg, chunk )
 588                 end
 589         )
 590
 591         -- Create a throttling LTN12 source
 592         -- See explaination in mimedecode_message_body().
 593         local tsrc = function()
 594                 if msg._urldecbuffer ~= null and #msg._urldecbuffer > 0 then
 595                         return ""
 596                 else
 597                         return source()
 598                 end
 599         end
 600
 601         -- Pump input data...
 602         while true do
 603                 -- get data
 604                 local ok, err = ltn12.pump.step( tsrc, sink )
 605
 606                 -- step
 607                 if not ok and err then
 608                         return nil, err
 609
 610                 -- eof
 611                 elseif not ok then
 612                         return true
 613                 end
 614         end
 615 end
 616
 617
 618 -- Parse a http message
 619 function parse_message( data, filecb )
 620
 621         local reader  = _linereader( data, HTTP_MAX_READBUF )
 622         local message = parse_message_header( reader )
 623
 624         if message then
 625                 parse_message_body( reader, message, filecb )
 626         end
 627
 628         return message
 629 end
 630
 631
 632 -- Parse a http message header
 633 function parse_message_header( source )
 634
 635         local ok   = true
 636         local msg  = { }
 637
 638         local sink = ltn12.sink.simplify(
 639                 function( chunk )
 640                         return process_states['magic']( msg, chunk )
 641                 end
 642         )
 643
 644         -- Pump input data...
 645         while ok do
 646
 647                 -- get data
 648                 ok, err = ltn12.pump.step( source, sink )
 649
 650                 -- error
 651                 if not ok and err then
 652                         return nil, err
 653
 654                 -- eof
 655                 elseif not ok then
 656
 657                         -- Process get parameters
 658                         if ( msg.request_method == "get" or msg.request_method == "post" ) and
 659                            msg.request_uri:match("?")
 660                         then
 661                                 msg.params = urldecode_params( msg.request_uri )
 662                         else
 663                                 msg.params = { }
 664                         end
 665
 666                         -- Populate common environment variables
 667                         msg.env = {
 668                                 CONTENT_LENGTH    = msg.headers['Content-Length'];
 669                                 CONTENT_TYPE      = msg.headers['Content-Type'];
 670                                 REQUEST_METHOD    = msg.request_method:upper();
 671                                 REQUEST_URI       = msg.request_uri;
 672                                 SCRIPT_NAME       = msg.request_uri:gsub("?.+$","");
 673                                 SCRIPT_FILENAME   = "";         -- XXX implement me
 674                                 SERVER_PROTOCOL   = "HTTP/" .. msg.http_version
 675                         }
 676
 677                         -- Populate HTTP_* environment variables
 678                         for i, hdr in ipairs( {
 679                                 'Accept',
 680                                 'Accept-Charset',
 681                                 'Accept-Encoding',
 682                                 'Accept-Language',
 683                                 'Connection',
 684                                 'Cookie',
 685                                 'Host',
 686                                 'Referer',
 687                                 'User-Agent',
 688                         } ) do
 689                                 local var = 'HTTP_' .. hdr:upper():gsub("%-","_")
 690                                 local val = msg.headers[hdr]
 691
 692                                 msg.env[var] = val
 693                         end
 694                 end
 695         end
 696
 697         return msg
 698 end
 699
 700
 701 -- Parse a http message body
 702 function parse_message_body( source, msg, filecb )
 703
 704         -- Is it multipart/mime ?
 705         if msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
 706            msg.env.CONTENT_TYPE:match("^multipart/form%-data")
 707         then
 708
 709                 return mimedecode_message_body( source, msg, filecb )
 710
 711         -- Is it application/x-www-form-urlencoded ?
 712         elseif msg.env.REQUEST_METHOD == "POST" and msg.env.CONTENT_TYPE and
 713                msg.env.CONTENT_TYPE == "application/x-www-form-urlencoded"
 714         then
 715                 return urldecode_message_body( source, msg, filecb )
 716
 717
 718         -- Unhandled encoding
 719         -- If a file callback is given then feed it line by line, else
 720         -- store whole buffer in message.content
 721         else
 722
 723                 local sink
 724                 local length = 0
 725
 726                 -- If we have a file callback then feed it
 727                 if type(filecb) == "function" then
 728                         sink = filecb
 729
 730                 -- ... else append to .content
 731                 else
 732                         msg.content = ""
 733                         msg.content_length = 0
 734
 735                         sink = function( chunk )
 736                                 if ( msg.content_length ) + #chunk <= HTTP_MAX_CONTENT then
 737
 738                                         msg.content        = msg.content        .. chunk
 739                                         msg.content_length = msg.content_length + #chunk
 740
 741                                         return true
 742                                 else
 743                                         return nil, "POST data exceeds maximum allowed length"
 744                                 end
 745                         end
 746                 end
 747
 748                 -- Pump data...
 749                 while true do
 750                         local ok, err = ltn12.pump.step( source, sink )
 751
 752                         if not ok and err then
 753                                 return nil, err
 754                         elseif not err then
 755                                 return true
 756                         end
 757                 end
 758         end
 759 end