Sat, 02 Jan 2010 06:06:48 +0000
multihttp: Don't attempt to return progress if download hasn't started yet
package.preload['util.logger']=(function(...) local t; if os.getenv("LAHTTP_DEBUG")then t=_G.print; else t=function()end; end local i,a=select,tostring; module"logger" local function e(t,...) local e,o=0,#arg; return(t:gsub("%%(.)",function(t)if t~="%"and e<=o then e=e+1;return a(arg[e]);end end)); end local function s(n,...) local e,t=0,i('#',...); local o={...}; return(n:gsub("%%(.)",function(i)if e<=t then e=e+1;return a(o[e]);end end)); end function init(e) return function(a,e,...) t(a,s(e,...)); end end return _M; end) package.preload['net.server']=(function(...) local l=function(e) return _G[e] end local J=function(e) for t,a in pairs(e)do e[t]=nil end end local w,e=require("util.logger").init("socket"),table.concat; local n=function(...)return w("debug",e{...});end local H=function(...)return w("warn",e{...});end local e=collectgarbage local ie=1 local A=l"type" local T=l"pairs" local he=l"ipairs" local h=l"tostring" local e=l"collectgarbage" local o=l"os" local a=l"table" local t=l"string" local e=l"coroutine" local V=o.time local R=o.difftime local te=a.concat local a=a.remove local B=t.len local de=t.sub local ue=e.wrap local le=e.yield local I=select(2,pcall(require,"ssl")) local L=require"socket" local ee=(I and I.wrap) local fe=L.bind local me=L.sleep local ce=L.select local e=(I and I.newcontext) local G local X local Z local W local Y local ne local re local se local ae local oe local P local d local Q local t local D local K local p local s local C local r local i local k local b local f local c local a local o local v local U local S local _ local j local M local u local z local x local O local E local q local N local F local g p={} s={} r={} C={} i={} b={} f={} k={} a=0 o=0 v=0 U=0 S=0 _=1 j=0 z=51e3*1024 x=25e3*1024 O=12e5 E=6e4 q=6*60*60 N=false g=1e3 _maxsslhandshake=30 ae=function(y,l,k,u,b,t,p,v) p=p or g local f=0 local g,c=y.onincoming,y.ondisconnect local m local c=false if t then c=true if not e then H"luasec not found" c=false end if A(t)~="table"then H"server.lua: wrong server sslctx" c=false end local o; o,m=e(t) if not o then m=m or"wrong sslctx parameters" local a; a=m:match("^error loading (.-) %("); if a then if a=="private key"then a=t.key or"your private key"; elseif a=="certificate"then a=t.certificate or"your certificate file"; end local e=m:match("%((.+)%)$")or"some reason"; if e=="Permission denied"then e="Check that the permissions allow Prosody to read this file."; elseif e=="No such file or directory"then e="Check that the path is correct, and the file exists."; elseif e=="system lib"then e="Previous error (see logs), or other system error."; else e="Reason: "..h(e or"unknown"):lower(); end w("error","SSL/TLS: Failed to load %s: %s",a,e); else w("error","SSL/TLS: Error initialising for port %d: %s",u,m); end c=false end t=o; end if not c then t=false; if v then w("error","Failed to listen on port %d due to SSL/TLS to SSL/TLS initialisation errors (see logs)",u) return nil,"Cannot start ssl, see log for details" end end local m=l.accept local e={} e.shutdown=function()end e.ssl=function() return c end e.sslctx=function() return t end e.remove=function() f=f-1 end e.close=function() for t,e in T(i)do if e.serverport==u then e.disconnect(e,"server closed") e:close(true) end end l:close() o=d(r,l,o) a=d(s,l,a) i[l]=nil e=nil l=nil n"server.lua: closed server handler and removed sockets from list" end e.ip=function() return k end e.serverport=function() return u end e.socket=function() return l end e.readbuffer=function() if f>p then n("server.lua: refused new client connection: server full") return false end local a,s=m(l) if a then local i,o=a:getpeername() a:settimeout(0) local e,a,t=D(e,y,a,i,u,o,b,t,v) if t then return false end f=f+1 n("server.lua: accepted new client connection from ",h(i),":",h(o)," to ",h(u)) return g(e) elseif s then n("server.lua: error with new client connection: ",h(s)) return false end end return e end D=function(M,e,t,I,Y,L,_,p,P) t:settimeout(0) local w local T local q local v local O=e.onincoming local F=e.status local g=e.ondisconnect local y={} local m=0 local C local A local D local l=0 local j=false local E=false local H,R=0,0 local z=z local x=x local e=y e.dispatch=function() return O end e.disconnect=function() return g end e.setlistener=function(a,t) O=t.onincoming g=t.ondisconnect end e.getstats=function() return R,H end e.ssl=function() return v end e.sslctx=function() return p end e.send=function(n,i,o,a) return w(t,i,o,a) end e.receive=function(a,o) return T(t,a,o) end e.shutdown=function(a) return q(t,a) end e.close=function(u,h) if not e then return true;end a=d(s,t,a) b[e]=nil if m~=0 then if not(h or A)then e.sendbuffer() if m~=0 then if e then e.write=nil end C=true return false end else w(t,te(y,"",1,m),1,l) end end if t then c=q and q(t) t:close() o=d(r,t,o) i[t]=nil t=nil else n"server.lua: socket already closed" end if e then f[e]=nil k[e]=nil e=nil end if M then M.remove() end n"server.lua: closed client handler and removed socket from list" return true end e.ip=function() return I end e.serverport=function() return Y end e.clientport=function() return L end local k=function(i,a) l=l+B(a) if l>z then k[e]="send buffer exceeded" e.write=W return false elseif t and not r[t]then o=addsocket(r,t,o) end m=m+1 y[m]=a if e then f[e]=f[e]or u end return true end e.write=k e.bufferqueue=function(t) return y end e.socket=function(a) return t end e.pattern=function(a,t) _=t or _ return _ end e.set_send=function(a,t) w=t or w return w end e.bufferlen=function(o,t,a) z=a or z x=t or x return l,x,z end e.lock_read=function(i,o) if o==true then local o=a a=d(s,t,a) b[e]=nil if a~=o then j=true end elseif o==false then if j then j=false a=addsocket(s,t,a) b[e]=u end end return j end e.lock=function(i,a) e.lock_read(a) if a==true then e.write=W local a=o o=d(r,t,o) f[e]=nil if o~=a then E=true end elseif a==false then e.write=k if E then E=false k("") end end return j,E end local b=function() local o,t,a=T(t,_) if not t or(t=="wantread"or t=="timeout")or B(a)>0 then local o=o or a or"" local a=B(o) if a>x then g(e,"receive buffer exceeded") e.close(true) return false end local a=a*ie R=R+a S=S+a b[e]=u return O(e,o,t) else n("server.lua: client ",h(I),":",h(L)," read error: ",h(t)) A=true g(e,t) c=e and e.close() return false end end local f=function() local p,a,i,s,v; local v; if t then s=te(y,"",1,m) p,a,i=w(t,s,1,l) v=(p or i or 0)*ie H=H+v U=U+v c=N and J(y) else p,a,v=false,"closed",0; end if p then m=0 l=0 o=d(r,t,o) c=D and e:starttls(true) f[e]=nil c=C and e.close() return true elseif i and(a=="timeout"or a=="wantwrite")then s=de(s,i+1,l) y[1]=s m=1 l=l-i f[e]=u return true else n("server.lua: client ",h(I),":",h(L)," write error: ",h(a)) A=true g(e,a) c=e and e.close() return false end end local l; function e.set_sslctx(i,t) v=true p=t; local u local m l=ue(function(t) local i for l=1,_maxsslhandshake do o=(u and d(r,t,o))or o a=(m and d(s,t,a))or a m,u=nil,nil c,i=t:dohandshake() if not i then n("server.lua: ssl handshake done") e.readbuffer=b e.sendbuffer=f c=F and F(e,"ssl-handshake-complete") a=addsocket(s,t,a) return true else n("server.lua: error during ssl handshake: ",h(i)) if i=="wantwrite"and not u then o=addsocket(r,t,o) u=true elseif i=="wantread"and not m then a=addsocket(s,t,a) m=true else break; end le() end end g(e,"ssl handshake failed") c=e and e:close(true) return false end ) end if p then e:set_sslctx(p); if P then local a t,a=ee(t,p) if a then n("server.lua: ssl error: ",h(a)) return nil,nil,a end t:settimeout(0) e.readbuffer=l e.sendbuffer=l l(t) if not t then return nil,nil,"ssl handshake failed"; end else v=false e.starttls=function(c,u) if not u then D=true return end local c,u=t t,u=ee(t,p) if u then n("server.lua: error while starting tls on client: ",h(u)) return nil,u end t:settimeout(0) w=t.send T=t.receive q=G i[t]=e a=addsocket(s,t,a) a=d(s,c,a) o=d(r,c,o) i[c]=nil e.starttls=nil D=nil v=true e.readbuffer=l e.sendbuffer=l l(t) end e.readbuffer=b e.sendbuffer=f end else v=false e.readbuffer=b e.sendbuffer=f end w=t.send T=t.receive q=(v and G)or t.shutdown i[t]=e a=addsocket(s,t,a) return e,t end G=function() end W=function() return false end addsocket=function(a,t,e) if not a[t]then e=e+1 a[e]=t a[t]=e end return e; end d=function(e,a,t) local o=e[a] if o then e[a]=nil local i=e[t] e[t]=nil if i~=a then e[i]=o e[o]=i end return t-1 end return t end P=function(e) o=d(r,e,o) a=d(s,e,a) i[e]=nil e:close() end re=function(o,e,d,u,r,l) local t if A(d)~="table"then t="invalid listener table" end if not A(e)=="number"or not(e>=0 and e<=65535)then t="invalid port" elseif p[e]then t="listeners on port '"..e.."' already exist" elseif r and not I then t="luasec not found" end if t then H("server.lua, port ",e,": ",t) return nil,t end o=o or"*" local t,h=fe(o,e) if h then H("server.lua, port ",e,": ",h) return nil,h end local h,r=ae(d,t,o,e,u,r,g,l) if not h then t:close() return nil,r end t:settimeout(0) a=addsocket(s,t,a) p[e]=h i[t]=h n("server.lua: new server listener on '",o,":",e,"'") return h end se=function(e) return p[e]; end Q=function(e) local t=p[e] if not t then return nil,"no server found on port '"..h(e).."'" end t:close() p[e]=nil return true end ne=function() for t,e in T(i)do e:close() i[t]=nil end a=0 o=0 v=0 p={} s={} r={} C={} i={} end oe=function() return _,j,z,x,O,E,q,N,g,_maxsslhandshake end K=function(e) if A(e)~="table"then return nil,"invalid settings table" end _=tonumber(e.timeout)or _ j=tonumber(e.sleeptime)or j z=tonumber(e.maxsendlen)or z x=tonumber(e.maxreadlen)or x O=tonumber(e.checkinterval)or O E=tonumber(e.sendtimeout)or E q=tonumber(e.readtimeout)or q N=e.cleanqueue g=e._maxclientsperserver or g _maxsslhandshake=e._maxsslhandshake or _maxsslhandshake return true end Y=function(e) if A(e)~="function"then return nil,"invalid listener function" end v=v+1 C[v]=e return true end Z=function() return S,U,a,o,v end local e=true; setquitting=function(t) e=not t; return; end X=function() while e do local a,e,t=ce(s,r,_) for e,t in he(e)do local e=i[t] if e then e.sendbuffer() else P(t) n"server.lua: found no handler and closed socket (writelist)" end end for t,e in he(a)do local t=i[e] if t then t.readbuffer() else P(e) n"server.lua: found no handler and closed socket (readlist)" end end for e,t in T(k)do e.disconnect()(e,t) e:close(true) end J(k) u=V() if R(u-F)>=1 then for e=1,v do C[e](u) end F=u end me(j) end return"quitting" end local function s() return"select"; end local n=function(e,s,n,t,a,h,d) local t=D(nil,t,e,s,n,"clientport",a,h,d) i[e]=t o=addsocket(r,e,o) return t,e end local t=function(a,o,i,s,r,h) local t,e=L.tcp() if e then return nil,e end t:settimeout(0) c,e=t:connect(a,o) if e then local e=n(t,a,o,i) else D(nil,i,t,a,o,"clientport",s,r,h) end end l"setmetatable"(i,{__mode="k"}) l"setmetatable"(b,{__mode="k"}) l"setmetatable"(f,{__mode="k"}) F=V() M=V() Y(function() local e=R(u-M) if e>O then M=u for e,t in T(f)do if R(u-t)>E then e.disconnect()(e,"send timeout") e:close(true) end end for e,t in T(b)do if R(u-t)>q then e.disconnect()(e,"read timeout") e:close() end end end end ) return{ addclient=t, wrapclient=n, loop=X, stats=Z, closeall=ne, addtimer=Y, addserver=re, getserver=se, getsettings=oe, setquitting=setquitting, removeserver=Q, get_backend=s, changesettings=K, } end) package.preload['net.httpclient_listener']=(function(...) local a=require"util.logger".init("httpclient_listener"); local i=require"net.connlisteners".register; local e={}; local t={}; local t={default_port=80,default_mode="*a"}; function t.onincoming(t,o) local e=e[t]; if not e then a("warn","Received response from connection %s with no request attached!",tostring(t)); return; end if o and e.reader then e:reader(o); end end function t.ondisconnect(a,t) local t=e[a]; if t then t:reader(nil); end e[a]=nil; end function t.register_request(o,t) a("debug","Attaching request %s to connection %s",tostring(t.id or t),tostring(o)); e[o]=t; end i("httpclient",t); end) package.preload['net.connlisteners']=(function(...) local r=(CFG_SOURCEDIR or".").."/net/"; local h=require"net.server"; local o=require"util.logger".init("connlisteners"); local i=tostring; local s,n,a= dofile,pcall,error module"connlisteners" local e={}; function register(t,a) if e[t]and e[t]~=a then o("debug","Listener %s is already registered, not registering any more",t); return false; end e[t]=a; o("debug","Registered connection listener %s",t); return true; end function deregister(t) e[t]=nil; end function get(t) local a=e[t]; if not a then local s,n=n(s,r..t:gsub("[^%w%-]","_").."_listener.lua"); if not s then o("error","Error while loading listener '%s': %s",i(t),i(n)); return nil,n; end a=e[t]; end return a; end function start(o,e) local t,n=get(o); if not t then a("No such connection module: "..o..(n and(" ("..n..")")or""),0); end if e then if(e.type=="ssl"or e.type=="tls")and not e.ssl then a("No SSL context supplied for a "..i(e.type):upper().." connection!",0); elseif e.ssl and e.type=="tcp"then a("SSL context supplied for a TCP connection!",0); end end local i=(e and e.interface)or t.default_interface or"*"; local o=(e and e.port)or t.default_port or a("Can't start listener "..o.." because no port was specified, and it has no default port",0); local a=(e and e.mode)or t.default_mode or 1; local n=(e and e.ssl)or nil; local s=99999999; local e=e and e.type=="ssl"; return h.addserver(i,o,t,a,n,e); end return _M; end) package.preload['net.http']=(function(...) local b=require"socket" local v=require"mime" local y=require"socket.url" local p=require"net.server" local e=require"net.connlisteners".get; local d=e("httpclient")or error("No httpclient listener!"); local r,i=table.insert,table.concat; local s,u,c,m,f,l,t,a= tonumber,tostring,pairs,xpcall,select,debug.traceback,string.char,string.format; local n=require"util.logger".init("http"); local o=function()end module"http" function urlencode(e)return e and(e:gsub("%W",function(e)return a("%%%02x",e:byte());end));end function urldecode(e)return e and(e:gsub("%%(%x%x)",function(e)return t(s(e,16));end));end local function w(t,e) if t.method=="HEAD"then return nil end if e==204 or e==304 or e==301 then return nil end if e>=100 and e<200 then return nil end return 1 end local function h(e,t,a) if not t then if e.body then n("debug","Connection closed, but we have data, calling callback..."); e.callback(i(e.body),e.code,e); elseif e.state~="completed"then e.callback("connection-closed",0,e); end destroy_request(e); e.body=nil; e.state="completed"; return; end if e.state=="body"and e.state~="completed"then o("Reading body...") if not e.body then e.body={};e.havebodylength,e.bodylength=0,s(e.responseheaders["content-length"]);end if a then t=t:sub(a,-1) end r(e.body,t); if e.bodylength then e.havebodylength=e.havebodylength+#t; if e.havebodylength>=e.bodylength then n("debug","Have full body, calling callback"); if e.callback then e.callback(i(e.body),e.code,e); end e.body=nil; e.state="completed"; else o("","Have "..e.havebodylength.." bytes out of "..e.bodylength); end end elseif e.state=="headers"then o("Reading headers...") local i=a; local n=e.responseheaders or{}; for t in t:sub(a,-1):gmatch("(.-)\r\n")do a=a+#t+2; local a,i=t:match("(%S+): (.+)"); if a and i then n[a:lower()]=i; o("Header: "..a:lower().." = "..i); elseif#t==0 then e.responseheaders=n; break; else o("Unhandled header line: "..t); end end e.state="body"; if#t>a then return h(e,t,a); end elseif e.state=="status"then o("Reading status...") local i,a,n,o=t:match("^HTTP/(%S+) (%d+) (.-)\r\n()",a); a=s(a); if not a then return e.callback("invalid-status-line",0,e); end e.code,e.responseversion=a,i; if e.onlystatus or not w(e,a)then if e.callback then e.callback(nil,a,e); end destroy_request(e); return; end e.state="headers"; if#t>o then return h(e,t,o); end end end local function w(e)n("error","Traceback[http]: %s: %s",u(e),l());end function request(e,a,s) local e=y.parse(e); if not(e and e.host)then s(nil,0,e); return nil,"invalid-url"; end if not e.path then e.path="/"; end local l,o; local t={["Host"]=e.host,["User-Agent"]="Prosody XMPP Server"} if e.userinfo then t["Authorization"]="Basic "..v.b64(e.userinfo); end if a then l=a.headers; e.onlystatus=a.onlystatus; o=a.body; if o then e.method="POST "; t["Content-Length"]=u(#o); t["Content-Type"]="application/x-www-form-urlencoded"; end if a.method then e.method=a.method;end end e.handler,e.conn=p.wrapclient(b.tcp(),e.host,e.port or 80,d,"*a"); e.write=function(...)return e.handler:write(...);end e.conn:settimeout(0); local u,a=e.conn:connect(e.host,e.port or 80); if not u and a~="timeout"then s(nil,0,e); return nil,a; end local a={e.method or"GET"," ",e.path," HTTP/1.1\r\n"}; if e.query then r(a,4,"?"); r(a,5,e.query); end e.write(i(a)); local a={[2]=": ",[4]="\r\n"}; if l then for o,n in c(l)do a[1],a[3]=o,n; e.write(i(a)); t[o]=nil; end end for o,n in c(t)do a[1],a[3]=o,n; e.write(i(a)); t[o]=nil; end e.write("\r\n"); if o then e.write(o); end e.callback=function(a,t,o)n("debug","Calling callback, status %s",t or"---");return f(2,m(function()return s(a,t,o)end,w));end e.reader=h; e.state="status"; d.register_request(e.handler,e); return e; end function destroy_request(e) if e.conn then e.handler.close() d.ondisconnect(e.conn,"closed"); end end _M.urlencode=urlencode; return _M; end) package.preload['util.timer']=(function(...) local h=require"net.server".addtimer; local a=require"net.server".event; local d=require"net.server".event_base; local s=os.time; local n=table.insert; local e=table.remove; local e,i=ipairs,pairs; local r=type; local o={}; local t={}; module"timer" local e; if not a then function e(e,a) local o=s(); e=e+o; if e>=o then n(t,{e,a}); else a(); end end h(function() local s=s(); if#t>0 then for a,e in i(t)do n(o,e); end t={}; end for i,t in i(o)do local t,a=t[1],t[2]; if t<=s then o[i]=nil; local t=a(s); if r(t)=="number"then e(t,a);end end end end); else local t=(a.core and a.core.LEAVE)or-1; function e(a,e) d:addevent(nil,0,function() local e=e(); if e then return 0,e; else return t; end end ,a); end end add_task=e; return _M; end) package.preload['multihttp']=(function(...) local s=require"net.http"; local o=require"net.server"; local i=require"util.timer"; local n,h,r,d= pairs,ipairs,type,setmetatable; module"multihttp" __index=_M; function set_callback(t,e) t.callback=e; end function set_progress_callback(e,a,t) t=t or 1; e.progress_callback=a; i.add_task(t,function() if e.progress_callback then e.progress_callback(e); if e.progress_callback then return t; end end end); end function add_url(t,e) t.urls[e]=true; end function remove_url(t,e) t.urls[e]=nil; end function download(e,i) e.status="downloading"; local t=0; for a in n(e.urls)do t=t+1; e.urls[a]= s.request(a,nil,function(n,t,s) if e.callback then e.downloading_count=e.downloading_count-1; e.callback(a,t,n,s); if e.downloading_count==0 and i then o.setquitting(true); end end end); end e.download_count=t; e.downloading_count=t; if i then o.loop(); o.setquitting(false); end return t; end function progress(e) local a={total={}}; for o,t in n(e.urls)do local e={}; e.bytes_downloaded=t.havebodylength; e.bytes_total=t.bodylength; if e.bytes_total then e.percent=100/(e.bytes_total/e.bytes_downloaded); end a[o]=e; end return a; end function new(t,e) local t={callback=t,urls={}}; if r(e)=="table"then for a,e in h(e)do t.urls[e]=true; end end return d(t,_M); end return _M; end) require"net.connlisteners" require"net.httpclient_listener" return require"net.http";