Stream = require('stream') Zlib = require('zlib') Request = require('request') SocketIoStream = require('socket.io-stream') Base = require('../base') Logger = require('./logger') debug = Base.logger('http_proxy') PROXY_HTTP_HOST_USE_PROXY = 'USE_PROXY' upstreamUrl = null isUpstreamHttps = null # Rewriting vars. isRewriteHostOn = null isRewritePostBodyOn = null upstreamHost = null upstreamProto = null upstreamHostReStr = null upstreamHostRe = null upstreamDomainCookieRe = null rewriteProtoDelimReStr = ":\\/\\/|:\\\\\\/\\\\\\/|%3A%2F%2F" # supports: '://', ':\/\/', '%3A%2F%2F' rewriteCustomReplacers = null rewriteHeadersCustomReplacers = null rewriteUseRealDeflate = null rewriteResponseDomainsReStr = null rewriteResponseDomainsRe = null rewriteResponseDomainsBase = null rewriteResponseDomainsCookieRe = null proxyableDomainsRe = null rewriteCustomGenerator = null exports.setupHttpProxy = (upstream) -> upstreamUrl = upstream || Base.env.IX_HTTP_PROXY upstreamUrl = upstreamUrl?.trim().replace(/\/$/, '') # CASB rewriteResponseDomainsReStr = Base.env.IX_PROXY_REWRITE_RESPONSE_DOMAINS if rewriteResponseDomainsReStr upstreamUrl ||= '' rewriteResponseDomainsReStr = "[-0-9A-Za-z]+(?:\\.[-0-9A-Za-z]+)+" if rewriteResponseDomainsReStr.toLowerCase() == 'on' console.log("Rewriting response domains matching /#{rewriteResponseDomainsReStr}/") rewriteResponseDomainsRe = new RegExp("((?:https?(?::|%3A))?(?://|\\/\\/|%2F%2F))(#{rewriteResponseDomainsReStr})((?:[^-.\\w]|$)[-\\w.~:/?#\\[\\]@!$&'()*+,;=%]*)", 'gi') rewriteResponseDomainsCookieRe = new RegExp("(;\\s+domain=\\s*\\.?)(#{rewriteResponseDomainsReStr})(\\s|;|$)", "ig") proxyableDomainsRe = new RegExp("^#{rewriteResponseDomainsReStr}$") rewriteResponseDomainsBase = (Base.env.IX_PROXY_REWRITE_RESPONSE_DOMAINS_BASE || "#{Base.env.FAAS_API_KEY}.http-proxy-cert.faas.io").toLowerCase() throw new Error("No upstream server defined.") unless upstreamUrl || rewriteResponseDomainsRe debug('setupHttpProxy upstreamUrl', upstreamUrl) isUpstreamHttps = upstreamUrl.slice(0,6) == 'https:' isRewriteHostOn = Base.env.IX_PROXY_REWRITE_HOST?.toLowerCase() == 'on' isRewritePostBodyOn = Base.env.IX_PROXY_REWRITE_POST_BODY?.toLowerCase() == 'on' isRewriteHostOn ||= rewriteResponseDomainsRe? if isRewriteHostOn || isRewritePostBodyOn upstreamHost = upstreamUrl.replace(/^https?:\/\//i,'') upstreamProto = if isUpstreamHttps then 'https' else 'http' rewriteUseRealDeflate = Base.env.IX_PROXY_REWRITE_REAL_DEFLATE?.toLowerCase() == 'on' upstreamHostReStr = regexEscape(upstreamHost) upstreamHostRe = new RegExp("#{upstreamHostReStr}\\b", "ig") upstreamDomainCookieRe = new RegExp("(;\\s+domain=\\s*\\.?)#{upstreamHostReStr}(\\s|;|$)", "ig") if Base.env.IX_PROXY_REWRITE_PROTO_DELIM_REGEX rewriteProtoDelimReStr += "|"+Base.env.IX_PROXY_REWRITE_PROTO_DELIM_REGEX customRes = Base.env.IX_PROXY_REWRITE_CUSTOM_REGEXES if customRes console.log("Parsing IX_PROXY_REWRITE_CUSTOM_REGEXES: #{customRes}") replacers = JSON.parse(customRes) rewriteCustomReplacers = [] for replacer in replacers flags = "g" flags += replacer[2] if replacer.length > 2 rewriteCustomReplacers.push([new RegExp(replacer[0], flags), replacer[1]]) debug('rewriteCustomReplacers', rewriteCustomReplacers) customHeaderRes = Base.env.IX_PROXY_REWRITE_HEADERS_REGEXES if customHeaderRes console.log("Parsing IX_PROXY_REWRITE_HEADERS_REGEXES: #{customHeaderRes}") replacers = JSON.parse(customHeaderRes) rewriteHeadersCustomReplacers = [] for replacer in replacers # [ nameRe, search, replace [, flags] ] flags = "g" flags += replacer[3] if replacer.length > 3 rewriteHeadersCustomReplacers.push([new RegExp(replacer[0]), new RegExp(replacer[1], flags), replacer[2]]) debug('rewriteHeadersCustomReplacers', rewriteHeadersCustomReplacers) # Custom rewriter module. rewriteCustomModuleFile = Base.env.IX_PROXY_REWRITE_MODULE if rewriteCustomModuleFile rewriteCustomModuleFile = process.cwd()+'/'+rewriteCustomModuleFile console.log("Loading IX_PROXY_REWRITE_MODULE: '#{rewriteCustomModuleFile}'") try rewriteCustomModule = require(rewriteCustomModuleFile) catch e console.error(e) throw new Error("Error encountered while trying to load IX_PROXY_REWRITE_MODULE: "+("#{e.stack}".split("\n").slice(0,2).join("\n")||"#{e}")) rewriteCustomGenerator = rewriteCustomModule.newRewriter if typeof rewriteCustomGenerator == 'function' console.log("IX_PROXY_REWRITE_MODULE newRewriter() function loaded.") else throw new Error("IX_PROXY_REWRITE_MODULE did not export newRewriter() function") exports.onHttpProxy = (faasReq, raw_cb_proxyResponse) -> try proxyResponse = {} debug('onHttpProxy.faasReq.headers', faasReq?.headers) faasReq.origHost = faasReq.headers.host faasReq.origProto = faasReq.headers['x-forwarded-proto'] Logger.logHttpRequest(faasReq) cb_proxyResponse = (pr) -> debug('cb_proxyResponse', { headers: pr?.headers, lag: pr?.lag, statusCode: pr?.statusCode, statusMessage: pr?.statusMessage, error: pr?.error }) # debug('cp_proxyResponse.responseStream', pr?.responseStream) raw_cb_proxyResponse(pr) upOrigin = upstreamUrl if rewriteResponseDomainsRe hostCut = faasReq.origHost.indexOf("--#{rewriteResponseDomainsBase}") if hostCut > 0 newHost = faasReq.origHost.slice(0,hostCut) newProto = 'https' if newHost.slice(0,6) == 'http--' newHost = newHost.slice(6) newProto = 'http' newHost = newHost.replace(/--/g,'*').replace(/-/g,'.').replace(/\*/g,'-') throw new Error("Host is not proxyable: #{newHost} does not match #{proxyableDomainsRe}") unless proxyableDomainsRe.test(newHost) upOrigin = "#{newProto}://#{newHost}" throw new Error("No upstream server defined (CASB).") unless upOrigin url = upOrigin + faasReq.url fwdOptions = url: url headers: faasReq.headers method: faasReq.method followRedirect: false httpHost = Base.env.IX_PROXY_HTTP_HOST if isUpstreamHttps || httpHost == PROXY_HTTP_HOST_USE_PROXY delete fwdOptions.headers.host if httpHost && httpHost != PROXY_HTTP_HOST_USE_PROXY fwdOptions.headers.host = httpHost if isRewriteHostOn delete fwdOptions.headers.host delete fwdOptions.headers['x-forwarded-host'] delete fwdOptions.headers['x-forwarded-server'] # Some apps might use this (Wix) fwdReq = null reqStream = faasReq.requestStream reqCounter = new Stream.Transform() reqCounter.numBytes = 0 reqCounter._transform = (chunk, encoding, callback) -> debug('request chunk', encoding, chunk) # debug('request chunk body', chunk.toString()) reqCounter.numBytes += chunk.length callback(null, chunk) reqStream = safePipe(reqStream, reqCounter, 'reqCounter') if isRewritePostBodyOn && (fwdOptions.headers['content-length'] || fwdOptions.headers['content-type'] || fwdOptions.headers['transfer-encoding']) && !rewriteResponseDomainsRe? # 2017-05-05: Not supported for CASB ct = fwdOptions.headers['content-type']?.toLowerCase() || '' if ct.match(/\bapplication\/x-www-form-urlencoded\b|\bmultipart\/form-data\b|\bjson\b|\btext\/|\bxml\b/i) delete fwdOptions.headers['content-length'] fwdOptions.headers['transfer-encoding'] = 'chunked' reqStream = safePipe(reqStream, newRequestRewriterStream(faasReq.origHost, faasReq.origProto), 'RequestRewriter') reqCustomSetup = { url: fwdOptions.url, method: fwdOptions.method, followRedirect: fwdOptions.followRedirect } rewriteCustomInstance = rewriteCustomGenerator?(fwdOptions.headers, reqCustomSetup) if rewriteCustomInstance if rewriteCustomInstance.rewriteRequestHeaders rewriteCustomInstance.rewriteRequestHeaders(fwdOptions.headers, reqCustomSetup) fwdOptions.url = reqCustomSetup.url fwdOptions.method = reqCustomSetup.method fwdOptions.followRedirect = reqCustomSetup.followRedirect customReqTransform = safeTransform(rewriteCustomInstance.newRequestStreamTransform?(), 'IX_PROXY_REWRITE_MODULE#newRequestStreamTransform') reqStream = safePipe(reqStream, customReqTransform, 'rewriteCustomInstance.request') if customReqTransform debug('request', fwdOptions) fwdReq = Request(fwdOptions) catch e # Error: Invalid protocol: localhost: proxyResponse.statusCode = 500 proxyResponse.error = e.toString()+"\n" Logger.logHttpResponseError(faasReq, proxyResponse) cb_proxyResponse(proxyResponse) faasReq.requestStream.on('data', (data) -> debug('fwdReq.error data', data) ) # faasReq.requestStream.end(e.toString()) return responseStarted = false responseStream = SocketIoStream.createStream() proxyResponse.responseStream = responseStream faasReq.responseBytes = 0 respCounter = new Stream.Transform() respCounter._transform = (chunk, encoding, callback) -> debug('response chunk', encoding, chunk) faasReq.responseBytes += chunk.length callback(null, chunk) fwdReq.on 'error', (err) -> debug('fwdReq.error', err, proxyResponse) if responseStarted console.error('duplicate ERROR response attempted for '+url) responseStream.emit('error', err) else responseStarted = true proxyResponse.error = err.toString()+"\n" cb_proxyResponse(proxyResponse) try fwdReq.abort() fwdReq.destroy() faasReq.requestStream.unpipe() faasReq.requestStream.on('data', (data) -> debug('fwdReq.error data', data) ) responseStream.end() catch e console.error(e) Logger.logHttpResponseError(faasReq, proxyResponse) fwdReq.on 'response', (response) -> Logger.logHttpResponseStart(faasReq, response) debug('fwdReq.response', response.headers, response.statusCode, response.statusMessage) # debug('fwdReq.response.proxyResponse', proxyResponse) if responseStarted console.error('duplicate SUCCESS response attempted for '+url) return responseStarted = true proxyResponse.lag = faasReq.ts.responseStart - faasReq.ts.requestStart proxyResponse.headers = response.headers proxyResponse.statusCode = response.statusCode proxyResponse.statusMessage = response.statusMessage stream = fwdReq try if isRewriteHostOn || rewriteCustomInstance?.rewriteResponseHeaders || rewriteCustomInstance?.newResponseStreamTransform # Rewrite 301/302 redirects (Location), cookie domains (Set-Cookie), etc. rewriteHeaders(proxyResponse.headers, faasReq.origHost, faasReq.origProto) if isRewriteHostOn if rewriteCustomInstance?.rewriteResponseHeaders respCustomStatus = { statusCode: proxyResponse.statusCode, statusMessage: proxyResponse.statusMessage } rewriteCustomInstance.rewriteResponseHeaders(proxyResponse.headers, respCustomStatus) proxyResponse.statusCode = respCustomStatus.statusCode proxyResponse.statusMessage = respCustomStatus.statusMessage ce = proxyResponse.headers['content-encoding']?.toLowerCase() isGzip = (ce == 'gzip' || ce == 'x-gzip') isDeflate = (ce == 'deflate') ct = proxyResponse.headers['content-type']?.toLowerCase() || '' canRewrite = (ct.indexOf('text/') > -1 || ct.indexOf('javascript') > -1 || ct.indexOf('js') > -1) canRewrite &&= (!ce || ce == 'identity' || isGzip || isDeflate) customRespTransform = safeTransform(rewriteCustomInstance.newResponseStreamTransform(canRewrite), 'IX_PROXY_REWRITE_MODULE#newResponseStreamTransform') if rewriteCustomInstance?.newResponseStreamTransform if canRewrite && (isRewriteHostOn || customRespTransform) # We may change the size during rewrite, # so kill content-length and accept-ranges # (as the Range request would be forwarded), # and send it as chunked (opposite of content-length). delete proxyResponse.headers['content-length'] proxyResponse.headers['transfer-encoding'] = 'chunked' delete proxyResponse.headers['accept-ranges'] if isGzip stream = safePipe(stream, Zlib.createGunzip(), 'Gunzip') else if isDeflate # 'deflate' was screwed up by Microsoft. # Assume it's raw. Shouldn't be according to spec, # but we operate in the real world. # http://stackoverflow.com/questions/883841/why-do-real-world-servers-prefer-gzip-over-deflate-encoding if rewriteUseRealDeflate stream = safePipe(stream, Zlib.createInflate(), 'Inflate') else stream = safePipe(stream, Zlib.createInflateRaw(), 'InflateRaw') stream = safePipe(stream, newRewriterStream(faasReq.origHost, faasReq.origProto), 'Rewriter') if isRewriteHostOn stream = safePipe(stream, customRespTransform, 'rewriteCustomInstance.response') if customRespTransform # TODO: Error handling if custom _transform is buggy! if isGzip stream = safePipe(stream, Zlib.createGzip(), 'Gzip') else if isDeflate if rewriteUseRealDeflate stream = safePipe(stream, Zlib.createDeflate(), 'Deflate') else stream = safePipe(stream, Zlib.createDeflateRaw(), 'DeflateRaw') else stream = safePipe(stream, customRespTransform, 'rewriteCustomInstance.response') if customRespTransform # TODO: Error handling if custom _transform is buggy! catch e # exception in rewriteCustomInstance.rewriteResponseHeaders console.error(e) proxyResponse.error = e.toString()+"\n" proxyResponse.statusCode = 500 proxyResponse.statusMessage = 'Internal Server Error' cb_proxyResponse(proxyResponse) ### # TODO: Better handling of errors! # To test, manually force a wrong decoder (e.g. "if isGzip || true"). # Need to incorporate fwdReq.on('error',...) logic at any point in chain. ### if stream == fwdReq stream.pipe(respCounter).pipe(responseStream) else safePipe(safePipe(stream, respCounter, 'respCounter'), responseStream, 'responseStream') fwdReq.on 'end', -> Logger.logHttpResponseEnd(faasReq) reqStream.pipe(fwdReq) newRewriterStream = (host, proto) -> queue = '' changeProtoOriginsRe = null upProto = upstreamProto if rewriteResponseDomainsRe && host.indexOf("--#{rewriteResponseDomainsBase}") > 0 upProto = if host.slice(0,6) == 'http--' then 'http' else 'https' if upProto != proto hostsReStr = regexEscape(host) if Base.env.IX_PROXY_REWRITE_PROTO_HOSTS_REGEX hostsReStr += "|"+Base.env.IX_PROXY_REWRITE_PROTO_HOSTS_REGEX changeProtoOriginsRe = new RegExp("#{upProto}(#{rewriteProtoDelimReStr})(#{hostsReStr})\\b", "ig") debug('changeProtoOriginsRe', changeProtoOriginsRe) rewriter = new Stream.Transform() rewriter._transform = (chunk, encoding, callback) -> debug('rewriter chunk', encoding, chunk) queue += chunk.toString('utf8') queue = queue.replace(upstreamHostRe, host) if upstreamUrl queue = queue.replace(changeProtoOriginsRe, "#{proto}$1$2") if changeProtoOriginsRe if rewriteCustomReplacers for replacer in rewriteCustomReplacers queue = queue.replace(replacer[0], replacer[1]) queue = rewriteResponseDomains(queue) if rewriteResponseDomainsRe if queue.length > 100 splitPoint = queue.length - 100 oldChunk = queue.slice(0, splitPoint) queue = queue.slice(splitPoint) rewriter.push(new Buffer(oldChunk, 'utf8')) callback() rewriter._flush = (callback) -> debug('rewriter flush') rewriter.push(new Buffer(queue, 'utf8')) callback() rewriter newRequestRewriterStream = (host, proto) -> queue = '' isBinary = false downstreamHostReStr = regexEscape(host) if upstreamProto != proto changeProtoOriginsRe = new RegExp("#{proto}(#{rewriteProtoDelimReStr})(#{downstreamHostReStr})\\b", "ig") debug('changeProtoOriginsRe', changeProtoOriginsRe) downstreamHostRe = new RegExp("#{downstreamHostReStr}\\b", "ig") debug('downstreamHostRe', downstreamHostRe) rewriter = new Stream.Transform() rewriter._transform = (chunk, encoding, callback) -> debug('requestRewriter chunk', encoding, chunk) if isBinary rewriter.push(chunk) callback() return chunkStr = chunk.toString() if chunk.equals(new Buffer(chunkStr)) queue += chunkStr # Do replacements in the reverse order of the response replacements. if upstreamProto != proto queue = queue.replace(changeProtoOriginsRe, "#{upstreamProto}$1$2") queue = queue.replace(downstreamHostRe, upstreamHost) if queue.length > 100 splitPoint = queue.length - 100 oldChunk = queue.slice(0, splitPoint) queue = queue.slice(splitPoint) rewriter.push(new Buffer(oldChunk)) callback() else # Switch to binary pass-thru mode (image uploads, etc.) debug('requestRewriter switch to binary mode') isBinary = true rewriter.push(new Buffer(queue)) if queue.length > 0 rewriter._transform(chunk, encoding, callback) rewriter._flush = (callback) -> debug('requestRewriter flush') rewriter.push(new Buffer(queue)) unless isBinary callback() rewriter regexEscape = (str) -> # http://stackoverflow.com/questions/2593637/how-to-escape-regular-expression-in-javascript/17326679#17326679 str.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, "\\$1") rewriteHeaders = (headers, host, proto) -> # Always rewrite Location: (for 301/302 redirects). if headers.location if upstreamUrl headers.location = headers.location.replace(upstreamHostRe, host) if upstreamProto != proto hostReStr = regexEscape(host) headers.location = headers.location.replace(new RegExp("#{upstreamProto}(#{rewriteProtoDelimReStr})(#{hostReStr})\\b", "ig"), "#{proto}$1$2") headers.location = rewriteResponseDomains(headers.location) # Always rewrite Set-Cookie: domain=... if headers['set-cookie'] for val, i in headers['set-cookie'] val = val.replace(upstreamDomainCookieRe, "$1#{host}$2") if upstreamUrl headers['set-cookie'][i] = rewriteResponseDomains(val, true) # Possibly rewrite other things - higher-up Set-Cookie domains, etc. if rewriteHeadersCustomReplacers for replacer in rewriteHeadersCustomReplacers nameRe = replacer[0] search = replacer[1] replace = replacer[2] for own name, value of headers if name.match(nameRe) if Array.isArray(value) for val, i in value value[i] = rewriteResponseDomains(val.replace(search, replace)) else headers[name] = rewriteResponseDomains(value.replace(search, replace)) rewriteResponseDomains = (str, forCookie) -> return str unless rewriteResponseDomainsRe re = if forCookie then rewriteResponseDomainsCookieRe else rewriteResponseDomainsRe str.replace re, (replaceCbArgs...) -> # Support groups in user-specified domain regexp with replaceCbArgs match = replaceCbArgs[0] before = replaceCbArgs[1] domain = replaceCbArgs[2] after = replaceCbArgs[replaceCbArgs.length-3] debug("rewriteResponseDomains match", match) replacement = match unless domain == rewriteResponseDomainsBase || domain.indexOf("--#{rewriteResponseDomainsBase}") > 0 newDomain = domain.replace(/-/g,'--').replace(/\./g,'-')+'--'+rewriteResponseDomainsBase if !forCookie && before.match(/^http\b/i) newDomain = "http--#{newDomain}" replacement = before+newDomain replacement += after replacement safePipe = (inpipe, outpipe, lbl) -> outpipe.label = lbl label = "stream.#{inpipe.label}" inpipe.pipe(outpipe) inpipe.on 'error', (err) -> console.error(["#{label}.error", err]) ### # responseStream (SocketIoStream) doesn't seem to respond well # to ".emit('error',...)". So .end() it instead of propogating error. ### # try outpipe.emit('error', err) catch e then debug("#{label}.outpipe.emitError.error", e) try outpipe.end() catch e then debug("#{label}.end.error", e) try inpipe.unpipe(outpipe) catch e then debug("#{label}.unpipe.error", e) try inpipe.on('data', -> debug("#{label}.data", data)) catch e then debug("#{label}.data.error", e) inpipe.on('end', -> debug("#{label}.end")) outpipe safeTransform = (transform, lbl) -> return transform unless transform handleError = (e, callback) -> errMsg = "Error during transform #{lbl}: "+("#{e.stack}".split("\n").slice(0,2).join(" ")||"#{e}") console.error(errMsg) transform.push("[#{errMsg}]") callback() origTransform = transform._transform transform._transform = (chunk, encoding, callback) -> try origTransform.call(transform, chunk, encoding, callback) catch e handleError(e, callback) origFlush = transform._flush transform._flush = (callback) -> try origFlush.call(transform, callback) catch e handleError(e, callback) transform