Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | 20x 20x 20x 20x 50x 50x 50x 5x 3x 3x 12x 3x 3x 12x 2x 2x 2x 6x 5x 2x 30x 4x 4x 15x 4x 4x 7x 4x 18x 15x 15x 15x | import request from 'superagent'
import * as cj from 'cookiejar'
/**
* Cookie-rewriting details:
*
* ScrapingConnector uses a cookie-jar (through superagent) to
* handle sessions on traditional HTML backends. This lets
* us make authenticated request client- *and* server-side,
* for Desktop, etc.
*
* To maintain sessions when running server-side, ScrapingConnector
* supports cookie re-writing. Eg:
*
* 1) Imagine you are running shop.com (the real backend) and
* upwa-shop.com (a server-side proxy using this Connector).
*
* 2) A browser requests a resource from the proxy, which
* forwards a request to shop.com. The shop.com response
* contains the header "set-cookie: session=1; domain=shop.com"
*
* 3) Before responding to the browser, this connector
* rewrites the cookie domain to proxy-shop.com, so that
* it will be included on the subsequent request from the
* browser to the proxy.
*/
/**
* A base class for screen-scraping connectors.
*
* ScrapingConnector does three things:
*
* 1) Allows `window` to be injected, so we can swap it for JSDOM
* and run the Connector outside of a browser.
*
* 2) Supports cookie-based sessions server-side, using a
* cookie-jar and domain-name rewriting.
*
* 3) Lets us get/set default headers for requests.
*
*/
export class ScrapingConnector {
/**
* @param {Window} window A browser-native window or JSDOM implementation.
* @param {Object.<string, string>} cookieDomainRewrites the domains to rewrite
* on any cookies received from the backend, eg.`{"shop.com": "proxy-shop.com"}`
*/
constructor({window, cookieDomainRewrites}) {
this.window = window
this.cookieDomainRewrites = cookieDomainRewrites || {}
this.agent = request.agent()
this.cookieJarSnapshot = null
}
/**
* Return a Document constructed from a superagent HTTP response. Use
* to parse content out of the DOM.
*
* @param {HTTPResponse} res a superagent http response, expected to be HTML.
* @returns {Document}
*/
buildDocument(res) {
const doc = this.window.document.implementation.createHTMLDocument()
doc.documentElement.innerHTML = res.text
return doc
}
/**
* @inheritDoc
*/
getDefaultHeaders() {
const headers = (this.agent._defaults
.filter((defaultCall) => defaultCall.fn === 'set')
.map((header) => ({[header.arguments[0]]: header.arguments[1]}))
)
return Object.assign({}, headers)
}
/**
* @inheritDoc
*/
setDefaultHeaders(headers) {
Object.keys(headers).forEach((k) => {
this.agent.set(k, headers[k])
})
}
inBrowser() {
return typeof window !== 'undefined'
}
/**
* Get all *new or changed* cookies in the Connector's cookie jar,
* since creation or the last call to `setCookies()`.
*
* Each returned cookie can be set directly as a header on an HTTP
* response in the "set-cookie" header.
*
* This is only for use *outside* a browser. In the browser, cookies
* are handled automatically.
*
* Example return value:
*
* `["a=b; domain=example.com; path=/", "c=d; domain=example2.com; path=/"]`
*
* @returns {Array.<String>}
*/
getCookies() {
Eif (!this.inBrowser()) {
const rewrites = this.cookieDomainRewrites || {}
const rewriter = (cookie) => ScrapingConnector.rewriteCookieDomain(rewrites, cookie)
const snapshot = this.cookieJarSnapshot || new cj.CookieJar()
return ScrapingConnector.diffCookieJars(snapshot, this.agent.jar)
.map(rewriter)
.map((cookie) => cookie.toString())
} else {
return []
}
}
/**
* Set the initial cookies on the connector.
*
* The `cookies` parameter must be in the format for the `Cookie` header
* taken directly from an HTTP request, eg: `"a=b; c=d"`
*
* @param {String} cookies
*/
setCookies(cookies) {
Eif (!this.inBrowser()) {
cookies = cookies || ''
cookies.split('; ')
.filter((x) => x !== '')
.forEach((c) => this.agent.jar.setCookies(c))
this.cookieJarSnapshot = ScrapingConnector.cloneCookieJar(this.agent.jar)
}
}
static cloneCookie(cookie) {
return Object.assign(new cj.Cookie(), cookie)
}
static cloneCookieJar(jar) {
const clone = new cj.CookieJar()
jar.getCookies(cj.CookieAccessInfo.All).forEach((c) => {
clone.setCookie(ScrapingConnector.cloneCookie(c))
})
return clone
}
/**
* Returns the cookies that are new or changed in `newJar`.
*
* @param oldJar {CookieJar}
* @param newJar {CookieJar}
* @returns {Array<Cookie>}
*/
static diffCookieJars(oldJar, newJar) {
const oldCookies = oldJar.getCookies(cj.CookieAccessInfo.All)
.map((c) => c.toString())
return newJar.getCookies(cj.CookieAccessInfo.All)
.filter((c) => oldCookies.indexOf(c.toString()) < 0)
}
/**
* Clone `cookie` replace `originalDomain` with `proxyDomain` if the
* original cookie had an explicit domain set.
*
* This is for server-side usage, eg. when running the IM as a proxy
* on a different domain than the original ecommerce backend.
*
* @param {Object<String, String>} rewrites
* @param {Array<Cookie>} cookie
* @returns {Array<Cookie>}
*/
static rewriteCookieDomain(rewrites, cookie) {
const clone = ScrapingConnector.cloneCookie(cookie)
clone.domain = rewrites[clone.domain] || clone.domain
return clone
}
}
|