/** @module merlins */
import {ScrapingConnector} from './scraping-connector'
const ROOT_CATEGORY = 'root'
/**
* Takes commerce-integrations Integer type and returns the URL
* representation used to view a product details page on the website.
*
* @param {module:types.Integer} id
* @returns {String}
* @private
*/
const getProductUrl = (id) => {
return `/${id}.html`
}
// This regex is used to get a product's ID from its "Wishlist" or "Compare"
// button. It's stored in a string in a data attribute of the button.
const PRODUCT_REGEXP = new RegExp(/product":"([0-9]*)","/)
/**
* Takes a Magento product attribute value object and parses out a
* commerce-integrations VariationPropertyValue type
*
* @param {Object} data
* @returns {module:types.VariationPropertyValue}
* @private
*/
const parseVariationPropertyValue = ({label, id}) => ({
name: label,
value: id
})
/**
* Takes a Magento product variation attribute object and parses out a
* commerce-integrations VariationProperty type
*
* @param {Object} data
* @returns {module:types.VariationProperty}
* @private
*/
const parseVariationProperty = ({code, label, options = []}) => ({
id: code,
label,
values: options.map(parseVariationPropertyValue)
})
/**
* Takes a Magento product object and parses out an
* commerce-integrations Variation type
*
* @param {Object} data
* @returns {module:types.Variation}
* @private
*/
const parseVariation = (id, value, config) => {
const convertedValues = Object.keys(value).reduce((acc, curr) => {
acc[config.attributes[curr].code] = value[curr]
return acc
}, {})
return {
id,
price: parseFloat(config.optionPrices[id].basePrice.amount),
orderable: true, // Magento doesn't show out of stock variations... I think...
values: convertedValues
}
}
/**
* Takes a takes a Magento image object and returns a set of commerce-integrations
* Image type.
*
* @param {Object} data
* @returns {Array.<module:types.Image>}
* @private
*/
const parseImageSet = ({caption, full, thumb}) => {
const sets = []
// Parse `full` image
sets.push({
images: [{title: caption, src: full, alt: caption}],
variationProperties: [],
sizeType: 'full'
})
// Parse `thumb` image
sets.push({
images: [{title: caption, src: thumb, alt: caption}],
variationProperties: [],
sizeType: 'thumb'
})
return sets
}
/**
* Takes a document object and parses out a commerce-integrations Product type
*
* @param {Object} htmlDoc a {@link https://developer.mozilla.org/en-US/docs/Web/API/Document|Document} object
* @returns {Promise<module:types.Product>}
* @private
*/
const parseProduct = (htmlDoc, id) => {
const name = htmlDoc.querySelector('.page-title').textContent
const description = htmlDoc.querySelector('.product.description').textContent
const categoryEl = htmlDoc.querySelector('.breadcrumbs .item:nth-last-child(2) a')
const priceEl = htmlDoc.querySelector('meta[property="product:price:amount"]')
const skuEl = htmlDoc.querySelector('.sku .value')
// Parse the categoryId
let categoryId
if (categoryEl) {
const categoryIdMatch = categoryEl.pathname.match(/\/([^.]+)\.html/)
categoryId = categoryIdMatch && categoryIdMatch[1]
}
// Parse the currency, price, prices, upc
let price
let upc
if (priceEl) { price = parseFloat(priceEl.getAttribute('content')) }
if (skuEl) { upc = skuEl.textContent }
// We'll get alot of the product information from an inline script which contains
// product details in the form of JSON.
let productConfig
const productScriptEls = htmlDoc.querySelectorAll('.product-add-form script')
if (productScriptEls && productScriptEls.length > 1) { // NOTE: If there are variations there will be more than 1 script
const productScriptJSON = JSON.parse(productScriptEls[0].textContent)
productConfig = productScriptJSON['#product_addtocart_form'].configurable.spConfig
}
let galleryConfig
const galleryScriptEl = htmlDoc.querySelector('.product.media script')
if (galleryScriptEl) {
const galleryScriptJSON = JSON.parse(galleryScriptEl.textContent)
galleryConfig = galleryScriptJSON['[data-gallery-role=gallery-placeholder]']['mage/gallery/gallery'].data
}
return {
id,
name,
imageSets: galleryConfig &&
galleryConfig.reduce((acc, curr) => ([...acc, ...parseImageSet(curr)]), []),
description,
categoryId, // NOTE: Without the category in the path, it won't show up in the bread crumbs.
brand: `Merlin's Potions`,
minimumOrderQuantity: 1,
stepQuantity: 1,
upc,
unit: 'ea',
price,
prices: productConfig &&
Object.keys(productConfig.prices).reduce((acc, curr) => ({...acc, ...{[curr]: productConfig.prices[curr].amount}}), {}),
variations: productConfig &&
Object.keys(productConfig.index).map((key) => parseVariation(key, productConfig.index[key], productConfig)),
variationProperties: productConfig &&
Object.keys(productConfig.attributes).map((attributeId) => parseVariationProperty(productConfig.attributes[attributeId])),
variationValues: productConfig && productConfig.defaultValues
}
}
/**
* A connector for www.merlinspotions.com, using screen-scraping.
*
* @implements {module:interfaces.CommerceConnector}
*/
export class MerlinsConnector extends ScrapingConnector {
constructor({window, basePath, dondeGeoBasePath, dondeApiBasePath, cookieDomainRewrites}) {
super({window, cookieDomainRewrites})
this.basePath = basePath
this.dondeGeoBasePath = dondeGeoBasePath
this.dondeApiBasePath = dondeApiBasePath
}
/**
* @private
*/
loginUrl() {
return `${this.basePath}/customer/account/login/`
}
/**
* @inheritDoc
*/
login(username, password) {
return this.agent.get(this.loginUrl())
.then((res) => this.buildDocument(res))
.then((doc) => doc.querySelector('#login-form input[name="form_key"]').value)
.then((formKey) => ({
form_key: formKey,
'login[username]': username,
'login[password]': password,
persistent_remember_me: 'on',
send: '',
}))
.then((data) => (
this.agent.post(`${this.basePath}/customer/account/loginPost/`)
.type('form')
.send(data)
)
)
.then((res) => this.buildDocument(res))
.then(() => this.getCustomer('me'))
}
/**
* @inheritDoc
*/
logout() {
return this.agent.post(`${this.basePath}/customer/account/logout/`)
.then(() => undefined)
}
/**
* @inheritDoc
*/
getCustomer(id) {
if (id !== 'me') {
return Promise.reject(new Error('This backend only supports getting the current customer ' +
'(pass "me" as the ID)'))
}
const ensureLoggedIn = (res) => {
const loggedOut = res.redirects.indexOf(this.loginUrl()) >= 0
return loggedOut ? Promise.reject(new Error('logged out')) : Promise.resolve(res)
}
return this.agent.get(`${this.basePath}/customer/account/edit/`)
.then((res) => ensureLoggedIn(res))
.then((res) => this.buildDocument(res))
.then((doc) => {
const form = doc.getElementById('form-validate')
return {
id: 'me',
firstName: form.querySelector('input[name="firstname"]').value,
lastName: form.querySelector('input[name="lastname"]').value,
email: form.querySelector('input[name="email"]').value,
}
})
}
/**
* @inheritDoc
*/
refreshSession() {
// This is for interface compatibility - it's not necessary
// on a screen-scraping connector, generally.
return this.getCustomer('me')
}
/**
* Given a configuration in the form of a plain object, this method returns
* a new MerlinsConnector instance.
*
* @param {Object} config
*
* @returns {module:types.MerlinsConnector} The new MerlinsConnector instance.
*/
static fromConfig(config) {
return new this.prototype.constructor(config)
}
/**
* @inheritDoc
*/
getProduct(id) {
const url = `${this.basePath}${getProductUrl(id)}`
return this.agent.get(url)
.then((res) => this.buildDocument(res))
.then((htmlDoc) => {
if (htmlDoc.title === '404 Not Found') {
throw new Error('Product Not Found')
}
return htmlDoc
})
.then((htmlDoc) => this.parseProduct(htmlDoc, id))
}
/**
* @inheritDoc
*/
getProducts(ids = []) {
const getProductPromises = ids.map((id) => this.getProduct(id).catch(() => null))
return Promise.all(getProductPromises)
.then((products) => {
const validProducts = products.filter((product) => !!product)
return {
count: validProducts.length,
data: validProducts,
total: validProducts.length
}
})
}
/**
* Takes a OCAPI Product object and parses it into a commerce-integrations Product type.
* @param {Object} data a OCAPI {@link https://documentation.demandware.com/DOC1/topic/com.demandware.dochelp/OCAPI/18.3/shop/Documents/Product.html|Product} document
*
* @returns {Promise<module:types.Store>}
*/
parseProduct(data, id) {
return parseProduct(data, id)
}
/**
* @inheritDoc
*/
searchProducts(productSearchRequest) {
const url = `${this.basePath}${this.getSearchUrl(productSearchRequest)}`
if (!productSearchRequest.count || [5, 10, 15].includes(productSearchRequest.count)) {
console.warn(`The Merlins backend only supports 'count' values of 5, 10, 15. Defaulting to 5.`)
productSearchRequest.count = 5
}
return this.agent.get(url)
.then((res) => this.buildDocument(res))
.then((htmlDoc) => this.parseSearchProducts(htmlDoc, productSearchRequest))
}
parseGetStore(htmlDoc) {
// Parse the page into the models unique parts and assign them to the result.
const mainContent = htmlDoc.getElementById('maincontent')
const name = mainContent.querySelector('.location-title').textContent
const addressLine1 = mainContent.querySelector('[itemprop="streetAddress"]').textContent.trim()
const city = mainContent.querySelector('[itemprop="addressLocality"]').textContent
const country = mainContent.querySelector('.location-crumb.first-crumb').textContent
const phone = mainContent.querySelector('[itemprop="telephone"]').textContent.replace(new RegExp('[(]|[)]|[.]|[ ]|[-]|[#]|[x]', 'g'), '')
const postalCode = mainContent.querySelector('[itemprop="postalCode"]').textContent
const hoursContent = this.window.document.createElement('div')
hoursContent.innerHTML = mainContent.querySelector('#hoursBlock').textContent
const hours = {}
const hoursNodeList = hoursContent.querySelectorAll('.fh-item')
hoursNodeList.forEach((hour) => {
const content = hour.querySelector('[itemprop="name"]')
if (content) {
const day = content.textContent.slice(0, -1)
const openingTime = hour.querySelector('[itemprop="opens"]').textContent
const closingTime = hour.querySelector('[itemprop="closes"]').textContent
hours[day] = {
openingTime,
closingTime
}
}
})
return {
addressLine1,
city,
country,
name,
phone: phone ? phone : undefined,
postalCode: postalCode ? postalCode : undefined,
hours: hours ? hours : undefined
}
}
/**
* @inheritDoc
*/
getStore(id) {
const storesBasePath = this.basePath.replace('www', 'locations')
const url = `${storesBasePath}/${id}.html?utm_source=storelocator`
return this.agent.get(url)
.then((res) => this.buildDocument(res))
.then((htmlDoc) => this.parseGetStore(htmlDoc))
.then((data) => ({...data, id}))
}
/**
* @inheritDoc
*/
getCategory(id) {
// Always request the basePath for getting all categories
return this.agent.get(this.basePath)
.then((res) => this.buildDocument(res))
.then((htmlDoc) => {
let category
if (id === ROOT_CATEGORY) {
const categoryEls = [...htmlDoc.querySelectorAll('.navigation a')].map((el) => el.parentElement)
const categories = categoryEls.map(this.parseCategory)
category = {
id: ROOT_CATEGORY,
categories
}
} else {
const categoryAnchorEl = htmlDoc.querySelector(`.navigation a[href*="${id}"]`)
if (!categoryAnchorEl) {
throw new Error('Category Not Found')
}
category = this.parseCategory(categoryAnchorEl.parentElement)
}
return category
})
}
/**
* @inheritDoc
*/
getCategories(ids = []) {
const getCategoryPromises = ids.map((id) => this.getCategory(id).catch(() => null))
return Promise.all(getCategoryPromises)
.then((categories) => {
const validCategories = categories.filter((category) => !!category)
return {
count: validCategories.length,
data: validCategories,
total: validCategories.length
}
})
}
/**
* Takes a html element that represents a category and parses it into a commerce-integrations
* Category type.
*
* @param {Object} htmlDoc a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
*
* @returns {Promise<module:types.Category>}
*/
parseCategory(categoryEl) {
const anchorEl = categoryEl.querySelector('a')
const idMatch = anchorEl.pathname.match(/[^\\](\w+).html$/)
return {
id: idMatch && idMatch[1],
name: anchorEl.innerText
}
}
/**
* Takes commerce-integrations ProductSearchRequest type and returns the URL
* representation used to query the website.
*
* @param {module:types.ProductSearchRequest} productSearchRequest
*
* @returns {String}
*/
getSearchUrl({
count,
sort,
start,
total,
query,
filters = {},
}) {
const params = []
const pathname = query ?
'/catalogsearch/result/' :
`/${filters.categoryId}.html`
if (count) { params.push(`product_list_limit=${count}`) }
if (total) { params.push(`product_list_total=${total}`) }
if (sort) { params.push(`product_list_order=${sort}`) }
if (start) { params.push(`p=${Math.ceil(total / start)}`) }
if (query) { params.push(`q=${query}`) }
// Compose the filters part of the url
Object.keys(filters)
.filter((key) => !['categoryId'].includes(key))
.forEach((key) => {
params.push(`${key}=${filters[key]}`)
})
return `${pathname}?${params.sort().join('&')}`
}
/**
* Takes a dom element representing a merlins search result product tile and parses out an
* commerce-integrations ProductSearchResult type.
*
* @param {Object} productEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
*
* @returns {module:types.ProductSearchResult}
*/
parseProductSearchResult(productEl) {
const linkEl = productEl.querySelector('.product-item-link')
const imageEl = productEl.querySelector('.product-image-photo')
const priceEl = productEl.querySelector('.price')
const stockEl = productEl.querySelector('.stock')
const productName = linkEl.textContent
const actionButtonData = productEl.querySelector('.towishlist').getAttribute('data-post')
const productId = PRODUCT_REGEXP.exec(actionButtonData)[1]
const available = !stockEl
let price
if (available) {
const priceMatch = priceEl.textContent.match(/\$(.+)/)
price = priceMatch ? parseFloat(priceEl.textContent.match(/\$(.+)/)[1]) : ''
} else {
price = -1 // Something obviously invalid
}
const image = {
alt: imageEl.getAttribute('alt'),
description: imageEl.getAttribute('alt'),
src: imageEl.getAttribute('src'),
title: imageEl.getAttribute('alt')
}
return {
available,
productId,
productName,
defaultImage: image,
price,
variationProperties: []
}
}
/**
* Takes a dom element representing a merlins sorting select and parses out an
* array od commerce-integrations SortOption type.
*
* @param {Object} sortEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
*
* @returns {Array.<module:types.SortOption>}
*/
parseSortingOptions(sortEl) {
const optionEls = sortEl.querySelectorAll('option')
return [...optionEls].map((el) => ({
id: el.value,
label: el.innerHTML
}))
}
/**
* Takes a dom element representing a merlins filter and parses out an
* commerce-integrations Filter type.
*
* @param {Object} optionEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
*
* @returns {module:types.Filter}
*/
parseFilter(optionEl) {
const label = optionEl.querySelector('.filter-options-title').textContent
const propertyId = label.toLowerCase()
const valueEls = optionEl.querySelectorAll('.items .item')
const values = [...valueEls].map((valueEl) => {
const countEl = valueEl.querySelector('.count')
const anchorEl = valueEl.querySelector('a')
// Parse out the count and then remove the element so we can get the label text.
const count = parseInt(countEl.textContent)
countEl.remove()
return {
count,
label: anchorEl.textContent.trim(),
value: anchorEl.search.match(/\w=([^&]+)/)[1]
}
})
return {
propertyId,
label,
values
}
}
/**
* Takes a dom element representing a merlins selected filter and parses out a
* commerce-integrations Filter type.
*
* @param {Object} selectedEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
* @param {String} currentUrl The current page url used to help determine filter values.
*
* @returns {module:types.Filter}
*/
parseSelectedFilter(selectedEl, currentUrl) {
// Use an anchor to parse out the search string
const anchorEl = this.window.document.createElement('a')
anchorEl.href = currentUrl
const label = selectedEl.querySelector('.filter-label').textContent
const propertyId = label.toLowerCase()
const valueLabel = selectedEl.querySelector('.filter-value').textContent
const valueValue = anchorEl.search.match(new RegExp(`${propertyId}=([^&]+)`))[1]
return {
propertyId,
label,
values: [{
label: valueLabel,
value: valueValue
}]
}
}
/**
* Takes a dom element and parses out an array of commerce-integrations
* Filter type representing those filters that are selected.
*
* @param {Object} selectedFiltersEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
* @param {String} currentUrl The current page url used to help determine filter values.
*
* @returns {Array.<module:types.Filter>}
*/
parseSelectedFilters(selectedFiltersEl, currentUrl) {
const selectedEls = selectedFiltersEl.querySelectorAll('.items .item')
return [...selectedEls].map((el) => this.parseSelectedFilter(el, currentUrl))
}
/**
* Takes a dom element and parses out an array of commerce-integrations
* Filter type.
*
* @param {Object} filterOptionsEl a {@link https://developer.mozilla.org/en-US/docs/Web/API/Element|Element} object
* @param {String} currentUrl The current page url used to help determine filter values.
*
* @returns {Array.<module:types.Filter>}
*/
parseFilters(filterOptionsEl, currentUrl) {
// Note: Merlin's Potion's is unique in the way that it handles filters, normally
// you'd see checkboxes of all the filters with those that are selected being filled
// in. That makes it easy to parse. But in this case they are split up in two sections,
// one is the filters that aren't selected and a list of links, and the other is a
// section of selected filters with it's value.
// we'll have to parse these both to get as close to a complete list of all the filters.
const optionsEls = filterOptionsEl.querySelectorAll('.filter-options-item')
return [...optionsEls].map((el) => this.parseFilter(el, currentUrl))
}
parseSearchProducts(htmlDoc, searchRequest) {
const {filters, query} = searchRequest
const is404 = htmlDoc.title === '404 Not Found'
const isEmpty = !!htmlDoc.querySelector('.message.info.empty')
if (is404 || isEmpty) {
return {
count: 0,
filters: [],
results: [],
selectedFilters: filters,
sortingOptions: [],
start: 0,
total: 0
}
}
// This is the current url of the page, it'll be used to get filter values, etc.
const currentUrl =
JSON.parse(htmlDoc.querySelector('.toolbar-products')
.getAttribute('data-mage-init'))
.productListToolbarForm
.url
// Parse the page into the models unique parts and assign them to the result.
const mainContent = htmlDoc.getElementById('maincontent')
const resultEls = mainContent.querySelectorAll('.products.product-items > .product-item')
const pageSize = parseInt(mainContent.querySelector('#limiter').value)
const pageEl = mainContent.querySelector('.pages .current span:last-child')
const page = pageEl ? parseInt(pageEl.textContent) - 1 : 0
const filterOptionsEl = mainContent.querySelector('.filter-options')
const selectedFiltersEl = mainContent.querySelector('.filter-current')
const filterOptions = filterOptionsEl ?
this.parseFilters(filterOptionsEl, currentUrl) : []
const selectedFilters = selectedFiltersEl ?
this.parseSelectedFilters(selectedFiltersEl, currentUrl) : []
const sortEl = mainContent.querySelector('#sorter')
const totalEl = mainContent.querySelector('#toolbar-amount .toolbar-number')
return {
count: resultEls.length,
query: htmlDoc.querySelector('#search').value || query,
filters: [...filterOptions, ...selectedFilters],
results: [...resultEls].map(this.parseProductSearchResult),
selectedFilters: selectedFilters.reduce((obj, curr) => {
obj[curr.propertyId] = curr.values[0].value
return obj
}, {}),
selectedSortingOption: sortEl ? sortEl.value : '',
sortingOptions: sortEl ? this.parseSortingOptions(sortEl) : [],
start: page * pageSize,
total: totalEl ? parseInt(totalEl.textContent) : 0,
}
}
/**
* @inheritDoc
*/
searchStores(storeSearchRequest) {
let count = storeSearchRequest.count > 0 ? storeSearchRequest.count : 20
const start = storeSearchRequest.start > 0 ? storeSearchRequest.start : 0
const latitude = storeSearchRequest.latlon.latitude
const longitude = storeSearchRequest.latlon.longitude
let total
if (isNaN(latitude) || isNaN(longitude)) {
throw new Error('Provide Latitude and Longitude coordinates')
}
const getStoresUrl = `${this.dondeApiBasePath}/near?center=${latitude},${longitude}&dondeKey=56c3681586312e000e000001&_=1527268304391`
return this.agent.get(getStoresUrl)
.then((res) => res.body)
.then((jsonRes) => {
total = jsonRes.length
const stores = jsonRes.map((store) => {
const storeId = store.llp_url.replace('.html', '').substring(1)
return this.getStore(storeId)
.then((data) => {
return data
})
})
return Promise.all(stores).then((stores) => {
let storesToReturn
const totalPages = Math.ceil(total / count)
if (totalPages === start + 1) { // on last page
const remainder = total % count
if (remainder > 0) {
count = remainder
}
storesToReturn = stores.slice(stores.length - remainder)
} else if (totalPages < start + 1) {
throw new Error('No stores found')
} else { // middle pages
storesToReturn = stores.slice(start * count, (start + 1) * count)
}
return {
count,
start,
total,
stores: storesToReturn,
}
})
})
}
}