import { URL } from "url"; import _ from "lodash"; const DEFAULT_REMOVED_QUERY_PARAMS = [ "utm_campaign", "utm_source", "utm_content", "utm_term", "Utm_viral", "utm_medium", "utm_identifier", "utm_key", "fb_source", "referrer", "referer", "ref", "Rand", "rnd", "randid", ".?rand", ".?partner", "cb", "count", "adid", "session", "sessionid", "session_redirect", "paramd", "userinfo", "uid", "refresh", "ocid", "ncid", "clickenc", "fbclid", "Amp_js_v", "amp_gsa", "ns_source", "ns_campaign", "ns_mchanel", "ns_mchannel", "Ns_linkname", "ns_fee", "src", "ex_cid", "usqp", "source", "xid", "trkid", "utm_social-type", "mbid", "utm_brand", "__twitter_impression", "utm_referrer", "CMP", ]; const paramRemovalPatterns: Record = _.mapValues( { default: DEFAULT_REMOVED_QUERY_PARAMS, "trivia.com": ["guid", "dicbo", ...DEFAULT_REMOVED_QUERY_PARAMS], }, createQueryParamRegExp ); const pathRemovalPatterns: Record = { default: [/\/amp(?:\/[?&]*)?$/], }; const protocolRemovalPatterns: Record = { default: [/^(https?:\/\/)?(www\.)?/], }; const domainSanitizationPattern = /^www\./; export const sanitizeURL = (url: string): string => { const parsed = new URL(url); const domain = parsed.hostname.replace(domainSanitizationPattern, ""); const pathRemovalPattern = pathRemovalPatterns[domain] || pathRemovalPatterns.default; const paramRemovalPattern = paramRemovalPatterns[domain] || paramRemovalPatterns.default; for (const pattern of [...pathRemovalPattern, paramRemovalPattern]) { url = url.replace(pattern, ""); } // ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ // │ href │ // ├──────────┬──┬─────────────────────┬────────────────────────┬───────────────────────────┬───────┤ // │ protocol │ │ auth │ host │ path │ hash │ // │ │ │ ├─────────────────┬──────┼──────────┬────────────────┤ │ // │ │ │ │ hostname │ port │ pathname │ search │ │ // │ │ │ │ │ │ ├─┬──────────────┤ │ // │ │ │ │ │ │ │ │ query │ │ // " https: // user : pass @ sub.example.com : 8080 /p/a/t/h ? query=string #hash " // │ │ │ │ │ hostname │ port │ │ │ │ // │ │ │ │ ├─────────────────┴──────┤ │ │ │ // │ protocol │ │ username │ password │ host │ │ │ │ // ├──────────┴──┼──────────┴──────────┼────────────────────────┤ │ │ │ // │ origin │ │ origin │ pathname │ search │ hash │ // ├─────────────┴─────────────────────┴────────────────────────┴──────────┴────────────────┴───────┤ // │ href │ // └────────────────────────────────────────────────────────────────────────────────────────────────┘ // add a / for homepage url => no pathname or only /, no search, no hash, not ends with /, ?, # const urlWithProtocol = new URL(url); if ( urlWithProtocol.pathname === "/" && urlWithProtocol.search === "" && urlWithProtocol.hash === "" && !url.endsWith("/") ) { // remove ? if no query string if (url.endsWith("?")) { url = url.slice(0, -1); } url += "/"; } const protocolRemovalPattern = protocolRemovalPatterns[domain] || protocolRemovalPatterns.default; for (const pattern of [...protocolRemovalPattern]) { url = url.replace(pattern, ""); } return url; }; function createQueryParamRegExp(params: string[]) { return new RegExp(`(((?<=\\?)|&|%3F|%26|;)(?:${params.join("|")})=[^&]*)`, "g"); }