Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 | 1x 1x 1x 1x 1x 1x 1x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 3x 2x 2x 1x 2x 2x 2x 2x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 4x 1x | /**
* @file fetch-terms-from-index.js
* @description Fetches term definitions from an external specification repository using either the published GitHub Pages site or the raw repository fallback.
*/
const fs = require('node:fs');
const path = require('node:path');
const { JSDOM } = require('jsdom');
const axios = require('axios');
const { getPath } = require('../../../config/paths');
const Logger = require('../../utils/logger');
const CACHE_DIR = getPath('githubcache');
/**
* Retrieves the latest commit hash for a specific file in a GitHub repository.
*
* @param {string} token - GitHub API token for authentication
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {string} filePath - Path to the file within the repository
* @param {Object} headers - HTTP headers for the request
* @returns {Promise<string|null>} The commit SHA or null if not found
*/
async function getFileCommitHash(token, owner, repo, filePath, headers) {
try {
const normalizedPath = filePath.replace(/^\//, '');
const commitsUrl = `https://api.github.com/repos/${owner}/${repo}/commits?path=${normalizedPath}&per_page=1`;
Logger.process(`Fetching latest commit for file: ${commitsUrl}`);
const response = await axios.get(commitsUrl, { headers });
if (response.status !== 200 || !Array.isArray(response.data) || response.data.length === 0) {
Logger.error(`Could not find commit information for ${filePath}`);
return null;
}
return response.data[0].sha;
} catch (error) {
Logger.error(`Error fetching commit hash: ${error.message}`);
return null;
}
}
/**
* Fetches all term definitions from an external specification repository.
* Extracts terms from the generated index.html file and includes CSS classes
* to identify whether terms are local definitions or external references.
*
* @param {string} token - GitHub API token for authentication
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {Object} [options={}] - Additional options
* @param {string} [options.ghPageUrl] - GitHub Pages URL for the repository
* @returns {Promise<Object|null>} Object containing:
* - {number} timestamp - Unix timestamp when terms were fetched
* - {string} repository - Full repository path (owner/repo)
* - {Array<Object>} terms - Array of term objects, each containing:
* - {string} term - The term identifier
* - {string} definition - HTML definition content
* - {Array<string>} classes - CSS classes from the dt element ('term-local' or 'term-external')
* - {string|null} sha - Commit hash
* - {string|null} avatarUrl - Avatar URL
* - {string} outputFileName - Name of the cached file
*/
async function fetchAllTermsFromIndex(token, owner, repo, options = {}) {
try {
const headers = token ? { Authorization: `token ${token}` } : {};
let indexHtmlUrl;
let commitHash = null;
if (options.ghPageUrl) {
indexHtmlUrl = options.ghPageUrl.endsWith('/')
? `${options.ghPageUrl}index.html`
: `${options.ghPageUrl}/index.html`;
Logger.process(`Fetching index.html from GitHub Pages: ${indexHtmlUrl}`);
try {
const mainBranchUrl = `https://api.github.com/repos/${owner}/${repo}/branches/main`;
let branchResponse;
try {
branchResponse = await axios.get(mainBranchUrl, { headers });
} catch (authError) {
if (authError.response && authError.response.status === 401) {
// Token is present but invalid; retry without auth since the repo is public.
Logger.warn(`GITHUB_API_TOKEN is invalid or expired — retrying branch lookup without authentication`);
branchResponse = await axios.get(mainBranchUrl);
} else {
throw authError;
}
}
Eif (branchResponse && branchResponse.status === 200) {
commitHash = branchResponse.data.commit.sha;
Logger.success(`Got commit hash from main branch: ${commitHash}`);
}
} catch (error) {
// Commit hash is optional — warn and continue rather than blocking the build.
Logger.warn(`Could not get commit hash from main branch: ${error.message}`);
}
} else E{
Logger.warn('No GitHub Pages URL provided, falling back to repository method');
const specsJsonUrl = `https://api.github.com/repos/${owner}/${repo}/contents/specs.json`;
Logger.process(`Fetching specs.json from: ${specsJsonUrl}`);
const specsJsonResponse = await axios.get(specsJsonUrl, { headers });
if (specsJsonResponse.status !== 200) {
Logger.error(`Could not find specs.json in repository ${owner}/${repo}`);
return null;
}
const specsJsonContent = Buffer.from(specsJsonResponse.data.content, 'base64').toString('utf8');
const specsJson = JSON.parse(specsJsonContent);
const outputPath = specsJson.specs?.[0]?.output_path;
if (!outputPath) {
Logger.error(`No output_path found in specs.json for repository ${owner}/${repo}`);
return null;
}
const normalizedOutputPath = outputPath.replace(/^\.\//, '').replace(/\/$/, '');
const indexHtmlPath = `${normalizedOutputPath}/index.html`;
indexHtmlUrl = `https://raw.githubusercontent.com/${owner}/${repo}/main/${indexHtmlPath}`;
Logger.process(`Fetching index.html from raw repository: ${indexHtmlUrl}`);
commitHash = await getFileCommitHash(token, owner, repo, indexHtmlPath, headers);
if (!commitHash) {
Logger.warn('Could not get commit hash for index.html, continuing without it');
}
}
const indexHtmlResponse = await axios.get(indexHtmlUrl, { headers });
Iif (indexHtmlResponse.status !== 200) {
Logger.error(`Could not find index.html at ${indexHtmlUrl}`);
return null;
}
const dom = new JSDOM(indexHtmlResponse.data);
const document = dom.window.document;
const termDlList = document.querySelector('dl.terms-and-definitions-list');
Iif (!termDlList) {
Logger.error(`No terms-and-definitions-list found in ${indexHtmlUrl}`);
return null;
}
const terms = [];
const dtElements = termDlList.querySelectorAll('dt');
dtElements.forEach(dt => {
const termSpan = dt.querySelector('span[id^="term:"]');
Iif (!termSpan) {
return;
}
// Extract the canonical term identifier from the term-local-original-term span.
// This contains the original term identifier as used in the source file naming
// and tref/xref references. If this span doesn't exist, skip the term entirely.
const originalTermSpan = dt.querySelector('span.term-local-original-term');
Iif (!originalTermSpan) {
return;
}
const termText = originalTermSpan.textContent.trim();
Iif (!termText) {
return;
}
// Extract classes from the <dt> element to determine if it's a local or external term.
// This helps identify if a tref to an external resource is itself a tref (term-external)
// or a local definition (term-local).
const dtClasses = dt.className ? dt.className.split(/\s+/).filter(Boolean) : [];
const termClasses = dtClasses.filter(cls => cls === 'term-local' || cls === 'term-external');
const definitions = [];
let pointer = dt.nextElementSibling;
while (pointer && pointer.tagName.toLowerCase() === 'dd') {
definitions.push(pointer.outerHTML);
pointer = pointer.nextElementSibling;
}
terms.push({
term: termText,
definition: definitions.join('\n'),
classes: termClasses
});
});
const timestamp = Date.now();
fs.mkdirSync(CACHE_DIR, { recursive: true });
const outputFileName = `${timestamp}-${owner}-${repo}-terms.json`;
const outputFilePath = path.join(CACHE_DIR, outputFileName);
const result = {
timestamp,
repository: `${owner}/${repo}`,
terms,
sha: commitHash,
avatarUrl: null,
outputFileName
};
fs.writeFileSync(outputFilePath, JSON.stringify(result, null, 2));
Logger.success(`Saved ${terms.length} terms to ${outputFilePath}`);
return result;
} catch (error) {
if (error.response) {
if (error.response.status === 404) {
Logger.error(`Resource not found: ${error.config.url}`);
} else if (error.response.status === 403 && error.response.headers['x-ratelimit-remaining'] === '0') {
const resetTime = new Date(Number(error.response.headers['x-ratelimit-reset']) * 1000);
Logger.error(`GitHub API rate limit exceeded. Try again after ${resetTime.toLocaleString()}`);
} else if (error.response.status === 401) {
Logger.error(`GitHub API authentication failed. Verify that GITHUB_API_TOKEN is valid and has read access to the repository.`);
} else {
Logger.error(`Error fetching data: ${error.response.status} ${error.response.statusText}`);
}
} else {
Logger.error(`Error fetching term: ${error.message}`);
}
return null;
}
}
/**
* Fetches a specific term definition from an external specification repository.
* This is a convenience wrapper around fetchAllTermsFromIndex that returns
* only a single matching term.
*
* @param {string} token - GitHub API token for authentication
* @param {string} term - The specific term to fetch
* @param {string} owner - Repository owner
* @param {string} repo - Repository name
* @param {string} termsDir - Terms directory (currently unused but kept for compatibility)
* @param {Object} [options={}] - Additional options
* @param {string} [options.ghPageUrl] - GitHub Pages URL for the repository
* @returns {Promise<Object|null>} Object containing:
* - {string} term - The term identifier
* - {string} content - HTML definition content
* - {Array<string>} classes - CSS classes ('term-local' or 'term-external')
* - {string|null} sha - Commit hash
* - {Object} repository - Repository metadata
*/
async function fetchTermsFromIndex(token, term, owner, repo, termsDir, options = {}) {
const allTermsData = await fetchAllTermsFromIndex(token, owner, repo, options);
if (!allTermsData || !Array.isArray(allTermsData.terms)) {
return null;
}
const foundTerm = allTermsData.terms.find(t => t.term.toLowerCase() === term.toLowerCase());
if (!foundTerm) {
Logger.error(`Term "${term}" not found in repository ${owner}/${repo}`);
return null;
}
Logger.success(`Found term '${term}' in repository ${owner}/${repo}`);
return {
term: foundTerm.term,
content: foundTerm.definition,
classes: foundTerm.classes || [],
sha: allTermsData.sha,
repository: {
owner: {
login: owner,
avatar_url: allTermsData.avatarUrl
},
name: repo
}
};
}
module.exports = {
fetchTermsFromIndex,
fetchAllTermsFromIndex
};
|