From bed7cfcaff500da54171e9e8acc4d2423635bfcd Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Fri, 15 Nov 2024 15:05:25 +0530 Subject: [PATCH 1/7] minor: Add versioning to product docs Signed-off-by: Vipul Gupta (@vipulgupta2048) --- tools/github-parser.js | 173 ++++++++++++++++++++++++++++ tools/versioning.js | 248 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 421 insertions(+) create mode 100644 tools/github-parser.js create mode 100644 tools/versioning.js diff --git a/tools/github-parser.js b/tools/github-parser.js new file mode 100644 index 0000000000..e1650dae91 --- /dev/null +++ b/tools/github-parser.js @@ -0,0 +1,173 @@ +/*! + * parse-github-url + * + * Copyright (c) 2015-2017, Jon Schlinkert. + * Released under the MIT License. + */ + +// GitHub repository URL parser - example +// > ghParser('https://github.com/balena-io/balena-cli/blob/master/docs/balena-cli.md') +// Url { +// protocol: 'https:', +// slashes: true, +// auth: null, +// host: 'github.com', +// port: null, +// hostname: 'github.com', +// hash: null, +// search: null, +// query: null, +// pathname: 'balena-io/balena-cli/blob/master/docs/balena-cli.md', +// path: 'balena-io/balena-cli/blob/master/docs/balena-cli.md', +// href: 'https://github.com/balena-io/balena-cli/blob/master/docs/balena-cli.md', +// filepath: 'docs/balena-cli.md', +// branch: 'master', +// blob: 'master/docs/balena-cli.md', +// owner: 'balena-io', +// name: 'balena-cli', +// repo: 'balena-io/balena-cli', +// repository: 'balena-io/balena-cli' +// } + +'use strict'; + +var url = require('url'); +var cache = { __proto__: null }; + +function isChecksum(str) { + return (/^[a-f0-9]{40}$/i).test(str); +} + +function getBranch(str, obj) { + var segs = str.split('#'); + var branch; + if (segs.length > 1) { + branch = segs[segs.length - 1]; + } + if (!branch && obj.hash && obj.hash.charAt(0) === '#') { + branch = obj.hash.slice(1); + } + return branch || 'master'; +} + +function trimSlash(path) { + return path.charAt(0) === '/' ? path.slice(1) : path; +} + +function name(str) { + return str ? str.replace(/\.git$/, '') : null; +} + +function owner(str) { + if (!str) { + return null; + } + var idx = str.indexOf(':'); + if (idx > -1) { + return str.slice(idx + 1); + } + return str; +} + +function parse(str) { + if (typeof str !== 'string' || !str.length) { + return null; + } + + if (str.indexOf('git@gist') !== -1 || str.indexOf('//gist') !== -1) { + return null; + } + + // parse the URL + var obj = url.parse(str); + if (typeof obj.path !== 'string' || !obj.path.length || typeof obj.pathname !== 'string' || !obj.pathname.length) { + return null; + } + + if (!obj.host && (/^git@/).test(str) === true) { + // return the correct host for git@ URLs + obj.host = url.parse('http://' + str.replace(/git@([^:]+):/, '$1/')).host; + } + + obj.path = trimSlash(obj.path); + obj.pathname = trimSlash(obj.pathname); + obj.filepath = null; + + if (obj.path.indexOf('repos') === 0) { + obj.path = obj.path.slice(6); + } + + var seg = obj.path.split('/').filter(Boolean); + var hasBlob = seg[2] === 'blob'; + if (hasBlob && !isChecksum(seg[3])) { + obj.branch = seg[3]; + if (seg.length > 4) { + obj.filepath = seg.slice(4).join('/'); + } + } + + var blob = str.indexOf('blob'); + if (hasBlob && blob !== -1) { + obj.blob = str.slice(blob + 5); + } + + var hasTree = seg[2] === 'tree'; + var tree = str.indexOf('tree'); + if (hasTree && tree !== -1) { + var idx = tree + 5; + var branch = str.slice(idx); + var slash = branch.indexOf('/'); + if (slash !== -1) { + branch = branch.slice(0, slash); + } + obj.branch = branch; + } + + obj.owner = owner(seg[0]); + obj.name = name(seg[1]); + + if (seg.length > 1 && obj.owner && obj.name) { + obj.repo = obj.owner + '/' + obj.name; + } else { + var href = obj.href.split(':'); + if (href.length === 2 && obj.href.indexOf('//') === -1) { + obj.repo = obj.repo || href[href.length - 1]; + var repoSegments = obj.repo.split('/'); + obj.owner = repoSegments[0]; + obj.name = repoSegments[1]; + + } else { + var match = obj.href.match(/\/([^/]*)$/); + obj.owner = match ? match[1] : null; + obj.repo = null; + } + + if (obj.repo && (!obj.owner || !obj.name)) { + var segs = obj.repo.split('/'); + if (segs.length === 2) { + obj.owner = segs[0]; + obj.name = segs[1]; + } + } + } + + if (!obj.branch) { + obj.branch = seg[2] || getBranch(obj.path, obj); + if (seg.length > 3) { + obj.filepath = seg.slice(3).join('/'); + } + } + + obj.host = obj.host || 'github.com'; + obj.owner = obj.owner || null; + obj.name = obj.name || null; + obj.repository = obj.repo; + return obj; +} + +module.exports = function parseGithubUrl(str) { + if (!cache[str]) { + cache[str] = parse(str); + } + return cache[str]; +}; \ No newline at end of file diff --git a/tools/versioning.js b/tools/versioning.js new file mode 100644 index 0000000000..0156b26202 --- /dev/null +++ b/tools/versioning.js @@ -0,0 +1,248 @@ +#!/usr/bin/env node + +const https = require('https'); +const fs = require('fs'); +const fsPromises = require('fs/promises'); +const path = require('path'); +const parseGithubUrl = require('./github-parser') + +// Retrieve GitHub API token from environment variable (optional) +const githubToken = process.env.GITHUB_TOKEN || null; + +/** + * Configures GitHub API request options with authentication and headers + * @param {string} endpoint - GitHub API endpoint path + * @returns {Object} HTTPS request configuration options + */ +function githubRequestOptions(endpoint) { + const options = { + hostname: 'api.github.com', + path: endpoint, + method: 'GET', + headers: { + 'User-Agent': 'Node.js GitHub Tags Fetcher', + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28' + } + }; + + // Add authentication token if available + if (githubToken) { + options.headers['Authorization'] = `Bearer ${githubToken}`; + } + + return options +} + +/** + * Processes and filters GitHub repository tags + * @param {Array} tagsWithDates - List of tags with their release dates + * @returns {Array} Curated list of compatible version tags + */ +function findComplyingTags(tagsWithDates) { + // Filter semantic version tags (e.g., v1.2.3 or 1.2.3) + const semanticTags = tagsWithDates + .filter(tag => /^v?\d+\.\d+\.\d+$/.test(tag.name)) + .sort((a, b) => { + // Sort tags by release date in descending order + return b.date - a.date; + }); + + // Identify the latest major version + const latestMajorVersion = semanticTags[0].name.split('.')[0].replace('v', ''); + const latestMajorTag = semanticTags[0]; + + // Calculate the one-year cutoff date + const oneYearAgo = new Date(latestMajorTag.date); + oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1); + + // Initialize compatible tags with the latest major version + const compatibleTags = [latestMajorTag]; + const majorVersions = new Set([latestMajorVersion]); + + // Find additional compatible major versions within one year + for (const tag of semanticTags.slice(1)) { + const majorVersion = tag.name.split('.')[0].replace('v', ''); + + // Skip already processed major versions + if (majorVersions.has(majorVersion)) continue; + + // Include tags within one year of the latest major version + if (tag.date >= oneYearAgo) { + compatibleTags.push(tag); + majorVersions.add(majorVersion); + } + } + + // Ensure at least 5 tags are included if possible + while (compatibleTags.length < 5 && semanticTags.length > compatibleTags.length) { + for (const tag of semanticTags) { + const majorVersion = tag.name.split('.')[0].replace('v', ''); + if (!majorVersions.has(majorVersion)) { + compatibleTags.push(tag); + majorVersions.add(majorVersion); + if (compatibleTags.length === 5) break; + } + } + } + + // Sort compatible tags by date + compatibleTags.sort((a, b) => b.date - a.date); + + // Create final tagged list with additional metadata + const result = compatibleTags.map((tag, index) => { + let displayName = tag.name; + let tagDate = new Date(tag.date) + + // Mark first tag as latest + if (index === 0) { + displayName = `${tag.name} latest`; + } + // Mark older tags as deprecated + else if (tagDate < oneYearAgo) { + displayName = `${tag.name} deprecated`; + } + + return { id: tag.name, name: displayName, releaseDate: tagDate }; + }); + + return result; +} + +/** + * Fetches all release tags from a GitHub repository + * @param {string} owner - Repository owner + * @param {string} repo - Repository name + * @returns {Promise} List of processed version tags + */ +async function fetchGitHubTags(owner, repo) { + // Recursive function to handle GitHub API pagination + async function fetchAllTagsWithDates(page = 1, allTags = []) { + return new Promise((resolve, reject) => { + const req = https.request(githubRequestOptions(`/repos/${owner}/${repo}/releases?per_page=100&page=${page}`, owner, repo), (res) => { + let data = ''; + + res.on('data', (chunk) => { + data += chunk; + }); + + res.on('end', () => { + try { + const tags = JSON.parse(data); + // Handle API rate limit errors + if (tags?.message?.includes("API rate limit exceeded")) { + throw new Error(`GitHub API Rate Limit exceeded, please authenticate ${tags.message}`) + } + + // Extract tag details + const tagsWithDetails = tags.map(tag => ({ + name: tag.name, + date: tag.published_at + })); + + // Check for additional pages + const linkHeader = res.headers.link; + if (linkHeader && linkHeader.includes('rel="next"')) { + resolve(fetchAllTagsWithDates(page + 1, [...allTags, ...tagsWithDetails])); + } else { + resolve([...allTags, ...tagsWithDetails]); + } + } catch (error) { + reject(error); + } + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + req.end(); + }); + } + + return findComplyingTags(await fetchAllTagsWithDates()) +} + +/** + * Downloads a specific file version from GitHub + * @param {string} apiUrl - GitHub API endpoint for file + * @param {string} version - Version/tag of the file + * @param {string} versionedDocsFolder - Output directory for downloaded files + * @param {string} githubToken - GitHub authentication token + * @returns {Promise} Path to downloaded file + */ +async function fetchFileForVersion(apiUrl, version, versionedDocsFolder, githubToken) { + return new Promise((resolve, reject) => { + const req = https.request(githubRequestOptions(apiUrl), (res) => { + // Create output path for versioned file + const outputPath = path.join(versionedDocsFolder, `${version}.md`); + + // Stream file download + const writeStream = fs.createWriteStream(outputPath); + res.pipe(writeStream); + + writeStream.on('finish', () => { + writeStream.close(); + resolve(outputPath); + }); + + writeStream.on('error', (err) => { + reject(err); + }); + }); + + req.on('error', (error) => { + reject(error); + }); + + req.end(); + }); +} + +/** + * Main script execution function + * Fetches and versions documentation for a GitHub repository + */ +async function main() { + // Retrieve GitHub repository URL from command line argument + const repoUrl = process.argv[2]; + + // Validate repository URL input + if (!repoUrl) { + console.error('Usage: node versioning.js '); + console.error('Please provide a valid GitHub repository URL'); + process.exit(1); + } + + // Parse repository details + const { owner, name: repoName, filepath } = parseGithubUrl(repoUrl); + const versionsConfigFile = `./config/dictionaries/${(repoName).replaceAll(/-/g, "")}.json` + const versionedDocsFolder = path.join(__dirname, `../shared/${repoName}-versions`) + + try { + // Fetch and process repository versions + const versions = await fetchGitHubTags(owner, repoName, githubToken); + + // Write versions configuration + await fsPromises.writeFile(versionsConfigFile, JSON.stringify(versions, null, 2)); + await fsPromises.mkdir(versionedDocsFolder, { recursive: true }); + + // Download documentation for each version + for (const version of versions) { + await fetchFileForVersion( + `/repos/${owner}/${repoName}/contents/${filepath}?ref=${version.id}`, + version.id, + versionedDocsFolder, + githubToken + ); + } + console.log(`Versioned ${repoName} docs successfully`); + } catch (error) { + console.error('Error:', error.message); + process.exit(1); + } +} + +// Execute the main function +main(); \ No newline at end of file From 8d9d15b31d43dccb687b9163f80d3c8afef4fe4a Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Fri, 15 Nov 2024 15:05:59 +0530 Subject: [PATCH 2/7] Add versioned balenaCLI docs Signed-off-by: Vipul Gupta (@vipulgupta2048) --- .cspell/balena-words.txt | 1 + .gitignore | 4 ++++ config/navigation.txt | 2 +- templates/balena-cli.html | 8 ++++++++ tools/build.sh | 3 +++ tools/fetch-external.sh | 2 +- tools/versioning.js | 4 ++-- 7 files changed, 20 insertions(+), 4 deletions(-) create mode 100644 templates/balena-cli.html diff --git a/.cspell/balena-words.txt b/.cspell/balena-words.txt index d9cd47c290..60654e1cf0 100644 --- a/.cspell/balena-words.txt +++ b/.cspell/balena-words.txt @@ -4,6 +4,7 @@ balena-api balena-base-ui balena-build balena-builder +balenacli balenahup balenalib balenista diff --git a/.gitignore b/.gitignore index ad5c475c4f..2c3eae0f6c 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,10 @@ pages/reference/base-images/base-images-ref.md # Ignore dynamic docs created for masterclasses shared/masterclass +# Ignore dynamic balenaCLI version docs +config/dictionaries/balenacli.json +shared/balena-cli-versions/ + # Ignore dynamic assets generated for Getting Started static/img/device/** config/dictionaries/device.json diff --git a/config/navigation.txt b/config/navigation.txt index a99ee0dbd8..670aff8aed 100644 --- a/config/navigation.txt +++ b/config/navigation.txt @@ -149,7 +149,7 @@ Reference [/reference/api/overview] Resources[/reference/api/resources/$resource] - CLI[/reference/balena-cli] + CLI[/reference/balena-cli/$balenacli] SDKs Node.js SDK[/reference/sdk/node-sdk] Python SDK[/reference/sdk/python-sdk] diff --git a/templates/balena-cli.html b/templates/balena-cli.html new file mode 100644 index 0000000000..a05b70b2b0 --- /dev/null +++ b/templates/balena-cli.html @@ -0,0 +1,8 @@ +{% extends "default.html" %} + +{% block dynamicSwitchCustom %} +

+

+{% endblock %} + +
diff --git a/tools/build.sh b/tools/build.sh index a949142001..1051ed98ea 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -16,6 +16,9 @@ node ./tools/generate-docs-contracts.js # Generate base images docs ./tools/build-base-images.sh +# Generate balena-cli docs +node tools/versioning.js https://github.com/balena-io/balena-cli/blob/master/docs/balena-cli.md + # Convert .jpg, .jpeg, .png images to .webp format ./tools/convert-images-to-webp.sh diff --git a/tools/fetch-external.sh b/tools/fetch-external.sh index 22457c07a5..3155115593 100755 --- a/tools/fetch-external.sh +++ b/tools/fetch-external.sh @@ -14,7 +14,7 @@ mkdir -p shared/masterclass/debugging/ which jq && JQ="$(which jq)" || JQ="../../node_modules/node-jq/bin/jq" # get latest CLI docs -curl --fail --show-error -o pages/reference/balena-cli.md -L https://github.com/balena-io/balena-cli/raw/master/docs/balena-cli.md & +# curl --fail --show-error -o pages/reference/balena-cli.md -L https://github.com/balena-io/balena-cli/raw/master/docs/balena-cli.md & # Engine # get latest balena-engine debugging docs diff --git a/tools/versioning.js b/tools/versioning.js index 0156b26202..d4f09109d8 100644 --- a/tools/versioning.js +++ b/tools/versioning.js @@ -217,12 +217,12 @@ async function main() { // Parse repository details const { owner, name: repoName, filepath } = parseGithubUrl(repoUrl); - const versionsConfigFile = `./config/dictionaries/${(repoName).replaceAll(/-/g, "")}.json` + const versionsConfigFile = `./config/dictionaries/${repoName.replaceAll(/-/g, "")}.json` const versionedDocsFolder = path.join(__dirname, `../shared/${repoName}-versions`) try { // Fetch and process repository versions - const versions = await fetchGitHubTags(owner, repoName, githubToken); + const versions = await fetchGitHubTags(owner, repoName); // Write versions configuration await fsPromises.writeFile(versionsConfigFile, JSON.stringify(versions, null, 2)); From 391bfd29efa22babf0f40ef109157f4227283acb Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Fri, 15 Nov 2024 15:51:52 +0530 Subject: [PATCH 3/7] Add balena-cli docs Signed-off-by: Vipul Gupta (@vipulgupta2048) --- .gitignore | 3 --- pages/reference/balena-cli.md | 12 ++++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 pages/reference/balena-cli.md diff --git a/.gitignore b/.gitignore index 2c3eae0f6c..87461c4d2d 100644 --- a/.gitignore +++ b/.gitignore @@ -14,9 +14,6 @@ tmp .lycheecache # Ignore external docs that are pulled via tools/fetch-external.sh -pages/reference/cli.md -pages/reference/balena-cli.md - pages/reference/sdk/node-sdk.md pages/reference/sdk/python-sdk.md diff --git a/pages/reference/balena-cli.md b/pages/reference/balena-cli.md new file mode 100644 index 0000000000..f7ef4f98de --- /dev/null +++ b/pages/reference/balena-cli.md @@ -0,0 +1,12 @@ +--- +title: balena CLI Documentation + +layout: balena-cli.html + +dynamic: + variables: [ $balenacli ] + ref: $original_ref/$balenacli + $switch_text: balena CLI version $balenacli +--- + +{{import "balena-cli-versions"}} From 352e815131f09a6d0cc8cc5a1fcf7e7c725adc2f Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Sat, 16 Nov 2024 02:52:59 +0530 Subject: [PATCH 4/7] Use raw GitHub url instead of contents API Signed-off-by: Vipul Gupta (@vipulgupta2048) --- tools/build.sh | 6 +++--- tools/versioning.js | 16 +++++++--------- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tools/build.sh b/tools/build.sh index 1051ed98ea..eb1a348df1 100755 --- a/tools/build.sh +++ b/tools/build.sh @@ -10,15 +10,15 @@ cd "$SCRIPT_DIR/.." # Generate Getting Started assets node ./tools/generate-docs-contracts.js +# Generate versioned balena-cli docs +node ./tools/versioning.js https://github.com/balena-io/balena-cli/blob/master/docs/balena-cli.md + # Generate Masterclasses Dynamically ./tools/build-masterclass.sh # Generate base images docs ./tools/build-base-images.sh -# Generate balena-cli docs -node tools/versioning.js https://github.com/balena-io/balena-cli/blob/master/docs/balena-cli.md - # Convert .jpg, .jpeg, .png images to .webp format ./tools/convert-images-to-webp.sh diff --git a/tools/versioning.js b/tools/versioning.js index d4f09109d8..cee90d3788 100644 --- a/tools/versioning.js +++ b/tools/versioning.js @@ -22,7 +22,6 @@ function githubRequestOptions(endpoint) { headers: { 'User-Agent': 'Node.js GitHub Tags Fetcher', 'Accept': 'application/vnd.github+json', - 'X-GitHub-Api-Version': '2022-11-28' } }; @@ -119,7 +118,7 @@ async function fetchGitHubTags(owner, repo) { // Recursive function to handle GitHub API pagination async function fetchAllTagsWithDates(page = 1, allTags = []) { return new Promise((resolve, reject) => { - const req = https.request(githubRequestOptions(`/repos/${owner}/${repo}/releases?per_page=100&page=${page}`, owner, repo), (res) => { + const req = https.request(githubRequestOptions(`/repos/${owner}/${repo}/releases?per_page=100&page=${page}`), (res) => { let data = ''; res.on('data', (chunk) => { @@ -169,17 +168,17 @@ async function fetchGitHubTags(owner, repo) { * @param {string} apiUrl - GitHub API endpoint for file * @param {string} version - Version/tag of the file * @param {string} versionedDocsFolder - Output directory for downloaded files - * @param {string} githubToken - GitHub authentication token * @returns {Promise} Path to downloaded file */ -async function fetchFileForVersion(apiUrl, version, versionedDocsFolder, githubToken) { +async function fetchFileForVersion(apiUrl, version, versionedDocsFolder) { return new Promise((resolve, reject) => { - const req = https.request(githubRequestOptions(apiUrl), (res) => { - // Create output path for versioned file + const req = https.request(apiUrl, (res) => { + // Ensure output directory exists const outputPath = path.join(versionedDocsFolder, `${version}.md`); - // Stream file download + // Create write stream const writeStream = fs.createWriteStream(outputPath); + res.pipe(writeStream); writeStream.on('finish', () => { @@ -231,10 +230,9 @@ async function main() { // Download documentation for each version for (const version of versions) { await fetchFileForVersion( - `/repos/${owner}/${repoName}/contents/${filepath}?ref=${version.id}`, + `https://raw.githubusercontent.com/${owner}/${repoName}/refs/tags/${version.id}/${filepath}`, version.id, versionedDocsFolder, - githubToken ); } console.log(`Versioned ${repoName} docs successfully`); From 2a71cd7f73b5ae4fd92c85ed98f873af43bed822 Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Sat, 16 Nov 2024 03:20:25 +0530 Subject: [PATCH 5/7] Redirect CLI docs to latest version page Signed-off-by: Vipul Gupta (@vipulgupta2048) --- config/redirects.txt | 16 ++++++++++------ tools/versioning.js | 4 +++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/config/redirects.txt b/config/redirects.txt index 6a66962c2a..6b65ca975e 100644 --- a/config/redirects.txt +++ b/config/redirects.txt @@ -88,12 +88,12 @@ /deployment/wifi /reference/OS/network/2.x/ /runtime/terminal/ /learn/manage/ssh-access/ /runtime/terminal /learn/manage/ssh-access/ -/using/cli/ /reference/balena-cli/ -/using/cli /reference/balena-cli/ +/using/cli/ /reference/balena-cli/latest/ +/using/cli /reference/balena-cli/latest/ /learn/deploy/release-strategy/update-locking/API/ /reference/supervisor/supervisor-api/ /learn/deploy/release-strategy/update-locking/API /reference/supervisor/supervisor-api/ -/reference/cli/ /reference/balena-cli/ -/reference/cli /reference/balena-cli/ +/reference/cli/ /reference/balena-cli/latest/ +/reference/cli /reference/balena-cli/latest/ /docs/learn/deploy/release-strategy/update-locking/API/ /reference/supervisor/supervisor-api/ /docs/learn/deploy/release-strategy/update-locking/API /reference/supervisor/supervisor-api/ /learn/more/masterclass/ /learn/more/masterclasses/overview/ @@ -192,8 +192,8 @@ /runtime/supervisor-api /reference/supervisor/supervisor-api/ /runtime/data-api/ /reference/api/overview/ /runtime/data-api /reference/api/overview/ -/tools/cli/ /reference/balena-cli/ -/tools/cli /reference/balena-cli/ +/tools/cli/ /reference/balena-cli/latest/ +/tools/cli /reference/balena-cli/latest/ /tools/sdk/ /reference/sdk/node-sdk/ /tools/sdk /reference/sdk/node-sdk/ /tools/python-sdk/ /reference/sdk/python-sdk/ @@ -266,6 +266,10 @@ /reference/base-images/base-images/ /reference/base-images/balena-base-images/ /reference/base-images/base-images /reference/base-images/balena-base-images/ +# Versioned CLI docs +/reference/balena-cli/ /reference/balena-cli/latest/ +/reference/balena-cli /reference/balena-cli/latest/ + # Important: keep dynamic redirect below the static redirects # https://developers.cloudflare.com/pages/platform/redirects/ diff --git a/tools/versioning.js b/tools/versioning.js index cee90d3788..109ce0be3f 100644 --- a/tools/versioning.js +++ b/tools/versioning.js @@ -90,19 +90,21 @@ function findComplyingTags(tagsWithDates) { // Create final tagged list with additional metadata const result = compatibleTags.map((tag, index) => { + let tagId = tag.name let displayName = tag.name; let tagDate = new Date(tag.date) // Mark first tag as latest if (index === 0) { displayName = `${tag.name} latest`; + tagId = "latest" } // Mark older tags as deprecated else if (tagDate < oneYearAgo) { displayName = `${tag.name} deprecated`; } - return { id: tag.name, name: displayName, releaseDate: tagDate }; + return { id: tagId, name: displayName, releaseDate: tagDate }; }); return result; From 6acfb684bff5a87b3a7e8ceab2049b4720fdabe9 Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Sat, 16 Nov 2024 03:45:41 +0530 Subject: [PATCH 6/7] Added logging for checking GITHUB_TOKEN Signed-off-by: Vipul Gupta (@vipulgupta2048) --- tools/versioning.js | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/tools/versioning.js b/tools/versioning.js index 109ce0be3f..eb48408a95 100644 --- a/tools/versioning.js +++ b/tools/versioning.js @@ -4,10 +4,14 @@ const https = require('https'); const fs = require('fs'); const fsPromises = require('fs/promises'); const path = require('path'); -const parseGithubUrl = require('./github-parser') +const parseGithubUrl = require('./github-parser'); +const { version } = require('os'); // Retrieve GitHub API token from environment variable (optional) const githubToken = process.env.GITHUB_TOKEN || null; +if (!githubToken) { + console.log('WARNING: GITHUB_TOKEN not provided in the environment. Versioning scripts might get rate-limited.'); +} /** * Configures GitHub API request options with authentication and headers @@ -90,21 +94,18 @@ function findComplyingTags(tagsWithDates) { // Create final tagged list with additional metadata const result = compatibleTags.map((tag, index) => { - let tagId = tag.name - let displayName = tag.name; let tagDate = new Date(tag.date) // Mark first tag as latest if (index === 0) { - displayName = `${tag.name} latest`; - tagId = "latest" + return { id: "latest", name: `${tag.name} latest`, version: tag.name, releaseDate: tagDate }; } // Mark older tags as deprecated else if (tagDate < oneYearAgo) { - displayName = `${tag.name} deprecated`; + return { id: tag.name, name: `${tag.name} deprecated`, version: tag.name, releaseDate: tagDate }; } - return { id: tagId, name: displayName, releaseDate: tagDate }; + return { id: tag.name, name: tag.name, version: tag.name, releaseDate: tagDate }; }); return result; @@ -220,20 +221,25 @@ async function main() { const { owner, name: repoName, filepath } = parseGithubUrl(repoUrl); const versionsConfigFile = `./config/dictionaries/${repoName.replaceAll(/-/g, "")}.json` const versionedDocsFolder = path.join(__dirname, `../shared/${repoName}-versions`) + + console.log(`Started versioning ${repoName} docs`) try { // Fetch and process repository versions - const versions = await fetchGitHubTags(owner, repoName); + const tagVersions = await fetchGitHubTags(owner, repoName); // Write versions configuration - await fsPromises.writeFile(versionsConfigFile, JSON.stringify(versions, null, 2)); + await fsPromises.writeFile(versionsConfigFile, JSON.stringify(tagVersions, null, 2)); + if (fs.existsSync(versionedDocsFolder)) { + await fsPromises.rm(versionedDocsFolder, { recursive: true }); + } await fsPromises.mkdir(versionedDocsFolder, { recursive: true }); // Download documentation for each version - for (const version of versions) { + for (const tagVersion of tagVersions) { await fetchFileForVersion( - `https://raw.githubusercontent.com/${owner}/${repoName}/refs/tags/${version.id}/${filepath}`, - version.id, + `https://raw.githubusercontent.com/${owner}/${repoName}/refs/tags/${tagVersion.version}/${filepath}`, + tagVersion.id, versionedDocsFolder, ); } From a7823e3c886464a4bb9756ffc83b4968555be37f Mon Sep 17 00:00:00 2001 From: "Vipul Gupta (@vipulgupta2048)" Date: Mon, 18 Nov 2024 19:07:20 +0530 Subject: [PATCH 7/7] Added versioned headings for balena-cli Signed-off-by: Vipul Gupta (@vipulgupta2048) --- package-lock.json | 10 ++++++++ package.json | 1 + pages/reference/balena-cli.md | 2 ++ tools/fetch-external.sh | 3 --- tools/versioning.js | 48 ++++++++++++++++++++++++++++++----- 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 6cdcdf4205..37ab3e7b6d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "cspell": "^8.9.1", "css-loader": "5.2.7", "file-loader": "6.2.0", + "line-by-line": "^0.1.6", "mini-css-extract-plugin": "^1.6.2", "node-html-markdown": "^1.3.0", "node-jq": "^4.4.0", @@ -6480,6 +6481,15 @@ "node": ">=0.10.0" } }, + "node_modules/line-by-line": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/line-by-line/-/line-by-line-0.1.6.tgz", + "integrity": "sha512-MmwVPfOyp0lWnEZ3fBA8Ah4pMFvxO6WgWovqZNu7Y4J0TNnGcsV4S1LzECHbdgqk1hoHc2mFP1Axc37YUqwafg==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/lines-and-columns": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", diff --git a/package.json b/package.json index 6cd2436bc6..3e34f6aae5 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "cspell": "^8.9.1", "css-loader": "5.2.7", "file-loader": "6.2.0", + "line-by-line": "^0.1.6", "mini-css-extract-plugin": "^1.6.2", "node-html-markdown": "^1.3.0", "node-jq": "^4.4.0", diff --git a/pages/reference/balena-cli.md b/pages/reference/balena-cli.md index f7ef4f98de..6fadb0e65f 100644 --- a/pages/reference/balena-cli.md +++ b/pages/reference/balena-cli.md @@ -9,4 +9,6 @@ dynamic: $switch_text: balena CLI version $balenacli --- +# balena CLI {{ $balenacli.version }} Documentation + {{import "balena-cli-versions"}} diff --git a/tools/fetch-external.sh b/tools/fetch-external.sh index 3155115593..621f8bc35e 100755 --- a/tools/fetch-external.sh +++ b/tools/fetch-external.sh @@ -13,9 +13,6 @@ mkdir -p shared/masterclass/debugging/ # Use node-jq if jq is not pre-installed in the environment nor set in path which jq && JQ="$(which jq)" || JQ="../../node_modules/node-jq/bin/jq" -# get latest CLI docs -# curl --fail --show-error -o pages/reference/balena-cli.md -L https://github.com/balena-io/balena-cli/raw/master/docs/balena-cli.md & - # Engine # get latest balena-engine debugging docs curl --fail --show-error -o shared/masterclass/debugging/engine.md -L https://github.com/balena-os/balena-engine/raw/master/balena-docs/engine-debugging.md & diff --git a/tools/versioning.js b/tools/versioning.js index eb48408a95..1bb875471a 100644 --- a/tools/versioning.js +++ b/tools/versioning.js @@ -5,7 +5,7 @@ const fs = require('fs'); const fsPromises = require('fs/promises'); const path = require('path'); const parseGithubUrl = require('./github-parser'); -const { version } = require('os'); +const LineByLineReader = require('line-by-line'); // Retrieve GitHub API token from environment variable (optional) const githubToken = process.env.GITHUB_TOKEN || null; @@ -78,7 +78,7 @@ function findComplyingTags(tagsWithDates) { } // Ensure at least 5 tags are included if possible - while (compatibleTags.length < 5 && semanticTags.length > compatibleTags.length) { + if (compatibleTags.length < 5 && semanticTags.length > compatibleTags.length) { for (const tag of semanticTags) { const majorVersion = tag.name.split('.')[0].replace('v', ''); if (!majorVersions.has(majorVersion)) { @@ -177,15 +177,18 @@ async function fetchFileForVersion(apiUrl, version, versionedDocsFolder) { return new Promise((resolve, reject) => { const req = https.request(apiUrl, (res) => { // Ensure output directory exists - const outputPath = path.join(versionedDocsFolder, `${version}.md`); + const outputPathWithHeading = path.join(versionedDocsFolder, `${version}-withheading.md`); // Create write stream - const writeStream = fs.createWriteStream(outputPath); + const writeStream = fs.createWriteStream(outputPathWithHeading); res.pipe(writeStream); - writeStream.on('finish', () => { + writeStream.on('finish', async () => { writeStream.close(); + const outputPath = path.join(versionedDocsFolder, `${version}.md`); + await removeFirstLine(outputPathWithHeading, outputPath) + await fsPromises.unlink(outputPathWithHeading); resolve(outputPath); }); @@ -198,13 +201,44 @@ async function fetchFileForVersion(apiUrl, version, versionedDocsFolder) { reject(error); }); - req.end(); + req.end(); + }); +} + +// Function to remove first line from a file +async function removeFirstLine(srcPath, destPath) { + return new Promise((resolve, reject) => { + const lr = new LineByLineReader(srcPath); + const output = fs.createWriteStream(destPath); + let isFirstLine = true; + + lr.on('error', (err) => { + reject(err); + }); + + lr.on('line', (line) => { + // Skip the first line + if (isFirstLine) { + isFirstLine = false; + return; + } + + // Write subsequent lines to the output file + output.write(line + '\n'); + }); + + lr.on('end', () => { + output.end(); + resolve(destPath); + }); }); } + /** * Main script execution function * Fetches and versions documentation for a GitHub repository + * Add versionheadings flag to add version headings to docs */ async function main() { // Retrieve GitHub repository URL from command line argument @@ -212,7 +246,7 @@ async function main() { // Validate repository URL input if (!repoUrl) { - console.error('Usage: node versioning.js '); + console.error('Usage: node versioning.js [noversionheadings]'); console.error('Please provide a valid GitHub repository URL'); process.exit(1); }