feat: replace update-icon-list.js with update-icons.js, update build pipeline and docs

This commit is contained in:
2026-04-28 10:22:33 +08:00
parent eeba687551
commit 109b48ec20
7 changed files with 790 additions and 280 deletions
+3 -3
View File
@@ -11,9 +11,9 @@ The output is a drop-in CSS font system - users link `fonts/ficons.css` and use
## Development Commands
```bash
# Fetch latest icon metadata (writes to data/all_icons.json)
# Fetch latest icon metadata and sync SVGs (writes to data/all_icons.json + svgs/*/*.svg)
npm run update:icons
# or: node update-icon-list.js
# or: node update-icons.js
# Convert JSON to browser-ready JS (writes to data/all_icons.js)
npm run build:icons
@@ -44,7 +44,7 @@ Examples: `fi-rs-bookmark`, `fi-br-home`, `fi-brands-instagram`
### Data Pipeline
1. **`update-icon-list.js`** - Fetches icon metadata with pagination. Rate-limited (100ms delay). Outputs `data/all_icons.json`.
1. **`update-icons.js`** - Fetches icon metadata page-by-page and immediately syncs SVG files for each page into `svgs/{prefix}/` (to avoid expiring SVG tokens), then writes `data/all_icons.json`. Existing files are skipped unless icon metadata changed.
2. **`build-icons-js.js`** - Converts JSON to `window.FICONS_DATA` global for browser use in explorer.
+11 -148
View File
@@ -1,171 +1,34 @@
# @invisi/ficons
A local web-compatible icon font package with **thousands of icons** across multiple style variations.
A local icon font package with webfonts, a small icon dataset, and a built-in explorer.
## Features
## Install
- **Thousands of icons** across 15 style variations (stroke weights × corner styles)
- **Self-hosted** - no CDN dependencies
- **Interactive explorer** - browse, search, and filter icons locally
- **Web fonts** - WOFF2, WOFF, TTF formats
- **Easy integration** - drop-in CSS with class-based icons
## Installation
```shell
```sh
npm i @invisi/ficons
```
## Quick Start
Include the CSS in your HTML:
## Basic usage
```html
<link rel="stylesheet" href="fonts/ficons.css">
```
Use icons with `<span>` or `<i>` elements:
```html
<i class="fi fi-rs-user"></i>
<span class="fi fi-brs-home"></span>
<i class="fi fi-brands-instagram"></i>
```
## Icon Styles
## CLI
| Weight | Corner | Prefix | Example |
|:-----------|:---------|:-------|:-------------------------------|
| Regular | Straight | fi-rs | `<i class="fi fi-rs-user"></i>` |
| Regular | Rounded | fi-rr | `<i class="fi fi-rr-user"></i>` |
| Bold | Straight | fi-bs | `<i class="fi fi-bs-user"></i>` |
| Bold | Rounded | fi-br | `<i class="fi fi-br-user"></i>` |
| Solid | Straight | fi-ss | `<i class="fi fi-ss-user"></i>` |
| Solid | Rounded | fi-sr | `<i class="fi fi-sr-user"></i>` |
| Thin | Straight | fi-ts | `<i class="fi fi-ts-user"></i>` |
| Thin | Rounded | fi-tr | `<i class="fi fi-tr-user"></i>` |
| Brands | - | fi-brands | `<i class="fi fi-brands-facebook"></i>` |
## Icon Count
| Variant | Icons |
|-------------------|--------|
| Regular Straight | 5,043 |
| Regular Rounded | 5,039 |
| Bold Straight | 5,055 |
| Bold Rounded | 5,041 |
| Solid Straight | 5,044 |
| Solid Rounded | 5,051 |
| Thin Straight | 5,053 |
| Thin Rounded | 5,032 |
| Regular Chubby | 3,093 |
| Solid Chubby | 3,093 |
| Thin Chubby | 3,093 |
| Duotone Straight | 180 |
| Duotone Rounded | 160 |
| Duotone Chubby | 270 |
| Brands | 245 |
**Thousands of icon variations**
## Styling
Icons inherit `font-size` and `color` from their parent:
```html
<!-- Size -->
<i class="fi fi-rs-heart" style="font-size: 24px;"></i>
<i class="fi fi-rs-heart" style="font-size: 48px;"></i>
<!-- Color -->
<i class="fi fi-rs-star" style="color: #f1c40f;"></i>
<i class="fi fi-rs-heart" style="color: #e74c3c;"></i>
```
## Package Structure
```
fonts/
ficons.css # Unified CSS (imports all styles)
css/
flaticon-regular-straight.css
flaticon-regular-rounded.css
flaticon-bold-straight.css
...
webfonts/ # Font files (woff2, woff, ttf)
data/
all_icons.js # Icon data for explorer
explorer.html # Interactive icon browser
```
## CLI Search
Search for icons directly from the command line using `npx`:
```bash
# Basic search
```sh
npx @invisi/ficons explore
npx @invisi/ficons search camera
# Search with multiple keywords
npx @invisi/ficons search arrow left
# Filter by variation (prefix)
npx @invisi/ficons search user rr
npx @invisi/ficons search home --variation ss
npx @invisi/ficons search camera --variation rr
```
Outputs JSON with icon names and available variations:
## Development
```json
{
"camera": ["rs", "rr", "bs", "br", "ss", "sr", "ts", "tr"],
"camera-phone": ["rs", "rr", "bs", "br"]
}
```
### Variation Prefixes
| Prefix | Style |
|:-------|:------|
| `rs` | Regular Straight |
| `rr` | Regular Rounded |
| `rc` | Regular Chubby |
| `bs` | Bold Straight |
| `br` | Bold Rounded |
| `bc` | Bold Chubby |
| `ss` | Solid Straight |
| `sr` | Solid Rounded |
| `sc` | Solid Chubby |
| `ts` | Thin Straight |
| `tr` | Thin Rounded |
| `tc` | Thin Chubby |
| `ds` | Duotone Straight |
| `dr` | Duotone Rounded |
| `dc` | Duotone Chubby |
| `brands` | Brand Logos |
## Icon Explorer
Open `explorer.html` in a browser to:
- Browse all icons
- Search by name or tags
- Filter by weight, corner style, and type
- Copy HTML snippets and CSS classes
- Adjust icon preview size
## Development Scripts
```bash
# Fetch latest icon metadata
```sh
# Fetch metadata and sync SVGs page-by-page
npm run update:icons
# or: node update-icon-list.js
# Convert JSON to browser-ready JS
npm run build:icons
# or: node build-icons-js.js
# Generate webfonts from SVGs using FontForge
npm run build:fonts
# or: node build-fonts.js
```
+15 -6
View File
@@ -1,6 +1,20 @@
const fs = require('fs');
const path = require('path');
function toEssentialIcon(icon = {}) {
const name = typeof icon.name === 'string' ? icon.name : '';
const prefix = typeof icon.prefix === 'string' ? icon.prefix : '';
if (!name || !prefix) return null;
const minimal = { name, prefix };
if (typeof icon.tags === 'string' && icon.tags.trim()) {
minimal.tags = icon.tags;
}
return minimal;
}
function buildIconsJs({
inputPath = path.join(__dirname, 'data', 'all_icons.json'),
outputPath = path.join(__dirname, 'data', 'all_icons.js')
@@ -10,12 +24,7 @@ function buildIconsJs({
}
const raw = fs.readFileSync(inputPath, 'utf8');
const icons = JSON.parse(raw).map(({ name, prefix, tags, is_brand }) => ({
name,
prefix,
tags,
is_brand
}));
const icons = JSON.parse(raw).map(toEssentialIcon).filter(Boolean);
const payload = `window.FICONS_DATA = ${JSON.stringify(icons)};\n`;
fs.writeFileSync(outputPath, payload, 'utf8');
+1 -1
View File
File diff suppressed because one or more lines are too long
+5 -2
View File
@@ -18,7 +18,7 @@
],
"scripts": {
"build:icons": "node build-icons-js.js",
"update:icons": "node update-icon-list.js",
"update:icons": "node update-icons.js",
"build:fonts": "node build-fonts.js"
},
"keywords": [
@@ -26,5 +26,8 @@
"fonts",
"icon-font",
"webfonts"
]
],
"dependencies": {
"cli-progress": "^3.12.0"
}
}
-120
View File
@@ -1,120 +0,0 @@
const fs = require('fs');
const path = require('path');
const https = require('https');
const buildIconsJs = require('./build-icons-js');
const API_URL = 'https://www.flaticon.com/ajax/icon-fonts-most-downloaded/';
const HEADERS = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none'
};
function parseArgs(argv) {
const args = {};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (!arg.startsWith('--')) continue;
const key = arg.slice(2);
const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : true;
if (value !== true) i += 1;
args[key] = value;
}
return args;
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function fetchJson(page) {
const url = `${API_URL}${page}`;
return new Promise((resolve, reject) => {
const request = https.request(url, { headers: HEADERS }, response => {
let data = '';
response.on('data', chunk => {
data += chunk;
});
response.on('end', () => {
try {
resolve(JSON.parse(data));
} catch (error) {
reject(error);
}
});
});
request.on('error', reject);
request.end();
});
}
async function run() {
const args = parseArgs(process.argv.slice(2));
const start = Number(args.start || 1);
let end = Number(args.end || 0);
const delay = Number(args.delay || 100);
if (Number.isNaN(start) || start < 1) {
throw new Error('Invalid --start value');
}
console.log(`Fetching icons starting at page ${start}...`);
const first = await fetchJson(start);
if (!first || !Array.isArray(first.items)) {
throw new Error('Unexpected response from icon API');
}
if (!end || Number.isNaN(end)) {
end = Number(first.pages || start);
}
const itemsById = new Map();
const addItems = items => {
(items || []).forEach(item => {
if (item && item.id) {
itemsById.set(item.id, item);
}
});
};
addItems(first.items);
console.log(`Page ${start}/${end}: ${first.items.length} items`);
for (let page = start + 1; page <= end; page += 1) {
const payload = await fetchJson(page);
addItems(payload.items);
if (page % 25 === 0 || page === end) {
console.log(`Page ${page}/${end}: ${payload.items.length} items`);
}
if (delay > 0) {
await sleep(delay);
}
}
const outputDir = path.join(__dirname, 'data');
if (!fs.existsSync(outputDir)) {
fs.mkdirSync(outputDir, { recursive: true });
}
const allIcons = Array.from(itemsById.values());
const outputPath = path.join(outputDir, 'all_icons.json');
fs.writeFileSync(outputPath, JSON.stringify(allIcons), 'utf8');
console.log(`Wrote ${outputPath} (${allIcons.length} icons)`);
buildIconsJs({
inputPath: outputPath,
outputPath: path.join(outputDir, 'all_icons.js')
});
}
run().catch(error => {
console.error(error.message || error);
process.exit(1);
});
+755
View File
@@ -0,0 +1,755 @@
const fs = require('fs');
const path = require('path');
const https = require('https');
const buildIconsJs = require('./build-icons-js');
let cliProgress = null;
try {
// Optional at runtime for environments not bootstrapped with package manager hooks.
cliProgress = require('cli-progress');
} catch (error) {
if (error && error.code !== 'MODULE_NOT_FOUND') {
throw error;
}
}
const API_URL = 'https://www.flaticon.com/ajax/icon-fonts-most-downloaded/';
const HEADERS = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.9',
'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121"',
'Sec-Ch-Ua-Mobile': '?0',
'Sec-Ch-Ua-Platform': '"macOS"',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none'
};
function parseArgs(argv) {
const args = {};
for (let i = 0; i < argv.length; i += 1) {
const arg = argv[i];
if (!arg.startsWith('--')) continue;
const key = arg.slice(2);
const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : true;
if (value !== true) i += 1;
args[key] = value;
}
return args;
}
function createNoopBar() {
return {
start() {},
update() {},
increment() {},
stop() {}
};
}
function createProgressManager({ enabled = Boolean(process.stdout.isTTY) } = {}) {
if (!enabled || !cliProgress || !cliProgress.MultiBar) {
return {
enabled: false,
createBar() {
return createNoopBar();
},
stopAll() {}
};
}
const bars = new Set();
const multiBar = new cliProgress.MultiBar(
{
hideCursor: true,
clearOnComplete: true,
stopOnComplete: false
},
cliProgress.Presets.shades_classic
);
const createBar = options => {
let bar = null;
let started = false;
return {
start(total, startValue = 0, payload = {}) {
if (started) return;
const safeTotal = Math.max(1, Number(total) || 1);
const safeStart = Math.max(0, Math.min(Number(startValue) || 0, safeTotal));
bar = multiBar.create(safeTotal, safeStart, payload, options);
started = true;
bars.add(bar);
},
update(value, payload) {
if (!started || !bar) return;
bar.update(value, payload);
},
increment(value, payload) {
if (!started || !bar) return;
bar.increment(value, payload);
},
stop() {
if (!started || !bar) return;
try {
bar.stop();
} catch (error) {
// Ignore progress bar shutdown errors.
}
// Keep bar instances attached to preserve stable line ordering.
started = false;
bars.delete(bar);
bar = null;
}
};
};
return {
enabled: true,
createBar,
stopAll() {
bars.forEach(bar => {
try {
bar.stop();
} catch (error) {
// Ignore progress bar shutdown errors.
}
});
bars.clear();
try {
multiBar.stop();
} catch (error) {
// Ignore progress bar shutdown errors.
}
}
};
}
function sleep(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
function requestBuffer(url, options = {}) {
const {
method = 'GET',
headers = {},
allowNotModified = false,
redirectCount = 0
} = options;
const MAX_REDIRECTS = 5;
return new Promise((resolve, reject) => {
const request = https.request(url, { method, headers: { ...HEADERS, ...headers } }, response => {
const statusCode = response.statusCode || 0;
const location = response.headers.location;
if (allowNotModified && statusCode === 304) {
response.resume();
resolve({
statusCode,
headers: response.headers,
buffer: Buffer.alloc(0)
});
return;
}
if (statusCode >= 300 && statusCode < 400 && statusCode !== 304 && location) {
response.resume();
if (redirectCount >= MAX_REDIRECTS) {
reject(new Error(`Too many redirects for ${url}`));
return;
}
const redirectUrl = new URL(location, url).toString();
requestBuffer(redirectUrl, {
method,
headers,
allowNotModified,
redirectCount: redirectCount + 1
}).then(resolve).catch(reject);
return;
}
if (statusCode < 200 || statusCode >= 300) {
let preview = '';
response.setEncoding('utf8');
response.on('data', chunk => {
if (preview.length < 200) {
preview += chunk;
}
});
response.on('end', () => {
reject(new Error(`Request failed (${statusCode}) for ${url}: ${preview.slice(0, 200)}`));
});
return;
}
const chunks = [];
response.on('data', chunk => {
chunks.push(chunk);
});
response.on('end', () => {
resolve({
statusCode,
headers: response.headers,
buffer: Buffer.concat(chunks)
});
});
});
request.on('error', reject);
request.setTimeout(30000, () => {
request.destroy(new Error(`Request timeout for ${url}`));
});
request.end();
});
}
function fetchJson(page) {
const url = `${API_URL}${page}`;
return requestBuffer(url).then(response => {
const data = response.buffer.toString('utf8');
try {
return JSON.parse(data);
} catch (error) {
throw new Error(`Failed to parse JSON response for page ${page}: ${error.message}`);
}
});
}
function getIconKey(icon) {
return `${icon.prefix}/${icon.name}`;
}
function toEssentialIcon(icon = {}) {
const name = typeof icon.name === 'string' ? icon.name : '';
const prefix = typeof icon.prefix === 'string' ? icon.prefix : '';
if (!name || !prefix) return null;
const minimal = { name, prefix };
if (typeof icon.tags === 'string' && icon.tags.trim()) {
minimal.tags = icon.tags;
}
return minimal;
}
function normalizeSvgUrl(svgUrl) {
if (!svgUrl || typeof svgUrl !== 'string') return '';
try {
const url = new URL(svgUrl);
return `${url.origin}${url.pathname}`;
} catch (error) {
return svgUrl.split('?')[0];
}
}
function getIconFingerprint(icon) {
const normalizedSvg = normalizeSvgUrl(icon.svg);
if (!icon.id && !normalizedSvg) {
return '';
}
return JSON.stringify({
id: icon.id || null,
svg: normalizedSvg
});
}
function buildLatestIconsByKey(icons) {
const map = new Map();
icons.forEach(icon => {
if (!icon || !icon.prefix || !icon.name) return;
map.set(getIconKey(icon), icon);
});
return map;
}
function loadPreviousFingerprints(inputPath) {
if (!fs.existsSync(inputPath)) {
return new Map();
}
try {
const raw = fs.readFileSync(inputPath, 'utf8');
const icons = JSON.parse(raw);
const map = new Map();
icons.forEach(icon => {
if (!icon || !icon.prefix || !icon.name) return;
map.set(getIconKey(icon), getIconFingerprint(icon));
});
return map;
} catch (error) {
console.warn(`Warning: failed to read previous icon metadata from ${inputPath}: ${error.message}`);
return new Map();
}
}
function normalizeHeaderValue(value) {
if (value === null || value === undefined) return '';
if (Array.isArray(value)) return value[0] || '';
return String(value);
}
function parseHttpDate(value) {
const raw = normalizeHeaderValue(value).trim();
if (!raw) return 0;
const ms = Date.parse(raw);
return Number.isNaN(ms) ? 0 : ms;
}
function parseContentLength(value, fallback = 0) {
const raw = normalizeHeaderValue(value).trim();
if (!raw) return fallback;
const parsed = Number(raw);
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
}
function extractSvgHeaders(headers, bufferLength = 0) {
return {
lastModified: normalizeHeaderValue(headers['last-modified']).trim(),
contentLength: parseContentLength(headers['content-length'], bufferLength)
};
}
function applyRemoteMtime(filePath, lastModified) {
const ms = parseHttpDate(lastModified);
if (!ms) return;
try {
fs.utimesSync(filePath, new Date(), new Date(ms));
} catch (error) {
// Non-fatal: file contents are already written.
}
}
async function isSvgUpToDate(icon, outputPath, retries = 2) {
if (!icon.svg) {
return false;
}
const stat = fs.statSync(outputPath);
const localMtimeMs = stat.mtimeMs;
const localSize = stat.size;
const ifModifiedSince = new Date(localMtimeMs).toUTCString();
const requestHeaders = {};
if (ifModifiedSince) {
requestHeaders['If-Modified-Since'] = ifModifiedSince;
}
for (let attempt = 0; attempt <= retries; attempt += 1) {
try {
const response = await requestBuffer(icon.svg, {
method: 'HEAD',
headers: requestHeaders,
allowNotModified: true
});
if (response.statusCode === 304) {
return true;
}
const remoteHeaders = extractSvgHeaders(response.headers);
const remoteModifiedMs = parseHttpDate(remoteHeaders.lastModified);
// Server can return 200 even when unchanged. Compare remote headers with local file stats.
if (remoteModifiedMs && remoteModifiedMs <= localMtimeMs) {
if (!remoteHeaders.contentLength || remoteHeaders.contentLength === localSize) {
return true;
}
}
if (!remoteModifiedMs && remoteHeaders.contentLength > 0 && remoteHeaders.contentLength === localSize) {
return true;
}
return false;
} catch (error) {
if (attempt >= retries) {
throw error;
}
await sleep((attempt + 1) * 250);
}
}
return false;
}
async function downloadSvg(icon, outputPath, retries = 2) {
if (!icon.svg) {
throw new Error('Icon has no SVG URL');
}
for (let attempt = 0; attempt <= retries; attempt += 1) {
try {
const response = await requestBuffer(icon.svg);
const svg = response.buffer.toString('utf8');
if (!svg.includes('<svg')) {
throw new Error('Downloaded content is not valid SVG');
}
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.writeFileSync(outputPath, svg, 'utf8');
const headers = extractSvgHeaders(response.headers, response.buffer.length);
applyRemoteMtime(outputPath, headers.lastModified);
return;
} catch (error) {
if (attempt >= retries) {
throw error;
}
await sleep((attempt + 1) * 250);
}
}
}
async function syncSvgAssets({
icons,
previousFingerprints,
svgRoot,
concurrency,
retries,
progressEnabled = false,
onProgress = null
}) {
const latestByKey = buildLatestIconsByKey(icons);
const queue = Array.from(latestByKey.entries()).map(([key, icon]) => ({ key, icon }));
const total = queue.length;
const appliedFingerprints = new Map();
const counters = {
total,
downloaded: 0,
updated: 0,
newOrMissing: 0,
skipped: 0,
checkedHeaders: 0,
failed: 0,
failures: []
};
if (total === 0) {
return {
...counters,
appliedFingerprints
};
}
const prefixes = new Set(queue.map(item => item.icon.prefix).filter(Boolean));
prefixes.forEach(prefix => {
fs.mkdirSync(path.join(svgRoot, prefix), { recursive: true });
});
let cursor = 0;
let processed = 0;
const emitProgress = (action, key) => {
if (typeof onProgress === 'function') {
onProgress({
processed,
total,
action,
key,
counters: { ...counters }
});
}
if (!progressEnabled && (processed % 500 === 0 || processed === total)) {
console.log(`SVG sync ${processed}/${total} (downloaded ${counters.downloaded}, skipped ${counters.skipped}, failed ${counters.failed})`);
}
};
async function worker() {
while (true) {
const index = cursor;
cursor += 1;
if (index >= total) return;
const { key, icon } = queue[index];
const outputPath = path.join(svgRoot, icon.prefix, `${icon.name}.svg`);
const hasFile = fs.existsSync(outputPath);
const previousFingerprint = previousFingerprints.get(key);
const currentFingerprint = getIconFingerprint(icon);
const metadataUpdated = previousFingerprint !== undefined && previousFingerprint !== currentFingerprint;
let isUpdate = hasFile && metadataUpdated;
if (hasFile && !metadataUpdated) {
try {
const upToDate = await isSvgUpToDate(icon, outputPath, retries);
counters.checkedHeaders += 1;
if (upToDate) {
counters.skipped += 1;
appliedFingerprints.set(key, currentFingerprint);
processed += 1;
emitProgress('skipped', key);
continue;
}
isUpdate = true;
} catch (error) {
counters.failed += 1;
if (counters.failures.length < 20) {
counters.failures.push(`${key}: ${error.message}`);
}
processed += 1;
emitProgress('failed', key);
continue;
}
}
try {
await downloadSvg(icon, outputPath, retries);
counters.downloaded += 1;
let action = 'new';
if (isUpdate) {
counters.updated += 1;
action = 'updated';
} else {
counters.newOrMissing += 1;
}
processed += 1;
appliedFingerprints.set(key, currentFingerprint);
emitProgress(action, key);
} catch (error) {
counters.failed += 1;
if (counters.failures.length < 20) {
counters.failures.push(`${key}: ${error.message}`);
}
processed += 1;
emitProgress('failed', key);
}
}
}
const workerCount = Math.max(1, Math.min(concurrency, total));
await Promise.all(Array.from({ length: workerCount }, () => worker()));
return {
...counters,
appliedFingerprints
};
}
async function run() {
const args = parseArgs(process.argv.slice(2));
const start = Number(args.start || 1);
let end = Number(args.end || 0);
const delay = Number(args.delay || 100);
const svgConcurrency = Number(args['svg-concurrency'] || 8);
const svgRetries = Number(args['svg-retries'] || 2);
if (Number.isNaN(start) || start < 1) {
throw new Error('Invalid --start value');
}
if (Number.isNaN(svgConcurrency) || svgConcurrency < 1) {
throw new Error('Invalid --svg-concurrency value');
}
if (Number.isNaN(svgRetries) || svgRetries < 0) {
throw new Error('Invalid --svg-retries value');
}
const progress = createProgressManager();
const fetchBar = progress.createBar({
format: 'Fetch pages |{bar}| {percentage}% | {value}/{total} pages | icons: {icons}',
barsize: 28
});
const buildBar = progress.createBar({
format: 'Build data |{bar}| {percentage}% | {value}/{total} steps | {action}',
barsize: 28
});
const svgBar = progress.createBar({
format: 'Sync SVGs |{bar}| {percentage}% | {value}/{total} pages | dl:{downloaded} upd:{updated} new:{newOrMissing} skip:{skipped} fail:{failed} | {action}',
barsize: 28
});
try {
const outputDir = path.join(__dirname, 'data');
fs.mkdirSync(outputDir, { recursive: true });
const outputPath = path.join(outputDir, 'all_icons.json');
const previousFingerprints = loadPreviousFingerprints(outputPath);
const currentFingerprints = new Map(previousFingerprints);
const svgRoot = path.join(__dirname, 'svgs');
const syncSummary = {
total: 0,
downloaded: 0,
updated: 0,
newOrMissing: 0,
skipped: 0,
checkedHeaders: 0,
failed: 0,
failures: []
};
console.log(`Fetching icons starting at page ${start}...`);
const first = await fetchJson(start);
if (!first || !Array.isArray(first.items)) {
throw new Error('Unexpected response from icon API');
}
if (!end || Number.isNaN(end)) {
end = Number(first.pages || start);
}
const itemsById = new Map();
const addItems = items => {
(items || []).forEach(item => {
if (item && item.id) {
itemsById.set(item.id, item);
}
});
};
const totalPages = Math.max(1, end - start + 1);
if (progress.enabled) {
fetchBar.start(totalPages, 0, { icons: 0 });
svgBar.start(totalPages, 0, {
downloaded: 0,
updated: 0,
newOrMissing: 0,
skipped: 0,
failed: 0,
action: 'starting'
});
}
for (let page = start; page <= end; page += 1) {
const payload = page === start ? first : await fetchJson(page);
if (!payload || !Array.isArray(payload.items)) {
throw new Error(`Unexpected response from icon API for page ${page}`);
}
addItems(payload.items);
if (progress.enabled) {
fetchBar.update(page - start + 1, { icons: itemsById.size });
} else if (page % 25 === 0 || page === end || page === start) {
console.log(`Page ${page}/${end}: ${payload.items.length} items`);
}
const pageIcons = Array.from(buildLatestIconsByKey(payload.items).values());
const pageSync = await syncSvgAssets({
icons: pageIcons,
previousFingerprints: currentFingerprints,
svgRoot,
concurrency: svgConcurrency,
retries: svgRetries,
progressEnabled: true
});
pageSync.appliedFingerprints.forEach((fingerprint, key) => {
currentFingerprints.set(key, fingerprint);
});
syncSummary.total += pageSync.total;
syncSummary.downloaded += pageSync.downloaded;
syncSummary.updated += pageSync.updated;
syncSummary.newOrMissing += pageSync.newOrMissing;
syncSummary.skipped += pageSync.skipped;
syncSummary.checkedHeaders += pageSync.checkedHeaders;
syncSummary.failed += pageSync.failed;
if (syncSummary.failures.length < 20 && pageSync.failures.length > 0) {
const remaining = 20 - syncSummary.failures.length;
syncSummary.failures.push(...pageSync.failures.slice(0, remaining));
}
if (progress.enabled) {
svgBar.update(page - start + 1, {
downloaded: syncSummary.downloaded,
updated: syncSummary.updated,
newOrMissing: syncSummary.newOrMissing,
skipped: syncSummary.skipped,
failed: syncSummary.failed,
action: `page ${page}/${end}`
});
} else if (page % 25 === 0 || page === end) {
console.log(
`SVG sync ${page}/${end}: downloaded ${syncSummary.downloaded}, ` +
`updated ${syncSummary.updated}, new_or_missing ${syncSummary.newOrMissing}, ` +
`skipped ${syncSummary.skipped}, failed ${syncSummary.failed}`
);
}
if (delay > 0 && page < end) {
await sleep(delay);
}
}
fetchBar.stop();
svgBar.stop();
const allIcons = Array.from(itemsById.values())
.map(toEssentialIcon)
.filter(Boolean);
if (progress.enabled) {
buildBar.start(2, 0, { action: 'starting' });
}
fs.writeFileSync(outputPath, JSON.stringify(allIcons), 'utf8');
if (progress.enabled) {
buildBar.update(1, { action: 'wrote data/all_icons.json' });
} else {
console.log(`Wrote ${outputPath} (${allIcons.length} icons)`);
}
// Avoid mixed output while rendering progress bars.
const originalConsoleLog = console.log;
if (progress.enabled) {
console.log = () => {};
}
try {
buildIconsJs({
inputPath: outputPath,
outputPath: path.join(outputDir, 'all_icons.js')
});
} finally {
if (progress.enabled) {
console.log = originalConsoleLog;
}
}
if (progress.enabled) {
buildBar.update(2, { action: 'built data/all_icons.js' });
buildBar.stop();
}
console.log(
`SVG sync complete: total=${syncSummary.total}, downloaded=${syncSummary.downloaded}, ` +
`updated=${syncSummary.updated}, new_or_missing=${syncSummary.newOrMissing}, ` +
`skipped=${syncSummary.skipped}, checked_headers=${syncSummary.checkedHeaders}, failed=${syncSummary.failed}`
);
if (syncSummary.failures.length > 0) {
console.error('Sample SVG download failures:');
syncSummary.failures.forEach(failure => {
console.error(`- ${failure}`);
});
}
if (syncSummary.failed > 0) {
throw new Error(`Failed to download ${syncSummary.failed} SVG assets`);
}
} finally {
progress.stopAll();
}
}
run().catch(error => {
console.error(error.message || error);
process.exit(1);
});