feat: replace update-icon-list.js with update-icons.js, update build pipeline and docs
This commit is contained in:
@@ -11,9 +11,9 @@ The output is a drop-in CSS font system - users link `fonts/ficons.css` and use
|
||||
## Development Commands
|
||||
|
||||
```bash
|
||||
# Fetch latest icon metadata (writes to data/all_icons.json)
|
||||
# Fetch latest icon metadata and sync SVGs (writes to data/all_icons.json + svgs/*/*.svg)
|
||||
npm run update:icons
|
||||
# or: node update-icon-list.js
|
||||
# or: node update-icons.js
|
||||
|
||||
# Convert JSON to browser-ready JS (writes to data/all_icons.js)
|
||||
npm run build:icons
|
||||
@@ -44,7 +44,7 @@ Examples: `fi-rs-bookmark`, `fi-br-home`, `fi-brands-instagram`
|
||||
|
||||
### Data Pipeline
|
||||
|
||||
1. **`update-icon-list.js`** - Fetches icon metadata with pagination. Rate-limited (100ms delay). Outputs `data/all_icons.json`.
|
||||
1. **`update-icons.js`** - Fetches icon metadata page-by-page and immediately syncs SVG files for each page into `svgs/{prefix}/` (to avoid expiring SVG tokens), then writes `data/all_icons.json`. Existing files are skipped unless icon metadata changed.
|
||||
|
||||
2. **`build-icons-js.js`** - Converts JSON to `window.FICONS_DATA` global for browser use in explorer.
|
||||
|
||||
|
||||
@@ -1,171 +1,34 @@
|
||||
# @invisi/ficons
|
||||
|
||||
A local web-compatible icon font package with **thousands of icons** across multiple style variations.
|
||||
A local icon font package with webfonts, a small icon dataset, and a built-in explorer.
|
||||
|
||||
## Features
|
||||
## Install
|
||||
|
||||
- **Thousands of icons** across 15 style variations (stroke weights × corner styles)
|
||||
- **Self-hosted** - no CDN dependencies
|
||||
- **Interactive explorer** - browse, search, and filter icons locally
|
||||
- **Web fonts** - WOFF2, WOFF, TTF formats
|
||||
- **Easy integration** - drop-in CSS with class-based icons
|
||||
|
||||
## Installation
|
||||
|
||||
```shell
|
||||
```sh
|
||||
npm i @invisi/ficons
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
Include the CSS in your HTML:
|
||||
## Basic usage
|
||||
|
||||
```html
|
||||
<link rel="stylesheet" href="fonts/ficons.css">
|
||||
```
|
||||
|
||||
Use icons with `<span>` or `<i>` elements:
|
||||
|
||||
```html
|
||||
<i class="fi fi-rs-user"></i>
|
||||
<span class="fi fi-brs-home"></span>
|
||||
<i class="fi fi-brands-instagram"></i>
|
||||
```
|
||||
|
||||
## Icon Styles
|
||||
## CLI
|
||||
|
||||
| Weight | Corner | Prefix | Example |
|
||||
|:-----------|:---------|:-------|:-------------------------------|
|
||||
| Regular | Straight | fi-rs | `<i class="fi fi-rs-user"></i>` |
|
||||
| Regular | Rounded | fi-rr | `<i class="fi fi-rr-user"></i>` |
|
||||
| Bold | Straight | fi-bs | `<i class="fi fi-bs-user"></i>` |
|
||||
| Bold | Rounded | fi-br | `<i class="fi fi-br-user"></i>` |
|
||||
| Solid | Straight | fi-ss | `<i class="fi fi-ss-user"></i>` |
|
||||
| Solid | Rounded | fi-sr | `<i class="fi fi-sr-user"></i>` |
|
||||
| Thin | Straight | fi-ts | `<i class="fi fi-ts-user"></i>` |
|
||||
| Thin | Rounded | fi-tr | `<i class="fi fi-tr-user"></i>` |
|
||||
| Brands | - | fi-brands | `<i class="fi fi-brands-facebook"></i>` |
|
||||
|
||||
## Icon Count
|
||||
|
||||
| Variant | Icons |
|
||||
|-------------------|--------|
|
||||
| Regular Straight | 5,043 |
|
||||
| Regular Rounded | 5,039 |
|
||||
| Bold Straight | 5,055 |
|
||||
| Bold Rounded | 5,041 |
|
||||
| Solid Straight | 5,044 |
|
||||
| Solid Rounded | 5,051 |
|
||||
| Thin Straight | 5,053 |
|
||||
| Thin Rounded | 5,032 |
|
||||
| Regular Chubby | 3,093 |
|
||||
| Solid Chubby | 3,093 |
|
||||
| Thin Chubby | 3,093 |
|
||||
| Duotone Straight | 180 |
|
||||
| Duotone Rounded | 160 |
|
||||
| Duotone Chubby | 270 |
|
||||
| Brands | 245 |
|
||||
|
||||
**Thousands of icon variations**
|
||||
|
||||
## Styling
|
||||
|
||||
Icons inherit `font-size` and `color` from their parent:
|
||||
|
||||
```html
|
||||
<!-- Size -->
|
||||
<i class="fi fi-rs-heart" style="font-size: 24px;"></i>
|
||||
<i class="fi fi-rs-heart" style="font-size: 48px;"></i>
|
||||
|
||||
<!-- Color -->
|
||||
<i class="fi fi-rs-star" style="color: #f1c40f;"></i>
|
||||
<i class="fi fi-rs-heart" style="color: #e74c3c;"></i>
|
||||
```
|
||||
|
||||
## Package Structure
|
||||
|
||||
```
|
||||
fonts/
|
||||
ficons.css # Unified CSS (imports all styles)
|
||||
css/
|
||||
flaticon-regular-straight.css
|
||||
flaticon-regular-rounded.css
|
||||
flaticon-bold-straight.css
|
||||
...
|
||||
webfonts/ # Font files (woff2, woff, ttf)
|
||||
data/
|
||||
all_icons.js # Icon data for explorer
|
||||
explorer.html # Interactive icon browser
|
||||
```
|
||||
|
||||
## CLI Search
|
||||
|
||||
Search for icons directly from the command line using `npx`:
|
||||
|
||||
```bash
|
||||
# Basic search
|
||||
```sh
|
||||
npx @invisi/ficons explore
|
||||
npx @invisi/ficons search camera
|
||||
|
||||
# Search with multiple keywords
|
||||
npx @invisi/ficons search arrow left
|
||||
|
||||
# Filter by variation (prefix)
|
||||
npx @invisi/ficons search user rr
|
||||
npx @invisi/ficons search home --variation ss
|
||||
npx @invisi/ficons search camera --variation rr
|
||||
```
|
||||
|
||||
Outputs JSON with icon names and available variations:
|
||||
## Development
|
||||
|
||||
```json
|
||||
{
|
||||
"camera": ["rs", "rr", "bs", "br", "ss", "sr", "ts", "tr"],
|
||||
"camera-phone": ["rs", "rr", "bs", "br"]
|
||||
}
|
||||
```
|
||||
|
||||
### Variation Prefixes
|
||||
|
||||
| Prefix | Style |
|
||||
|:-------|:------|
|
||||
| `rs` | Regular Straight |
|
||||
| `rr` | Regular Rounded |
|
||||
| `rc` | Regular Chubby |
|
||||
| `bs` | Bold Straight |
|
||||
| `br` | Bold Rounded |
|
||||
| `bc` | Bold Chubby |
|
||||
| `ss` | Solid Straight |
|
||||
| `sr` | Solid Rounded |
|
||||
| `sc` | Solid Chubby |
|
||||
| `ts` | Thin Straight |
|
||||
| `tr` | Thin Rounded |
|
||||
| `tc` | Thin Chubby |
|
||||
| `ds` | Duotone Straight |
|
||||
| `dr` | Duotone Rounded |
|
||||
| `dc` | Duotone Chubby |
|
||||
| `brands` | Brand Logos |
|
||||
|
||||
## Icon Explorer
|
||||
|
||||
Open `explorer.html` in a browser to:
|
||||
|
||||
- Browse all icons
|
||||
- Search by name or tags
|
||||
- Filter by weight, corner style, and type
|
||||
- Copy HTML snippets and CSS classes
|
||||
- Adjust icon preview size
|
||||
|
||||
## Development Scripts
|
||||
|
||||
```bash
|
||||
# Fetch latest icon metadata
|
||||
```sh
|
||||
# Fetch metadata and sync SVGs page-by-page
|
||||
npm run update:icons
|
||||
# or: node update-icon-list.js
|
||||
|
||||
# Convert JSON to browser-ready JS
|
||||
npm run build:icons
|
||||
# or: node build-icons-js.js
|
||||
|
||||
# Generate webfonts from SVGs using FontForge
|
||||
npm run build:fonts
|
||||
# or: node build-fonts.js
|
||||
```
|
||||
|
||||
+15
-6
@@ -1,6 +1,20 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
function toEssentialIcon(icon = {}) {
|
||||
const name = typeof icon.name === 'string' ? icon.name : '';
|
||||
const prefix = typeof icon.prefix === 'string' ? icon.prefix : '';
|
||||
|
||||
if (!name || !prefix) return null;
|
||||
|
||||
const minimal = { name, prefix };
|
||||
if (typeof icon.tags === 'string' && icon.tags.trim()) {
|
||||
minimal.tags = icon.tags;
|
||||
}
|
||||
|
||||
return minimal;
|
||||
}
|
||||
|
||||
function buildIconsJs({
|
||||
inputPath = path.join(__dirname, 'data', 'all_icons.json'),
|
||||
outputPath = path.join(__dirname, 'data', 'all_icons.js')
|
||||
@@ -10,12 +24,7 @@ function buildIconsJs({
|
||||
}
|
||||
|
||||
const raw = fs.readFileSync(inputPath, 'utf8');
|
||||
const icons = JSON.parse(raw).map(({ name, prefix, tags, is_brand }) => ({
|
||||
name,
|
||||
prefix,
|
||||
tags,
|
||||
is_brand
|
||||
}));
|
||||
const icons = JSON.parse(raw).map(toEssentialIcon).filter(Boolean);
|
||||
|
||||
const payload = `window.FICONS_DATA = ${JSON.stringify(icons)};\n`;
|
||||
fs.writeFileSync(outputPath, payload, 'utf8');
|
||||
|
||||
+1
-1
File diff suppressed because one or more lines are too long
+5
-2
@@ -18,7 +18,7 @@
|
||||
],
|
||||
"scripts": {
|
||||
"build:icons": "node build-icons-js.js",
|
||||
"update:icons": "node update-icon-list.js",
|
||||
"update:icons": "node update-icons.js",
|
||||
"build:fonts": "node build-fonts.js"
|
||||
},
|
||||
"keywords": [
|
||||
@@ -26,5 +26,8 @@
|
||||
"fonts",
|
||||
"icon-font",
|
||||
"webfonts"
|
||||
]
|
||||
],
|
||||
"dependencies": {
|
||||
"cli-progress": "^3.12.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
const buildIconsJs = require('./build-icons-js');
|
||||
|
||||
const API_URL = 'https://www.flaticon.com/ajax/icon-fonts-most-downloaded/';
|
||||
const HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121"',
|
||||
'Sec-Ch-Ua-Mobile': '?0',
|
||||
'Sec-Ch-Ua-Platform': '"macOS"',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none'
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const key = arg.slice(2);
|
||||
const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : true;
|
||||
if (value !== true) i += 1;
|
||||
args[key] = value;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function fetchJson(page) {
|
||||
const url = `${API_URL}${page}`;
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.request(url, { headers: HEADERS }, response => {
|
||||
let data = '';
|
||||
response.on('data', chunk => {
|
||||
data += chunk;
|
||||
});
|
||||
response.on('end', () => {
|
||||
try {
|
||||
resolve(JSON.parse(data));
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
request.on('error', reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const start = Number(args.start || 1);
|
||||
let end = Number(args.end || 0);
|
||||
const delay = Number(args.delay || 100);
|
||||
|
||||
if (Number.isNaN(start) || start < 1) {
|
||||
throw new Error('Invalid --start value');
|
||||
}
|
||||
|
||||
console.log(`Fetching icons starting at page ${start}...`);
|
||||
const first = await fetchJson(start);
|
||||
if (!first || !Array.isArray(first.items)) {
|
||||
throw new Error('Unexpected response from icon API');
|
||||
}
|
||||
|
||||
if (!end || Number.isNaN(end)) {
|
||||
end = Number(first.pages || start);
|
||||
}
|
||||
|
||||
const itemsById = new Map();
|
||||
const addItems = items => {
|
||||
(items || []).forEach(item => {
|
||||
if (item && item.id) {
|
||||
itemsById.set(item.id, item);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
addItems(first.items);
|
||||
console.log(`Page ${start}/${end}: ${first.items.length} items`);
|
||||
|
||||
for (let page = start + 1; page <= end; page += 1) {
|
||||
const payload = await fetchJson(page);
|
||||
addItems(payload.items);
|
||||
if (page % 25 === 0 || page === end) {
|
||||
console.log(`Page ${page}/${end}: ${payload.items.length} items`);
|
||||
}
|
||||
if (delay > 0) {
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
const outputDir = path.join(__dirname, 'data');
|
||||
if (!fs.existsSync(outputDir)) {
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
}
|
||||
|
||||
const allIcons = Array.from(itemsById.values());
|
||||
const outputPath = path.join(outputDir, 'all_icons.json');
|
||||
fs.writeFileSync(outputPath, JSON.stringify(allIcons), 'utf8');
|
||||
console.log(`Wrote ${outputPath} (${allIcons.length} icons)`);
|
||||
|
||||
buildIconsJs({
|
||||
inputPath: outputPath,
|
||||
outputPath: path.join(outputDir, 'all_icons.js')
|
||||
});
|
||||
}
|
||||
|
||||
run().catch(error => {
|
||||
console.error(error.message || error);
|
||||
process.exit(1);
|
||||
});
|
||||
+755
@@ -0,0 +1,755 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const https = require('https');
|
||||
const buildIconsJs = require('./build-icons-js');
|
||||
|
||||
let cliProgress = null;
|
||||
try {
|
||||
// Optional at runtime for environments not bootstrapped with package manager hooks.
|
||||
cliProgress = require('cli-progress');
|
||||
} catch (error) {
|
||||
if (error && error.code !== 'MODULE_NOT_FOUND') {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const API_URL = 'https://www.flaticon.com/ajax/icon-fonts-most-downloaded/';
|
||||
const HEADERS = {
|
||||
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Sec-Ch-Ua': '"Not A(Brand";v="99", "Google Chrome";v="121"',
|
||||
'Sec-Ch-Ua-Mobile': '?0',
|
||||
'Sec-Ch-Ua-Platform': '"macOS"',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none'
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {};
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i];
|
||||
if (!arg.startsWith('--')) continue;
|
||||
const key = arg.slice(2);
|
||||
const value = argv[i + 1] && !argv[i + 1].startsWith('--') ? argv[i + 1] : true;
|
||||
if (value !== true) i += 1;
|
||||
args[key] = value;
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function createNoopBar() {
|
||||
return {
|
||||
start() {},
|
||||
update() {},
|
||||
increment() {},
|
||||
stop() {}
|
||||
};
|
||||
}
|
||||
|
||||
function createProgressManager({ enabled = Boolean(process.stdout.isTTY) } = {}) {
|
||||
if (!enabled || !cliProgress || !cliProgress.MultiBar) {
|
||||
return {
|
||||
enabled: false,
|
||||
createBar() {
|
||||
return createNoopBar();
|
||||
},
|
||||
stopAll() {}
|
||||
};
|
||||
}
|
||||
|
||||
const bars = new Set();
|
||||
const multiBar = new cliProgress.MultiBar(
|
||||
{
|
||||
hideCursor: true,
|
||||
clearOnComplete: true,
|
||||
stopOnComplete: false
|
||||
},
|
||||
cliProgress.Presets.shades_classic
|
||||
);
|
||||
|
||||
const createBar = options => {
|
||||
let bar = null;
|
||||
let started = false;
|
||||
|
||||
return {
|
||||
start(total, startValue = 0, payload = {}) {
|
||||
if (started) return;
|
||||
const safeTotal = Math.max(1, Number(total) || 1);
|
||||
const safeStart = Math.max(0, Math.min(Number(startValue) || 0, safeTotal));
|
||||
bar = multiBar.create(safeTotal, safeStart, payload, options);
|
||||
started = true;
|
||||
bars.add(bar);
|
||||
},
|
||||
update(value, payload) {
|
||||
if (!started || !bar) return;
|
||||
bar.update(value, payload);
|
||||
},
|
||||
increment(value, payload) {
|
||||
if (!started || !bar) return;
|
||||
bar.increment(value, payload);
|
||||
},
|
||||
stop() {
|
||||
if (!started || !bar) return;
|
||||
try {
|
||||
bar.stop();
|
||||
} catch (error) {
|
||||
// Ignore progress bar shutdown errors.
|
||||
}
|
||||
// Keep bar instances attached to preserve stable line ordering.
|
||||
started = false;
|
||||
bars.delete(bar);
|
||||
bar = null;
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
return {
|
||||
enabled: true,
|
||||
createBar,
|
||||
stopAll() {
|
||||
bars.forEach(bar => {
|
||||
try {
|
||||
bar.stop();
|
||||
} catch (error) {
|
||||
// Ignore progress bar shutdown errors.
|
||||
}
|
||||
});
|
||||
bars.clear();
|
||||
try {
|
||||
multiBar.stop();
|
||||
} catch (error) {
|
||||
// Ignore progress bar shutdown errors.
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function requestBuffer(url, options = {}) {
|
||||
const {
|
||||
method = 'GET',
|
||||
headers = {},
|
||||
allowNotModified = false,
|
||||
redirectCount = 0
|
||||
} = options;
|
||||
|
||||
const MAX_REDIRECTS = 5;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = https.request(url, { method, headers: { ...HEADERS, ...headers } }, response => {
|
||||
const statusCode = response.statusCode || 0;
|
||||
const location = response.headers.location;
|
||||
|
||||
if (allowNotModified && statusCode === 304) {
|
||||
response.resume();
|
||||
resolve({
|
||||
statusCode,
|
||||
headers: response.headers,
|
||||
buffer: Buffer.alloc(0)
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusCode >= 300 && statusCode < 400 && statusCode !== 304 && location) {
|
||||
response.resume();
|
||||
|
||||
if (redirectCount >= MAX_REDIRECTS) {
|
||||
reject(new Error(`Too many redirects for ${url}`));
|
||||
return;
|
||||
}
|
||||
|
||||
const redirectUrl = new URL(location, url).toString();
|
||||
requestBuffer(redirectUrl, {
|
||||
method,
|
||||
headers,
|
||||
allowNotModified,
|
||||
redirectCount: redirectCount + 1
|
||||
}).then(resolve).catch(reject);
|
||||
return;
|
||||
}
|
||||
|
||||
if (statusCode < 200 || statusCode >= 300) {
|
||||
let preview = '';
|
||||
response.setEncoding('utf8');
|
||||
response.on('data', chunk => {
|
||||
if (preview.length < 200) {
|
||||
preview += chunk;
|
||||
}
|
||||
});
|
||||
response.on('end', () => {
|
||||
reject(new Error(`Request failed (${statusCode}) for ${url}: ${preview.slice(0, 200)}`));
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const chunks = [];
|
||||
response.on('data', chunk => {
|
||||
chunks.push(chunk);
|
||||
});
|
||||
response.on('end', () => {
|
||||
resolve({
|
||||
statusCode,
|
||||
headers: response.headers,
|
||||
buffer: Buffer.concat(chunks)
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
request.on('error', reject);
|
||||
request.setTimeout(30000, () => {
|
||||
request.destroy(new Error(`Request timeout for ${url}`));
|
||||
});
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function fetchJson(page) {
|
||||
const url = `${API_URL}${page}`;
|
||||
return requestBuffer(url).then(response => {
|
||||
const data = response.buffer.toString('utf8');
|
||||
try {
|
||||
return JSON.parse(data);
|
||||
} catch (error) {
|
||||
throw new Error(`Failed to parse JSON response for page ${page}: ${error.message}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function getIconKey(icon) {
|
||||
return `${icon.prefix}/${icon.name}`;
|
||||
}
|
||||
|
||||
function toEssentialIcon(icon = {}) {
|
||||
const name = typeof icon.name === 'string' ? icon.name : '';
|
||||
const prefix = typeof icon.prefix === 'string' ? icon.prefix : '';
|
||||
|
||||
if (!name || !prefix) return null;
|
||||
|
||||
const minimal = { name, prefix };
|
||||
if (typeof icon.tags === 'string' && icon.tags.trim()) {
|
||||
minimal.tags = icon.tags;
|
||||
}
|
||||
|
||||
return minimal;
|
||||
}
|
||||
|
||||
function normalizeSvgUrl(svgUrl) {
|
||||
if (!svgUrl || typeof svgUrl !== 'string') return '';
|
||||
try {
|
||||
const url = new URL(svgUrl);
|
||||
return `${url.origin}${url.pathname}`;
|
||||
} catch (error) {
|
||||
return svgUrl.split('?')[0];
|
||||
}
|
||||
}
|
||||
|
||||
function getIconFingerprint(icon) {
|
||||
const normalizedSvg = normalizeSvgUrl(icon.svg);
|
||||
if (!icon.id && !normalizedSvg) {
|
||||
return '';
|
||||
}
|
||||
|
||||
return JSON.stringify({
|
||||
id: icon.id || null,
|
||||
svg: normalizedSvg
|
||||
});
|
||||
}
|
||||
|
||||
function buildLatestIconsByKey(icons) {
|
||||
const map = new Map();
|
||||
icons.forEach(icon => {
|
||||
if (!icon || !icon.prefix || !icon.name) return;
|
||||
map.set(getIconKey(icon), icon);
|
||||
});
|
||||
return map;
|
||||
}
|
||||
|
||||
function loadPreviousFingerprints(inputPath) {
|
||||
if (!fs.existsSync(inputPath)) {
|
||||
return new Map();
|
||||
}
|
||||
|
||||
try {
|
||||
const raw = fs.readFileSync(inputPath, 'utf8');
|
||||
const icons = JSON.parse(raw);
|
||||
const map = new Map();
|
||||
|
||||
icons.forEach(icon => {
|
||||
if (!icon || !icon.prefix || !icon.name) return;
|
||||
map.set(getIconKey(icon), getIconFingerprint(icon));
|
||||
});
|
||||
|
||||
return map;
|
||||
} catch (error) {
|
||||
console.warn(`Warning: failed to read previous icon metadata from ${inputPath}: ${error.message}`);
|
||||
return new Map();
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeHeaderValue(value) {
|
||||
if (value === null || value === undefined) return '';
|
||||
if (Array.isArray(value)) return value[0] || '';
|
||||
return String(value);
|
||||
}
|
||||
|
||||
function parseHttpDate(value) {
|
||||
const raw = normalizeHeaderValue(value).trim();
|
||||
if (!raw) return 0;
|
||||
const ms = Date.parse(raw);
|
||||
return Number.isNaN(ms) ? 0 : ms;
|
||||
}
|
||||
|
||||
function parseContentLength(value, fallback = 0) {
|
||||
const raw = normalizeHeaderValue(value).trim();
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function extractSvgHeaders(headers, bufferLength = 0) {
|
||||
return {
|
||||
lastModified: normalizeHeaderValue(headers['last-modified']).trim(),
|
||||
contentLength: parseContentLength(headers['content-length'], bufferLength)
|
||||
};
|
||||
}
|
||||
|
||||
function applyRemoteMtime(filePath, lastModified) {
|
||||
const ms = parseHttpDate(lastModified);
|
||||
if (!ms) return;
|
||||
|
||||
try {
|
||||
fs.utimesSync(filePath, new Date(), new Date(ms));
|
||||
} catch (error) {
|
||||
// Non-fatal: file contents are already written.
|
||||
}
|
||||
}
|
||||
|
||||
async function isSvgUpToDate(icon, outputPath, retries = 2) {
|
||||
if (!icon.svg) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const stat = fs.statSync(outputPath);
|
||||
const localMtimeMs = stat.mtimeMs;
|
||||
const localSize = stat.size;
|
||||
const ifModifiedSince = new Date(localMtimeMs).toUTCString();
|
||||
|
||||
const requestHeaders = {};
|
||||
if (ifModifiedSince) {
|
||||
requestHeaders['If-Modified-Since'] = ifModifiedSince;
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
||||
try {
|
||||
const response = await requestBuffer(icon.svg, {
|
||||
method: 'HEAD',
|
||||
headers: requestHeaders,
|
||||
allowNotModified: true
|
||||
});
|
||||
|
||||
if (response.statusCode === 304) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const remoteHeaders = extractSvgHeaders(response.headers);
|
||||
const remoteModifiedMs = parseHttpDate(remoteHeaders.lastModified);
|
||||
|
||||
// Server can return 200 even when unchanged. Compare remote headers with local file stats.
|
||||
if (remoteModifiedMs && remoteModifiedMs <= localMtimeMs) {
|
||||
if (!remoteHeaders.contentLength || remoteHeaders.contentLength === localSize) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!remoteModifiedMs && remoteHeaders.contentLength > 0 && remoteHeaders.contentLength === localSize) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
} catch (error) {
|
||||
if (attempt >= retries) {
|
||||
throw error;
|
||||
}
|
||||
await sleep((attempt + 1) * 250);
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
async function downloadSvg(icon, outputPath, retries = 2) {
|
||||
if (!icon.svg) {
|
||||
throw new Error('Icon has no SVG URL');
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt <= retries; attempt += 1) {
|
||||
try {
|
||||
const response = await requestBuffer(icon.svg);
|
||||
const svg = response.buffer.toString('utf8');
|
||||
|
||||
if (!svg.includes('<svg')) {
|
||||
throw new Error('Downloaded content is not valid SVG');
|
||||
}
|
||||
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, svg, 'utf8');
|
||||
|
||||
const headers = extractSvgHeaders(response.headers, response.buffer.length);
|
||||
applyRemoteMtime(outputPath, headers.lastModified);
|
||||
return;
|
||||
} catch (error) {
|
||||
if (attempt >= retries) {
|
||||
throw error;
|
||||
}
|
||||
await sleep((attempt + 1) * 250);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function syncSvgAssets({
|
||||
icons,
|
||||
previousFingerprints,
|
||||
svgRoot,
|
||||
concurrency,
|
||||
retries,
|
||||
progressEnabled = false,
|
||||
onProgress = null
|
||||
}) {
|
||||
const latestByKey = buildLatestIconsByKey(icons);
|
||||
const queue = Array.from(latestByKey.entries()).map(([key, icon]) => ({ key, icon }));
|
||||
const total = queue.length;
|
||||
const appliedFingerprints = new Map();
|
||||
|
||||
const counters = {
|
||||
total,
|
||||
downloaded: 0,
|
||||
updated: 0,
|
||||
newOrMissing: 0,
|
||||
skipped: 0,
|
||||
checkedHeaders: 0,
|
||||
failed: 0,
|
||||
failures: []
|
||||
};
|
||||
|
||||
if (total === 0) {
|
||||
return {
|
||||
...counters,
|
||||
appliedFingerprints
|
||||
};
|
||||
}
|
||||
|
||||
const prefixes = new Set(queue.map(item => item.icon.prefix).filter(Boolean));
|
||||
prefixes.forEach(prefix => {
|
||||
fs.mkdirSync(path.join(svgRoot, prefix), { recursive: true });
|
||||
});
|
||||
|
||||
let cursor = 0;
|
||||
let processed = 0;
|
||||
|
||||
const emitProgress = (action, key) => {
|
||||
if (typeof onProgress === 'function') {
|
||||
onProgress({
|
||||
processed,
|
||||
total,
|
||||
action,
|
||||
key,
|
||||
counters: { ...counters }
|
||||
});
|
||||
}
|
||||
|
||||
if (!progressEnabled && (processed % 500 === 0 || processed === total)) {
|
||||
console.log(`SVG sync ${processed}/${total} (downloaded ${counters.downloaded}, skipped ${counters.skipped}, failed ${counters.failed})`);
|
||||
}
|
||||
};
|
||||
|
||||
async function worker() {
|
||||
while (true) {
|
||||
const index = cursor;
|
||||
cursor += 1;
|
||||
if (index >= total) return;
|
||||
|
||||
const { key, icon } = queue[index];
|
||||
const outputPath = path.join(svgRoot, icon.prefix, `${icon.name}.svg`);
|
||||
const hasFile = fs.existsSync(outputPath);
|
||||
const previousFingerprint = previousFingerprints.get(key);
|
||||
const currentFingerprint = getIconFingerprint(icon);
|
||||
const metadataUpdated = previousFingerprint !== undefined && previousFingerprint !== currentFingerprint;
|
||||
|
||||
let isUpdate = hasFile && metadataUpdated;
|
||||
|
||||
if (hasFile && !metadataUpdated) {
|
||||
try {
|
||||
const upToDate = await isSvgUpToDate(icon, outputPath, retries);
|
||||
counters.checkedHeaders += 1;
|
||||
|
||||
if (upToDate) {
|
||||
counters.skipped += 1;
|
||||
appliedFingerprints.set(key, currentFingerprint);
|
||||
processed += 1;
|
||||
emitProgress('skipped', key);
|
||||
continue;
|
||||
}
|
||||
|
||||
isUpdate = true;
|
||||
} catch (error) {
|
||||
counters.failed += 1;
|
||||
if (counters.failures.length < 20) {
|
||||
counters.failures.push(`${key}: ${error.message}`);
|
||||
}
|
||||
processed += 1;
|
||||
emitProgress('failed', key);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await downloadSvg(icon, outputPath, retries);
|
||||
|
||||
counters.downloaded += 1;
|
||||
let action = 'new';
|
||||
if (isUpdate) {
|
||||
counters.updated += 1;
|
||||
action = 'updated';
|
||||
} else {
|
||||
counters.newOrMissing += 1;
|
||||
}
|
||||
|
||||
processed += 1;
|
||||
appliedFingerprints.set(key, currentFingerprint);
|
||||
emitProgress(action, key);
|
||||
} catch (error) {
|
||||
counters.failed += 1;
|
||||
if (counters.failures.length < 20) {
|
||||
counters.failures.push(`${key}: ${error.message}`);
|
||||
}
|
||||
processed += 1;
|
||||
emitProgress('failed', key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workerCount = Math.max(1, Math.min(concurrency, total));
|
||||
await Promise.all(Array.from({ length: workerCount }, () => worker()));
|
||||
return {
|
||||
...counters,
|
||||
appliedFingerprints
|
||||
};
|
||||
}
|
||||
|
||||
async function run() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const start = Number(args.start || 1);
|
||||
let end = Number(args.end || 0);
|
||||
const delay = Number(args.delay || 100);
|
||||
const svgConcurrency = Number(args['svg-concurrency'] || 8);
|
||||
const svgRetries = Number(args['svg-retries'] || 2);
|
||||
|
||||
if (Number.isNaN(start) || start < 1) {
|
||||
throw new Error('Invalid --start value');
|
||||
}
|
||||
if (Number.isNaN(svgConcurrency) || svgConcurrency < 1) {
|
||||
throw new Error('Invalid --svg-concurrency value');
|
||||
}
|
||||
if (Number.isNaN(svgRetries) || svgRetries < 0) {
|
||||
throw new Error('Invalid --svg-retries value');
|
||||
}
|
||||
|
||||
const progress = createProgressManager();
|
||||
const fetchBar = progress.createBar({
|
||||
format: 'Fetch pages |{bar}| {percentage}% | {value}/{total} pages | icons: {icons}',
|
||||
barsize: 28
|
||||
});
|
||||
const buildBar = progress.createBar({
|
||||
format: 'Build data |{bar}| {percentage}% | {value}/{total} steps | {action}',
|
||||
barsize: 28
|
||||
});
|
||||
const svgBar = progress.createBar({
|
||||
format: 'Sync SVGs |{bar}| {percentage}% | {value}/{total} pages | dl:{downloaded} upd:{updated} new:{newOrMissing} skip:{skipped} fail:{failed} | {action}',
|
||||
barsize: 28
|
||||
});
|
||||
|
||||
try {
|
||||
const outputDir = path.join(__dirname, 'data');
|
||||
fs.mkdirSync(outputDir, { recursive: true });
|
||||
|
||||
const outputPath = path.join(outputDir, 'all_icons.json');
|
||||
const previousFingerprints = loadPreviousFingerprints(outputPath);
|
||||
const currentFingerprints = new Map(previousFingerprints);
|
||||
const svgRoot = path.join(__dirname, 'svgs');
|
||||
const syncSummary = {
|
||||
total: 0,
|
||||
downloaded: 0,
|
||||
updated: 0,
|
||||
newOrMissing: 0,
|
||||
skipped: 0,
|
||||
checkedHeaders: 0,
|
||||
failed: 0,
|
||||
failures: []
|
||||
};
|
||||
|
||||
console.log(`Fetching icons starting at page ${start}...`);
|
||||
const first = await fetchJson(start);
|
||||
if (!first || !Array.isArray(first.items)) {
|
||||
throw new Error('Unexpected response from icon API');
|
||||
}
|
||||
|
||||
if (!end || Number.isNaN(end)) {
|
||||
end = Number(first.pages || start);
|
||||
}
|
||||
|
||||
const itemsById = new Map();
|
||||
const addItems = items => {
|
||||
(items || []).forEach(item => {
|
||||
if (item && item.id) {
|
||||
itemsById.set(item.id, item);
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
const totalPages = Math.max(1, end - start + 1);
|
||||
if (progress.enabled) {
|
||||
fetchBar.start(totalPages, 0, { icons: 0 });
|
||||
svgBar.start(totalPages, 0, {
|
||||
downloaded: 0,
|
||||
updated: 0,
|
||||
newOrMissing: 0,
|
||||
skipped: 0,
|
||||
failed: 0,
|
||||
action: 'starting'
|
||||
});
|
||||
}
|
||||
|
||||
for (let page = start; page <= end; page += 1) {
|
||||
const payload = page === start ? first : await fetchJson(page);
|
||||
if (!payload || !Array.isArray(payload.items)) {
|
||||
throw new Error(`Unexpected response from icon API for page ${page}`);
|
||||
}
|
||||
|
||||
addItems(payload.items);
|
||||
|
||||
if (progress.enabled) {
|
||||
fetchBar.update(page - start + 1, { icons: itemsById.size });
|
||||
} else if (page % 25 === 0 || page === end || page === start) {
|
||||
console.log(`Page ${page}/${end}: ${payload.items.length} items`);
|
||||
}
|
||||
|
||||
const pageIcons = Array.from(buildLatestIconsByKey(payload.items).values());
|
||||
const pageSync = await syncSvgAssets({
|
||||
icons: pageIcons,
|
||||
previousFingerprints: currentFingerprints,
|
||||
svgRoot,
|
||||
concurrency: svgConcurrency,
|
||||
retries: svgRetries,
|
||||
progressEnabled: true
|
||||
});
|
||||
|
||||
pageSync.appliedFingerprints.forEach((fingerprint, key) => {
|
||||
currentFingerprints.set(key, fingerprint);
|
||||
});
|
||||
|
||||
syncSummary.total += pageSync.total;
|
||||
syncSummary.downloaded += pageSync.downloaded;
|
||||
syncSummary.updated += pageSync.updated;
|
||||
syncSummary.newOrMissing += pageSync.newOrMissing;
|
||||
syncSummary.skipped += pageSync.skipped;
|
||||
syncSummary.checkedHeaders += pageSync.checkedHeaders;
|
||||
syncSummary.failed += pageSync.failed;
|
||||
|
||||
if (syncSummary.failures.length < 20 && pageSync.failures.length > 0) {
|
||||
const remaining = 20 - syncSummary.failures.length;
|
||||
syncSummary.failures.push(...pageSync.failures.slice(0, remaining));
|
||||
}
|
||||
|
||||
if (progress.enabled) {
|
||||
svgBar.update(page - start + 1, {
|
||||
downloaded: syncSummary.downloaded,
|
||||
updated: syncSummary.updated,
|
||||
newOrMissing: syncSummary.newOrMissing,
|
||||
skipped: syncSummary.skipped,
|
||||
failed: syncSummary.failed,
|
||||
action: `page ${page}/${end}`
|
||||
});
|
||||
} else if (page % 25 === 0 || page === end) {
|
||||
console.log(
|
||||
`SVG sync ${page}/${end}: downloaded ${syncSummary.downloaded}, ` +
|
||||
`updated ${syncSummary.updated}, new_or_missing ${syncSummary.newOrMissing}, ` +
|
||||
`skipped ${syncSummary.skipped}, failed ${syncSummary.failed}`
|
||||
);
|
||||
}
|
||||
|
||||
if (delay > 0 && page < end) {
|
||||
await sleep(delay);
|
||||
}
|
||||
}
|
||||
|
||||
fetchBar.stop();
|
||||
svgBar.stop();
|
||||
|
||||
const allIcons = Array.from(itemsById.values())
|
||||
.map(toEssentialIcon)
|
||||
.filter(Boolean);
|
||||
|
||||
if (progress.enabled) {
|
||||
buildBar.start(2, 0, { action: 'starting' });
|
||||
}
|
||||
|
||||
fs.writeFileSync(outputPath, JSON.stringify(allIcons), 'utf8');
|
||||
|
||||
if (progress.enabled) {
|
||||
buildBar.update(1, { action: 'wrote data/all_icons.json' });
|
||||
} else {
|
||||
console.log(`Wrote ${outputPath} (${allIcons.length} icons)`);
|
||||
}
|
||||
|
||||
// Avoid mixed output while rendering progress bars.
|
||||
const originalConsoleLog = console.log;
|
||||
if (progress.enabled) {
|
||||
console.log = () => {};
|
||||
}
|
||||
|
||||
try {
|
||||
buildIconsJs({
|
||||
inputPath: outputPath,
|
||||
outputPath: path.join(outputDir, 'all_icons.js')
|
||||
});
|
||||
} finally {
|
||||
if (progress.enabled) {
|
||||
console.log = originalConsoleLog;
|
||||
}
|
||||
}
|
||||
|
||||
if (progress.enabled) {
|
||||
buildBar.update(2, { action: 'built data/all_icons.js' });
|
||||
buildBar.stop();
|
||||
}
|
||||
|
||||
console.log(
|
||||
`SVG sync complete: total=${syncSummary.total}, downloaded=${syncSummary.downloaded}, ` +
|
||||
`updated=${syncSummary.updated}, new_or_missing=${syncSummary.newOrMissing}, ` +
|
||||
`skipped=${syncSummary.skipped}, checked_headers=${syncSummary.checkedHeaders}, failed=${syncSummary.failed}`
|
||||
);
|
||||
|
||||
if (syncSummary.failures.length > 0) {
|
||||
console.error('Sample SVG download failures:');
|
||||
syncSummary.failures.forEach(failure => {
|
||||
console.error(`- ${failure}`);
|
||||
});
|
||||
}
|
||||
|
||||
if (syncSummary.failed > 0) {
|
||||
throw new Error(`Failed to download ${syncSummary.failed} SVG assets`);
|
||||
}
|
||||
} finally {
|
||||
progress.stopAll();
|
||||
}
|
||||
}
|
||||
|
||||
run().catch(error => {
|
||||
console.error(error.message || error);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user