import {createHash} from 'node:crypto'; import {createWriteStream} from 'node:fs'; import {cp, mkdir, readFile, readdir, rm, stat, rename, unlink, writeFile} from 'node:fs/promises'; import {join} from 'node:path'; import {Readable, Transform} from 'node:stream'; import {pipeline} from 'node:stream/promises'; import {assetPrefix, requiredOnnxFiles, voiceFiles, voiceRepository, voiceRevision} from './assets.mjs'; const root = process.cwd(); const output = join(root, 'public', 'static', assetPrefix); const onnxSource = join(root, 'node_modules', 'onnxruntime-web', 'dist'); const piperSource = join(root, 'node_modules', '@diffusionstudio', 'piper-wasm', 'build'); const localAiPrefix = 'wllama-gemma3-v2'; const wllamaUrl = 'https://registry.npmjs.org/@wllama/wllama/-/wllama-3.8.1.tgz'; const wllamaSha512 = 'Pl7BEJhPU8OHnV/yhn3DFZYhLLOdSHh+4TND79WCftHnlUAdQPAsyc/CfIOY+4mVxvf5VW3Xw3Uf7BBfotS1gw=='; const gemmaUrl = 'https://huggingface.co/ggml-org/gemma-3-1b-it-GGUF/resolve/f9c28bcd85737ffc5aef028638d3341d49869c27/gemma-3-1b-it-Q4_K_M.gguf'; const gemmaSize = 806058240; const gemmaSha256 = '8ccc5cd1f1b3602548715ae25a66ed73fd5dc68a210412eea643eb20eb75a135'; async function downloadVoiceFile(file) { const destination = join(output, 'voices', file.path); const temporary = `${destination}.part`; await mkdir(join(destination, '..'), {recursive: true}); const response = await fetch(`${voiceRepository}/${voiceRevision}/${file.path}`); if (!response.ok || !response.body) throw new Error(`Unable to download ${file.path}: HTTP ${response.status}`); const sha256 = createHash('sha256'); const sha1 = createHash('sha1').update(Buffer.from(`blob ${file.size}\0`)); let size = 0; const verify = new Transform({ transform(chunk, _encoding, callback) { size += chunk.length; sha256.update(chunk); sha1.update(chunk); callback(null, chunk); }, }); try { await pipeline(Readable.fromWeb(response.body), verify, createWriteStream(temporary)); if (size !== file.size || (file.sha256 && sha256.digest('hex') !== file.sha256) || (file.gitBlobSha1 && sha1.digest('hex') !== file.gitBlobSha1)) { throw new Error(`Integrity check failed for ${file.path} (${size} bytes)`); } await rename(temporary, destination); } catch (error) { await unlink(temporary).catch(() => {}); throw error; } } await rm(join(root, 'public', 'static'), {recursive: true, force: true}); const localAiRoot = join(root, 'public', 'static', localAiPrefix); const wllamaRoot = join(localAiRoot, 'wllama'); const modelRoot = join(localAiRoot, 'models'); const licenseRoot = join(localAiRoot, 'licenses'); await mkdir(wllamaRoot, {recursive: true}); await mkdir(modelRoot, {recursive: true}); await mkdir(licenseRoot, {recursive: true}); // Build-time downloads are pinned and integrity checked. Browser clients use only our /static URLs. const wllamaResponse = await fetch(wllamaUrl); if (!wllamaResponse.ok || !wllamaResponse.body) throw new Error(`Unable to download Wllama: HTTP ${wllamaResponse.status}`); const tgzPath = join(root, '.wllama-3.8.1.tgz'); const wllamaHash = createHash('sha512'); await pipeline(Readable.fromWeb(wllamaResponse.body), new Transform({transform(chunk, _encoding, callback) { wllamaHash.update(chunk); callback(null, chunk); }}), createWriteStream(tgzPath)); if (wllamaHash.digest('base64') !== wllamaSha512) throw new Error('Wllama package integrity check failed'); const {execFile} = await import('node:child_process'); const {promisify} = await import('node:util'); await promisify(execFile)('tar', ['-xzf', tgzPath, '-C', wllamaRoot, '--strip-components=1']); await unlink(tgzPath); for (const relative of ['esm/index.js', 'esm/wasm/wllama.wasm']) { if (!(await stat(join(wllamaRoot, relative))).isFile()) throw new Error(`Missing Wllama runtime asset: ${relative}`); } for (const file of ['GEMMA_USER_TERMS.html', 'GEMMA_TERMS.html', 'GEMMA_PROHIBITED_USE_POLICY.html', 'GEMMA_NOTICE.txt', 'GEMMA_SOURCE.txt', 'WLLAMA_LICENSE.txt', 'LLAMA_CPP_LICENSE.txt']) { const source = join(root, 'licenses', file); if (!(await stat(source).catch(() => null))?.isFile()) throw new Error(`Missing Gemma legal asset: ${file}`); await cp(source, join(licenseRoot, file)); } await downloadVerified(gemmaUrl, join(modelRoot, 'gemma-3-1b-it-Q4_K_M.gguf'), gemmaSize, gemmaSha256); await cp(join(root, 'prompts', 'browser-assistance.yaml'), join(localAiRoot, 'browser-assistance.yaml')); await cp(join(root, 'client', 'local-ai.js'), join(localAiRoot, 'local-ai.js')); const termsSha256 = createHash('sha256').update(await readFile(join(licenseRoot, 'GEMMA_TERMS.html'))).digest('hex'); const policySha256 = createHash('sha256').update(await readFile(join(licenseRoot, 'GEMMA_PROHIBITED_USE_POLICY.html'))).digest('hex'); const userTermsSha256 = createHash('sha256').update(await readFile(join(licenseRoot, 'GEMMA_USER_TERMS.html'))).digest('hex'); const acceptanceId = createHash('sha256').update(`${termsSha256}:${policySha256}:${userTermsSha256}`).digest('hex'); await writeFile(join(localAiRoot, 'manifest.json'), JSON.stringify({ asset_version: '1', model: {id: 'google/gemma-3-1b-it', quantization: 'Q4_K_M', bytes: gemmaSize, sha256: gemmaSha256, url: `/static/${localAiPrefix}/models/gemma-3-1b-it-Q4_K_M.gguf`}, runtime: {name: 'wllama', version: '3.8.1', module: `/static/${localAiPrefix}/wllama/esm/index.js`, wasm: `/static/${localAiPrefix}/wllama/esm/wasm/wllama.wasm`}, prompt: {id: 'browser-assistance', version: 1, url: `/static/${localAiPrefix}/browser-assistance.yaml`}, license: { acceptance_id: acceptanceId, user_terms_sha256: userTermsSha256, terms_sha256: termsSha256, policy_sha256: policySha256, terms: `/static/${localAiPrefix}/licenses/GEMMA_TERMS.html`, user_terms: `/static/${localAiPrefix}/licenses/GEMMA_USER_TERMS.html`, prohibited_use_policy: `/static/${localAiPrefix}/licenses/GEMMA_PROHIBITED_USE_POLICY.html`, notice: `/static/${localAiPrefix}/licenses/GEMMA_NOTICE.txt`, source: `/static/${localAiPrefix}/licenses/GEMMA_SOURCE.txt`, runtime: `/static/${localAiPrefix}/licenses/WLLAMA_LICENSE.txt`, llama_cpp_runtime: `/static/${localAiPrefix}/licenses/LLAMA_CPP_LICENSE.txt`, }, }, null, 2)); await mkdir(join(output, 'onnx'), {recursive: true}); await mkdir(join(output, 'piper'), {recursive: true}); for (const file of voiceFiles) await downloadVoiceFile(file); await cp(join(root, 'PIPER_VOICE_NOTICES.txt'), join(output, 'voices', 'ATTRIBUTION.txt')); const onnxFiles = (await readdir(onnxSource)).filter(name => name.endsWith('.wasm') || name.endsWith('.mjs')); for (const file of requiredOnnxFiles) { if (!onnxFiles.includes(file)) throw new Error(`Missing ONNX Runtime asset: ${file}`); } for (const file of onnxFiles) await cp(join(onnxSource, file), join(output, 'onnx', file)); for (const file of ['piper_phonemize.wasm', 'piper_phonemize.data']) { const source = join(piperSource, file); const fileStat = await stat(source).catch(() => null); if (!fileStat?.isFile() || fileStat.size === 0) throw new Error(`Missing Piper runtime file: ${file}`); await cp(source, join(output, 'piper', file)); } for (const file of voiceFiles) { const downloaded = join(output, 'voices', file.path); if ((await stat(downloaded)).size !== file.size) throw new Error(`Missing verified voice file: ${file.path}`); } console.log(`Static voice and local AI assets prepared at ${output}`); async function downloadVerified(url, destination, expectedSize, expectedSha256) { await mkdir(join(destination, '..'), {recursive: true}); const response = await fetch(url); if (!response.ok || !response.body) throw new Error(`Unable to download ${destination}: HTTP ${response.status}`); const hash = createHash('sha256'); let size = 0; const verify = new Transform({transform(chunk, _encoding, callback) { size += chunk.length; hash.update(chunk); callback(null, chunk); }}); await pipeline(Readable.fromWeb(response.body), verify, createWriteStream(destination)); if (size !== expectedSize || hash.digest('hex') !== expectedSha256) throw new Error(`Integrity check failed for ${destination} (${size} bytes)`); }