// s200-upload.mjs: the Status 200 Uploads uploader, version 1, for Node 18 or newer (no packages). // Sends one file from this computer to a Status 200 Uploads upload link (resumable, TUS 1.0.0). // Docs and the SHA-256 of every published version: https://status200uploads.com/docs/api // A published version never changes; a new one gets a new folder (/uploader/v2/...). // // It reads its inputs from the environment only. The command that status200_create_upload_link (or // POST /api/v2/media/uploads) hands out sets them all; only the S200_FILE line is filled in by hand. // S200_UPLOAD_URL the upload address; nothing but Status 200 Uploads' storage host is accepted // S200_UPLOAD_TOKEN the signed upload token (about 2 hours, for this one upload) // S200_CHECK the first 8 hex of SHA-256(S200_UPLOAD_URL + "\n" + S200_UPLOAD_TOKEN) // S200_MAX_SECONDS how long this run may take (default 100); it always ends within that + 10 s // S200_FILE the file to send // Every run asks storage how many bytes it already has and continues from there. // Prints "progress N% (X.X of Y.Y MiB)" every 15 s and at the end, then exactly one JSON line. // Exit: 0 done | 1 failed (network: run again once) | 2 refused (a rerun cannot help) | 3 partial (time used: // run again) | 4 token_rejected (ask for a fresh token for the same file_id) | 5 busy (another uploader on // this computer is sending this file) | 6 bad_command (the command was not copied exactly) import { createHash } from 'node:crypto' import { mkdir, open, readFile, rm, stat } from 'node:fs/promises' import { homedir } from 'node:os' import { join } from 'node:path' const ADDRESS = /^https:\/\/yglvofckrdutesfzxwyb\.storage\.supabase\.co\/storage\/v1\/upload\/resumable\/sign\/[A-Za-z0-9_-]+$/ const TOKEN = /^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$/ const CHUNK = 6 * 1024 * 1024 // storage takes 6 MiB pieces; the last one may be shorter const HEAD_MS = 30_000 const PATCH_MS = 180_000 const REPORT_MS = 15_000 const LOCK_TOUCH_MS = 15_000 const LOCK_STALE_MS = 60_000 // a lock file nobody touched for this long is left over, whatever pid it names const env = process.env const url = env.S200_UPLOAD_URL ?? '' const token = env.S200_UPLOAD_TOKEN ?? '' const started = Date.now() let deadline = started + 100_000 let lastOffset = null // what storage last said it has let fileSize = null // the file's size, once the upload starts let shown = null // the offset the last progress line showed const live = new Set() // requests in flight, aborted before the process ends const sha256 = (text) => createHash('sha256').update(text, 'utf8').digest('hex') const sleep = (ms) => new Promise((r) => setTimeout(r, ms)) const remaining = () => deadline - Date.now() const mib = (n) => (n / 1048576).toFixed(1) const progress = (offset, size) => { shown = offset console.log(`progress ${size ? Math.floor((offset * 100) / size) : 100}% (${mib(offset)} of ${mib(size)} MiB)`) } /** Ends the run with an exit code and its JSON line. No process.exit here: mid-fetch it can crash Node on Windows. */ class Stop extends Error { constructor(code, info) { super(info.state) this.code = code this.info = info } } const stop = (code, info) => { throw new Stop(code, info) } const withOffset = (state, size, message) => lastOffset === null ? { state, size, message } : { state, bytes_sent: lastOffset, size, message } /** * The wait after the n-th failure in a row: 1, 2, 4, 8, then 15 s, as in the PowerShell and sh uploaders. Capped at * 15 s so that 8 refused HEADs take about a minute: a blocked network ends with exit 1 inside the default 100 s * (exit 1 twice = stop), not with an exit 3 that says "run again" for ever. A 30 s Wi-Fi drop is still bridged. */ const backoff = (n) => Math.min(15_000, 1000 * 2 ** (n - 1)) /** Server text that may go into the JSON line: never the token, the address or its id; printable ASCII, short. */ function clean(text) { let t = String(text) const id = url.slice(url.lastIndexOf('/') + 1) if (token) t = t.split(token).join('[token]') if (url) t = t.split(url).join('[upload address]') if (id.length >= 16) t = t.split(id).join('[upload id]') return t.replace(/[^\x20-\x7e]/g, ' ').replace(/ +/g, ' ').trim().slice(0, 300) } /** * One request to the upload address, with a timeout that never runs past the time budget. Returns * {status, headers, text} or null for a network failure or timeout. Redirects are not followed: storage never * redirects, and a followed redirect would carry x-signature to another host. */ async function request(method, headers, body, normalMs) { const ac = new AbortController() const timer = setTimeout(() => ac.abort(), Math.max(1000, Math.min(normalMs, remaining() - 5000))) live.add(ac) try { const res = await fetch(url, { method, headers, body, signal: ac.signal, redirect: 'manual' }) let text = '' if (res.body) { const reader = res.body.getReader() const parts = [] for (let n = 0; n < 65536; ) { const { done, value } = await reader.read() if (done) break parts.push(value) n += value.length } reader.cancel().catch(() => {}) text = Buffer.concat(parts).toString('utf8') } return { status: res.status, headers: res.headers, text } } catch { return null } finally { clearTimeout(timer) live.delete(ac) } } /** Upload-Offset as a number. A missing or odd value must never read as "finished". */ function offsetFrom(r, size) { const raw = r.headers.get('upload-offset') const n = raw !== null && /^[0-9]{1,16}$/.test(raw) ? Number(raw) : NaN if (!Number.isSafeInteger(n) || n > size) stop(1, withOffset('failed', size, 'the server answered without a valid Upload-Offset')) return n } async function upload(fh, size) { fileSize = size const base = { 'Tus-Resumable': '1.0.0', 'x-signature': token } const outOfTime = () => stop(3, withOffset('partial', size, 'this run used its time; run the same command again to continue')) /** * The wait before the next try. It never reaches into the last 10 s of the run (DESIGN 9.2): a wait that does not * fit ends the run now, as it would end after a shortened one. Shortened waits are not used because a timer may * fire a little early, and every early wake-up just above 10 s left would send one more try at once. */ const pause = async (ms) => { if (remaining() - ms < 10_000) outOfTime() await sleep(ms) } const tokenRejected = () => stop(4, withOffset('token_rejected', size, 'storage refused the token (expired or not valid); ask for a fresh token for the same file_id')) /** How many bytes storage really has. Stops on answers that a retry cannot change. */ async function serverOffset() { for (let attempt = 1; ; attempt++) { if (remaining() < 10_000) outOfTime() const r = await request('HEAD', base, undefined, HEAD_MS) if (r && (r.status === 200 || r.status === 204)) { const length = r.headers.get('upload-length') if (length !== null) { const n = /^[0-9]{1,16}$/.test(length) ? Number(length) : NaN if (!Number.isSafeInteger(n)) stop(1, withOffset('failed', size, 'the server answered without a valid Upload-Length')) if (n !== size) stop(2, { state: 'refused', size, message: `this link expects ${n} bytes, the file has ${size}` }) } lastOffset = offsetFrom(r, size) return lastOffset } if (r?.status === 400) tokenRejected() // a HEAD answer has no body to read if (r && [401, 403, 404, 410].includes(r.status)) { stop(2, { state: 'refused', status: r.status, size, message: `storage refused this upload (HTTP ${r.status}): it may be finished, cancelled or older than 24 hours` }) } if (attempt >= 8) stop(1, withOffset('failed', size, 'the upload server cannot be reached')) await pause(backoff(attempt)) } } const shorter = () => stop(2, withOffset('refused', size, 'the file got shorter while it was being uploaded')) let offset = await serverOffset() let failures = 0 let lastReport = 0 const report = () => { lastReport = Date.now() progress(offset, size) } while (offset < size) { if (remaining() < 10_000) outOfTime() // The link was made for this exact size; a file that changes size now cannot match it any more. const now = (await fh.stat()).size if (now < size) shorter() if (now > size) stop(2, withOffset('refused', size, `the file changed while it was being uploaded (it has ${now} bytes now, the link expects ${size})`)) const want = Math.min(CHUNK, size - offset) const chunk = Buffer.allocUnsafe(want) // a fresh buffer: a failed request may still hold the previous one for (let got = 0; got < want; ) { const { bytesRead } = await fh.read(chunk, got, want - got, offset + got) if (!bytesRead) shorter() got += bytesRead } const r = await request('PATCH', { ...base, 'Upload-Offset': String(offset), 'Content-Type': 'application/offset+octet-stream' }, chunk, PATCH_MS) const status = r?.status ?? 0 let moved = false let why = status ? `HTTP ${status}` : 'no answer' if (status === 204) { const next = offsetFrom(r, size) lastOffset = next if (next > offset) { offset = next failures = 0 moved = true } else { // Storage took nothing: if the file shrank, say so; otherwise it counts as a failure. if ((await fh.stat()).size < size) shorter() why = 'storage took none of the bytes' } } else { const text = r?.text ?? '' // Storage answers 400 to ExpiredSignature, InvalidSignature and InvalidJWT ('"exp" claim timestamp check // failed'). "exp" counts only as a word of its own: a 400 that says "expected" is not about the token. if (status === 400 && /\bexp\b|signature|jwt|jws|token/i.test(text)) tokenRejected() if ([400, 401, 403, 404, 410, 413, 415].includes(status) || (status === 409 && /exist|duplicate/i.test(text))) { // The same words in every uploader, so the AI app reads "storage refused" whichever one ran. const said = clean(text) stop(2, { state: 'refused', status, size, message: `storage refused this upload (HTTP ${status})${said ? `: ${said}` : ''}` }) } } if (!moved) { if (++failures > 10) stop(1, withOffset('failed', size, `gave up after 10 failed tries in a row (last: ${why})`)) // 409 = our offset may be wrong: ask at once. Anything else: back off first. if (status !== 409) await pause(backoff(failures)) // A failed PATCH may still have stored part of the chunk, or all of it with the answer lost, so ask storage. const before = offset offset = await serverOffset() if (offset > before) { // Storage has more than before: the upload moves and only answers were lost. The count starts again, or a // line that cuts every PATCH but keeps its bytes would end in "failed" while it makes progress. failures = 0 } else if (status === 409) { // The same offset after a 409: not a wrong offset but another request still holding the upload (storage // serves one at a time), such as a PATCH cut on our side that storage has not dropped yet. Wait for it. await pause(backoff(failures)) } } if (Date.now() - lastReport >= REPORT_MS || offset === size) report() } if (shown !== size) report() return { state: 'done', size } } // --------------------------------------------------------------------------------------------------------------- // One uploader per upload on this computer (DESIGN 9.2 step 4). On Windows the lock is the same file the // PowerShell uploader holds, so the two runners exclude each other too. // --------------------------------------------------------------------------------------------------------------- function lockPath() { const dir = process.platform === 'win32' ? join(env.LOCALAPPDATA || join(homedir(), 'AppData', 'Local'), 's200-upload') : join(env.HOME || homedir(), '.s200-upload') return { dir, file: join(dir, `lock-${sha256(url).slice(0, 16)}`) } } function alive(pid) { try { process.kill(pid, 0) return true } catch (e) { return e?.code === 'EPERM' // it exists, it is just not ours to signal } } /** True when the lock at `file` was left behind by an uploader that is gone. */ async function leftOver(file) { let st try { st = await stat(file) } catch (e) { return e?.code === 'ENOENT' } const old = Date.now() - st.mtimeMs > LOCK_STALE_MS // A lock naming this very process was left by one that is gone: its pid has been reused, by us. const gone = (pid) => Number(pid) === process.pid || !alive(Number(pid)) if (st.isDirectory()) { // The sh uploader's lock: a folder holding its pid and its deadline ("until", seconds since 1970). It is not // touched while it runs, so it is left over when that pid is gone or the deadline is 30 s past (sh's own rule: // pids get reused). A folder without a pid yet may be one sh is still filling in. const read = (name) => readFile(join(file, name), 'utf8').catch(() => '') const pid = /^\s*([0-9]{1,10})\s*$/.exec(await read('pid'))?.[1] const until = /^\s*([0-9]{1,12})\s*$/.exec(await read('until'))?.[1] if (until && Date.now() / 1000 > Number(until) + 30) return true return pid ? gone(pid) : old } let text try { text = await readFile(file, 'utf8') } catch (e) { return e?.code === 'ENOENT' // unreadable: PowerShell holds it with FileShare.None, so it is in use } const pid = /^\s*([0-9]{1,10})\s*$/.exec(text)?.[1] // A running Node uploader touches its lock every 15 s, so an old file names a dead or reused pid. return pid ? gone(pid) || old : old } /** {release} when this run holds the lock, null when another uploader does. */ async function takeLock() { const { dir, file } = lockPath() const without = (why) => { // A computer where the lock cannot be written (a read-only home in a sandbox) still uploads; storage itself // refuses a second client on the same upload. console.error(`note: running without the lock (${why})`) return { release: async () => {} } } try { await mkdir(dir, { recursive: true, mode: 0o700 }) } catch (e) { return without(e?.code ?? 'mkdir failed') } for (let attempt = 0; attempt < 5; attempt++) { let fh try { fh = await open(file, 'wx', 0o600) } catch (e) { if (e?.code !== 'EEXIST') return without(e?.code ?? 'open failed') if (!(await leftOver(file))) return null await rm(file, { recursive: true, force: true }).catch(() => {}) continue } await fh.write(`${process.pid}\n`) const touch = setInterval(() => { const t = new Date() fh.utimes(t, t).catch(() => {}) }, LOCK_TOUCH_MS) touch.unref() return { async release() { clearInterval(touch) await fh.close().catch(() => {}) const text = await readFile(file, 'utf8').catch(() => '') if (text.trim() === String(process.pid)) await rm(file, { force: true }).catch(() => {}) }, } } return null } async function main() { if (typeof fetch !== 'function') stop(2, { state: 'refused', message: 'this uploader needs Node 18 or newer; ask for the native runner instead' }) // Before any request: the copy check, the time budget, our address only, the file, the lock. The copy check comes // first: a slip anywhere in the command (a letter of the address or the token, a lost line) ends with exit 6, "copy // it again", and never reads as a refusal. It binds the address to the token, so an address that passes it and is // still not ours was changed on purpose, together with its check: that one is refused (exit 2). if (String(env.S200_CHECK ?? '').toLowerCase() !== sha256(`${url}\n${token}`).slice(0, 8) || !TOKEN.test(token) || token.length > 4096) { stop(6, { state: 'bad_command', message: url ? 'the command was not copied exactly; copy it again and change only ' : 'the command was not copied exactly: the upload address did not reach the uploader (in sh, every S200_ line must end with a space and a backslash); copy it again and change only ', }) } const max = env.S200_MAX_SECONDS ?? '' if (max !== '' && !/^[1-9][0-9]{0,5}$/.test(max)) { stop(6, { state: 'bad_command', message: 'the command was not copied exactly: S200_MAX_SECONDS must be a whole number of seconds' }) } deadline = started + (max === '' ? 100 : Number(max)) * 1000 if (!ADDRESS.test(url)) stop(2, { state: 'refused', message: 'this is not a Status 200 Uploads upload address' }) let fh try { fh = await open(env.S200_FILE ?? '', 'r') } catch { stop(2, { state: 'refused', message: 'file not found (S200_FILE must be the full path of a file this user can read)' }) } try { const st = await fh.stat() if (!st.isFile()) stop(2, { state: 'refused', message: 'file not found (S200_FILE is not a file)' }) if (st.size === 0) stop(2, { state: 'refused', message: 'the file is empty' }) const lock = await takeLock() if (!lock) stop(5, { state: 'busy', message: 'another uploader on this computer is already sending this file' }) try { return await upload(fh, st.size) } finally { await lock.release() } } finally { await fh.close().catch(() => {}) } } let code = 0 let info try { info = await main() } catch (e) { code = e instanceof Stop ? e.code : 1 info = e instanceof Stop ? e.info : { state: 'failed', message: `unexpected error (${e?.code ?? e?.name ?? 'unknown'})` } } for (const ac of live) ac.abort() // A run that stops mid-upload: when storage's count moved since the last progress line, one more comes first, so // the last line before the JSON says where the upload is (as in the other uploaders). if (fileSize !== null && lastOffset !== null && lastOffset !== shown) progress(lastOffset, fileSize) console.log(JSON.stringify(info)) process.exitCode = code // The run must end on time even if something still holds the event loop open. setTimeout(() => process.exit(code), 3000).unref()