#!/bin/sh # s200-upload.sh, version v1: sends one file from this computer to a Status 200 Uploads upload link. # Status 200 Uploads: https://status200uploads.com/docs/api # A published version never changes. The command that runs it checks its SHA-256 first. # # POSIX sh and curl: macOS, Linux and Git Bash on Windows. Besides curl it uses only tools those systems already # have: tail, head, wc, awk, sed, tr, cut, grep, mkdir, mktemp, date, and sha256sum, shasum or openssl. # curl runs with -q, so ~/.curlrc is not read: a proxy is taken from HTTPS_PROXY or ALL_PROXY. # # The command from status200_create_upload_link (or POST /api/v2/media/uploads) sets these, and only these: # S200_UPLOAD_URL the upload address (it must be on Status 200 Uploads' storage host) # S200_UPLOAD_TOKEN the signed token for that one address # S200_CHECK the first 8 hex of SHA-256(address + "\n" + token): catches a command not copied exactly # S200_MAX_SECONDS how long this run may take (default 100); it always ends within that plus 10 s # S200_FILE the full path of the file # The upload is resumable (TUS 1.0.0): every run asks storage how many bytes it has and continues from there, # so running the same command again after any stop continues the upload. # # Output: "progress N% (X.X of Y.Y MiB)" every 15 s and at the end, then exactly one JSON line. Exit codes: # 0 done the whole file is stored # 1 failed gave up after retries (network) # 2 refused a rerun cannot help (wrong size, file missing or changed, foreign address, refused by storage) # 3 partial the time for this run is used up: run the same command again to continue # 4 token_rejected the token expired or was refused: get a fresh token for the same file_id, then run again # 5 busy another uploader on this computer is already sending this file # 6 bad_command the command was not copied exactly # Byte-wise text tools and a "." decimal point, whatever the computer's language is. No file-name patterns, the # default word splitting, and no errexit or nounset carried in from the environment (bash imports SHELLOPTS): # every failure below is handled where it happens. LC_ALL=C export LC_ALL unset IFS set -f +e +u # Supabase takes resumable uploads in pieces of exactly 6 MiB (the last one may be shorter). CHUNK=6291456 # The only place this script sends anything to. UPLOAD_PREFIX=https://yglvofckrdutesfzxwyb.storage.supabase.co/storage/v1/upload/resumable/sign/ UA='s200-upload/v1 (sh)' URL=${S200_UPLOAD_URL-} TOKEN=${S200_UPLOAD_TOKEN-} F=${S200_FILE-} OFF= SIZE= DEADLINE= TMPD= HDRS= BODY= OUT= AUTH= LOCK= HAVE_LOCK= JOB= CODE=000 RC=0 SENT= LAST_ERR= FAILS=0 LAST=0 SHOWN= # T = now, in seconds. date +%s is not POSIX but every system named above has it; awk's srand() is the fallback. clock() { T=$(date +%s 2>/dev/null) case $T in '' | *[!0-9]*) T=$(awk 'BEGIN { srand(); print srand() }') ;; esac } # Any text (server answers included) as the inside of one JSON string: printable ASCII, at most 300 characters. jstr() { printf '%s' "$1" | tr -d '\000-\037\177-\377' | cut -c1-300 | sed 's/\\/\\\\/g; s/"/\\"/g' } # Ends the run: exactly one JSON line, then the exit code. $1 code, $2 state, $3 message (optional). When storage's # count moved since the last progress line, one more comes first, so the last line before the JSON says where the # upload is (as in the other uploaders). stop() { trap '' HUP INT TERM if [ -n "$OFF" ] && [ "$OFF" != "$SHOWN" ]; then progress; fi counts= if [ -n "$OFF" ]; then counts=",\"bytes_sent\":$OFF"; fi if [ -n "$SIZE" ]; then counts="$counts,\"size\":$SIZE"; fi if [ -n "${3-}" ]; then printf '{"state":"%s","message":"%s"%s}\n' "$2" "$(jstr "$3")" "$counts" else printf '{"state":"%s"%s}\n' "$2" "$counts" fi exit "$1" } busy() { trap '' HUP INT TERM printf '%s\n' '{"state":"busy","message":"another uploader on this computer is already sending this file"}' exit 5 } # A stop from outside (Ctrl+C, or the app ending the command) still says where the upload is, at once: requests # and pauses run as background jobs (see finish_job), and the one still running is ended here. on_signal() { trap '' HUP INT TERM if [ -n "$JOB" ]; then kill "$JOB" 2>/dev/null; fi stop 3 partial 'stopped before the end: run the same command again to continue' } cleanup() { if [ -n "$HAVE_LOCK" ] && [ "$(cat "$LOCK/pid" 2>/dev/null)" = "$$" ]; then rm -rf "$LOCK"; fi if [ -n "$TMPD" ]; then rm -rf "$TMPD"; fi } trap cleanup EXIT trap on_signal HUP INT TERM # A byte count from storage or from wc: digits only, no leading zero, at most 15 digits. is_count() { case $1 in 0) return 0 ;; '' | 0* | *[!0-9]*) return 1 ;; esac [ "${#1}" -le 15 ] } # FS = the file's size right now, or nothing when it cannot be read. It runs before every piece, so it starts one # process only (wc; its blanks are split off by set --, with globbing off). file_size() { set -- $(wc -c 2>/dev/null < "$F") FS= if [ "$#" -eq 1 ] && is_count "$1"; then FS=$1; fi } # SHA-256 of standard input as lower-case hex, with whichever tool this system has. Only short strings are hashed # here (S200_CHECK and the lock name); the file itself is never hashed or read into memory by this script. sha256_hex() { if command -v sha256sum >/dev/null 2>&1; then sha256sum elif command -v shasum >/dev/null 2>&1; then shasum -a 256 else openssl dgst -sha256 2>/dev/null | sed 's/^.*= *//' fi | cut -c1-64 | tr 'ABCDEF' 'abcdef' } # Waits for the job just started in the background (a request or a pause); RC = its exit code. A shell runs a trap # only when its foreground command has ended, so a request in the foreground would hold back a Ctrl+C or a stop for # up to 180 s; a wait is cut short at once instead, and on_signal ends the job. finish_job() { JOB=$! wait "$JOB" RC=$? JOB= } # Before every request: under 10 s left ends the run (exit 3, run again); otherwise TMO is the request's timeout, # its normal value ($1) cut to the time left minus 5 s. This is what makes the run end on time, whatever the network. budget() { clock REM=$((DEADLINE - T)) if [ "$REM" -lt 10 ]; then out_of_time; fi TMO=$((REM - 5)) if [ "$TMO" -gt "$1" ]; then TMO=$1; fi } # Exit 3. When the last try failed, the line says how, so a run that never reached storage (a firewall, a proxy) # does not read like one that was only slow. LAST_ERR is cleared whenever storage answers as expected. out_of_time() { if [ -n "$LAST_ERR" ]; then stop 3 partial "the time for this run is used up (last try: $LAST_ERR): run the same command again to continue" fi stop 3 partial 'the time for this run is used up: run the same command again to continue' } # Waits $1 seconds between tries, but never into the last 10 s of the run. When no wait fits any more, the run # ends (exit 3): the clock counts whole seconds, and trying again at once would turn the backoff into a burst of # requests in the run's last second. pause() { clock s=$((DEADLINE - T - 10)) if [ "$s" -le 0 ]; then out_of_time; fi if [ "$s" -gt "$1" ]; then s=$1; fi sleep "$s" > /dev/null 2>&1 & finish_job } # B = the wait before try number $1: 1, 2, 4, 8, then 15 s. A network that never answers gives up (exit 1) after # about a minute of quick failures, while a 30-second Wi-Fi drop is still bridged. backoff() { B=1 i=1 while [ "$i" -lt "$1" ] && [ "$B" -lt 15 ]; do B=$((B * 2)) i=$((i + 1)) done if [ "$B" -gt 15 ]; then B=15; fi } # LAST_ERR = the last failure in plain words, for the JSON line. why() { if [ "$CODE" != 000 ]; then LAST_ERR="HTTP $CODE" return fi case $RC in 5) LAST_ERR='the proxy could not be found' ;; 6) LAST_ERR='the host name could not be resolved (no internet, or DNS is blocked)' ;; 7) LAST_ERR='no connection (a firewall, proxy or sandbox may block the storage host)' ;; 28) LAST_ERR='no answer in time' ;; 35) LAST_ERR='the secure connection failed' ;; 52 | 55 | 56) LAST_ERR='the connection was cut' ;; 60) LAST_ERR='the certificate was not trusted (a proxy or antivirus may be inspecting HTTPS)' ;; *) LAST_ERR="curl error $RC" ;; esac } # UO and UL = Upload-Offset and Upload-Length of the last answer ("-" when absent). Header names in any case, # CR removed; inner blanks become "_" so a value like "12 34" can never pass as a number. read_headers() { hv=$(awk ' function val(s) { s = substr(s, index(s, ":") + 1); gsub(/^[ \t]+|[ \t]+$/, "", s); gsub(/[ \t]/, "_", s); return s == "" ? "-" : s } { sub(/\r$/, ""); n = tolower($0) } n ~ /^upload-offset:/ && o == "" { o = val($0) } n ~ /^upload-length:/ && l == "" { l = val($0) } END { printf "%s %s\n", (o == "" ? "-" : o), (l == "" ? "-" : l) } ' "$HDRS" 2>/dev/null) UO=${hv%% *} UL=${hv#* } } # The server's answer body, made safe for the JSON line, which lands in the AI app's transcript: printable ASCII, # at most 300 characters, and never the token, the upload address or its id (an error page may echo the request). # They are taken out before the text is cut, so no cut-off part of one can remain. awk's index() matches them as # plain text; none of them holds a backslash (steps 1 and 2), so -v passes them unchanged. body_text() { head -c 65536 "$BODY" 2>/dev/null | tr -d '\000-\037\177-\377' | awk -v t="$TOKEN" -v u="$URL" -v i="$ID" ' function hide(s, x, r, p, o) { o = "" while (x != "" && (p = index(s, x)) > 0) { o = o substr(s, 1, p - 1) r; s = substr(s, p + length(x)) } return o s } { s = s $0 } END { s = hide(s, t, "[token]") s = hide(s, u, "[upload address]") if (length(i) >= 16) s = hide(s, i, "[upload id]") print substr(s, 1, 300) }' } # Both requests start curl with -q, which must come first: it keeps ~/.curlrc out. A "retry" there would multiply # every time limit, a "fail" would drop the answer that says the token expired, and others (insecure, trace, # output) would weaken or log the request. The token comes from $AUTH, never from curl's command line. The -w line # has no newline at its end (curl for Windows writes one as CR LF); read still takes the line at the end of the file. # HEAD: how many bytes storage has. CODE = the status (000 when nothing answered), RC = curl's exit code. ask_offset() { : > "$HDRS" : > "$OUT" curl -q -sS -I -K "$AUTH" -A "$UA" --connect-timeout 20 --max-time "$TMO" -H 'Tus-Resumable: 1.0.0' \ -D "$HDRS" -o "$BODY" -w '%{http_code}' "$URL" > "$OUT" 2>/dev/null & finish_job CODE= read -r CODE < "$OUT" case $CODE in [0-9][0-9][0-9]) ;; *) CODE=000 ;; esac } # PATCH the next piece, read straight from the file at the offset: never more than one piece in memory, whatever # the file's size. SENT = the bytes curl really sent (fewer than the piece means the file got shorter). send_chunk() { : > "$HDRS" : > "$BODY" : > "$OUT" tail -c +"$((OFF + 1))" -- "$F" 2>/dev/null | head -c "$WANT" | curl -q -sS -X PATCH -K "$AUTH" -A "$UA" \ --connect-timeout 20 --max-time "$TMO" --data-binary @- \ -H 'Tus-Resumable: 1.0.0' -H "Upload-Offset: $OFF" -H 'Content-Type: application/offset+octet-stream' \ -H 'Expect:' -D "$HDRS" -o "$BODY" -w '%{http_code} %{size_upload}' "$URL" > "$OUT" 2>/dev/null & finish_job CODE= SENT= read -r CODE SENT < "$OUT" SENT=${SENT%%.*} case $CODE in [0-9][0-9][0-9]) ;; *) CODE=000 SENT= ;; esac } # UO, from the last answer, must be a byte count no larger than the file: a missing, odd or too large # Upload-Offset must never read as "finished". check_offset() { if is_count "$UO" && [ "$UO" -le "$SIZE" ]; then return 0; fi stop 1 failed 'the server answered without a valid Upload-Offset' } # Sets OFF to the byte count storage really has. Stops on answers that no retry can change. server_offset() { try=0 while :; do try=$((try + 1)) budget 30 ask_offset case $CODE in 200 | 204) read_headers if [ "$UL" != - ]; then is_count "$UL" || stop 1 failed 'storage answered without a valid Upload-Length' if [ "$UL" != "$SIZE" ]; then stop 2 refused "this link expects $UL bytes, the file has $SIZE"; fi fi check_offset OFF=$UO LAST_ERR= return 0 ;; 400) stop 4 token_rejected 'storage refused the token (expired, or not the one for this upload)' ;; 401 | 403 | 404 | 410) stop 2 refused "storage refused this upload (HTTP $CODE): it may be finished, cancelled or older than 24 hours" ;; esac why if [ "$try" -ge 8 ]; then stop 1 failed "the upload server cannot be reached: $LAST_ERR"; fi backoff "$try" pause "$B" done } # A PATCH that did not move the upload: one more failure in a row. After 10 the run gives up; otherwise it waits # (not after a 409, which says our offset may be wrong) and asks storage for its offset, since a failed PATCH may # still have stored part of the piece, or all of it with the answer lost. The same rules as the other uploaders. failed_try() { FAILS=$((FAILS + 1)) if [ "$FAILS" -gt 10 ]; then stop 1 failed "gave up after 10 failed tries in a row: $LAST_ERR"; fi pcode=$CODE if [ "$pcode" != 409 ]; then backoff "$FAILS" pause "$B" fi before=$OFF server_offset if [ "$OFF" -gt "$before" ]; then # Storage has more than before: the upload moves and only answers were lost. The count starts again, or a line # that cuts every PATCH but keeps its bytes would end in "failed" while it makes progress. FAILS=0 elif [ "$pcode" = 409 ]; then # The same offset after a 409: not a wrong offset but another request still holding the upload (storage serves # one at a time), such as a PATCH cut on our side that storage has not dropped yet. Wait for it. backoff "$FAILS" pause "$B" fi } # A 2xx answer to a PATCH: storage's Upload-Offset is the truth from here on. took() { read_headers check_offset if [ $((UO - OFF)) -lt "$WANT" ] && is_count "$SENT" && [ "$SENT" -lt "$WANT" ]; then # Storage has less than the piece because curl sent less: the file is shorter now, which no rerun can finish. OFF=$UO stop 2 refused 'the file got shorter while it was being uploaded' fi if [ "$UO" -gt "$OFF" ]; then # Progress, also when storage kept less than the whole piece: the next piece starts from its offset. OFF=$UO FAILS=0 LAST_ERR= return 0 fi OFF=$UO LAST_ERR='storage took none of the bytes' failed_try } # Any other answer to a PATCH: stop when a retry cannot help, otherwise ask storage for its offset and go on. not_taken() { why # Storage answers 400 to ExpiredSignature, InvalidSignature and InvalidJWT ('"exp" claim timestamp check failed'). # "exp" counts only as a word of its own: a 400 that says "expected" is not about the token. if [ "$CODE" = 400 ] && grep -qiE '(^|[^A-Za-z0-9_])exp([^A-Za-z0-9_]|$)|signature|jwt|jws|token' "$BODY" 2>/dev/null; then stop 4 token_rejected "storage refused the token: $(body_text)" fi case $CODE in 400 | 401 | 403 | 404 | 410 | 413 | 415) storage_refused ;; 409) if grep -qiE 'exist|duplicate' "$BODY" 2>/dev/null; then storage_refused; fi ;; esac failed_try } # Exit 2 for an answer no retry can change, with storage's own text when it has one. The same words in every # uploader, so the AI app reads "storage refused" whichever one ran. storage_refused() { said=$(body_text) if [ -n "$said" ]; then stop 2 refused "storage refused this upload (HTTP $CODE): $said"; fi stop 2 refused "storage refused this upload (HTTP $CODE)" } progress() { SHOWN=$OFF awk -v o="$OFF" -v s="$SIZE" 'BEGIN { printf "progress %d%% (%.1f of %.1f MiB)\n", int(o * 100 / s), o / 1048576, s / 1048576 }' } # --------------------------------------------------------------------------------------------------------------- # Before any request # --------------------------------------------------------------------------------------------------------------- clock START=$T # 1. The address and the token must be exactly the ones the command was made with, and this comes before anything # else: a slip anywhere in the command (a letter of the address or the token, a lost line) ends here with exit 6, # "copy it again", and never reads as a refusal: the copy checks (exit 6) come before the address and the file # (exit 2). A slip in a long token would otherwise reach storage as "invalid signature" and look like an expired # token. In sh a lost line is usually a lost backslash: every S200_ line of the command ends with a space and a # backslash, and without it the values up to that line never reach this script. The token has the shape of # storage's signed token (three non-empty parts of A-Z a-z 0-9 _ - joined by dots), the same rule as the other # uploaders, so nothing in it can break the header it goes into. SUM=$(printf '%s\n%s' "$URL" "$TOKEN" | sha256_hex) case $SUM in *[!0-9a-f]* | '') stop 2 refused 'no SHA-256 tool was found (sha256sum, shasum or openssl)' ;; esac if [ "${#SUM}" -ne 64 ]; then stop 2 refused 'no SHA-256 tool was found (sha256sum, shasum or openssl)'; fi CHECK=$(printf '%s' "${S200_CHECK-}" | tr 'ABCDEF' 'abcdef') BAD= if [ "$(printf '%s' "$SUM" | cut -c1-8)" != "$CHECK" ]; then BAD=1; fi case $TOKEN in '' | *[!A-Za-z0-9._-]* | .* | *. | *..* | *.*.*.*) BAD=1 ;; *.*.*) ;; *) BAD=1 ;; esac if [ "${#TOKEN}" -gt 4096 ]; then BAD=1; fi if [ -n "$BAD" ]; then if [ -z "$URL" ]; then stop 6 bad_command 'the command was not copied exactly: the upload address did not reach the uploader (in sh, every S200_ line must end with a space and a backslash); copy it again and change only ' fi stop 6 bad_command 'the command was not copied exactly; copy it again and change only ' fi # 1 to 999999 seconds (at most 6 digits, no leading zero), the same rule as the other uploaders; the command sets # 30 to 7000. MAX=${S200_MAX_SECONDS:-100} case $MAX in 0* | *[!0-9]* | ???????*) stop 6 bad_command 'the command was not copied exactly: S200_MAX_SECONDS must be a whole number of seconds' ;; esac DEADLINE=$((START + MAX)) # 2. Status 200 Uploads' storage only: the token can never be sent anywhere else, and this script can never serve # as a general upload tool. The check above binds the address to the token, so an address that passed it and is # still not ours was changed on purpose, together with its check. ID= case $URL in "$UPLOAD_PREFIX"*) ID=${URL#"$UPLOAD_PREFIX"} ;; esac case $ID in '' | *[!A-Za-z0-9_-]*) stop 2 refused 'this is not a Status 200 Uploads upload address' ;; esac # 3. The file. A path that does not exist will not start working on a rerun. if [ -z "$F" ] || [ ! -f "$F" ] || [ ! -r "$F" ]; then if [ "$F" = '' ]; then stop 2 refused 'file not found: S200_FILE still says ; put the full path of the file there'; fi stop 2 refused 'file not found' fi file_size SIZE=$FS if [ -z "$SIZE" ]; then stop 2 refused 'file not found'; fi if [ "$SIZE" -eq 0 ]; then stop 2 refused 'the file is empty'; fi command -v curl >/dev/null 2>&1 || stop 2 refused 'curl was not found on this computer' # 4. One uploader per upload on this computer, so a rerun never fights a run that is still going (for example one # left behind when a command was stopped). The lock is a directory named after the address, holding the pid of # its holder in "pid" and the holder's deadline in "until"; it is taken over when that process is gone, or is # past its deadline (pids get reused). The Node uploader (s200-upload.mjs) uses the same name on macOS and Linux # as a file holding its pid, touched every 15 s while it runs, and reads this directory's "pid" the same way. # Without a writable home folder the run goes on without the lock: the lock spares wasted retries, it does not # protect the bytes (storage's offset and the SHA-256 check after the upload do). KEY=$(printf '%s' "$URL" | sha256_hex | cut -c1-16) take_lock() { if [ -z "${HOME-}" ] || [ ! -d "$HOME" ]; then return 0; fi if [ ! -d "$HOME/.s200-upload" ]; then (umask 077 && mkdir -p "$HOME/.s200-upload") 2>/dev/null; fi LOCK=$HOME/.s200-upload/lock-$KEY tries=0 while [ "$tries" -lt 3 ]; do tries=$((tries + 1)) if mkdir "$LOCK" 2>/dev/null; then HAVE_LOCK=1 printf '%s\n' "$DEADLINE" > "$LOCK/until" printf '%s\n' "$$" > "$LOCK/pid" return 0 fi if [ -d "$LOCK" ]; then held=$(cat "$LOCK/pid" 2>/dev/null) if [ -z "$held" ]; then # Its holder may be between making the directory and writing the pid. sleep 1 held=$(cat "$LOCK/pid" 2>/dev/null) fi if lock_alive "$held" "$(cat "$LOCK/until" 2>/dev/null)"; then busy; fi # Gone: remove the lock, unless another run took it over in the meantime. if [ "$(cat "$LOCK/pid" 2>/dev/null)" = "$held" ]; then rm -rf "$LOCK"; fi elif [ -f "$LOCK" ]; then # The Node uploader's lock: in use while its pid lives and it was touched in the last minute or two. held=$(cat "$LOCK" 2>/dev/null) if lock_alive "$held" '' && [ -n "$(find "$LOCK" -mmin -2 2>/dev/null)" ]; then busy; fi rm -f "$LOCK" else LOCK= return 0 fi done busy } # Is the holder of a lock still running? $1 its pid, $2 its deadline (may be empty). lock_alive() { case $1 in '' | *[!0-9]*) return 1 ;; esac if [ "$1" = "$$" ]; then return 1; fi case $2 in '' | *[!0-9]*) ;; *) clock if [ "$T" -gt $(($2 + 30)) ]; then return 1; fi ;; esac kill -0 "$1" 2>/dev/null } TMPD=$(mktemp -d "${TMPDIR:-/tmp}/s200-upload.XXXXXX" 2>/dev/null || mktemp -d /tmp/s200-upload.XXXXXX 2>/dev/null) || TMPD= if [ -z "$TMPD" ] || [ ! -d "$TMPD" ]; then TMPD= stop 2 refused 'could not make a temporary folder' fi HDRS=$TMPD/headers BODY=$TMPD/body OUT=$TMPD/out AUTH=$TMPD/auth # curl reads the token from this file (-K) in a folder only this user can open: on a command line, any user of the # computer could read it (ps). The token holds no quote or backslash (step 2), so it is safe inside the quotes. if ! printf 'header = "x-signature: %s"\n' "$TOKEN" 2>/dev/null > "$AUTH"; then stop 2 refused 'could not write to a temporary folder' fi take_lock # --------------------------------------------------------------------------------------------------------------- # The upload: HEAD first, then 6 MiB PATCHes from storage's offset # --------------------------------------------------------------------------------------------------------------- server_offset while [ "$OFF" -lt "$SIZE" ]; do budget 180 # The link was made for this exact size, so a file that is shorter or longer now can never match it. file_size if [ -z "$FS" ]; then stop 2 refused 'the file can no longer be read (was it moved or deleted?)'; fi if [ "$FS" -lt "$SIZE" ]; then stop 2 refused 'the file got shorter while it was being uploaded'; fi if [ "$FS" -gt "$SIZE" ]; then stop 2 refused "the file changed while it was being uploaded (it has $FS bytes now, the link expects $SIZE)" fi if [ $((T - LAST)) -ge 15 ] && [ "$SHOWN" != "$OFF" ] && [ "$LAST" -gt 0 ]; then LAST=$T progress fi WANT=$((SIZE - OFF)) if [ "$WANT" -gt "$CHUNK" ]; then WANT=$CHUNK; fi send_chunk case $CODE in 200 | 204) took ;; *) not_taken ;; esac if [ "$LAST" -eq 0 ]; then # The first line comes right after the first piece, so a rerun shows at once that it went on from where # storage was, not from 0 %. clock LAST=$T progress fi done if [ "$SHOWN" != "$SIZE" ]; then progress; fi trap '' HUP INT TERM printf '{"state":"done","size":%s}\n' "$SIZE" exit 0