Share chunked upload engine across clients

Extract low-memory WebSocket upload logic into a new shared `script/chunkupload.js` module and wire both File Manager and Desktop to use it. The new uploader centralizes CRC32/checksum handling, pipelined windowed chunk sending, pause heartbeat, and retry/timeout behavior. Server-side low-memory upload handling is tightened to accept chunks only at the expected index, dropping duplicate or out-of-order chunk frames to keep progress/accounting and full-file checksums correct.

src/file_system.go

@@ -735,6 +735,23 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 			}
 			expectingBinary = false
 
+			/*
+				Chunks are accepted strictly in index order, and "next" is sent
+				once per accepted chunk - clients that keep several chunks in
+				flight count those acknowledgements to know which chunks landed.
+				Anything else is dropped without a reply:
+				- index < blockCounter: a copy of a chunk already written, resent
+				  by the client after an acknowledgement timeout. Accepting it
+				  again would count its bytes twice in the size and full-file
+				  checksum.
+				- index > blockCounter: sent ahead in a pipelined window before
+				  the chunk at blockCounter failed its CRC check. The client
+				  rewinds to the retry and sends it again.
+			*/
+			if pendingChunkIndex != blockCounter {
+				continue
+			}
+
 			// Verify chunk CRC32
 			chunkSum := crc32.ChecksumIEEE(message)
 			chunkSumBytes := []byte{byte(chunkSum >> 24), byte(chunkSum >> 16), byte(chunkSum >> 8), byte(chunkSum)}
@@ -748,15 +765,10 @@ func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
 				continue
 			}
 
-			// Chunk verified – write to tmp folder.
-			// Use pendingChunkIndex as the canonical filename so that a retry overwrites
-			// the previous (corrupted) attempt rather than creating a duplicate entry.
+			// Chunk verified – write to tmp folder
 			chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(pendingChunkIndex))
-			if pendingChunkIndex == blockCounter {
-				// First time this chunk index is successfully received
-				chunkName = append(chunkName, chunkFilepath)
-				blockCounter++
-			}
+			chunkName = append(chunkName, chunkFilepath)
+			blockCounter++
 
 			var writeErr error
 			if isHugeFile {

src/web/SystemAO/file_system/file_explorer.html

@@ -11,6 +11,7 @@
         <script type="text/javascript" src="../../script/semantic/semantic.min.js"></script>
         <script type="text/javascript" src="../../script/ao_module.js"></script>
         <script type="text/javascript" src="../../script/applocale.js"></script>
+        <script type="text/javascript" src="../../script/chunkupload.js"></script>
         <link rel="stylesheet" href="../../script/ao.css">
         <!-- Design tokens and components shared with the File Selector. Loaded
              before file_explorer.css so app rules can override them. -->

src/web/SystemAO/file_system/js/explorer/state.js

@@ -96,8 +96,9 @@ let uploadRetryMap = new Map(); //taskUUID -> {file, targetDir} for WebSocket up
 let lowMemoryMode = true;   //Upload with low memory mode channel
 let largeFileCutoffSize = 8192 * 1024 * 1024; //Any file larger than this size is consider "large file", default to 8GB
 let uploadFileChunkSize = 1024 * 1024 * 1; //1MB, 4MB not working quite well on slow network
+let uploadWindowSize = 8 * 1024 * 1024; //Bytes of chunks kept in flight per upload connection (see script/chunkupload.js)
 let postUploadModeCutoff = 25 * 1048576; //25MB, files smaller than this will upload using POST Mode
-const CHUNK_TIMEOUT_MS = 30000; //30s timeout waiting for server "next" ack before retrying a chunk
+const CHUNK_TIMEOUT_MS = 30000; //30s without any server "next" ack (once everything queued has been sent) before resending the oldest chunk
 
 /*
     Heartbeat interval while an upload is paused. Comfortably under the 60s idle
@@ -110,7 +111,7 @@ const CHUNK_TIMEOUT_MS = 30000; //30s timeout waiting for server "next" ack befo
 */
 const UPLOAD_PAUSE_PING_MS = 20000;
 const UPLOAD_PAUSE_TIMEOUT_CLOSE_CODE = 4001;
-const MAX_CHUNK_RETRIES = 3;    //Maximum retries per individual chunk before failing the upload
+const MAX_CHUNK_RETRIES = 3;    //Maximum retries without any progress before failing the upload
 
 /*
     Transfer panel state (see js/explorer/uploadui.js)

src/web/SystemAO/file_system/js/explorer/upload.js

@@ -116,42 +116,14 @@ function uploadFile(file, uuid=undefined, targetDir=undefined) {
             hugeFileMode = "&hugefile=true";
         }
 
-        let socket = new WebSocket(protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + encodeURIComponent(filename) + "&path=" + encodeURIComponent(uploadDir) + hugeFileMode);
-        let currentSendingIndex = 0;
-        let chunks = Math.ceil(file.size/uploadFileChunkSize);
-
-        // Per-chunk retry state
-        let chunkRetryCount = 0;
-        let chunkTimeoutTimer = null;
-
-        // Running CRC32 state across all chunks for full-file checksum
-        // Initialized to 0xFFFFFFFF (pre-conditioning), finalized with XOR at the end
-        let runningCRC32State = 0xFFFFFFFF;
-        // Track which chunk indices have already been factored into the running state
-        // so that retries do not corrupt the full-file CRC32
-        let chunkCRC32Committed = {};
-
         // Store file reference in retry map so the user can retry on failure
         uploadRetryMap.set(taskUUID, {file: file, targetDir: JSON.parse(JSON.stringify(uploadDir))});
 
-        /*
-            Pause / resume state.
-
-            Chunks are pulled by the server: every "next" acknowledgement asks
-            for one more. Pausing therefore just means not answering that ask
-            and remembering that one is outstanding, so nothing has to be
-            unwound and the transfer resumes exactly where it stopped.
-        */
-        let paused = false;
-        let resumeWaiting = false;
         let aborted = false;
         let completed = false;
-        let pausePingTimer = null;
 
         // Mark an upload task as failed and reveal the retry button
         function markUploadFailed(tUUID) {
-            clearTimeout(chunkTimeoutTimer);
-            stopPausePing();
             if (aborted) {
                 return;
             }
@@ -160,126 +132,88 @@ function uploadFile(file, uuid=undefined, targetDir=undefined) {
         }
 
         /*
-            Keep-alive while paused
-
-            A paused upload sends nothing at all, and neither an idle-timeout on
-            our own server nor a reverse proxy in front of it will keep such a
-            connection open. So the client keeps a slow heartbeat going for as
-            long as the pause lasts; the server answers each one, which puts
-            traffic on the wire in both directions.
+            The chunking, pipelining, checksums, retries and the pause
+            keep-alive all live in script/chunkupload.js, shared with the
+            desktop. This side only maps its events onto the transfer panel.
         */
-        function startPausePing() {
-            stopPausePing();
-            pausePingTimer = setInterval(function () {
-                if (socket.readyState != WebSocket.OPEN) {
-                    stopPausePing();
-                    return;
+        let transfer = ChunkUpload.start({
+            url: protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + encodeURIComponent(filename) + "&path=" + encodeURIComponent(uploadDir) + hugeFileMode,
+            file: file,
+            chunkSize: uploadFileChunkSize,
+            windowSize: uploadWindowSize,
+            ackTimeout: CHUNK_TIMEOUT_MS,
+            maxRetries: MAX_CHUNK_RETRIES,
+            pingInterval: UPLOAD_PAUSE_PING_MS,
+            onProgress: function(loaded, total){
+                setUploadTaskProgress(taskUUID, loaded, total);
+            },
+            onProcessing: function(){
+                setUploadTaskState(taskUUID, "processing");
+            },
+            onMove: function(status){
+                //File move from tmp to archive – show progress
+                setUploadTaskState(taskUUID, "processing");
+                setUploadTaskStatusText(taskUUID, status);
+            },
+            onDone: function(){
+                //Merge completed successfully
+                uploadRetryMap.delete(taskUUID);
+                unregisterUploadTransfer(taskUUID);
+                completed = true;
+                setUploadTaskState(taskUUID, "done");
+            },
+            onError: function(message, fromServer){
+                if (fromServer){
+                    msgbox("red remove", message);
                 }
-                socket.send(JSON.stringify({ping: true}));
-            }, UPLOAD_PAUSE_PING_MS);
-        }
-
-        function stopPausePing() {
-            if (pausePingTimer != null) {
-                clearInterval(pausePingTimer);
-                pausePingTimer = null;
-            }
-        }
-
-        // Send a specific chunk by index.
-        // Reads the slice as ArrayBuffer, computes CRC32, sends metadata then binary.
-        // Sets a CHUNK_TIMEOUT_MS timer; on expiry retries up to MAX_CHUNK_RETRIES times.
-        async function sendChunk(id) {
-            var offsetStart = id * uploadFileChunkSize;
-            var offsetEnd   = id * uploadFileChunkSize + uploadFileChunkSize;
-            var thisblob = file.slice(offsetStart, offsetEnd);
-
-            let arrayBuffer;
-            try {
-                arrayBuffer = await thisblob.arrayBuffer();
-            } catch(e) {
-                console.error("[Upload] Failed to read chunk " + id + ": " + e);
                 markUploadFailed(taskUUID);
-                return;
-            }
-
-            let bytes = new Uint8Array(arrayBuffer);
-
-            // Update the running full-file CRC32 only on the first attempt for each
-            // chunk index so that retries do not double-count the bytes
-            if (!chunkCRC32Committed[id]) {
-                runningCRC32State = crc32UpdateState(runningCRC32State, bytes);
-                chunkCRC32Committed[id] = true;
-            }
+            },
+            onClose: function(event){
+                unregisterUploadTransfer(taskUUID);
 
-            // Compute a standalone CRC32 for this chunk for transmission verification
-            let chunkCRCHex = crc32Hex(bytes);
-
-            // Protocol: text metadata frame, then binary data frame
-            socket.send(JSON.stringify({index: id, checksum: chunkCRCHex}));
-            socket.send(arrayBuffer);
-
-            // Report the bytes actually handed to the socket
-            setUploadTaskProgress(taskUUID, Math.min(file.size, offsetEnd), file.size);
-
-            // (Re)start the per-chunk acknowledgement timeout
-            clearTimeout(chunkTimeoutTimer);
-            chunkTimeoutTimer = setTimeout(function() {
-                if (chunkRetryCount < MAX_CHUNK_RETRIES) {
-                    chunkRetryCount++;
-                    console.warn("[Upload] Chunk " + id + " ACK timeout – retry " + chunkRetryCount + "/" + MAX_CHUNK_RETRIES);
-                    sendChunk(id);
-                } else {
-                    console.error("[Upload] Chunk " + id + " failed after " + MAX_CHUNK_RETRIES + " retries");
-                    markUploadFailed(taskUUID);
-                    socket.close();
+                /*
+                    Any close that is not the end of a finished upload and not the
+                    user cancelling leaves the task stranded mid-transfer, so offer
+                    retry rather than a row frozen at whatever percentage it reached.
+                    The server closes with uploadPauseCloseCode when a pause has been
+                    left running for too long, which is the case worth naming.
+                */
+                if (!completed && !aborted){
+                    if (event.code == UPLOAD_PAUSE_TIMEOUT_CLOSE_CODE){
+                        msgbox("caution", applocale.getString("upload/pauseExpired",
+                            "Upload cancelled: paused for too long"));
+                    }
+                    setUploadTaskState(taskUUID, "failed");
                 }
-            }, CHUNK_TIMEOUT_MS);
-        }
 
-        // Send whatever the server asked for next, or park the request if the
-        // user has paused. Every send path goes through here so pause only has
-        // to be handled once.
-        async function sendNext() {
-            if (paused || aborted) {
-                resumeWaiting = true;
-                return;
-            }
-            if (currentSendingIndex >= chunks){
-                // All chunks sent and acknowledged - send done + full-file checksum
-                let finalCRC32Hex = ((runningCRC32State ^ 0xFFFFFFFF) >>> 0).toString(16).padStart(8, '0');
-                socket.send(JSON.stringify({done: true, totalChunks: chunks, fileChecksum: finalCRC32Hex}));
-                setUploadTaskState(taskUUID, "processing");
-            }else{
-                await sendChunk(currentSendingIndex);
-                currentSendingIndex++;
+                uploadingFileCount--;
+                updateUploadFileCount();
+                //After the previous file has uploaded / errored, check if there are another file needed to be uploaded
+                setTimeout(function(){
+                    if (uploadPendingList.length > 0){
+                        let nextFile = uploadPendingList.shift();
+                        uploadFile(nextFile.File, nextFile.UUID, nextFile.TargetDir);
+                    }
+                }, 100)
+            },
+            onSocketError: function(error){
+                console.error("[Upload] WebSocket error:", error);
+                // Mark the task as failed and show the retry button
+                markUploadFailed(taskUUID);
             }
-        }
+        });
 
         registerUploadTransfer(taskUUID, {
             pausable: true,
             pause: function(){
-                paused = true;
-                //The chunk clock must stop too, or the outstanding chunk would
-                //"time out" while the upload is legitimately sitting idle
-                clearTimeout(chunkTimeoutTimer);
-                try { socket.send(JSON.stringify({pause: true})); } catch(e) {}
-                startPausePing();
+                transfer.pause();
             },
             resume: function(){
-                paused = false;
-                stopPausePing();
-                try { socket.send(JSON.stringify({resume: true})); } catch(e) {}
-                if (resumeWaiting){
-                    resumeWaiting = false;
-                    sendNext();
-                }
+                transfer.resume();
             },
             abort: function(){
                 aborted = true;
-                clearTimeout(chunkTimeoutTimer);
-                stopPausePing();
-                try { socket.close(); } catch(e) {}
+                transfer.abort();
             }
         });
 
@@ -287,114 +221,6 @@ function uploadFile(file, uuid=undefined, targetDir=undefined) {
         updateUploadFileCount();
         uploadingFileCount++;
 
-        //Start sending
-        socket.onopen = async function(e) {
-            /*
-                The pause button is live from the moment the row appears, which
-                can be before this socket finished connecting. A pause raised in
-                that window could not be sent, so replay it here - otherwise the
-                server never learns the upload is paused and reaps it as idle.
-            */
-            if (paused){
-                try { socket.send(JSON.stringify({pause: true})); } catch(e) {}
-                startPausePing();
-                return;
-            }
-            currentSendingIndex = 0;
-            await sendNext();
-        };
-
-        socket.onmessage = async function(event) {
-            var incomingValue = event.data;
-
-            if (incomingValue == `{"pong":true}`){
-                //Heartbeat reply while paused - nothing to do, the point is
-                //that a frame crossed the connection in each direction
-                return;
-            }
-
-            if (incomingValue == "next"){
-                // Server acknowledged the last chunk; clear timeout and reset retry counter
-                clearTimeout(chunkTimeoutTimer);
-                chunkRetryCount = 0;
-                await sendNext();
-
-            }else if (incomingValue == "OK"){
-                //Merge completed successfully
-                uploadRetryMap.delete(taskUUID);
-                unregisterUploadTransfer(taskUUID);
-                completed = true;
-                setUploadTaskState(taskUUID, "done");
-            }else{
-                //Try to parse it as JSON
-                try{
-                    var resp = JSON.parse(incomingValue);
-                    if (resp.error !== undefined){
-                        //Server reported an error
-                        msgbox("red remove", resp.error);
-                        markUploadFailed(taskUUID);
-                    }else if (resp.retryChunk !== undefined){
-                        // Server detected a CRC32 mismatch and requests a chunk re-send
-                        clearTimeout(chunkTimeoutTimer);
-                        if (chunkRetryCount < MAX_CHUNK_RETRIES){
-                            chunkRetryCount++;
-                            console.warn("[Upload] Server requested retry for chunk " + resp.retryChunk + " (CRC32 mismatch) – retry " + chunkRetryCount + "/" + MAX_CHUNK_RETRIES);
-                            await sendChunk(resp.retryChunk);
-                        } else {
-                            console.error("[Upload] Chunk " + resp.retryChunk + " CRC32 mismatch after max retries");
-                            markUploadFailed(taskUUID);
-                            socket.close();
-                        }
-                    }else if (resp.move !== undefined){
-                        //File move from tmp to archive – show progress
-                        setUploadTaskState(taskUUID, "processing");
-                        setUploadTaskStatusText(taskUUID, resp.move);
-                    }
-                }catch(ex){
-                    //Something else
-                    console.log(ex);
-                }
-            }
-
-        };
-
-        socket.onclose = function(event) {
-            clearTimeout(chunkTimeoutTimer);
-            stopPausePing();
-            unregisterUploadTransfer(taskUUID);
-
-            /*
-                Any close that is not the end of a finished upload and not the
-                user cancelling leaves the task stranded mid-transfer, so offer
-                retry rather than a row frozen at whatever percentage it reached.
-                The server closes with uploadPauseCloseCode when a pause has been
-                left running for too long, which is the case worth naming.
-            */
-            if (!completed && !aborted){
-                if (event.code == UPLOAD_PAUSE_TIMEOUT_CLOSE_CODE){
-                    msgbox("caution", applocale.getString("upload/pauseExpired",
-                        "Upload cancelled: paused for too long"));
-                }
-                setUploadTaskState(taskUUID, "failed");
-            }
-
-            uploadingFileCount--;
-            updateUploadFileCount();
-            //After the previous file has uploaded / errored, check if there are another file needed to be uploaded
-            setTimeout(function(){
-                if (uploadPendingList.length > 0){
-                    let nextFile = uploadPendingList.shift();
-                    uploadFile(nextFile.File, nextFile.UUID, nextFile.TargetDir);
-                }
-            }, 100)
-        };
-
-        socket.onerror = function(error) {
-            console.error("[Upload] WebSocket error:", error);
-            // Mark the task as failed and show the retry button
-            markUploadFailed(taskUUID);
-        };
-
     }else{
         /*
             Standard Upload Mode

src/web/SystemAO/file_system/js/explorer/util.js

@@ -7,26 +7,7 @@
     file_explorer.html - see the <script> block at the end of that file.
 */
 
-    //CRC32 lookup table and helpers for chunk/file integrity verification
-    const crc32Table = (() => {
-        const t = new Uint32Array(256);
-        for (let i = 0; i < 256; i++) {
-            let c = i;
-            for (let j = 0; j < 8; j++) c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
-            t[i] = c;
-        }
-        return t;
-    })();
-    //Update a running CRC32 state with new bytes (does not finalize)
-    function crc32UpdateState(state, bytes) {
-        for (let i = 0; i < bytes.length; i++)
-            state = (state >>> 8) ^ crc32Table[(state ^ bytes[i]) & 0xFF];
-        return state;
-    }
-    //Compute standalone CRC32 of a Uint8Array and return hex string
-    function crc32Hex(bytes) {
-        return ((crc32UpdateState(0xFFFFFFFF, bytes) ^ 0xFFFFFFFF) >>> 0).toString(16).padStart(8, '0');
-    }
+    //Upload chunk / file CRC32 lives with the upload protocol in script/chunkupload.js
 
     //Thumbnail base64 sniffing now lives in shared/filethumb.js as
     //FileThumb.thumbExtFromBase64 / FileThumb.base64ToDataURL

src/web/desktop.html

@@ -13,6 +13,7 @@
     <script type="text/javascript" src="./script/ao_module.js"></script>
     <script type="text/javascript" src="./SystemAO/desktop/script/jsCalendar/source/jsCalendar.js"></script>
     <script type="text/javascript" src="./script/applocale.js"></script>
+    <script type="text/javascript" src="./script/chunkupload.js"></script>
     <!-- Sentinel paths (%trashbin%, ...), shared with the File Manager -->
     <script type="text/javascript" src="./SystemAO/file_system/shared/specialpaths.js"></script>
     <!-- <script type="text/javascript" src="./script/html2canvas.min.js"></script> -->
@@ -1562,27 +1563,9 @@
         let uploadFileChunkSize = 1024 * 512; //512KB per chunk in low memory upload
         let largeFileCutoffSize = 8192 * 1024 * 1024; //Any file larger than this size is consider "large file", default to 8GB
         let postUploadModeCutoff = 25 * 1048576; //25MB, files smaller than this will upload using POST Mode
-        const CHUNK_TIMEOUT_MS = 30000; //30s to wait for server ack before retrying a chunk
-        const MAX_CHUNK_RETRIES = 3;    //Maximum per-chunk retries before marking the upload failed
-
-        //CRC32 lookup table and helpers (IEEE polynomial, matches Go's hash/crc32 package)
-        const crc32Table = (() => {
-            const t = new Uint32Array(256);
-            for (let i = 0; i < 256; i++) {
-                let c = i;
-                for (let j = 0; j < 8; j++) c = (c & 1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1);
-                t[i] = c;
-            }
-            return t;
-        })();
-        function crc32UpdateState(state, bytes) {
-            for (let i = 0; i < bytes.length; i++)
-                state = (state >>> 8) ^ crc32Table[(state ^ bytes[i]) & 0xFF];
-            return state;
-        }
-        function crc32Hex(bytes) {
-            return ((crc32UpdateState(0xFFFFFFFF, bytes) ^ 0xFFFFFFFF) >>> 0).toString(16).padStart(8, '0');
-        }
+        let uploadWindowSize = 8 * 1024 * 1024; //Bytes of chunks kept in flight per upload connection (see script/chunkupload.js)
+        const CHUNK_TIMEOUT_MS = 30000; //30s without any server ack (once everything queued has been sent) before resending the oldest chunk
+        const MAX_CHUNK_RETRIES = 3;    //Maximum retries without any progress before marking the upload failed
 
         //Others
         var monthNames = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
@@ -7366,148 +7349,61 @@
                 }
                 
 
-                let socket = new WebSocket(protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + filename + "&path=" + uploadDir + hugeFileMode);
-                let currentSendingIndex = 0;
-                let chunks = Math.ceil(file.size/uploadFileChunkSize);
-
-                // Per-chunk retry state
-                let chunkRetryCount = 0;
-                let chunkTimeoutTimer = null;
-
-                // Running CRC32 state across all chunks for full-file checksum
-                let runningCRC32State = 0xFFFFFFFF;
-                let chunkCRC32Committed = {};
-
-                async function sendChunk(id) {
-                    let offsetStart = id * uploadFileChunkSize;
-                    let offsetEnd   = id * uploadFileChunkSize + uploadFileChunkSize;
-                    let arrayBuffer;
-                    try {
-                        arrayBuffer = await file.slice(offsetStart, offsetEnd).arrayBuffer();
-                    } catch(e) {
-                        console.error("[Desktop Upload] Failed to read chunk " + id + ": " + e);
-                        return;
-                    }
-
-                    let bytes = new Uint8Array(arrayBuffer);
-
-                    // Update running full-file CRC32 only on first attempt for each chunk
-                    if (!chunkCRC32Committed[id]) {
-                        runningCRC32State = crc32UpdateState(runningCRC32State, bytes);
-                        chunkCRC32Committed[id] = true;
-                    }
-
-                    let chunkCRCHex = crc32Hex(bytes);
-
-                    // Send metadata frame then binary frame
-                    socket.send(JSON.stringify({index: id, checksum: chunkCRCHex}));
-                    socket.send(arrayBuffer);
+                //Chunking, pipelining, checksums and retries live in
+                //script/chunkupload.js, shared with the File Manager
+                let uploadIcon = function(){
+                    return $("." + uploadingIconUUID + ".launchIcon");
+                };
+                let showUploadFailed = function(){
+                    uploadIcon().find(".progress").removeClass("primary").addClass("error");
+                    uploadIcon().find(".launchIconText").text(applocale.getString("upload/message/failed", "Failed!"));
+                    setTimeout(function(){
+                        uploadIcon().fadeOut(3000, function() { $(this).remove(); });
+                    }, 2000);
+                };
 
-                    // Update desktop icon progress (cap at 95% to reserve room for merge)
-                    let progress = chunks <= 1 ? 50 : (id / (chunks - 1) * 95.0);
-                    if (progress > 95) progress = 95;
-                    if (uploadingIconUUID != undefined){
-                        $("." + uploadingIconUUID + ".launchIcon").find(".bar").css("width", progress + "%");
-                        if (progress >= 95){
-                            $("." + uploadingIconUUID + ".launchIcon").find(".progress").addClass("indeterminate");
+                ChunkUpload.start({
+                    url: protocol + window.location.hostname + ":" + port + "/system/file_system/lowmemUpload?filename=" + filename + "&path=" + uploadDir + hugeFileMode,
+                    file: file,
+                    chunkSize: uploadFileChunkSize,
+                    windowSize: uploadWindowSize,
+                    ackTimeout: CHUNK_TIMEOUT_MS,
+                    maxRetries: MAX_CHUNK_RETRIES,
+                    onProgress: function(loaded, total){
+                        //Cap at 95% to reserve room for the server side merge
+                        if (uploadingIconUUID == undefined){
+                            return;
                         }
-                    }
-
-                    // Start ack timeout; retry chunk on expiry
-                    clearTimeout(chunkTimeoutTimer);
-                    chunkTimeoutTimer = setTimeout(function() {
-                        if (chunkRetryCount < MAX_CHUNK_RETRIES) {
-                            chunkRetryCount++;
-                            console.warn("[Desktop Upload] Chunk " + id + " ACK timeout – retry " + chunkRetryCount + "/" + MAX_CHUNK_RETRIES);
-                            sendChunk(id);
-                        } else {
-                            console.error("[Desktop Upload] Chunk " + id + " failed after max retries");
-                            socket.close();
+                        let progress = Math.min(95, loaded / total * 95.0);
+                        uploadIcon().find(".bar").css("width", progress + "%");
+                        if (progress >= 95){
+                            uploadIcon().find(".progress").addClass("indeterminate");
                         }
-                    }, CHUNK_TIMEOUT_MS);
-                }
-
-                //Start sending
-                socket.onopen = async function(e) {
-                    if (filesize < uploadFileChunkSize){
+                    },
+                    onMove: function(){
+                        // Server is moving file from tmp to destination
                         if (uploadingIconUUID != undefined){
-                            $("." + uploadingIconUUID + ".launchIcon").find(".bar").css("width", "15%");
+                            uploadIcon().find(".bar").css("width", "100%");
                         }
-                    }
-                    await sendChunk(0);
-                    currentSendingIndex = 1;
-                };
-
-                socket.onmessage = async function(event) {
-                    var incomingValue = event.data;
-
-                    if (incomingValue == "next"){
-                        clearTimeout(chunkTimeoutTimer);
-                        chunkRetryCount = 0;
-
-                        if (currentSendingIndex >= chunks){
-                            // All chunks acknowledged – send done with full-file checksum
-                            let finalCRC32Hex = ((runningCRC32State ^ 0xFFFFFFFF) >>> 0).toString(16).padStart(8, '0');
-                            socket.send(JSON.stringify({done: true, totalChunks: chunks, fileChecksum: finalCRC32Hex}));
-                        }else{
-                            await sendChunk(currentSendingIndex);
-                            currentSendingIndex++;
+                    },
+                    onError: function(message, fromServer){
+                        if (fromServer){
+                            sendNotification("Upload Failed", message, "remove");
+                            showUploadFailed();
                         }
-
-                    }else if (incomingValue == "OK"){
-                        //Merge completed successfully – icon will be cleaned up by onclose
-                    }else{
-                        try{
-                            var resp = JSON.parse(incomingValue);
-                            if (resp.error !== undefined){
-                                sendNotification("Upload Failed", resp.error, "remove");
-                                clearTimeout(chunkTimeoutTimer);
-                                $("." + uploadingIconUUID + ".launchIcon").find(".progress").removeClass("primary").addClass("error");
-                                $("." + uploadingIconUUID + ".launchIcon").find(".launchIconText").text(applocale.getString("upload/message/failed", "Failed!"));
-                                setTimeout(function(){
-                                    $("." + uploadingIconUUID + ".launchIcon").fadeOut(3000, function() { $(this).remove(); });
-                                }, 2000);
-                            } else if (resp.retryChunk !== undefined){
-                                // Server-requested chunk retry (CRC32 mismatch)
-                                clearTimeout(chunkTimeoutTimer);
-                                if (chunkRetryCount < MAX_CHUNK_RETRIES) {
-                                    chunkRetryCount++;
-                                    console.warn("[Desktop Upload] Server requested retry for chunk " + resp.retryChunk);
-                                    await sendChunk(resp.retryChunk);
-                                } else {
-                                    console.error("[Desktop Upload] Chunk " + resp.retryChunk + " CRC32 mismatch after max retries");
-                                    socket.close();
-                                }
-                            } else if (resp.move !== undefined){
-                                // Server is moving file from tmp to destination
-                                if (uploadingIconUUID != undefined){
-                                    $("." + uploadingIconUUID + ".launchIcon").find(".bar").css("width", "100%");
-                                }
-                            }
-                        }catch(ex){
-                            console.log(incomingValue);
-                            console.log(ex);
+                    },
+                    onClose: function(){
+                        //Merge completed or failed - either way the icon goes
+                        $("." + uploadingIconUUID).remove();
+                        if (callback != undefined){
+                            callback();
                         }
+                    },
+                    onSocketError: function(error){
+                        console.error("[Desktop Upload] WebSocket error:", error);
+                        showUploadFailed();
                     }
-                };
-
-                socket.onclose = function(event) {
-                    clearTimeout(chunkTimeoutTimer);
-                    $("." + uploadingIconUUID).remove();
-                    if (callback != undefined){
-                        callback();
-                    }
-                };
-
-                socket.onerror = function(error) {
-                    console.error("[Desktop Upload] WebSocket error:", error);
-                    clearTimeout(chunkTimeoutTimer);
-                    $("." + uploadingIconUUID + ".launchIcon").find(".progress").removeClass("primary").addClass("error");
-                    $("." + uploadingIconUUID + ".launchIcon").find(".launchIconText").text(applocale.getString("upload/message/failed", "Failed!"));
-                    setTimeout(function(){
-                        $("." + uploadingIconUUID + ".launchIcon").fadeOut(3000, function() { $(this).remove(); });
-                    }, 2000);
-                };
+                });
             }else{
                 let url = 'system/file_system/upload'
                 let uploadCurrentPath = "user:/Desktop/";

src/web/script/chunkupload.js

@@ -0,0 +1,485 @@
+/*
+    chunkupload.js
+
+    Client side of the low-memory (WebSocket chunked) upload channel,
+    /system/file_system/lowmemUpload. Shared by the File Manager and the
+    desktop so both speak exactly the same protocol.
+
+    Protocol, per chunk: a text frame {"index": N, "checksum": "<crc32 hex>"}
+    followed by a binary frame with the chunk bytes. The server takes chunks
+    strictly in index order and answers each one it accepts with "next".
+    Anything else it silently drops: a copy of a chunk it already has, or a
+    chunk sent ahead of one that failed its CRC. On a CRC mismatch it asks for
+    {"retryChunk": N}. When every chunk is acknowledged the client sends
+    {"done": true, "totalChunks": N, "fileChecksum": "<crc32 hex>"} and the
+    server merges the chunks, reporting {"move": "..."} and finally "OK".
+
+    Pipelining: instead of waiting for "next" after every chunk, up to
+    windowSize bytes of chunks are kept in flight, so one connection is no
+    longer capped at one chunk per round trip. Because the server answers in
+    order, each "next" acknowledges the oldest chunk still in flight. A retry
+    request rewinds the sender to the chunk that failed (go-back-N); an
+    acknowledgement timeout resends just the oldest unacknowledged chunk. The
+    server drops whatever it receives out of order, so both are always safe.
+
+    Every chunk is read and checksummed exactly once per send. The whole-file
+    CRC32 is built from the chunk CRCs with crc32Combine instead of a second
+    pass over the bytes.
+*/
+
+(function (global) {
+    "use strict";
+
+    /*
+        CRC32 (IEEE, reflected, polynomial 0xEDB88320) - matches Go's
+        hash/crc32.ChecksumIEEE. Slicing-by-8 tables: CRC_TABLES[k] advances the
+        CRC by one byte followed by k zero bytes.
+    */
+    const CRC_POLY = 0xEDB88320;
+    const CRC_TABLES = (function () {
+        let tables = [];
+        for (let k = 0; k < 8; k++) {
+            tables.push(new Int32Array(256));
+        }
+        for (let i = 0; i < 256; i++) {
+            let c = i;
+            for (let j = 0; j < 8; j++) {
+                c = (c & 1) ? (CRC_POLY ^ (c >>> 1)) : (c >>> 1);
+            }
+            tables[0][i] = c;
+        }
+        for (let i = 0; i < 256; i++) {
+            let c = tables[0][i];
+            for (let k = 1; k < 8; k++) {
+                c = tables[0][c & 0xFF] ^ (c >>> 8);
+                tables[k][i] = c;
+            }
+        }
+        return tables;
+    })();
+
+    //CRC32 of a Uint8Array as an unsigned 32-bit number
+    function crc32(bytes) {
+        const t0 = CRC_TABLES[0], t1 = CRC_TABLES[1], t2 = CRC_TABLES[2], t3 = CRC_TABLES[3];
+        const t4 = CRC_TABLES[4], t5 = CRC_TABLES[5], t6 = CRC_TABLES[6], t7 = CRC_TABLES[7];
+        let crc = -1;
+        let i = 0;
+        const len = bytes.length;
+        const fast = len - (len % 8);
+        while (i < fast) {
+            const lo = crc ^ (bytes[i] | (bytes[i + 1] << 8) | (bytes[i + 2] << 16) | (bytes[i + 3] << 24));
+            crc = t7[lo & 0xFF] ^ t6[(lo >>> 8) & 0xFF] ^ t5[(lo >>> 16) & 0xFF] ^ t4[lo >>> 24] ^
+                  t3[bytes[i + 4]] ^ t2[bytes[i + 5]] ^ t1[bytes[i + 6]] ^ t0[bytes[i + 7]];
+            i += 8;
+        }
+        while (i < len) {
+            crc = t0[(crc ^ bytes[i]) & 0xFF] ^ (crc >>> 8);
+            i++;
+        }
+        return (crc ^ -1) >>> 0;
+    }
+
+    /*
+        crc32Combine(crcA, crcB, lenB) returns the CRC32 of A followed by B,
+        given only the CRC of each part and the byte length of B. Port of
+        zlib's crc32_combine (multmodp / x2nmodp).
+    */
+    function multmodp(a, b) {
+        let m = 0x80000000;
+        let p = 0;
+        for (;;) {
+            if (a & m) {
+                p ^= b;
+                if ((a & (m - 1)) === 0) {
+                    break;
+                }
+            }
+            m = m / 2;
+            b = (b & 1) ? ((b >>> 1) ^ CRC_POLY) : (b >>> 1);
+        }
+        return p >>> 0;
+    }
+
+    //X2N_TABLE[k] = x^(2^k) modulo the CRC polynomial
+    const X2N_TABLE = (function () {
+        let table = new Array(32);
+        let p = 0x40000000; //x^1
+        table[0] = p;
+        for (let n = 1; n < 32; n++) {
+            p = multmodp(p, p);
+            table[n] = p;
+        }
+        return table;
+    })();
+
+    //x^(n * 2^k) modulo the CRC polynomial
+    function x2nmodp(n, k) {
+        let p = 0x80000000; //x^0 == 1
+        while (n > 0) {
+            if (n % 2 === 1) {
+                p = multmodp(X2N_TABLE[k & 31], p);
+            }
+            n = Math.floor(n / 2);
+            k++;
+        }
+        return p;
+    }
+
+    function crc32Combine(crcA, crcB, lenB) {
+        return (multmodp(x2nmodp(lenB, 3), crcA) ^ crcB) >>> 0;
+    }
+
+    function crc32ToHex(crc) {
+        return (crc >>> 0).toString(16).padStart(8, "0");
+    }
+
+    function noop() {}
+
+    /*
+        Start uploading file over a new WebSocket to url.
+
+        Options (all callbacks optional):
+          url, file         - required
+          chunkSize         - bytes per chunk (default 1MB)
+          windowSize        - bytes allowed in flight (default 8MB, at least one chunk)
+          ackTimeout        - ms without any acknowledgement before rewinding (default 30s)
+          maxRetries        - rewinds allowed without progress before failing (default 3)
+          pingInterval      - heartbeat interval while paused (default 20s)
+          onProgress(ackedBytes, totalBytes)
+          onProcessing()    - every chunk acknowledged, server is merging
+          onMove(status)    - merge progress text from the server
+          onDone()          - server reported "OK"
+          onError(message, fromServer) - the upload failed
+          onClose(event)    - the WebSocket closed, for any reason
+          onSocketError(error)
+
+        Returns a controller: {pause(), resume(), abort(), isPaused(), socket}
+    */
+    function start(opts) {
+        const file = opts.file;
+        const chunkSize = opts.chunkSize || 1024 * 1024;
+        const windowChunks = Math.max(1, Math.floor((opts.windowSize || 8 * 1024 * 1024) / chunkSize));
+        const ackTimeout = opts.ackTimeout || 30000;
+        const maxRetries = (opts.maxRetries === undefined) ? 3 : opts.maxRetries;
+        const pingInterval = opts.pingInterval || 20000;
+        const chunks = Math.ceil(file.size / chunkSize);
+
+        const onProgress = opts.onProgress || noop;
+        const onProcessing = opts.onProcessing || noop;
+        const onMove = opts.onMove || noop;
+        const onDone = opts.onDone || noop;
+        const onError = opts.onError || noop;
+        const onClose = opts.onClose || noop;
+        const onSocketError = opts.onSocketError || noop;
+
+        const socket = new WebSocket(opts.url);
+
+        let acked = 0;          //chunks [0, acked) are accepted by the server
+        let nextIndex = 0;      //next chunk to put on the wire
+        let generation = 0;     //bumped on every rewind, invalidates a send in progress
+        let pumping = false;
+        let retries = 0;        //rewinds since the last acknowledgement
+
+        //Whole-file CRC, folded from each chunk's CRC the first time it is sent
+        let fileCRC = 0;
+        let crcFolded = 0;
+
+        let paused = false;
+        let doneSent = false;
+        let failed = false;
+        let aborted = false;
+        let ackTimer = null;
+        let pingTimer = null;
+
+        //No more chunk traffic once the upload is finished one way or another
+        function halted() {
+            return doneSent || failed || aborted;
+        }
+
+        function send(data) {
+            try {
+                socket.send(data);
+            } catch (e) {}
+        }
+
+        function stopAckTimer() {
+            clearTimeout(ackTimer);
+            ackTimer = null;
+        }
+
+        function armAckTimer() {
+            stopAckTimer();
+            if (paused || halted() || acked >= nextIndex) {
+                return;
+            }
+            ackTimer = setTimeout(onAckTimeout, ackTimeout);
+        }
+
+        function onAckTimeout() {
+            ackTimer = null;
+            if (paused || halted()) {
+                return;
+            }
+            if (socket.bufferedAmount > 0) {
+                //Still handing bytes to the network - a slow link, not a lost
+                //chunk. Resending now would only queue duplicates behind them.
+                armAckTimer();
+                return;
+            }
+            if (retries >= maxRetries) {
+                fail("Upload stalled: no acknowledgement from server", false);
+                return;
+            }
+            retries++;
+            console.warn("[Upload] Chunk " + acked + " not acknowledged - resending it (" + retries + "/" + maxRetries + ")");
+            /*
+                Only the oldest chunk, not the whole window. On a live socket
+                nothing is actually lost - the server is just slow - so every
+                resent chunk is a duplicate it has to read and drop before it
+                can acknowledge anything new. Rewinding a full window here would
+                bury the next acknowledgement under it and time out again.
+            */
+            resendChunk(acked);
+            armAckTimer();
+        }
+
+        //Put one chunk on the wire again without moving the send position
+        async function resendChunk(id) {
+            let bytes;
+            try {
+                bytes = await readChunk(id);
+            } catch (e) {
+                console.error("[Upload] Failed to read chunk " + id + ": " + e);
+                fail("Failed to read file", false);
+                return;
+            }
+            if (halted() || id < acked || socket.readyState != WebSocket.OPEN) {
+                return;
+            }
+            sendChunk(id, bytes, crc32(new Uint8Array(bytes)));
+        }
+
+        function readChunk(id) {
+            const offsetStart = id * chunkSize;
+            return file.slice(offsetStart, Math.min(file.size, offsetStart + chunkSize)).arrayBuffer();
+        }
+
+        //The header and its payload always leave back to back, with no await
+        //between them, so two senders can never interleave a pair
+        function sendChunk(id, buffer, chunkCRC) {
+            send(JSON.stringify({index: id, checksum: crc32ToHex(chunkCRC)}));
+            send(buffer);
+        }
+
+        function startPing() {
+            stopPing();
+            pingTimer = setInterval(function () {
+                if (socket.readyState != WebSocket.OPEN) {
+                    stopPing();
+                    return;
+                }
+                send(JSON.stringify({ping: true}));
+            }, pingInterval);
+        }
+
+        function stopPing() {
+            if (pingTimer != null) {
+                clearInterval(pingTimer);
+                pingTimer = null;
+            }
+        }
+
+        function fail(message, fromServer) {
+            if (failed || aborted) {
+                return;
+            }
+            failed = true;
+            stopAckTimer();
+            stopPing();
+            onError(message, fromServer);
+            if (!fromServer) {
+                try { socket.close(); } catch (e) {}
+            }
+        }
+
+        //Go back to chunk index and send everything from there again
+        function rewind(index) {
+            generation++;
+            nextIndex = Math.max(acked, Math.min(index, nextIndex));
+            armAckTimer();
+            pump();
+        }
+
+        //Fill the window. Only one pump runs at a time so chunks leave in order.
+        async function pump() {
+            if (pumping) {
+                return;
+            }
+            pumping = true;
+            try {
+                while (!paused && !halted() && socket.readyState == WebSocket.OPEN &&
+                        nextIndex < chunks && nextIndex - acked < windowChunks) {
+                    const id = nextIndex;
+                    const gen = generation;
+
+                    let buffer;
+                    try {
+                        buffer = await readChunk(id);
+                    } catch (e) {
+                        console.error("[Upload] Failed to read chunk " + id + ": " + e);
+                        fail("Failed to read file", false);
+                        return;
+                    }
+                    if (gen !== generation) {
+                        //Rewound while reading, re-evaluate from the top
+                        continue;
+                    }
+                    if (paused || halted() || socket.readyState != WebSocket.OPEN) {
+                        break;
+                    }
+
+                    const chunkCRC = crc32(new Uint8Array(buffer));
+                    if (id === crcFolded) {
+                        //First time this chunk is sent (a resend never runs ahead of it)
+                        fileCRC = crc32Combine(fileCRC, chunkCRC, buffer.byteLength);
+                        crcFolded++;
+                    }
+
+                    sendChunk(id, buffer, chunkCRC);
+                    nextIndex = id + 1;
+                    if (ackTimer == null) {
+                        armAckTimer();
+                    }
+                }
+
+                if (!paused && !halted() && acked >= chunks && socket.readyState == WebSocket.OPEN) {
+                    doneSent = true;
+                    stopAckTimer();
+                    send(JSON.stringify({done: true, totalChunks: chunks, fileChecksum: crc32ToHex(fileCRC)}));
+                    onProcessing();
+                }
+            } finally {
+                pumping = false;
+            }
+        }
+
+        socket.onopen = function () {
+            if (paused) {
+                //Paused before the socket connected - tell the server now, or
+                //it reaps the silent connection as idle
+                send(JSON.stringify({pause: true}));
+                startPing();
+                return;
+            }
+            pump();
+        };
+
+        socket.onmessage = function (event) {
+            const incoming = event.data;
+
+            if (incoming == "next") {
+                if (acked < chunks) {
+                    acked++;
+                    if (nextIndex < acked) {
+                        //The original copy got through after a rewind
+                        nextIndex = acked;
+                    }
+                }
+                retries = 0;
+                onProgress(Math.min(file.size, acked * chunkSize), file.size);
+                armAckTimer();
+                pump();
+                return;
+            }
+
+            if (incoming == "OK") {
+                onDone();
+                return;
+            }
+
+            let resp;
+            try {
+                resp = JSON.parse(incoming);
+            } catch (e) {
+                console.log("[Upload] Unexpected message: ", incoming);
+                return;
+            }
+
+            if (resp.pong !== undefined) {
+                //Heartbeat reply while paused
+                return;
+            } else if (resp.error !== undefined) {
+                fail(resp.error, true);
+            } else if (resp.retryChunk !== undefined) {
+                if (halted()) {
+                    return;
+                }
+                if (retries >= maxRetries) {
+                    console.error("[Upload] Chunk " + resp.retryChunk + " CRC32 mismatch after max retries");
+                    fail("Chunk checksum mismatch after " + maxRetries + " retries", false);
+                    return;
+                }
+                retries++;
+                console.warn("[Upload] Server requested retry for chunk " + resp.retryChunk + " (CRC32 mismatch) - retry " + retries + "/" + maxRetries);
+                rewind(resp.retryChunk);
+            } else if (resp.move !== undefined) {
+                onMove(resp.move);
+            }
+        };
+
+        socket.onclose = function (event) {
+            stopAckTimer();
+            stopPing();
+            onClose(event);
+        };
+
+        socket.onerror = function (error) {
+            onSocketError(error);
+        };
+
+        return {
+            socket: socket,
+            isPaused: function () {
+                return paused;
+            },
+            pause: function () {
+                if (paused || halted()) {
+                    return;
+                }
+                paused = true;
+                //Chunks already in flight still get acknowledged; only the ack
+                //clock stops, or a legitimately idle upload would "time out"
+                stopAckTimer();
+                if (socket.readyState == WebSocket.OPEN) {
+                    send(JSON.stringify({pause: true}));
+                    startPing();
+                }
+            },
+            resume: function () {
+                if (!paused) {
+                    return;
+                }
+                paused = false;
+                stopPing();
+                if (socket.readyState == WebSocket.OPEN) {
+                    send(JSON.stringify({resume: true}));
+                    armAckTimer();
+                    pump();
+                }
+            },
+            abort: function () {
+                aborted = true;
+                stopAckTimer();
+                stopPing();
+                try { socket.close(); } catch (e) {}
+            }
+        };
+    }
+
+    global.ChunkUpload = {
+        start: start,
+        crc32: crc32,
+        crc32Combine: crc32Combine,
+        crc32ToHex: crc32ToHex
+    };
+})(window);