Sfoglia il codice sorgente

Merge branch 'v3.0.5' of https://github.com/tobychui/arozos into v3.0.5

Toby Chui 3 settimane fa
parent
commit
ba4bc1d554

+ 23 - 10
src/file_system.go

@@ -2117,10 +2117,16 @@ func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFi
 		})
 
 		if err != nil {
-			systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
-		}
-
-		if destFsh.RequireBuffer {
+			//The half written archive is already removed by the zipper
+			if destFsh.RequireBuffer {
+				cleanFsBufferFileFromList(realSourceFiles)
+			}
+			if !fileOperationCancelled(oprId) {
+				systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
+				failTask(filepath.Base(outputFilename), err.Error())
+				return
+			}
+		} else if destFsh.RequireBuffer {
 			//Move the buffer result to remote
 			f, _ := os.Open(zipDestPath)
 			err = destFshAbs.WriteStream(outputFilename, f, 0775)
@@ -2267,17 +2273,24 @@ func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFi
 
 	//Check if the operation was cancelled by the user half way through
 	endingSignal := filesystem.FsOpr_Continue
-	if t, err := GetOngoingFileOperationByOprID(oprId); err == nil {
-		fileOprTaskLock.RLock()
-		if t.FileOperationSignal == filesystem.FsOpr_Cancel {
-			endingSignal = filesystem.FsOpr_Cancel
-		}
-		fileOprTaskLock.RUnlock()
+	if fileOperationCancelled(oprId) {
+		endingSignal = filesystem.FsOpr_Cancel
 	}
 
 	SetFileOperationTaskEnded(oprId, endingSignal, "")
 }
 
+// fileOperationCancelled reports whether the user asked to cancel the given operation
+func fileOperationCancelled(oprId string) bool {
+	t, err := GetOngoingFileOperationByOprID(oprId)
+	if err != nil {
+		return false
+	}
+	fileOprTaskLock.RLock()
+	defer fileOprTaskLock.RUnlock()
+	return t.FileOperationSignal == filesystem.FsOpr_Cancel
+}
+
 /*
 	Handle file operations
 

+ 25 - 0
src/file_system_task_test.go

@@ -230,6 +230,31 @@ func TestSetFileOperationTaskEnded(t *testing.T) {
 	}
 }
 
+func TestFileOperationCancelled(t *testing.T) {
+	tests := []struct {
+		name   string
+		signal int
+		want   bool
+	}{
+		{"running", filesystem.FsOpr_Continue, false},
+		{"paused", filesystem.FsOpr_Pause, false},
+		{"cancelled", filesystem.FsOpr_Cancel, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			thisTask := newTestFileOperationTask(t, "test_opr_cancelled_"+tt.name, []int64{100})
+			thisTask.FileOperationSignal = tt.signal
+			if got := fileOperationCancelled(thisTask.ID); got != tt.want {
+				t.Errorf("fileOperationCancelled() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+	if fileOperationCancelled("test_opr_not_exists") {
+		t.Error("an unknown operation should not be reported as cancelled")
+	}
+}
+
 // A file operation stopped by the user reports the abort as an error on its way
 // out. That must be recorded as a cancellation, not as a failure.
 func TestSetFileOperationTaskEndedAfterUserCancel(t *testing.T) {

+ 7 - 1
src/mod/filesystem/abstractions/sftpfs/sftpfs.go

@@ -328,7 +328,13 @@ func (s SFTPFileSystemAbstraction) Walk(root string, walkFn filepath.WalkFunc) e
 	root = arozfs.GenericPathFilter(root)
 	walker := s.client.Walk(root)
 	for walker.Step() {
-		walkFn(walker.Path(), walker.Stat(), walker.Err())
+		err := walkFn(walker.Path(), walker.Stat(), walker.Err())
+		if err == filepath.SkipDir {
+			walker.SkipDir()
+		} else if err != nil {
+			//The callback asked to stop, e.g. the user cancelled the operation
+			return err
+		}
 	}
 	return nil
 }

+ 2 - 2
src/mod/filesystem/abstractions/smbfs/smbfs.go

@@ -341,8 +341,8 @@ func (a ServerMessageBlockFileSystemAbstraction) Walk(root string, walkFn filepa
 		if err != nil {
 			return err
 		}
-		walkFn(filepath.ToSlash(filepath.Join(root, path)), statInfo, err)
-		return nil
+		//Hand the callback result back, so a walk can be stopped or a folder skipped
+		return walkFn(filepath.ToSlash(filepath.Join(root, path)), statInfo, err)
 	})
 	return err
 }

+ 134 - 475
src/mod/filesystem/fileOpr.go

@@ -104,6 +104,11 @@ func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHan
 		archive.Close()
 	}
 
+	//Keep listening to the user while a large file is being extracted
+	reporter := newProgressReporter(func(currentFile string, _ int64, _ int64) int {
+		return progressHandler(currentFile, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
+	}, 0)
+
 	//Start extracting
 	for _, srcFile := range filelist {
 		archive, err := zip.OpenReader(srcFile)
@@ -152,7 +157,8 @@ func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHan
 			if err != nil {
 				return err
 			}
-			_, err = io.Copy(writer, reader)
+			reporter.currentFile = file.Name
+			_, err = io.Copy(writer, newProgressReader(reader, reporter, 0))
 			if err != nil {
 				//Extraction failed. Remove this file if exists
 				writer.Close()
@@ -187,143 +193,7 @@ Returns the following progress: (current filename / current file count / total f
 if output is local path that is out of the scope of any fsh, leave outputFsh as nil
 */
 func ArozZipFileWithProgress(targetFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64) int) error {
-	//fmt.Println("WEBSOCKET ZIPPING", targetFshs, filelist)
-	//Get the file count from the filelist
-	totalFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := targetFshs[i]
-		fshAbs := thisFsh.FileSystemAbstraction
-		if thisFsh.FileSystemAbstraction.IsDir(srcpath) {
-			fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-				if !info.IsDir() {
-					totalFileCount++
-				}
-				return nil
-			})
-		} else {
-			totalFileCount++
-		}
-
-	}
-
-	//Create the target zip file
-	var file arozfs.File
-	var err error
-	if outputFsh != nil {
-		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
-	} else {
-		//Force local fs
-		file, err = os.Create(outputfile)
-	}
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	writer := zip.NewWriter(file)
-	defer writer.Close()
-
-	currentFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := targetFshs[i]
-		fshAbs := thisFsh.FileSystemAbstraction
-		//Local File System
-		if fshAbs.IsDir(srcpath) {
-			//This is a directory
-			topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-			err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-				if err != nil {
-					return err
-				}
-				if info.IsDir() {
-					return nil
-				}
-
-				if insideHiddenFolder(path) {
-					//This is hidden file / folder. Skip this
-					return nil
-				}
-
-				thisFile, err := fshAbs.ReadStream(path)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-				if includeTopLevelFolder {
-					relativePath = topLevelFolderName + "/" + relativePath
-				} else {
-					relativePath = arozfs.Base(srcpath) + "/" + relativePath
-				}
-
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
-					return err
-				}
-
-				//Update the zip progress
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-				for statusCode == 1 {
-					//Wait for the task to be resumed
-					time.Sleep(1 * time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-				}
-				if statusCode == 2 {
-					//Cancel
-					return errors.New("Operation cancelled by user")
-				}
-				return nil
-			})
-
-			if err != nil {
-				return err
-			}
-		} else {
-			//This is a file
-			topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-			thisFile, err := fshAbs.ReadStream(srcpath)
-			if err != nil {
-				return err
-			}
-			defer thisFile.Close()
-			relativePath := arozfs.Base(srcpath)
-			if includeTopLevelFolder {
-				relativePath = topLevelFolderName + "/" + relativePath
-			}
-
-			f, err := writer.Create(relativePath)
-			if err != nil {
-				return err
-			}
-
-			_, err = io.Copy(f, thisFile)
-			if err != nil {
-				return err
-			}
-
-			//Update the zip progress
-			currentFileCount++
-			statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-			for statusCode == 1 {
-				//Wait for the task to be resumed
-				time.Sleep(1 * time.Second)
-				statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-			}
-			if statusCode == 2 {
-				//Cancel
-				return errors.New("Operation cancelled by user")
-			}
-		}
-	}
-
-	return nil
+	return ArozZipFileWithProgressAndCompression(targetFshs, filelist, outputFsh, outputfile, includeTopLevelFolder, flate.DefaultCompression, progressHandler)
 }
 
 /*
@@ -340,6 +210,64 @@ func ArozZipFile(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *
 }
 
 func ArozZipFileWithCompressionLevel(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int) error {
+	return ArozZipFileWithProgressAndCompression(sourceFshs, filelist, outputFsh, outputfile, includeTopLevelFolder, compressionLevel, nil)
+}
+
+/*
+ArozZipFileWithProgressAndCompression combines compression level control with real-time progress updates.
+Accepts nil fsh entries for locally-buffered source paths, a nil outputFsh for a
+local output file and a nil progressHandler for zipping without progress updates.
+
+The progress handler returns the FsOpr_* control signal of the operation, and it
+is asked for it all the way through: while the source folders are scanned,
+after every file and while a large file streams into the archive. A pause holds
+the zipping where it stands and a cancel stops it within a fraction of a second,
+instead of only once the scan or the current file is done. An archive that is
+cancelled or fails half way is removed rather than left behind half written.
+*/
+func ArozZipFileWithProgressAndCompression(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int, progressHandler func(string, int, int, float64) int) error {
+	//The archive progress is counted in files. It goes through a progressReporter
+	//so the handler calls are rate limited and its pause / cancel signal applied
+	currentFileCount := 0
+	totalFileCount := 0
+	var reporter *progressReporter
+	if progressHandler != nil {
+		reporter = newProgressReporter(func(currentFile string, _ int64, _ int64) int {
+			progress := float64(0)
+			if totalFileCount > 0 {
+				progress = float64(currentFileCount) / float64(totalFileCount) * 100
+			}
+			return progressHandler(currentFile, currentFileCount, totalFileCount, progress)
+		}, 0)
+
+		//Count the files first so the progress has a total to measure against
+		for i, srcpath := range filelist {
+			fshAbs := zipSourceAbstraction(sourceFshs[i])
+			reporter.currentFile = arozfs.Base(srcpath)
+			if !fshAbs.IsDir(srcpath) {
+				totalFileCount++
+				continue
+			}
+
+			fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+				//Scanning a large folder takes a while, keep listening to the user
+				if reportErr := reporter.report(0, false); reportErr != nil {
+					return reportErr
+				}
+				if err != nil || info == nil || info.IsDir() || insideHiddenFolder(path) {
+					//Unreadable entries fail the zipping below, not the count
+					return nil
+				}
+				totalFileCount++
+				return nil
+			})
+
+			//The walk may have ended on a pause or a cancel, see where it stands
+			if err := reporter.report(0, true); err != nil {
+				return err
+			}
+		}
+	}
 
 	//Create the target zip file
 	var file arozfs.File
@@ -353,384 +281,115 @@ func ArozZipFileWithCompressionLevel(sourceFshs []*FileSystemHandler, filelist [
 	if err != nil {
 		return err
 	}
-	defer file.Close()
 
 	writer := zip.NewWriter(file)
-	defer writer.Close()
 	writer.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
 		return flate.NewWriter(out, compressionLevel)
 	})
 
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-		var fshAbs FileSystemAbstraction
-		if thisFsh == nil {
-			//Use local fs functions
-			if IsDir(srcpath) {
-				//This is a directory
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-
-					if insideHiddenFolder(path) {
-						//This is hidden file / folder. Skip this
-						return nil
-					}
-					thisFile, err := os.Open(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-
-					_, err = io.Copy(f, thisFile)
-					if err != nil {
-						return err
-					}
-
-					return nil
-				})
-
-				if err != nil {
-					return err
-				}
-			} else {
+	err = func() error {
+		for i, srcpath := range filelist {
+			fshAbs := zipSourceAbstraction(sourceFshs[i])
+			if !fshAbs.IsDir(srcpath) {
 				//This is a file
-				topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-				thisFile, err := os.Open(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
 				relativePath := arozfs.Base(srcpath)
 				if includeTopLevelFolder {
-					relativePath = topLevelFolderName + "/" + relativePath
+					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
 				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
+				if err := addZipEntry(writer, fshAbs, srcpath, relativePath, reporter); err != nil {
 					return err
 				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
+				currentFileCount++
+				if err := reporter.report(0, true); err != nil {
 					return err
 				}
-
+				continue
 			}
-		} else {
-			//Use file system abstraction
-			fshAbs = thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				//This is a directory
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-
-					if info.IsDir() {
-						return nil
-					}
-
-					if insideHiddenFolder(path) {
-						//This is hidden file / folder. Skip this
-						return nil
-					}
-
-					thisFile, err := fshAbs.ReadStream(path)
-					if err != nil {
-						fmt.Println(err)
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-
-					_, err = io.Copy(f, thisFile)
-					if err != nil {
-						return err
-					}
-
-					return nil
-				})
 
+			//This is a directory
+			topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
+			err := fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 					return err
 				}
-			} else {
-				//This is a file
-				topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-				thisFile, err := fshAbs.ReadStream(srcpath)
-				if err != nil {
-					return err
+				if info.IsDir() {
+					return nil
 				}
-				defer thisFile.Close()
-				relativePath := arozfs.Base(srcpath)
+				if insideHiddenFolder(path) {
+					//This is hidden file / folder. Skip this
+					return nil
+				}
+
+				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
 				if includeTopLevelFolder {
 					relativePath = topLevelFolderName + "/" + relativePath
+				} else {
+					relativePath = arozfs.Base(srcpath) + "/" + relativePath
 				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
+				if err := addZipEntry(writer, fshAbs, path, relativePath, reporter); err != nil {
 					return err
 				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
-					return err
-				}
-
+				currentFileCount++
+				return reporter.report(0, true)
+			})
+			if err != nil {
+				return err
 			}
 		}
+		return nil
+	}()
+
+	//Close the writer before the file, it still has the central directory to write
+	if closeErr := writer.Close(); err == nil {
+		err = closeErr
+	}
+	if closeErr := file.Close(); err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		//Never leave a half written archive behind
+		if outputFsh != nil {
+			outputFsh.FileSystemAbstraction.Remove(outputfile)
+		} else {
+			os.Remove(outputfile)
+		}
+		return err
 	}
 
 	return nil
 }
 
-// ArozZipFileWithProgressAndCompression combines compression level control with real-time progress updates.
-// Accepts nil fsh entries for locally-buffered source paths.
-func ArozZipFileWithProgressAndCompression(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int, progressHandler func(string, int, int, float64) int) error {
-	// Count total files for progress calculation
-	totalFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-		if thisFsh == nil {
-			if IsDir(srcpath) {
-				filepath.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-					if !info.IsDir() {
-						totalFileCount++
-					}
-					return nil
-				})
-			} else {
-				totalFileCount++
-			}
-		} else {
-			fshAbs := thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-					if !info.IsDir() {
-						totalFileCount++
-					}
-					return nil
-				})
-			} else {
-				totalFileCount++
-			}
-		}
+// zipSourceAbstraction returns the abstraction a zip source is read through.
+// A nil handler stands for a path on the local disk, like a remote file that
+// was buffered to local first.
+func zipSourceAbstraction(fsh *FileSystemHandler) FileSystemAbstraction {
+	if fsh == nil {
+		return localfs.NewLocalFileSystemAbstraction("", "", "", true)
 	}
+	return fsh.FileSystemAbstraction
+}
 
-	var file arozfs.File
-	var err error
-	if outputFsh != nil {
-		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
-	} else {
-		file, err = os.Create(outputfile)
+// addZipEntry streams one source file into the archive under relativePath.
+// The stream reports to the reporter, so a pause or cancel also reaches a file
+// that takes minutes to compress. Pass in a nil reporter to skip the reporting.
+func addZipEntry(writer *zip.Writer, fshAbs FileSystemAbstraction, path string, relativePath string, reporter *progressReporter) error {
+	if reporter != nil {
+		reporter.currentFile = arozfs.Base(path)
 	}
+
+	thisFile, err := fshAbs.ReadStream(path)
 	if err != nil {
 		return err
 	}
-	defer file.Close()
-
-	writer := zip.NewWriter(file)
-	defer writer.Close()
-	writer.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
-		return flate.NewWriter(out, compressionLevel)
-	})
-
-	currentFileCount := 0
-
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-
-		if thisFsh == nil {
-			// Local filesystem path (e.g. after buffering a remote FS)
-			if IsDir(srcpath) {
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-					if insideHiddenFolder(path) {
-						return nil
-					}
-
-					thisFile, err := os.Open(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-					if _, err = io.Copy(f, thisFile); err != nil {
-						return err
-					}
-
-					currentFileCount++
-					statusCode := progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					for statusCode == 1 {
-						time.Sleep(time.Second)
-						statusCode = progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					}
-					if statusCode == 2 {
-						return errors.New("Operation cancelled by user")
-					}
-					return nil
-				})
-				if err != nil {
-					return err
-				}
-			} else {
-				thisFile, err := os.Open(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := arozfs.Base(srcpath)
-				if includeTopLevelFolder {
-					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
-				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-				if _, err = io.Copy(f, thisFile); err != nil {
-					return err
-				}
-
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				for statusCode == 1 {
-					time.Sleep(time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				}
-				if statusCode == 2 {
-					return errors.New("Operation cancelled by user")
-				}
-			}
-		} else {
-			// FileSystemHandler abstraction (remote FS)
-			fshAbs := thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-					if insideHiddenFolder(path) {
-						return nil
-					}
-
-					thisFile, err := fshAbs.ReadStream(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-					if _, err = io.Copy(f, thisFile); err != nil {
-						return err
-					}
-
-					currentFileCount++
-					statusCode := progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					for statusCode == 1 {
-						time.Sleep(time.Second)
-						statusCode = progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					}
-					if statusCode == 2 {
-						return errors.New("Operation cancelled by user")
-					}
-					return nil
-				})
-				if err != nil {
-					return err
-				}
-			} else {
-				thisFile, err := fshAbs.ReadStream(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := arozfs.Base(srcpath)
-				if includeTopLevelFolder {
-					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
-				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-				if _, err = io.Copy(f, thisFile); err != nil {
-					return err
-				}
+	defer thisFile.Close()
 
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				for statusCode == 1 {
-					time.Sleep(time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				}
-				if statusCode == 2 {
-					return errors.New("Operation cancelled by user")
-				}
-			}
-		}
+	f, err := writer.Create(relativePath)
+	if err != nil {
+		return err
 	}
 
-	return nil
+	_, err = io.Copy(f, newProgressReader(thisFile, reporter, 0))
+	return err
 }
 
 func insideHiddenFolder(path string) bool {

+ 205 - 0
src/mod/filesystem/fileOprZip_test.go

@@ -0,0 +1,205 @@
+package filesystem
+
+import (
+	"archive/zip"
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+// makeZipSourceFolder creates a folder with fileCount small files and one
+// hidden file, and returns its path
+func makeZipSourceFolder(t *testing.T, fileCount int) string {
+	t.Helper()
+	srcFolder := filepath.Join(t.TempDir(), "source")
+	if err := os.MkdirAll(filepath.Join(srcFolder, "sub"), 0755); err != nil {
+		t.Fatalf("failed to create source folder: %v", err)
+	}
+	for i := 0; i < fileCount; i++ {
+		thisFile := filepath.Join(srcFolder, "file"+strconv.Itoa(i)+".txt")
+		if i%2 == 1 {
+			thisFile = filepath.Join(srcFolder, "sub", "file"+strconv.Itoa(i)+".txt")
+		}
+		if err := os.WriteFile(thisFile, []byte("content "+strconv.Itoa(i)), 0644); err != nil {
+			t.Fatalf("failed to write source file: %v", err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(srcFolder, ".hidden"), []byte("hidden"), 0644); err != nil {
+		t.Fatalf("failed to write hidden file: %v", err)
+	}
+	return srcFolder
+}
+
+func TestArozZipFileWithProgressCompletes(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 6)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	lastCurrent, lastTotal, lastProgress := 0, 0, float64(0)
+	err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false, func(_ string, current int, total int, progress float64) int {
+		lastCurrent, lastTotal, lastProgress = current, total, progress
+		return FsOpr_Continue
+	})
+	if err != nil {
+		t.Fatalf("ArozZipFileWithProgress() returned error: %v", err)
+	}
+
+	//The hidden file is neither zipped nor counted, so the progress ends at 100%
+	if lastTotal != 6 || lastCurrent != 6 {
+		t.Errorf("last progress report = %d / %d files, want 6 / 6", lastCurrent, lastTotal)
+	}
+	if lastProgress != 100 {
+		t.Errorf("last progress = %v, want 100", lastProgress)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 6 {
+		t.Errorf("archive holds %d entries, want 6", len(archive.File))
+	}
+}
+
+func TestArozZipFileWithoutProgressHandler(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 3)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	if err := ArozZipFile([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false); err != nil {
+		t.Fatalf("ArozZipFile() returned error: %v", err)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 3 {
+		t.Errorf("archive holds %d entries, want 3", len(archive.File))
+	}
+}
+
+func TestArozZipFileWithProgressCancel(t *testing.T) {
+	tests := []struct {
+		name string
+		//Whether the handler cancels at this report
+		cancelAt func(current int, total int) bool
+		makeSrc  func(t *testing.T) string
+	}{
+		{
+			//Nothing is counted yet while the source folder is still being scanned
+			name:     "while scanning",
+			cancelAt: func(current int, total int) bool { return total == 0 },
+			makeSrc:  func(t *testing.T) string { return makeZipSourceFolder(t, 20) },
+		},
+		{
+			//The only file is counted but not finished, so it is still streaming
+			name:     "inside a file",
+			cancelAt: func(current int, total int) bool { return total == 1 && current == 0 },
+			makeSrc: func(t *testing.T) string {
+				srcFile := filepath.Join(t.TempDir(), "large.bin")
+				if err := os.WriteFile(srcFile, bytes.Repeat([]byte("arozos"), 1<<20), 0644); err != nil {
+					t.Fatalf("failed to write source file: %v", err)
+				}
+				return srcFile
+			},
+		},
+		{
+			name:     "between files",
+			cancelAt: func(current int, total int) bool { return current == 2 },
+			makeSrc:  func(t *testing.T) string { return makeZipSourceFolder(t, 10) },
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src := tt.makeSrc(t)
+			output := filepath.Join(t.TempDir(), "out.zip")
+
+			cancelled := false
+			err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{src}, nil, output, false, func(_ string, current int, total int, _ float64) int {
+				if cancelled || tt.cancelAt(current, total) {
+					cancelled = true
+					return FsOpr_Cancel
+				}
+				return FsOpr_Continue
+			})
+
+			if !cancelled {
+				t.Fatal("the handler never reached the point where it cancels")
+			}
+			if err == nil {
+				t.Fatal("ArozZipFileWithProgress() should return an error once cancelled")
+			}
+			if _, statErr := os.Stat(output); !os.IsNotExist(statErr) {
+				t.Errorf("a cancelled archive should be removed, stat error = %v", statErr)
+			}
+		})
+	}
+}
+
+func TestArozZipFileWithProgressPauseWhileScanning(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 4)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	//Pause on the first report, which comes from the scan, then resume
+	signals := []int{FsOpr_Pause, FsOpr_Pause}
+	pausedReports := 0
+	err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false, func(_ string, current int, total int, _ float64) int {
+		if len(signals) > 0 {
+			if total != 0 {
+				t.Errorf("the pause should be applied while scanning, got a report of %d / %d files", current, total)
+			}
+			thisSignal := signals[0]
+			signals = signals[1:]
+			pausedReports++
+			return thisSignal
+		}
+		return FsOpr_Continue
+	})
+	if err != nil {
+		t.Fatalf("ArozZipFileWithProgress() returned error after resuming: %v", err)
+	}
+	if pausedReports != 2 {
+		t.Errorf("handler answered %d pause signals, want 2", pausedReports)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 4 {
+		t.Errorf("archive holds %d entries, want 4", len(archive.File))
+	}
+}
+
+func TestArozUnzipFileWithProgressCancelInsideFile(t *testing.T) {
+	//Build an archive holding one large file
+	tmp := t.TempDir()
+	srcFile := filepath.Join(tmp, "large.bin")
+	if err := os.WriteFile(srcFile, bytes.Repeat([]byte("arozos"), 1<<20), 0644); err != nil {
+		t.Fatalf("failed to write source file: %v", err)
+	}
+	archivePath := filepath.Join(tmp, "large.zip")
+	if err := ArozZipFile([]*FileSystemHandler{nil}, []string{srcFile}, nil, archivePath, false); err != nil {
+		t.Fatalf("ArozZipFile() returned error: %v", err)
+	}
+
+	//The file count only moves once a file is done, so 0 means mid stream
+	dest := filepath.Join(tmp, "extracted")
+	err := ArozUnzipFileWithProgress([]string{archivePath}, dest, func(_ string, current int, _ int, _ float64) int {
+		if current == 0 {
+			return FsOpr_Cancel
+		}
+		return FsOpr_Continue
+	})
+	if err == nil {
+		t.Fatal("ArozUnzipFileWithProgress() should return an error once cancelled")
+	}
+	if _, statErr := os.Stat(filepath.Join(dest, "large.bin")); !os.IsNotExist(statErr) {
+		t.Errorf("a cancelled extraction should not leave the partial file behind, stat error = %v", statErr)
+	}
+}