فهرست منبع

Merge branch 'v3.0.5' of https://github.com/tobychui/arozos into v3.0.5

Toby Chui 3 هفته پیش
والد
کامیت
ba4bc1d554

+ 23 - 10
src/file_system.go

@@ -2117,10 +2117,16 @@ func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFi
 		})
 		})
 
 
 		if err != nil {
 		if err != nil {
-			systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
-		}
-
-		if destFsh.RequireBuffer {
+			//The half written archive is already removed by the zipper
+			if destFsh.RequireBuffer {
+				cleanFsBufferFileFromList(realSourceFiles)
+			}
+			if !fileOperationCancelled(oprId) {
+				systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
+				failTask(filepath.Base(outputFilename), err.Error())
+				return
+			}
+		} else if destFsh.RequireBuffer {
 			//Move the buffer result to remote
 			//Move the buffer result to remote
 			f, _ := os.Open(zipDestPath)
 			f, _ := os.Open(zipDestPath)
 			err = destFshAbs.WriteStream(outputFilename, f, 0775)
 			err = destFshAbs.WriteStream(outputFilename, f, 0775)
@@ -2267,17 +2273,24 @@ func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFi
 
 
 	//Check if the operation was cancelled by the user half way through
 	//Check if the operation was cancelled by the user half way through
 	endingSignal := filesystem.FsOpr_Continue
 	endingSignal := filesystem.FsOpr_Continue
-	if t, err := GetOngoingFileOperationByOprID(oprId); err == nil {
-		fileOprTaskLock.RLock()
-		if t.FileOperationSignal == filesystem.FsOpr_Cancel {
-			endingSignal = filesystem.FsOpr_Cancel
-		}
-		fileOprTaskLock.RUnlock()
+	if fileOperationCancelled(oprId) {
+		endingSignal = filesystem.FsOpr_Cancel
 	}
 	}
 
 
 	SetFileOperationTaskEnded(oprId, endingSignal, "")
 	SetFileOperationTaskEnded(oprId, endingSignal, "")
 }
 }
 
 
+// fileOperationCancelled reports whether the user asked to cancel the given operation
+func fileOperationCancelled(oprId string) bool {
+	t, err := GetOngoingFileOperationByOprID(oprId)
+	if err != nil {
+		return false
+	}
+	fileOprTaskLock.RLock()
+	defer fileOprTaskLock.RUnlock()
+	return t.FileOperationSignal == filesystem.FsOpr_Cancel
+}
+
 /*
 /*
 	Handle file operations
 	Handle file operations
 
 

+ 25 - 0
src/file_system_task_test.go

@@ -230,6 +230,31 @@ func TestSetFileOperationTaskEnded(t *testing.T) {
 	}
 	}
 }
 }
 
 
+func TestFileOperationCancelled(t *testing.T) {
+	tests := []struct {
+		name   string
+		signal int
+		want   bool
+	}{
+		{"running", filesystem.FsOpr_Continue, false},
+		{"paused", filesystem.FsOpr_Pause, false},
+		{"cancelled", filesystem.FsOpr_Cancel, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			thisTask := newTestFileOperationTask(t, "test_opr_cancelled_"+tt.name, []int64{100})
+			thisTask.FileOperationSignal = tt.signal
+			if got := fileOperationCancelled(thisTask.ID); got != tt.want {
+				t.Errorf("fileOperationCancelled() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+
+	if fileOperationCancelled("test_opr_not_exists") {
+		t.Error("an unknown operation should not be reported as cancelled")
+	}
+}
+
 // A file operation stopped by the user reports the abort as an error on its way
 // A file operation stopped by the user reports the abort as an error on its way
 // out. That must be recorded as a cancellation, not as a failure.
 // out. That must be recorded as a cancellation, not as a failure.
 func TestSetFileOperationTaskEndedAfterUserCancel(t *testing.T) {
 func TestSetFileOperationTaskEndedAfterUserCancel(t *testing.T) {

+ 7 - 1
src/mod/filesystem/abstractions/sftpfs/sftpfs.go

@@ -328,7 +328,13 @@ func (s SFTPFileSystemAbstraction) Walk(root string, walkFn filepath.WalkFunc) e
 	root = arozfs.GenericPathFilter(root)
 	root = arozfs.GenericPathFilter(root)
 	walker := s.client.Walk(root)
 	walker := s.client.Walk(root)
 	for walker.Step() {
 	for walker.Step() {
-		walkFn(walker.Path(), walker.Stat(), walker.Err())
+		err := walkFn(walker.Path(), walker.Stat(), walker.Err())
+		if err == filepath.SkipDir {
+			walker.SkipDir()
+		} else if err != nil {
+			//The callback asked to stop, e.g. the user cancelled the operation
+			return err
+		}
 	}
 	}
 	return nil
 	return nil
 }
 }

+ 2 - 2
src/mod/filesystem/abstractions/smbfs/smbfs.go

@@ -341,8 +341,8 @@ func (a ServerMessageBlockFileSystemAbstraction) Walk(root string, walkFn filepa
 		if err != nil {
 		if err != nil {
 			return err
 			return err
 		}
 		}
-		walkFn(filepath.ToSlash(filepath.Join(root, path)), statInfo, err)
-		return nil
+		//Hand the callback result back, so a walk can be stopped or a folder skipped
+		return walkFn(filepath.ToSlash(filepath.Join(root, path)), statInfo, err)
 	})
 	})
 	return err
 	return err
 }
 }

+ 134 - 475
src/mod/filesystem/fileOpr.go

@@ -104,6 +104,11 @@ func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHan
 		archive.Close()
 		archive.Close()
 	}
 	}
 
 
+	//Keep listening to the user while a large file is being extracted
+	reporter := newProgressReporter(func(currentFile string, _ int64, _ int64) int {
+		return progressHandler(currentFile, unzippedFileCount, totalFileCounts, float64(unzippedFileCount)/float64(totalFileCounts)*100.0)
+	}, 0)
+
 	//Start extracting
 	//Start extracting
 	for _, srcFile := range filelist {
 	for _, srcFile := range filelist {
 		archive, err := zip.OpenReader(srcFile)
 		archive, err := zip.OpenReader(srcFile)
@@ -152,7 +157,8 @@ func ArozUnzipFileWithProgress(filelist []string, outputfile string, progressHan
 			if err != nil {
 			if err != nil {
 				return err
 				return err
 			}
 			}
-			_, err = io.Copy(writer, reader)
+			reporter.currentFile = file.Name
+			_, err = io.Copy(writer, newProgressReader(reader, reporter, 0))
 			if err != nil {
 			if err != nil {
 				//Extraction failed. Remove this file if exists
 				//Extraction failed. Remove this file if exists
 				writer.Close()
 				writer.Close()
@@ -187,143 +193,7 @@ Returns the following progress: (current filename / current file count / total f
 if output is local path that is out of the scope of any fsh, leave outputFsh as nil
 if output is local path that is out of the scope of any fsh, leave outputFsh as nil
 */
 */
 func ArozZipFileWithProgress(targetFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64) int) error {
 func ArozZipFileWithProgress(targetFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, progressHandler func(string, int, int, float64) int) error {
-	//fmt.Println("WEBSOCKET ZIPPING", targetFshs, filelist)
-	//Get the file count from the filelist
-	totalFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := targetFshs[i]
-		fshAbs := thisFsh.FileSystemAbstraction
-		if thisFsh.FileSystemAbstraction.IsDir(srcpath) {
-			fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-				if !info.IsDir() {
-					totalFileCount++
-				}
-				return nil
-			})
-		} else {
-			totalFileCount++
-		}
-
-	}
-
-	//Create the target zip file
-	var file arozfs.File
-	var err error
-	if outputFsh != nil {
-		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
-	} else {
-		//Force local fs
-		file, err = os.Create(outputfile)
-	}
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-
-	writer := zip.NewWriter(file)
-	defer writer.Close()
-
-	currentFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := targetFshs[i]
-		fshAbs := thisFsh.FileSystemAbstraction
-		//Local File System
-		if fshAbs.IsDir(srcpath) {
-			//This is a directory
-			topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-			err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-				if err != nil {
-					return err
-				}
-				if info.IsDir() {
-					return nil
-				}
-
-				if insideHiddenFolder(path) {
-					//This is hidden file / folder. Skip this
-					return nil
-				}
-
-				thisFile, err := fshAbs.ReadStream(path)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-				if includeTopLevelFolder {
-					relativePath = topLevelFolderName + "/" + relativePath
-				} else {
-					relativePath = arozfs.Base(srcpath) + "/" + relativePath
-				}
-
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
-					return err
-				}
-
-				//Update the zip progress
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-				for statusCode == 1 {
-					//Wait for the task to be resumed
-					time.Sleep(1 * time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-				}
-				if statusCode == 2 {
-					//Cancel
-					return errors.New("Operation cancelled by user")
-				}
-				return nil
-			})
-
-			if err != nil {
-				return err
-			}
-		} else {
-			//This is a file
-			topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-			thisFile, err := fshAbs.ReadStream(srcpath)
-			if err != nil {
-				return err
-			}
-			defer thisFile.Close()
-			relativePath := arozfs.Base(srcpath)
-			if includeTopLevelFolder {
-				relativePath = topLevelFolderName + "/" + relativePath
-			}
-
-			f, err := writer.Create(relativePath)
-			if err != nil {
-				return err
-			}
-
-			_, err = io.Copy(f, thisFile)
-			if err != nil {
-				return err
-			}
-
-			//Update the zip progress
-			currentFileCount++
-			statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-			for statusCode == 1 {
-				//Wait for the task to be resumed
-				time.Sleep(1 * time.Second)
-				statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, (float64(currentFileCount)/float64(totalFileCount))*float64(100))
-			}
-			if statusCode == 2 {
-				//Cancel
-				return errors.New("Operation cancelled by user")
-			}
-		}
-	}
-
-	return nil
+	return ArozZipFileWithProgressAndCompression(targetFshs, filelist, outputFsh, outputfile, includeTopLevelFolder, flate.DefaultCompression, progressHandler)
 }
 }
 
 
 /*
 /*
@@ -340,6 +210,64 @@ func ArozZipFile(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *
 }
 }
 
 
 func ArozZipFileWithCompressionLevel(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int) error {
 func ArozZipFileWithCompressionLevel(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int) error {
+	return ArozZipFileWithProgressAndCompression(sourceFshs, filelist, outputFsh, outputfile, includeTopLevelFolder, compressionLevel, nil)
+}
+
+/*
+ArozZipFileWithProgressAndCompression combines compression level control with real-time progress updates.
+Accepts nil fsh entries for locally-buffered source paths, a nil outputFsh for a
+local output file and a nil progressHandler for zipping without progress updates.
+
+The progress handler returns the FsOpr_* control signal of the operation, and it
+is asked for it all the way through: while the source folders are scanned,
+after every file and while a large file streams into the archive. A pause holds
+the zipping where it stands and a cancel stops it within a fraction of a second,
+instead of only once the scan or the current file is done. An archive that is
+cancelled or fails half way is removed rather than left behind half written.
+*/
+func ArozZipFileWithProgressAndCompression(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int, progressHandler func(string, int, int, float64) int) error {
+	//The archive progress is counted in files. It goes through a progressReporter
+	//so the handler calls are rate limited and its pause / cancel signal applied
+	currentFileCount := 0
+	totalFileCount := 0
+	var reporter *progressReporter
+	if progressHandler != nil {
+		reporter = newProgressReporter(func(currentFile string, _ int64, _ int64) int {
+			progress := float64(0)
+			if totalFileCount > 0 {
+				progress = float64(currentFileCount) / float64(totalFileCount) * 100
+			}
+			return progressHandler(currentFile, currentFileCount, totalFileCount, progress)
+		}, 0)
+
+		//Count the files first so the progress has a total to measure against
+		for i, srcpath := range filelist {
+			fshAbs := zipSourceAbstraction(sourceFshs[i])
+			reporter.currentFile = arozfs.Base(srcpath)
+			if !fshAbs.IsDir(srcpath) {
+				totalFileCount++
+				continue
+			}
+
+			fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
+				//Scanning a large folder takes a while, keep listening to the user
+				if reportErr := reporter.report(0, false); reportErr != nil {
+					return reportErr
+				}
+				if err != nil || info == nil || info.IsDir() || insideHiddenFolder(path) {
+					//Unreadable entries fail the zipping below, not the count
+					return nil
+				}
+				totalFileCount++
+				return nil
+			})
+
+			//The walk may have ended on a pause or a cancel, see where it stands
+			if err := reporter.report(0, true); err != nil {
+				return err
+			}
+		}
+	}
 
 
 	//Create the target zip file
 	//Create the target zip file
 	var file arozfs.File
 	var file arozfs.File
@@ -353,384 +281,115 @@ func ArozZipFileWithCompressionLevel(sourceFshs []*FileSystemHandler, filelist [
 	if err != nil {
 	if err != nil {
 		return err
 		return err
 	}
 	}
-	defer file.Close()
 
 
 	writer := zip.NewWriter(file)
 	writer := zip.NewWriter(file)
-	defer writer.Close()
 	writer.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
 	writer.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
 		return flate.NewWriter(out, compressionLevel)
 		return flate.NewWriter(out, compressionLevel)
 	})
 	})
 
 
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-		var fshAbs FileSystemAbstraction
-		if thisFsh == nil {
-			//Use local fs functions
-			if IsDir(srcpath) {
-				//This is a directory
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-
-					if insideHiddenFolder(path) {
-						//This is hidden file / folder. Skip this
-						return nil
-					}
-					thisFile, err := os.Open(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-
-					_, err = io.Copy(f, thisFile)
-					if err != nil {
-						return err
-					}
-
-					return nil
-				})
-
-				if err != nil {
-					return err
-				}
-			} else {
+	err = func() error {
+		for i, srcpath := range filelist {
+			fshAbs := zipSourceAbstraction(sourceFshs[i])
+			if !fshAbs.IsDir(srcpath) {
 				//This is a file
 				//This is a file
-				topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-				thisFile, err := os.Open(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
 				relativePath := arozfs.Base(srcpath)
 				relativePath := arozfs.Base(srcpath)
 				if includeTopLevelFolder {
 				if includeTopLevelFolder {
-					relativePath = topLevelFolderName + "/" + relativePath
+					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
 				}
 				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
+				if err := addZipEntry(writer, fshAbs, srcpath, relativePath, reporter); err != nil {
 					return err
 					return err
 				}
 				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
+				currentFileCount++
+				if err := reporter.report(0, true); err != nil {
 					return err
 					return err
 				}
 				}
-
+				continue
 			}
 			}
-		} else {
-			//Use file system abstraction
-			fshAbs = thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				//This is a directory
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-
-					if info.IsDir() {
-						return nil
-					}
-
-					if insideHiddenFolder(path) {
-						//This is hidden file / folder. Skip this
-						return nil
-					}
-
-					thisFile, err := fshAbs.ReadStream(path)
-					if err != nil {
-						fmt.Println(err)
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-
-					_, err = io.Copy(f, thisFile)
-					if err != nil {
-						return err
-					}
-
-					return nil
-				})
 
 
+			//This is a directory
+			topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
+			err := fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
 				if err != nil {
 				if err != nil {
 					return err
 					return err
 				}
 				}
-			} else {
-				//This is a file
-				topLevelFolderName := arozfs.Base(filepath.Dir(srcpath))
-				thisFile, err := fshAbs.ReadStream(srcpath)
-				if err != nil {
-					return err
+				if info.IsDir() {
+					return nil
 				}
 				}
-				defer thisFile.Close()
-				relativePath := arozfs.Base(srcpath)
+				if insideHiddenFolder(path) {
+					//This is hidden file / folder. Skip this
+					return nil
+				}
+
+				relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
 				if includeTopLevelFolder {
 				if includeTopLevelFolder {
 					relativePath = topLevelFolderName + "/" + relativePath
 					relativePath = topLevelFolderName + "/" + relativePath
+				} else {
+					relativePath = arozfs.Base(srcpath) + "/" + relativePath
 				}
 				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
+				if err := addZipEntry(writer, fshAbs, path, relativePath, reporter); err != nil {
 					return err
 					return err
 				}
 				}
-
-				_, err = io.Copy(f, thisFile)
-				if err != nil {
-					return err
-				}
-
+				currentFileCount++
+				return reporter.report(0, true)
+			})
+			if err != nil {
+				return err
 			}
 			}
 		}
 		}
+		return nil
+	}()
+
+	//Close the writer before the file, it still has the central directory to write
+	if closeErr := writer.Close(); err == nil {
+		err = closeErr
+	}
+	if closeErr := file.Close(); err == nil {
+		err = closeErr
+	}
+	if err != nil {
+		//Never leave a half written archive behind
+		if outputFsh != nil {
+			outputFsh.FileSystemAbstraction.Remove(outputfile)
+		} else {
+			os.Remove(outputfile)
+		}
+		return err
 	}
 	}
 
 
 	return nil
 	return nil
 }
 }
 
 
-// ArozZipFileWithProgressAndCompression combines compression level control with real-time progress updates.
-// Accepts nil fsh entries for locally-buffered source paths.
-func ArozZipFileWithProgressAndCompression(sourceFshs []*FileSystemHandler, filelist []string, outputFsh *FileSystemHandler, outputfile string, includeTopLevelFolder bool, compressionLevel int, progressHandler func(string, int, int, float64) int) error {
-	// Count total files for progress calculation
-	totalFileCount := 0
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-		if thisFsh == nil {
-			if IsDir(srcpath) {
-				filepath.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-					if !info.IsDir() {
-						totalFileCount++
-					}
-					return nil
-				})
-			} else {
-				totalFileCount++
-			}
-		} else {
-			fshAbs := thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				fshAbs.Walk(srcpath, func(_ string, info os.FileInfo, _ error) error {
-					if !info.IsDir() {
-						totalFileCount++
-					}
-					return nil
-				})
-			} else {
-				totalFileCount++
-			}
-		}
+// zipSourceAbstraction returns the abstraction a zip source is read through.
+// A nil handler stands for a path on the local disk, like a remote file that
+// was buffered to local first.
+func zipSourceAbstraction(fsh *FileSystemHandler) FileSystemAbstraction {
+	if fsh == nil {
+		return localfs.NewLocalFileSystemAbstraction("", "", "", true)
 	}
 	}
+	return fsh.FileSystemAbstraction
+}
 
 
-	var file arozfs.File
-	var err error
-	if outputFsh != nil {
-		file, err = outputFsh.FileSystemAbstraction.Create(outputfile)
-	} else {
-		file, err = os.Create(outputfile)
+// addZipEntry streams one source file into the archive under relativePath.
+// The stream reports to the reporter, so a pause or cancel also reaches a file
+// that takes minutes to compress. Pass in a nil reporter to skip the reporting.
+func addZipEntry(writer *zip.Writer, fshAbs FileSystemAbstraction, path string, relativePath string, reporter *progressReporter) error {
+	if reporter != nil {
+		reporter.currentFile = arozfs.Base(path)
 	}
 	}
+
+	thisFile, err := fshAbs.ReadStream(path)
 	if err != nil {
 	if err != nil {
 		return err
 		return err
 	}
 	}
-	defer file.Close()
-
-	writer := zip.NewWriter(file)
-	defer writer.Close()
-	writer.RegisterCompressor(zip.Deflate, func(out io.Writer) (io.WriteCloser, error) {
-		return flate.NewWriter(out, compressionLevel)
-	})
-
-	currentFileCount := 0
-
-	for i, srcpath := range filelist {
-		thisFsh := sourceFshs[i]
-
-		if thisFsh == nil {
-			// Local filesystem path (e.g. after buffering a remote FS)
-			if IsDir(srcpath) {
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = filepath.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-					if insideHiddenFolder(path) {
-						return nil
-					}
-
-					thisFile, err := os.Open(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-					if _, err = io.Copy(f, thisFile); err != nil {
-						return err
-					}
-
-					currentFileCount++
-					statusCode := progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					for statusCode == 1 {
-						time.Sleep(time.Second)
-						statusCode = progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					}
-					if statusCode == 2 {
-						return errors.New("Operation cancelled by user")
-					}
-					return nil
-				})
-				if err != nil {
-					return err
-				}
-			} else {
-				thisFile, err := os.Open(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := arozfs.Base(srcpath)
-				if includeTopLevelFolder {
-					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
-				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-				if _, err = io.Copy(f, thisFile); err != nil {
-					return err
-				}
-
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				for statusCode == 1 {
-					time.Sleep(time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				}
-				if statusCode == 2 {
-					return errors.New("Operation cancelled by user")
-				}
-			}
-		} else {
-			// FileSystemHandler abstraction (remote FS)
-			fshAbs := thisFsh.FileSystemAbstraction
-			if fshAbs.IsDir(srcpath) {
-				topLevelFolderName := filepath.ToSlash(arozfs.Base(filepath.Dir(srcpath)) + "/" + arozfs.Base(srcpath))
-				err = fshAbs.Walk(srcpath, func(path string, info os.FileInfo, err error) error {
-					if err != nil {
-						return err
-					}
-					if info.IsDir() {
-						return nil
-					}
-					if insideHiddenFolder(path) {
-						return nil
-					}
-
-					thisFile, err := fshAbs.ReadStream(path)
-					if err != nil {
-						return err
-					}
-					defer thisFile.Close()
-
-					relativePath := strings.ReplaceAll(filepath.ToSlash(path), filepath.ToSlash(filepath.Clean(srcpath))+"/", "")
-					if includeTopLevelFolder {
-						relativePath = topLevelFolderName + "/" + relativePath
-					} else {
-						relativePath = arozfs.Base(srcpath) + "/" + relativePath
-					}
-
-					f, err := writer.Create(relativePath)
-					if err != nil {
-						return err
-					}
-					if _, err = io.Copy(f, thisFile); err != nil {
-						return err
-					}
-
-					currentFileCount++
-					statusCode := progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					for statusCode == 1 {
-						time.Sleep(time.Second)
-						statusCode = progressHandler(arozfs.Base(path), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-					}
-					if statusCode == 2 {
-						return errors.New("Operation cancelled by user")
-					}
-					return nil
-				})
-				if err != nil {
-					return err
-				}
-			} else {
-				thisFile, err := fshAbs.ReadStream(srcpath)
-				if err != nil {
-					return err
-				}
-				defer thisFile.Close()
-
-				relativePath := arozfs.Base(srcpath)
-				if includeTopLevelFolder {
-					relativePath = arozfs.Base(filepath.Dir(srcpath)) + "/" + relativePath
-				}
-				f, err := writer.Create(relativePath)
-				if err != nil {
-					return err
-				}
-				if _, err = io.Copy(f, thisFile); err != nil {
-					return err
-				}
+	defer thisFile.Close()
 
 
-				currentFileCount++
-				statusCode := progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				for statusCode == 1 {
-					time.Sleep(time.Second)
-					statusCode = progressHandler(arozfs.Base(srcpath), currentFileCount, totalFileCount, float64(currentFileCount)/float64(totalFileCount)*100)
-				}
-				if statusCode == 2 {
-					return errors.New("Operation cancelled by user")
-				}
-			}
-		}
+	f, err := writer.Create(relativePath)
+	if err != nil {
+		return err
 	}
 	}
 
 
-	return nil
+	_, err = io.Copy(f, newProgressReader(thisFile, reporter, 0))
+	return err
 }
 }
 
 
 func insideHiddenFolder(path string) bool {
 func insideHiddenFolder(path string) bool {

+ 205 - 0
src/mod/filesystem/fileOprZip_test.go

@@ -0,0 +1,205 @@
+package filesystem
+
+import (
+	"archive/zip"
+	"bytes"
+	"os"
+	"path/filepath"
+	"strconv"
+	"testing"
+)
+
+// makeZipSourceFolder creates a folder with fileCount small files and one
+// hidden file, and returns its path
+func makeZipSourceFolder(t *testing.T, fileCount int) string {
+	t.Helper()
+	srcFolder := filepath.Join(t.TempDir(), "source")
+	if err := os.MkdirAll(filepath.Join(srcFolder, "sub"), 0755); err != nil {
+		t.Fatalf("failed to create source folder: %v", err)
+	}
+	for i := 0; i < fileCount; i++ {
+		thisFile := filepath.Join(srcFolder, "file"+strconv.Itoa(i)+".txt")
+		if i%2 == 1 {
+			thisFile = filepath.Join(srcFolder, "sub", "file"+strconv.Itoa(i)+".txt")
+		}
+		if err := os.WriteFile(thisFile, []byte("content "+strconv.Itoa(i)), 0644); err != nil {
+			t.Fatalf("failed to write source file: %v", err)
+		}
+	}
+	if err := os.WriteFile(filepath.Join(srcFolder, ".hidden"), []byte("hidden"), 0644); err != nil {
+		t.Fatalf("failed to write hidden file: %v", err)
+	}
+	return srcFolder
+}
+
+func TestArozZipFileWithProgressCompletes(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 6)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	lastCurrent, lastTotal, lastProgress := 0, 0, float64(0)
+	err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false, func(_ string, current int, total int, progress float64) int {
+		lastCurrent, lastTotal, lastProgress = current, total, progress
+		return FsOpr_Continue
+	})
+	if err != nil {
+		t.Fatalf("ArozZipFileWithProgress() returned error: %v", err)
+	}
+
+	//The hidden file is neither zipped nor counted, so the progress ends at 100%
+	if lastTotal != 6 || lastCurrent != 6 {
+		t.Errorf("last progress report = %d / %d files, want 6 / 6", lastCurrent, lastTotal)
+	}
+	if lastProgress != 100 {
+		t.Errorf("last progress = %v, want 100", lastProgress)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 6 {
+		t.Errorf("archive holds %d entries, want 6", len(archive.File))
+	}
+}
+
+func TestArozZipFileWithoutProgressHandler(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 3)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	if err := ArozZipFile([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false); err != nil {
+		t.Fatalf("ArozZipFile() returned error: %v", err)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 3 {
+		t.Errorf("archive holds %d entries, want 3", len(archive.File))
+	}
+}
+
+func TestArozZipFileWithProgressCancel(t *testing.T) {
+	tests := []struct {
+		name string
+		//Whether the handler cancels at this report
+		cancelAt func(current int, total int) bool
+		makeSrc  func(t *testing.T) string
+	}{
+		{
+			//Nothing is counted yet while the source folder is still being scanned
+			name:     "while scanning",
+			cancelAt: func(current int, total int) bool { return total == 0 },
+			makeSrc:  func(t *testing.T) string { return makeZipSourceFolder(t, 20) },
+		},
+		{
+			//The only file is counted but not finished, so it is still streaming
+			name:     "inside a file",
+			cancelAt: func(current int, total int) bool { return total == 1 && current == 0 },
+			makeSrc: func(t *testing.T) string {
+				srcFile := filepath.Join(t.TempDir(), "large.bin")
+				if err := os.WriteFile(srcFile, bytes.Repeat([]byte("arozos"), 1<<20), 0644); err != nil {
+					t.Fatalf("failed to write source file: %v", err)
+				}
+				return srcFile
+			},
+		},
+		{
+			name:     "between files",
+			cancelAt: func(current int, total int) bool { return current == 2 },
+			makeSrc:  func(t *testing.T) string { return makeZipSourceFolder(t, 10) },
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			src := tt.makeSrc(t)
+			output := filepath.Join(t.TempDir(), "out.zip")
+
+			cancelled := false
+			err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{src}, nil, output, false, func(_ string, current int, total int, _ float64) int {
+				if cancelled || tt.cancelAt(current, total) {
+					cancelled = true
+					return FsOpr_Cancel
+				}
+				return FsOpr_Continue
+			})
+
+			if !cancelled {
+				t.Fatal("the handler never reached the point where it cancels")
+			}
+			if err == nil {
+				t.Fatal("ArozZipFileWithProgress() should return an error once cancelled")
+			}
+			if _, statErr := os.Stat(output); !os.IsNotExist(statErr) {
+				t.Errorf("a cancelled archive should be removed, stat error = %v", statErr)
+			}
+		})
+	}
+}
+
+func TestArozZipFileWithProgressPauseWhileScanning(t *testing.T) {
+	srcFolder := makeZipSourceFolder(t, 4)
+	output := filepath.Join(t.TempDir(), "out.zip")
+
+	//Pause on the first report, which comes from the scan, then resume
+	signals := []int{FsOpr_Pause, FsOpr_Pause}
+	pausedReports := 0
+	err := ArozZipFileWithProgress([]*FileSystemHandler{nil}, []string{srcFolder}, nil, output, false, func(_ string, current int, total int, _ float64) int {
+		if len(signals) > 0 {
+			if total != 0 {
+				t.Errorf("the pause should be applied while scanning, got a report of %d / %d files", current, total)
+			}
+			thisSignal := signals[0]
+			signals = signals[1:]
+			pausedReports++
+			return thisSignal
+		}
+		return FsOpr_Continue
+	})
+	if err != nil {
+		t.Fatalf("ArozZipFileWithProgress() returned error after resuming: %v", err)
+	}
+	if pausedReports != 2 {
+		t.Errorf("handler answered %d pause signals, want 2", pausedReports)
+	}
+
+	archive, err := zip.OpenReader(output)
+	if err != nil {
+		t.Fatalf("output is not a readable zip: %v", err)
+	}
+	defer archive.Close()
+	if len(archive.File) != 4 {
+		t.Errorf("archive holds %d entries, want 4", len(archive.File))
+	}
+}
+
+func TestArozUnzipFileWithProgressCancelInsideFile(t *testing.T) {
+	//Build an archive holding one large file
+	tmp := t.TempDir()
+	srcFile := filepath.Join(tmp, "large.bin")
+	if err := os.WriteFile(srcFile, bytes.Repeat([]byte("arozos"), 1<<20), 0644); err != nil {
+		t.Fatalf("failed to write source file: %v", err)
+	}
+	archivePath := filepath.Join(tmp, "large.zip")
+	if err := ArozZipFile([]*FileSystemHandler{nil}, []string{srcFile}, nil, archivePath, false); err != nil {
+		t.Fatalf("ArozZipFile() returned error: %v", err)
+	}
+
+	//The file count only moves once a file is done, so 0 means mid stream
+	dest := filepath.Join(tmp, "extracted")
+	err := ArozUnzipFileWithProgress([]string{archivePath}, dest, func(_ string, current int, _ int, _ float64) int {
+		if current == 0 {
+			return FsOpr_Cancel
+		}
+		return FsOpr_Continue
+	})
+	if err == nil {
+		t.Fatal("ArozUnzipFileWithProgress() should return an error once cancelled")
+	}
+	if _, statErr := os.Stat(filepath.Join(dest, "large.bin")); !os.IsNotExist(statErr) {
+		t.Errorf("a cancelled extraction should not leave the partial file behind, stat error = %v", statErr)
+	}
+}