arozos-mirror/src/file_system.go

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"hash/crc32"
	"io"
	"io/fs"
	"math"
	"mime"
	"net/http"
	"net/url"
	"os"
	"sync"
	"sync/atomic"

	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/gorilla/websocket"
	uuid "github.com/satori/go.uuid"

	"imuslab.com/arozos/mod/compatibility"
	"imuslab.com/arozos/mod/filesystem"
	"imuslab.com/arozos/mod/filesystem/arozfs"
	fsp "imuslab.com/arozos/mod/filesystem/fspermission"
	"imuslab.com/arozos/mod/filesystem/fssort"
	"imuslab.com/arozos/mod/filesystem/fuzzy"
	hidden "imuslab.com/arozos/mod/filesystem/hidden"
	"imuslab.com/arozos/mod/filesystem/localversion"
	metadata "imuslab.com/arozos/mod/filesystem/metadata"
	"imuslab.com/arozos/mod/filesystem/shortcut"
	module "imuslab.com/arozos/mod/modules"
	prout "imuslab.com/arozos/mod/prouter"
	"imuslab.com/arozos/mod/share"
	"imuslab.com/arozos/mod/share/shareEntry"
	storage "imuslab.com/arozos/mod/storage"
	user "imuslab.com/arozos/mod/user"
	"imuslab.com/arozos/mod/utils"
)

var (
	thumbRenderHandler    *metadata.RenderHandler
	shareEntryTable       *shareEntry.ShareEntryTable
	shareManager          *share.Manager
	wsConnectionStore     sync.Map     //Storage of all the file operation tasks, keyed by operation id
	fileOprTaskLock       sync.RWMutex //Lock guarding the mutable fields of the task records above
	fileOprJanitorStarted atomic.Bool  //Whether the finished task record janitor is already running
)

// Status of a file operation task or of one of the files inside it
const (
	FsTask_Pending   = "pending"
	FsTask_Ongoing   = "ongoing"
	FsTask_Completed = "completed"
	FsTask_Error     = "error"
	FsTask_Cancelled = "cancelled"

	//How long an operation that finished without error is kept before the
	//janitor drops it, in seconds. Long enough for a connected dialog to render
	//the result and fire its completion callback, short enough that the listing
	//does not fill up with finished transfers.
	fileOprFinishedRecordTTL = 3

	//How many failed operations are kept per user. Failures are never dropped on
	//a timer so they can still be reviewed after reopening the desktop from
	//another browser or machine, but the backlog is bounded.
	fileOprErrorRecordLimit = 32

	//How often the janitor sweeps the finished task records. Kept well under
	//fileOprFinishedRecordTTL so a record is actually dropped near its deadline
	//rather than up to a whole sweep later.
	fileOprJanitorInterval = 2 * time.Second
)

type trashedFile struct {
	Filename         string
	Filepath         string
	FileExt          string
	IsDir            bool
	Filesize         int64
	RemoveTimestamp  int64
	RemoveDate       string
	OriginalPath     string
	OriginalFilename string
}

// A single source file inside a file operation task
type fileOperationSubtask struct {
	Filename string  //Base name of this source file
	Src      string  //Virtual path of this source file
	IsDir    bool    //Whether this source is a folder
	Size     int64   //Total size of this source file in bytes
	Done     int64   //Bytes of this source file that are already processed
	Progress float64 //Progress of this source file, in percentage
	Status   string  //Status of this source file, see the FsTask_* constants
	Error    string  //Error message of this source file, if any
}

type fileOperationTask struct {
	ID                  string  //Unique id for the task operation
	Owner               string  //Owner of the file opr
	Operation           string  //Type of the file opr: move / copy / zip / unzip
	Src                 string  //Source folder for opr
	Dest                string  //Destination folder for opr
	Progress            float64 //Progress for the operation
	LatestFile          string  //Latest file that is current transfering
	FileOperationSignal int     //Current control signal of the file opr

	Files     []*fileOperationSubtask //Per source file progress of this operation
	TotalSize int64                   //Total size of all the source files in bytes
	DoneSize  int64                   //Total bytes processed so far
	StartTime int64                   //Unix timestamp of when this operation started
	EndTime   int64                   //Unix timestamp of when this operation ended, 0 if still running
	Status    string                  //Status of this operation, see the FsTask_* constants
	Error     string                  //Error message of this operation, if any
}

func FileSystemInit() {
	router := prout.NewModuleRouter(prout.RouterOption{
		ModuleName:  "File Manager",
		AdminOnly:   false,
		UserHandler: userHandler,
		DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
			utils.SendErrorResponse(w, "Permission Denied")
		},
	})

	//Upload related functions
	router.HandleFunc("/system/file_system/upload", system_fs_handleUpload)
	router.HandleFunc("/system/file_system/lowmemUpload", system_fs_handleLowMemoryUpload)

	//Other file operations
	router.HandleFunc("/system/file_system/validateFileOpr", system_fs_validateFileOpr)
	router.HandleFunc("/system/file_system/fileOpr", system_fs_handleOpr)
	router.HandleFunc("/system/file_system/ws/fileOpr", system_fs_handleWebSocketOpr)
	router.HandleFunc("/system/file_system/fileOprAsync", system_fs_handleAsyncOpr)
	router.HandleFunc("/system/file_system/ws/fileOprStatus", system_fs_handleFileOprStatusWebSocket)
	router.HandleFunc("/system/file_system/listDir", system_fs_handleList)
	router.HandleFunc("/system/file_system/listDirHash", system_fs_handleDirHash)
	router.HandleFunc("/system/file_system/listRoots", system_fs_listRoot)
	router.HandleFunc("/system/file_system/listDrives", system_fs_listDrives)
	router.HandleFunc("/system/file_system/newItem", system_fs_handleNewObjects)
	router.HandleFunc("/system/file_system/preference", system_fs_handleUserPreference)
	router.HandleFunc("/system/file_system/listTrash", system_fs_scanTrashBin)
	router.HandleFunc("/system/file_system/ws/listTrash", system_fs_WebSocketScanTrashBin)
	router.HandleFunc("/system/file_system/clearTrash", system_fs_clearTrashBin)
	router.HandleFunc("/system/file_system/trashSettings", system_fs_handleTrashSettings)
	router.HandleFunc("/system/file_system/restoreTrash", system_fs_restoreFile)
	router.HandleFunc("/system/file_system/zipHandler", system_fs_zipHandler)
	router.HandleFunc("/system/file_system/getProperties", system_fs_getFileProperties)
	router.HandleFunc("/system/file_system/getStorageInfo", system_fs_getStorageInfo)
	router.HandleFunc("/system/file_system/versionHistory", system_fs_FileVersionHistory)

	router.HandleFunc("/system/file_system/handleFilePermission", system_fs_handleFilePermission)
	router.HandleFunc("/system/file_system/search", system_fs_handleFileSearch)

	//Thumbnail caching functions
	router.HandleFunc("/system/file_system/handleFolderCache", system_fs_handleFolderCache)
	router.HandleFunc("/system/file_system/handleCacheRender", system_fs_handleCacheRender)
	router.HandleFunc("/system/file_system/loadThumbnail", system_fs_handleThumbnailLoad)

	//Directory specific config
	router.HandleFunc("/system/file_system/sortMode", system_fs_handleFolderSortModePreference)

	//Register the module
	moduleHandler.RegisterModule(module.ModuleInfo{
		Name:        "File Manager",
		Group:       "System Tools",
		IconPath:    "SystemAO/file_system/img/small_icon.png",
		Version:     "1.0",
		StartDir:    "SystemAO/file_system/file_explorer.html",
		SupportFW:   true,
		InitFWSize:  []int{1075, 610},
		LaunchFWDir: "SystemAO/file_system/file_explorer.html",
		SupportEmb:  false,
	})

	//Register the Trashbin module
	moduleHandler.RegisterModule(module.ModuleInfo{
		Name:         "Trash Bin",
		Group:        "System Tools",
		IconPath:     "SystemAO/file_system/trashbin_img/small_icon.png",
		Version:      "1.0",
		StartDir:     "SystemAO/file_system/trashbin.html",
		SupportFW:    true,
		InitFWSize:   []int{400, 200},
		LaunchFWDir:  "SystemAO/file_system/trashbin.html",
		SupportEmb:   false,
		SupportedExt: []string{"*"},
	})

	//Register the Zip Extractor module
	moduleHandler.RegisterModule(module.ModuleInfo{
		Name:         "Zip Extractor",
		Group:        "System Tools",
		IconPath:     "SystemAO/file_system/img/zip_extractor.png",
		Version:      "1.0",
		SupportFW:    false,
		LaunchEmb:    "SystemAO/file_system/zip_extractor.html",
		SupportEmb:   true,
		InitEmbSize:  []int{260, 120},
		SupportedExt: []string{".zip"},
	})

	//Create user root if not exists
	err := os.MkdirAll(filepath.Join(*root_directory, "users/"), 0755)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Failed to create system storage root", err)
		panic(err)
	}

	//Create database table if not exists
	err = sysdb.NewTable("fs")
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
		panic(err)
	}

	//Create new table for sort preference
	err = sysdb.NewTable("fs-sortpref")
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
		panic(err)
	}

	//Create a RenderHandler for caching thumbnails
	thumbRenderHandler = metadata.NewRenderHandler()

	//Thumbnails of drives that render them where the file is stored (e.g.
	//cluster:/) are kept on this host, outside the drive and outside tmp:/
	//(whose files are cleared after a day)
	metadata.SetExternalCacheDir(filepath.Join("system", "cache", "thumbnails"))
	nightlyManager.RegisterNightlyTask(func() {
		metadata.PruneExternalCache(thumbnailCacheMaxAge)
	})

	/*
		Share Related Registering

		This section of functions create and register the file share service
		for the arozos

	*/
	//Create a share manager to handle user file sharae
	shareEntryTable = shareEntry.NewShareEntryTable(sysdb)
	shareManager = share.NewShareManager(share.Options{
		AuthAgent:       authAgent,
		ShareEntryTable: shareEntryTable,
		UserHandler:     userHandler,
		HostName:        *host_name,
		TmpFolder:       *tmp_directory,
	})

	//Share related functions
	router.HandleFunc("/system/file_system/share/new", shareManager.HandleCreateNewShare)
	router.HandleFunc("/system/file_system/share/delete", shareManager.HandleDeleteShare)
	router.HandleFunc("/system/file_system/share/edit", shareManager.HandleEditShare)
	router.HandleFunc("/system/file_system/share/checkShared", shareManager.HandleShareCheck)
	router.HandleFunc("/system/file_system/share/list", shareManager.HandleListAllShares)

	//Handle the main share function
	//Share function is now routed by the main router
	//http.HandleFunc("/share", shareManager.HandleShareAccess)

	/*
		File Operation Resume Functions
	*/
	//Create a sync map for file operation opened connections
	wsConnectionStore = sync.Map{}
	router.HandleFunc("/system/file_system/ongoing", system_fs_HandleOnGoingTasks)

	/*
		Nighly Tasks

		These functions allow file system to clear and maintain
		the arozos file system when no one is using the system
	*/

	//Trash bin retention and size limit, per user
	registerSetting(settingModule{
		Name:     "File Manager",
		Desc:     "Trash Bin Retention and Size",
		IconPath: "SystemAO/file_system/trashbin_img/small_icon.png",
		Group:    "Disk",
		StartDir: "SystemAO/disk/filemanager/trashsettings.html",
	})

	//Clear tmp folder if files is placed here too long
	nightlyManager.RegisterNightlyTask(system_fs_clearOldTmpFiles)

	//Clear shares that its parent file no longer exists in the system
	shareManager.ValidateAndClearShares()
	nightlyManager.RegisterNightlyTask(shareManager.ValidateAndClearShares)

	//Clear file version history that is more than 30 days
	go func() {
		//Start version history cleaning in background
		system_fs_clearVersionHistories()
		systemWideLogger.PrintAndLog("File System", "Startup File Version History Cleaning Completed", nil)

	}()
	systemWideLogger.PrintAndLog("File System", "Started File Version History Cleaning in background", nil)

	nightlyManager.RegisterNightlyTask(system_fs_clearVersionHistories)

	//Purge trashed files older than each user's retention setting
	nightlyManager.RegisterNightlyTask(system_fs_clearExpiredTrash)
}

/*
	File Search

	Handle file search in wildcard and recursive search

*/

func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
	//Get the user information
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Get the search target root path
	vpath, err := utils.PostPara(r, "path")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid vpath given")
		return
	}

	keyword, err := utils.PostPara(r, "keyword")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid keyword given")
		return
	}

	//Check if case sensitive is enabled
	casesensitve, _ := utils.PostPara(r, "casesensitive")

	vrootID, _, err := filesystem.GetIDFromVirtualPath(vpath)
	var targetFSH *filesystem.FileSystemHandler = nil
	if err != nil {
		utils.SendErrorResponse(w, "Invalid path given")
		return
	}
	targetFSH, err = GetFsHandlerByUUID(vrootID)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	//Translate the vpath to realpath if this is an actual path on disk
	resolvedPath, err := targetFSH.FileSystemAbstraction.VirtualPathToRealPath(vpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, "Invalid path given")
		return
	}
	rpath := resolvedPath

	//Check if the search mode is recursive keyword or wildcard
	if len(keyword) > 1 && keyword[:1] == "/" {
		//Wildcard

		//Updates 31-12-2021: Do not allow wildcard search on virtual type's FSH
		if targetFSH == nil {
			utils.SendErrorResponse(w, "Invalid path given")
			return
		}
		targetFshAbs := targetFSH.FileSystemAbstraction
		wildcard := keyword[1:]
		matchingFiles, err := targetFshAbs.Glob(filepath.Join(rpath, wildcard))
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		//Prepare result struct
		results := []filesystem.FileData{}

		escaped := false
		for _, matchedFile := range matchingFiles {
			thisVpath, _ := targetFSH.FileSystemAbstraction.RealPathToVirtualPath(matchedFile, userinfo.Username)
			isHidden, _ := hidden.IsHidden(thisVpath, true)
			if !isHidden {
				results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, matchedFile, 2))
			}

		}

		if escaped {
			utils.SendErrorResponse(w, "Search keywords contain escape character!")
			return
		}

		//OK. Tidy up the results
		js, _ := json.Marshal(results)
		utils.SendJSONResponse(w, string(js))
	} else {
		//Updates 2022-02-16: Build the fuzzy matcher if it is not a wildcard search
		matcher := fuzzy.NewFuzzyMatcher(keyword, casesensitve == "true")

		//Recursive keyword
		results := []filesystem.FileData{}
		var err error = nil

		fshAbs := targetFSH.FileSystemAbstraction
		err = fshAbs.Walk(rpath, func(path string, info os.FileInfo, err error) error {
			thisFilename := filepath.Base(path)
			if casesensitve != "true" {
				thisFilename = strings.ToLower(thisFilename)
			}

			if !filesystem.IsInsideHiddenFolder(path) {
				if matcher.Match(thisFilename) {
					//This is a matching file
					thisVpath, _ := fshAbs.RealPathToVirtualPath(path, userinfo.Username)
					results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, path, 2))
				}
			}

			return nil
		})

		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
		//OK. Tidy up the results
		js, _ := json.Marshal(results)
		utils.SendJSONResponse(w, string(js))
	}

}

/*
Handle low-memory upload operations

This function is specailly designed to work with low memory devices
(e.g. ZeroPi / Orange Pi Zero with 512MB RAM)

Two cases
1. Not Buffer FS + Huge File
=> Write chunks to fsa + merge to fsa

2. Else
=> write chunks to tmp (via os package) + merge to fsa
*/
/*
	Low memory (websocket) upload tunables

	uploadIdleTimeout is how long the server waits for the next chunk before it
	assumes the client is gone. A paused upload is deliberately exempt from it -
	the client is still there, it is just not sending - so uploadPauseTimeout
	takes over instead, to cover an upload someone paused and forgot about.

	uploadPauseCloseCode is a private-use websocket close code, so the client can
	tell "your pause expired" apart from an ordinary disconnect and offer retry.
*/
const (
	uploadIdleTimeout    = 300 * time.Second
	uploadPauseTimeout   = 2 * time.Hour
	uploadPauseCloseCode = 4001
)

func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
	//Get user info
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		w.WriteHeader(http.StatusUnauthorized)
		w.Write([]byte("401 - Unauthorized"))
		return
	}

	//Get filename and upload path
	filename, err := utils.GetPara(r, "filename")
	if filename == "" || err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Invalid filename given"))
		return
	}

	//Get upload target directory
	uploadTarget, err := utils.GetPara(r, "path")
	if uploadTarget == "" || err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Invalid path given"))
		return
	}

	//Unescape the upload target path
	unescapedPath, err := url.PathUnescape(uploadTarget)
	if err != nil {
		unescapedPath = uploadTarget
	}

	//Check if the user can write to this folder
	if !userinfo.CanWrite(unescapedPath) {
		//No permission
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("403 - Access Denied"))
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(unescapedPath)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Path translation failed"))
		return
	}
	fshAbs := fsh.FileSystemAbstraction

	//Translate the upload target directory
	realUploadPath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - Path translation failed"))
		return
	}

	//Check if it is huge file upload mode
	isHugeFile := false
	hugefile, _ := utils.GetPara(r, "hugefile")
	if hugefile == "true" && !fsh.RequireBuffer {
		//Huge file mode is only compatible with local file systems
		//For remote file system, use buffer to tmp then upload method
		isHugeFile = true
	}

	//Create destination folder if not exists
	targetUploadLocation := arozfs.ToSlash(filepath.Join(realUploadPath, filename))
	if !fshAbs.FileExists(realUploadPath) {
		fshAbs.MkdirAll(realUploadPath, 0755)
	}

	//Generate an UUID for this upload
	uploadUUID := uuid.NewV4().String()
	uploadFolder := filepath.Join(*tmp_directory, "uploads", uploadUUID)
	if isHugeFile {
		//Change to upload directly to target disk
		uploadFolder = filepath.Join(realUploadPath, ".metadata/.upload", uploadUUID)
		fshAbs.MkdirAll(uploadFolder, 0700)
	} else {
		os.MkdirAll(uploadFolder, 0700)
	}

	//Start websocket connection
	var upgrader = websocket.Upgrader{}
	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Failed to upgrade websocket connection: ", err)
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 WebSocket upgrade failed"))
		return
	}
	defer c.Close()

	//Handle WebSocket upload
	blockCounter := 0
	chunkName := []string{}

	/*
		Shared with the watchdog goroutine below, so these are read and written
		atomically. pausedSince is 0 while the upload is running, and holds the
		unix time the client paused at otherwise.
	*/
	var lastChunkArrivalTime int64 = time.Now().Unix()
	var pausedSince int64 = 0

	//Setup a timeout listener, check if connection still active every 1 minute
	ticker := time.NewTicker(60 * time.Second)
	done := make(chan bool)
	go func() {
		for {
			select {
			case <-done:
				return
			case <-ticker.C:
				if pausedAt := atomic.LoadInt64(&pausedSince); pausedAt > 0 {
					//Paused: silence is expected here, so only the pause clock
					//applies. WriteControl is the one write that is safe to make
					//from this goroutine while the reader holds the connection.
					if time.Since(time.Unix(pausedAt, 0)) > uploadPauseTimeout {
						systemWideLogger.PrintAndLog("File System", "Upload paused for too long. Disconnecting.", errors.New("upload pause timeout"))
						c.WriteControl(websocket.CloseMessage,
							websocket.FormatCloseMessage(uploadPauseCloseCode, "upload pause timeout"),
							time.Now().Add(time.Second))
						time.Sleep(1 * time.Second)
						c.Close()
						return
					}
					continue
				}

				if time.Now().Unix()-atomic.LoadInt64(&lastChunkArrivalTime) > int64(uploadIdleTimeout.Seconds()) {
					//Already 5 minutes without new data arraival. Stop connection
					systemWideLogger.PrintAndLog("File System", "Upload WebSocket connection timeout. Disconnecting.", errors.New("websocket connection timeout"))
					c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
					time.Sleep(1 * time.Second)
					c.Close()
					return
				}
			}
		}
	}()

	totalFileSize := int64(0)

	// Full-file CRC32 hasher (IEEE polynomial, matches JS crc32Table implementation)
	fileCRC32Hasher := crc32.NewIEEE()

	// Per-chunk state machine:
	// The client sends a text metadata frame {"index":N,"checksum":"hex"} before each binary frame.
	var pendingChunkIndex int
	var pendingChunkChecksum string
	expectingBinary := false

	for {
		mt, message, err := c.ReadMessage()
		if err != nil {
			//Connection closed by client. Clear the tmp folder and exit
			systemWideLogger.PrintAndLog("File System", "Upload terminated by client. Cleaning tmp folder", err)
			//Clear the tmp folder
			time.Sleep(1 * time.Second)
			if isHugeFile {
				fshAbs.RemoveAll(uploadFolder)
			} else {
				os.RemoveAll(uploadFolder)
			}
			return
		}

		if mt == 1 {
			// Text frame – either a control message, chunk metadata or done signal
			textMsg := strings.TrimSpace(string(message))

			/*
				Control messages are checked before anything else, and regardless
				of expectingBinary: a pause can land after the metadata frame but
				before its binary payload, and the flow control below would
				otherwise mistake it for a malformed chunk header.

				A chunk header or done signal unmarshals into this struct with
				every field false, so this is safe to try first.
			*/
			var ctrl struct {
				Pause  bool `json:"pause"`
				Resume bool `json:"resume"`
				Ping   bool `json:"ping"`
			}
			if jsonErr := json.Unmarshal([]byte(textMsg), &ctrl); jsonErr == nil &&
				(ctrl.Pause || ctrl.Resume || ctrl.Ping) {
				if ctrl.Pause {
					atomic.StoreInt64(&pausedSince, time.Now().Unix())
				} else if ctrl.Resume {
					atomic.StoreInt64(&pausedSince, 0)
					//Do not count the paused stretch as idle time
					atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
				}
				/*
					Any control message counts as the client still being there,
					so it also holds off the idle timeout. That matters when a
					pause is raised before this socket finished connecting: the
					pause frame is lost, but the heartbeat still arrives, and
					without this the upload would be reaped as idle anyway.

					Answering every control message is what puts traffic on the
					wire in both directions - a reverse proxy in front of us will
					drop a connection it sees nothing on.
				*/
				atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
				c.WriteMessage(1, []byte(`{"pong":true}`))
				continue
			}

			if !expectingBinary {
				// Check if this is the done signal
				var doneSignal struct {
					Done         bool   `json:"done"`
					TotalChunks  int    `json:"totalChunks"`
					FileChecksum string `json:"fileChecksum"`
				}
				if jsonErr := json.Unmarshal([]byte(textMsg), &doneSignal); jsonErr == nil && doneSignal.Done {
					// Verify the full-file CRC32 before merging
					if doneSignal.FileChecksum != "" {
						computedSum := fileCRC32Hasher.Sum32()
						computedSumBytes := []byte{byte(computedSum >> 24), byte(computedSum >> 16), byte(computedSum >> 8), byte(computedSum)}
						computedHex := hex.EncodeToString(computedSumBytes)
						if doneSignal.FileChecksum != computedHex {
							systemWideLogger.PrintAndLog("File System", "Upload file checksum mismatch: client="+doneSignal.FileChecksum+" server="+computedHex, nil)
							c.WriteMessage(1, []byte(`{"error":"File integrity check failed: full-file checksum mismatch"}`))
							c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
							time.Sleep(1 * time.Second)
							c.Close()
							if isHugeFile {
								fshAbs.RemoveAll(uploadFolder)
							} else {
								os.RemoveAll(uploadFolder)
							}
							return
						}
					}
					// Checksum verified – proceed to merge
					break
				}

				// Parse as chunk metadata
				var meta struct {
					Index    int    `json:"index"`
					Checksum string `json:"checksum"`
				}
				if jsonErr := json.Unmarshal([]byte(textMsg), &meta); jsonErr != nil {
					systemWideLogger.PrintAndLog("File System", "Invalid chunk metadata received: "+textMsg, jsonErr)
					continue
				}
				pendingChunkIndex = meta.Index
				pendingChunkChecksum = meta.Checksum
				expectingBinary = true
			}

		} else if mt == 2 {
			// Binary frame – the chunk data that follows a metadata frame
			if !expectingBinary {
				systemWideLogger.PrintAndLog("File System", "Received binary chunk without preceding metadata, ignoring", nil)
				continue
			}
			expectingBinary = false

			/*
				Chunks are accepted strictly in index order, and "next" is sent
				once per accepted chunk - clients that keep several chunks in
				flight count those acknowledgements to know which chunks landed.
				Anything else is dropped without a reply:
				- index < blockCounter: a copy of a chunk already written, resent
				  by the client after an acknowledgement timeout. Accepting it
				  again would count its bytes twice in the size and full-file
				  checksum.
				- index > blockCounter: sent ahead in a pipelined window before
				  the chunk at blockCounter failed its CRC check. The client
				  rewinds to the retry and sends it again.
			*/
			if pendingChunkIndex != blockCounter {
				continue
			}

			// Verify chunk CRC32
			chunkSum := crc32.ChecksumIEEE(message)
			chunkSumBytes := []byte{byte(chunkSum >> 24), byte(chunkSum >> 16), byte(chunkSum >> 8), byte(chunkSum)}
			chunkHex := hex.EncodeToString(chunkSumBytes)
			if pendingChunkChecksum != "" && pendingChunkChecksum != chunkHex {
				// CRC32 mismatch – ask the client to re-send this chunk
				systemWideLogger.PrintAndLog("File System", "Chunk "+strconv.Itoa(pendingChunkIndex)+" CRC32 mismatch: expected "+pendingChunkChecksum+" got "+chunkHex, nil)
				retryMsg, _ := json.Marshal(map[string]int{"retryChunk": pendingChunkIndex})
				c.WriteMessage(1, retryMsg)
				// Reset state; client will re-send the metadata+binary for this chunk
				continue
			}

			// Chunk verified – write to tmp folder
			chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(pendingChunkIndex))
			chunkName = append(chunkName, chunkFilepath)
			blockCounter++

			var writeErr error
			if isHugeFile {
				writeErr = fshAbs.WriteFile(chunkFilepath, message, 0700)
			} else {
				writeErr = os.WriteFile(chunkFilepath, message, 0700)
			}

			if writeErr != nil {
				systemWideLogger.PrintAndLog("File System", "Upload chunk write failed: "+writeErr.Error(), writeErr)
				c.WriteMessage(1, []byte(`{"error":"Write file chunk to disk failed"}`))
				c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
				time.Sleep(1 * time.Second)
				c.Close()
				if isHugeFile {
					fshAbs.RemoveAll(uploadFolder)
				} else {
					os.RemoveAll(uploadFolder)
				}
				return
			}

			// Update running full-file CRC32 with the verified chunk data
			fileCRC32Hasher.Write(message)

			// Update timing and quota tracking
			atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
			totalFileSize += int64(len(message))

			if totalFileSize > max_upload_size {
				c.WriteMessage(1, []byte(`{"error":"File size too large"}`))
				c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
				time.Sleep(1 * time.Second)
				c.Close()
				if isHugeFile {
					fshAbs.RemoveAll(uploadFolder)
				} else {
					os.RemoveAll(uploadFolder)
				}
				return
			} else if !userinfo.StorageQuota.HaveSpace(totalFileSize) {
				c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
				c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
				time.Sleep(1 * time.Second)
				c.Close()
				if isHugeFile {
					fshAbs.RemoveAll(uploadFolder)
				} else {
					os.RemoveAll(uploadFolder)
				}
				return
			}

			// Acknowledge the chunk; client will send the next metadata+binary pair
			c.WriteMessage(1, []byte("next"))
		}
	}

	//Try to decode the location if possible
	decodedUploadLocation, err := url.PathUnescape(targetUploadLocation)
	if err != nil {
		decodedUploadLocation = targetUploadLocation
	}

	//Do not allow % sign in filename. Replace all with underscore
	decodedUploadLocation = strings.ReplaceAll(decodedUploadLocation, "%", "_")

	//Merge the file. Merge file location must be on local machine
	mergeFileLocation := decodedUploadLocation
	var out arozfs.File
	if fsh.RequireBuffer {
		//The merge file location must be local buffer
		mergeFileLocation = getFsBufferFilepath(decodedUploadLocation, false)
		out, err = os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
	} else {
		//The merge file location can be local or remote that support OpenFile.
		out, err = fshAbs.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
	}
	defer out.Close()

	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Failed to open file:"+err.Error(), err)
		c.WriteMessage(1, []byte(`{"error":"Failed to open destination file"}`))
		c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
		time.Sleep(1 * time.Second)
		c.Close()
		return
	}

	for counter, filesrc := range chunkName {
		var srcChunkReader arozfs.File
		if isHugeFile {
			srcChunkReader, err = fshAbs.Open(filesrc)
		} else {
			srcChunkReader, err = os.Open(filesrc)
		}

		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Failed to open Source Chunk"+filesrc+" with error "+err.Error(), err)
			c.WriteMessage(1, []byte(`{"error":"Failed to open Source Chunk"}`))
			return
		}

		io.Copy(out, srcChunkReader)

		srcChunkReader.Close()

		//Delete file immediately to save space
		if isHugeFile {
			fshAbs.Remove(filesrc)
		} else {
			os.Remove(filesrc)
		}

		//Write to websocket for the percentage of upload is written fro tmp to dest
		moveProg := strconv.Itoa(int(math.Round(float64(counter)/float64(len(chunkName))*100))) + "%"
		c.WriteMessage(1, []byte(`{"move":"`+moveProg+`"}`))
	}

	out.Close()

	//Check if the size fit in user quota
	var fi fs.FileInfo
	if fsh.RequireBuffer {
		fi, err = os.Stat(mergeFileLocation)
	} else {
		fi, err = fshAbs.Stat(mergeFileLocation)
	}

	if err != nil {
		// Could not obtain stat, handle error
		systemWideLogger.PrintAndLog("File System", "Failed to validate uploaded file: "+mergeFileLocation+". Error Message: "+err.Error(), err)
		c.WriteMessage(1, []byte(`{"error":"Failed to validate uploaded file"}`))
		return
	}
	if !userinfo.StorageQuota.HaveSpace(fi.Size()) {
		c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
		if fsh.RequireBuffer {
			os.RemoveAll(mergeFileLocation)
		} else {
			fshAbs.RemoveAll(mergeFileLocation)
		}
		return
	}

	//Upload it to remote side if it fits the user quota && is buffer file
	if fsh.RequireBuffer {
		//This is local buffer file. Upload to dest fsh
		f, err := os.Open(mergeFileLocation)
		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Failed to open buffered file at "+mergeFileLocation+" with error "+err.Error(), err)
			c.WriteMessage(1, []byte(`{"error":"Failed to open buffered object"}`))
			f.Close()
			return
		}

		err = fsh.FileSystemAbstraction.WriteStream(decodedUploadLocation, f, 0775)
		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Failed to write to file system: "+fsh.UUID+" with error "+err.Error(), err)
			c.WriteMessage(1, []byte(`{"error":"Failed to upload to remote file system"}`))
			f.Close()
			return
		}

		//Remove the buffered file
		f.Close()
		os.Remove(mergeFileLocation)
	}

	//Log the upload filename
	//systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+filepath.Base(decodedUploadLocation), nil)

	//Set owner of the new uploaded file
	userinfo.SetOwnerOfFile(fsh, unescapedPath)

	//Return complete signal
	c.WriteMessage(1, []byte("OK"))

	//Stop the timeout listner
	done <- true

	//Clear the tmp folder
	time.Sleep(300 * time.Millisecond)
	if isHugeFile {
		fshAbs.RemoveAll(uploadFolder)
	} else {
		os.RemoveAll(uploadFolder)
	}

	//Close WebSocket connection after finished
	c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
	time.Sleep(300 * time.Second)
	c.Close()

}

/*
Handle FORM POST based upload

This function is design for general SBCs or computers with more than 2GB of RAM
(e.g. Raspberry Pi 4 / Linux Server)
*/
func system_fs_handleUpload(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Limit the max upload size to the user defined size
	if max_upload_size != 0 {
		r.Body = http.MaxBytesReader(w, r.Body, max_upload_size)
	}

	err = r.ParseMultipartForm(int64(*upload_buf) << 20)
	if err != nil {
		//Filesize too big
		systemWideLogger.PrintAndLog("File System", "Upload file size too big", err)
		utils.SendErrorResponse(w, "File too large")
		return
	}

	file, handler, err := r.FormFile("file")
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Error Retrieving File from upload by user: "+userinfo.Username, err)
		utils.SendErrorResponse(w, "Unable to parse file from upload")
		return
	}

	//Get upload target directory
	uploadTarget, _ := utils.PostPara(r, "path")
	if uploadTarget == "" {
		utils.SendErrorResponse(w, "Upload target cannot be empty.")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(uploadTarget)
	if err != nil {
		utils.SendErrorResponse(w, "Invalid upload target")
		return
	}

	targetFs := fsh.FileSystemAbstraction

	//Translate the upload target directory
	realUploadPath, err := targetFs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, "Upload target is invalid or permission denied.")
		return
	}

	storeFilename := handler.Filename //Filename of the uploaded file

	//Get request time
	uploadStartTime := time.Now().UnixNano() / int64(time.Millisecond)

	//Update for Firefox 94.0.2 (x64) -> Now firefox put its relative path inside Content-Disposition -> filename
	//Skip this handler logic if Firefox version is in between 84.0.2 to 94.0.2
	bypassMetaCheck := compatibility.FirefoxBrowserVersionForBypassUploadMetaHeaderCheck(r.UserAgent())
	if !bypassMetaCheck && strings.Contains(handler.Header["Content-Disposition"][0], "filename=") && strings.Contains(handler.Header["Content-Disposition"][0], "/") {
		//This is a firefox MIME Header for file inside folder. Look for the actual filename
		headerFields := strings.Split(handler.Header["Content-Disposition"][0], "; ")
		possibleRelativePathname := ""
		for _, hf := range headerFields {
			if strings.Contains(hf, "filename=") && len(hf) > 11 {
				//Found. Overwrite original filename with the latest one
				possibleRelativePathname = hf[10 : len(hf)-1]
				storeFilename = possibleRelativePathname
				break
			}
		}
	}

	destFilepath := arozfs.ToSlash(filepath.Join(realUploadPath, storeFilename))
	//fmt.Println(destFilepath, realUploadPath, storeFilename)
	if !targetFs.FileExists(filepath.Dir(destFilepath)) {
		targetFs.MkdirAll(filepath.Dir(destFilepath), 0775)
	}

	//Check if the upload target is read only.
	accmode := userinfo.GetPathAccessPermission(uploadTarget)
	if accmode == arozfs.FsReadOnly {
		utils.SendErrorResponse(w, "The upload target is Read Only.")
		return
	} else if accmode == arozfs.FsDenied {
		utils.SendErrorResponse(w, "Access Denied")
		return
	}

	//Check for storage quota
	uploadFileSize := handler.Size
	if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
		utils.SendErrorResponse(w, "User Storage Quota Exceeded")
		return
	}

	//Do not allow % sign in filename. Replace all with underscore
	destFilepath = strings.ReplaceAll(destFilepath, "%", "_")

	//Move the file to destination file location
	if *enable_asyncFileUpload {
		//Use Async upload method
		systemWideLogger.PrintAndLog("File System", "AsyncFileUpload flag has been deprecated. Falling back to blocking upload.", errors.New("call to deprecated flag: asyncFileUpload"))
	}

	err = targetFs.WriteStream(destFilepath, file, 0775)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Write stream to destination file system abstraction from upload failed", err)
		utils.SendErrorResponse(w, "Write upload to destination disk failed")
		return

	}
	file.Close()

	//Clear up buffered files
	r.MultipartForm.RemoveAll()

	//Set the ownership of file
	userinfo.SetOwnerOfFile(fsh, uploadTarget)

	//Finish up the upload
	/*
		fmt.Printf("Uploaded File: %+v\n", handler.Filename)
		fmt.Printf("File Size: %+v\n", handler.Size)
		fmt.Printf("MIME Header: %+v\n", handler.Header)
		fmt.Println("Upload target: " + realUploadPath)
	*/

	//Fnish upload. Fix the tmp filename
	//systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+handler.Filename, nil)

	//Do upload finishing stuff

	//Add a delay to the complete message to make sure browser catch the return value
	currentTimeMilli := time.Now().UnixNano() / int64(time.Millisecond)
	if currentTimeMilli-uploadStartTime < 100 {
		//Sleep until at least 300 ms
		time.Sleep(time.Duration(100 - (currentTimeMilli - uploadStartTime)))
	}
	//Completed
	utils.SendOK(w)
}

// Validate if the copy and target process will involve file overwriting problem.
func system_fs_validateFileOpr(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	vsrcFiles, _ := utils.PostPara(r, "src")
	vdestFile, _ := utils.PostPara(r, "dest")
	var duplicateFiles []string = []string{}

	//Loop through all files are see if there are duplication during copy and paste
	sourceFiles := []string{}
	decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
	err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
	if err != nil {
		utils.SendErrorResponse(w, "Source file JSON parse error.")
		return
	}

	destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
	if err != nil {
		utils.SendErrorResponse(w, "Operation Valid Failed: "+err.Error())
		return
	}

	rdestFile, _ := destFsh.FileSystemAbstraction.VirtualPathToRealPath(destSubpath, userinfo.Username)
	for _, file := range sourceFiles {
		srcFsh, srcSubpath, _ := GetFSHandlerSubpathFromVpath(string(file))
		rsrcFile, _ := srcFsh.FileSystemAbstraction.VirtualPathToRealPath(srcSubpath, userinfo.Username)
		if destFsh.FileSystemAbstraction.FileExists(filepath.Join(rdestFile, filepath.Base(rsrcFile))) {
			//File exists already.
			vpath, _ := srcFsh.FileSystemAbstraction.RealPathToVirtualPath(rsrcFile, userinfo.Username)
			duplicateFiles = append(duplicateFiles, vpath)
		}

	}

	jsonString, _ := json.Marshal(duplicateFiles)
	utils.SendJSONResponse(w, string(jsonString))
}

/*
Trash bin settings

Stored per user in the same preference table the File Manager already uses,
so they follow the account rather than the browser.

Both settings use zero as "no limit", which is also what an account that has
never opened the settings page reads back:

	trash/retentionDays   0 = never auto remove,  otherwise 1..365
	trash/quotaBytes      0 = unlimited
*/
const (
	trashRetentionPrefKey = "trash/retentionDays"
	trashQuotaPrefKey     = "trash/quotaBytes"
	trashMaxRetentionDays = 365

	//What an account that has never opened the settings page holds trash for.
	//An explicitly stored zero still means "never remove" - only the absence
	//of a stored value falls back to this.
	trashDefaultRetentionDays = 30
)

func system_fs_getTrashRetentionDays(username string) int {
	result := ""
	err := sysdb.Read("fs", "pref/"+trashRetentionPrefKey+"/"+username, &result)
	if err != nil {
		//Never set for this account
		return trashDefaultRetentionDays
	}
	days, err := strconv.Atoi(result)
	if err != nil || days < 0 {
		//Stored but unreadable - the default is a safer answer than "never"
		return trashDefaultRetentionDays
	}
	if days > trashMaxRetentionDays {
		days = trashMaxRetentionDays
	}
	return days
}

func system_fs_getTrashQuotaBytes(username string) int64 {
	result := ""
	err := sysdb.Read("fs", "pref/"+trashQuotaPrefKey+"/"+username, &result)
	if err != nil {
		return 0
	}
	quota, err := utils.StringToInt64(result)
	if err != nil || quota < 0 {
		return 0
	}
	return quota
}

/*
Space a single trashed entry takes up

A folder counts for everything inside it, not the zero bytes its own entry
reports - otherwise a user could fill the bin with folders and never reach
the quota. Hidden files are included because they were moved along with the
rest, and the trash itself lives inside a hidden folder.
*/
func system_fs_getTrashEntrySize(fsh *filesystem.FileSystemHandler, rpath string) int64 {
	if fsh.FileSystemAbstraction.IsDir(rpath) {
		size, _ := fsh.GetDirctorySizeFromRealPath(rpath, true)
		return size
	}
	return int64(fsh.FileSystemAbstraction.GetFileSize(rpath))
}

// Total bytes currently sitting in a user's trash across every file system
func system_fs_getTrashUsage(username string) int64 {
	files, fshs, err := system_fs_listTrash(username)
	if err != nil {
		return 0
	}
	total := int64(0)
	for c, file := range files {
		total += system_fs_getTrashEntrySize(fshs[c], file)
	}
	return total
}

/*
Read or write the trash settings for the logged in user.

GET  with no value  -> {"RetentionDays":n,"QuotaBytes":n,"MaxRetentionDays":365,"UsedBytes":n}
POST retentionDays / quotaBytes -> stores them
*/
func system_fs_handleTrashSettings(w http.ResponseWriter, r *http.Request) {
	username, err := authAgent.GetUserName(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	retention, retentionSet := utils.PostPara(r, "retentionDays")
	quota, quotaSet := utils.PostPara(r, "quotaBytes")

	if retentionSet == nil && retention != "" {
		days, err := strconv.Atoi(retention)
		if err != nil || days < 0 || days > trashMaxRetentionDays {
			utils.SendErrorResponse(w, "Invalid retention day given")
			return
		}
		sysdb.Write("fs", "pref/"+trashRetentionPrefKey+"/"+username, strconv.Itoa(days))
	}

	if quotaSet == nil && quota != "" {
		bytes, err := utils.StringToInt64(quota)
		if err != nil || bytes < 0 {
			utils.SendErrorResponse(w, "Invalid quota given")
			return
		}
		sysdb.Write("fs", "pref/"+trashQuotaPrefKey+"/"+username, utils.Int64ToString(bytes))
	}

	type trashSettings struct {
		RetentionDays    int
		QuotaBytes       int64
		MaxRetentionDays int
		UsedBytes        int64
	}
	js, _ := json.Marshal(trashSettings{
		RetentionDays:    system_fs_getTrashRetentionDays(username),
		QuotaBytes:       system_fs_getTrashQuotaBytes(username),
		MaxRetentionDays: trashMaxRetentionDays,
		UsedBytes:        system_fs_getTrashUsage(username),
	})
	utils.SendJSONResponse(w, string(js))
}

/*
Nightly cleanup

Removes trashed files past their owner's retention window. Users who have
not set a retention (or set it to zero) keep their trash indefinitely, which
is the behaviour every account had before this setting existed.
*/
func system_fs_clearExpiredTrash() {
	for _, username := range authAgent.ListUsers() {
		retentionDays := system_fs_getTrashRetentionDays(username)
		if retentionDays <= 0 {
			//Auto removal disabled for this user
			continue
		}

		userinfo, err := userHandler.GetUserInfoFromUsername(username)
		if err != nil {
			continue
		}

		cutoff := time.Now().Unix() - int64(retentionDays)*86400
		files, fshs, err := system_fs_listTrash(username)
		if err != nil {
			continue
		}

		removed := 0
		for c, file := range files {
			if !nightlyShouldMaintainFsh(fshs[c]) {
				//Trash inside a drive shared by the cluster is emptied by the
				//master node, so it is not purged once per member
				continue
			}

			/*
				The removal time is stored as the file extension when the file
				was recycled. Anything without a parsable timestamp is left
				alone rather than guessed at.
			*/
			ext := filepath.Ext(file)
			if len(ext) < 2 {
				continue
			}
			timestamp, err := utils.StringToInt64(ext[1:])
			if err != nil || timestamp > cutoff {
				continue
			}

			fshAbs := fshs[c].FileSystemAbstraction
			fileVpath, err := fshAbs.RealPathToVirtualPath(file, username)
			if err == nil && userinfo.IsOwnerOfFile(fshs[c], fileVpath) {
				userinfo.RemoveOwnershipFromFile(fshs[c], fileVpath)
			}
			fshAbs.RemoveAll(file)

			//Drop the .trash folder too once nothing is left in it
			remaining, _ := fshAbs.Glob(filepath.Dir(file) + "/*")
			if len(remaining) == 0 {
				fshAbs.Remove(filepath.Dir(file))
			}
			removed++
		}

		if removed > 0 {
			systemWideLogger.PrintAndLog("File System", "Removed "+strconv.Itoa(removed)+
				" expired trash item(s) for user "+username, nil)
		}
	}
}

// Scan all directory and get trash file and send back results with WebSocket
func system_fs_WebSocketScanTrashBin(w http.ResponseWriter, r *http.Request) {
	//Get and check user permission
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Upgrade to websocket
	var upgrader = websocket.Upgrader{}
	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - " + err.Error()))
		systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
		return
	}

	//Start Scanning
	scanningRoots := []*filesystem.FileSystemHandler{}
	//Get all roots to scan
	for _, storage := range userinfo.GetAllFileSystemHandler() {
		if storage.Hierarchy == "backup" {
			//Skip this fsh
			continue
		}
		scanningRoots = append(scanningRoots, storage)
	}

	for _, fsh := range scanningRoots {
		thisFshAbs := fsh.FileSystemAbstraction
		rootPath, err := thisFshAbs.VirtualPathToRealPath("", userinfo.Username)
		if err != nil {
			continue
		}
		err = thisFshAbs.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
			oneLevelUpper := filepath.Base(filepath.Dir(path))
			if oneLevelUpper == ".trash" {
				//This is a trashbin dir.
				file := path

				//Parse the trashFile struct
				timestamp := filepath.Ext(file)[1:]
				originalName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(filepath.Base(file)))
				originalExt := filepath.Ext(filepath.Base(originalName))
				virtualFilepath, _ := thisFshAbs.RealPathToVirtualPath(file, userinfo.Username)
				virtualOrgPath, _ := thisFshAbs.RealPathToVirtualPath(filepath.Dir(filepath.Dir(filepath.Dir(file))), userinfo.Username)
				rawsize := thisFshAbs.GetFileSize(file)
				timestampInt64, _ := utils.StringToInt64(timestamp)
				removeTimeDate := time.Unix(timestampInt64, 0)
				if thisFshAbs.IsDir(file) {
					originalExt = ""
				}

				thisTrashFileObject := trashedFile{
					Filename:         filepath.Base(file),
					Filepath:         virtualFilepath,
					FileExt:          originalExt,
					IsDir:            thisFshAbs.IsDir(file),
					Filesize:         int64(rawsize),
					RemoveTimestamp:  timestampInt64,
					RemoveDate:       removeTimeDate.Format("2006-01-02 15:04:05"),
					OriginalPath:     virtualOrgPath,
					OriginalFilename: originalName,
				}

				//Send out the result as JSON string
				js, _ := json.Marshal(thisTrashFileObject)
				err := c.WriteMessage(1, js)
				if err != nil {
					//Connection already closed
					return err
				}
			}

			return nil
		})

		if err != nil {
			//Scan or client connection error (Connection closed?)
			return
		}
	}

	//Close connection after finished
	c.Close()

}

// Scan all the directory and get trash files within the system
func system_fs_scanTrashBin(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	username := userinfo.Username

	results := []trashedFile{}
	files, fshs, err := system_fs_listTrash(username)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Get information of each files and process it into results
	for c, file := range files {
		fsAbs := fshs[c].FileSystemAbstraction
		timestamp := filepath.Ext(file)[1:]
		originalName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(filepath.Base(file)))
		originalExt := filepath.Ext(filepath.Base(originalName))
		virtualFilepath, _ := fsAbs.RealPathToVirtualPath(file, userinfo.Username)
		virtualOrgPath, _ := fsAbs.RealPathToVirtualPath(filepath.Dir(filepath.Dir(filepath.Dir(file))), userinfo.Username)
		rawsize := fsAbs.GetFileSize(file)
		timestampInt64, _ := utils.StringToInt64(timestamp)
		removeTimeDate := time.Unix(timestampInt64, 0)
		if fsAbs.IsDir(file) {
			originalExt = ""
		}
		results = append(results, trashedFile{
			Filename:         filepath.Base(file),
			Filepath:         virtualFilepath,
			FileExt:          originalExt,
			IsDir:            fsAbs.IsDir(file),
			Filesize:         int64(rawsize),
			RemoveTimestamp:  timestampInt64,
			RemoveDate:       removeTimeDate.Format("2006-01-02 15:04:05"),
			OriginalPath:     virtualOrgPath,
			OriginalFilename: originalName,
		})
	}

	//Sort the results by date, latest on top
	sort.Slice(results[:], func(i, j int) bool {
		return results[i].RemoveTimestamp > results[j].RemoveTimestamp
	})

	//Format and return the json results
	jsonString, _ := json.Marshal(results)
	utils.SendJSONResponse(w, string(jsonString))
}

// Restore a trashed file to its parent dir
func system_fs_restoreFile(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	targetTrashedFile, err := utils.PostPara(r, "src")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid src given")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetTrashedFile)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	fshAbs := fsh.FileSystemAbstraction

	//Translate it to realpath
	realpath, _ := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if !fshAbs.FileExists(realpath) {
		utils.SendErrorResponse(w, "File not exists")
		return
	}

	//Check if this is really a trashed file
	if filepath.Base(filepath.Dir(realpath)) != ".trash" {
		utils.SendErrorResponse(w, "File not in trashbin")
		return
	}

	//OK to proceed.
	originalFilename := strings.TrimSuffix(filepath.Base(realpath), filepath.Ext(filepath.Base(realpath)))
	restoreFolderRoot := filepath.Dir(filepath.Dir(filepath.Dir(realpath)))
	targetPath := filepath.ToSlash(filepath.Join(restoreFolderRoot, originalFilename))
	//systemWideLogger.PrintAndLog("File System", (targetPath)
	fshAbs.Rename(realpath, targetPath)

	//Check if the parent dir has no more fileds. If yes, remove it
	filescounter, _ := fshAbs.Glob(filepath.Dir(realpath) + "/*")
	if len(filescounter) == 0 {
		fshAbs.Remove(filepath.Dir(realpath))
	}

	utils.SendOK(w)
}

// Clear all trashed file in the system
func system_fs_clearTrashBin(w http.ResponseWriter, r *http.Request) {
	u, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	fileList, fshs, err := system_fs_listTrash(u.Username)

	if err != nil {
		utils.SendErrorResponse(w, "Unable to clear trash: "+err.Error())
		return
	}

	//Get list success. Remove each of them.
	for c, file := range fileList {
		fileVpath, _ := fshs[c].FileSystemAbstraction.RealPathToVirtualPath(file, u.Username)
		isOwner := u.IsOwnerOfFile(fshs[c], fileVpath)
		if isOwner {
			//This user own this system. Remove this file from his quota
			u.RemoveOwnershipFromFile(fshs[c], fileVpath)
		}
		fshAbs := fshs[c].FileSystemAbstraction
		fshAbs.RemoveAll(file)
		//Check if its parent directory have no files. If yes, remove the dir itself as well.
		filesInThisTrashBin, _ := fshAbs.Glob(filepath.Dir(file) + "/*")
		if len(filesInThisTrashBin) == 0 {
			fshAbs.Remove(filepath.Dir(file))
		}
	}

	utils.SendOK(w)
}

// Get all trash in a string list
func system_fs_listTrash(username string) ([]string, []*filesystem.FileSystemHandler, error) {
	userinfo, _ := userHandler.GetUserInfoFromUsername(username)
	scanningRoots := []*filesystem.FileSystemHandler{}
	//Get all roots to scan
	for _, storage := range userinfo.GetAllFileSystemHandler() {
		if storage.Hierarchy == "backup" {
			//Skip this fsh
			continue
		}

		scanningRoots = append(scanningRoots, storage)
	}

	files := []string{}
	fshs := []*filesystem.FileSystemHandler{}
	for _, thisFsh := range scanningRoots {
		thisFshAbs := thisFsh.FileSystemAbstraction
		rootPath, _ := thisFshAbs.VirtualPathToRealPath("", userinfo.Username)
		err := thisFshAbs.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
			oneLevelUpper := filepath.Base(filepath.Dir(path))
			if oneLevelUpper == ".trash" {
				//This is a trashbin dir.
				files = append(files, path)
				fshs = append(fshs, thisFsh)
			}
			return nil
		})
		if err != nil {
			continue
		}
	}

	return files, fshs, nil
}

/*
	Handle new file or folder functions

	Required information
	@type {folder / file}
	@ext {any that is listed in the template folder}
	if no paramter is passed in, default listing all the supported template file
*/

func system_fs_handleNewObjects(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Validate the token
	tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
	if !tokenValid {
		http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
		return
	}

	fileType, _ := utils.PostPara(r, "type")     //File creation type, {file, folder}
	vsrc, _ := utils.PostPara(r, "src")          //Virtual file source folder, do not include filename
	filename, _ := utils.PostPara(r, "filename") //Filename for the new file

	if fileType == "" && filename == "" {
		//List all the supported new filetype
		if !filesystem.FileExists("system/newitem/") {
			os.MkdirAll("system/newitem/", 0755)
		}

		type newItemObject struct {
			Desc string
			Ext  string
		}

		var newItemList []newItemObject
		newItemTemplate, _ := filepath.Glob("system/newitem/*")
		for _, file := range newItemTemplate {
			thisItem := new(newItemObject)
			thisItem.Desc = strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
			thisItem.Ext = filepath.Ext(file)[1:]
			newItemList = append(newItemList, *thisItem)
		}

		jsonString, err := json.Marshal(newItemList)
		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Unable to parse JSON string for new item list", err)
			utils.SendErrorResponse(w, "Unable to parse new item list. See server log for more information.")
			return
		}
		utils.SendJSONResponse(w, string(jsonString))
		return
	} else if fileType != "" && filename != "" {
		if vsrc == "" {
			utils.SendErrorResponse(w, "Missing paramter: 'src'")
			return
		}

		fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
		fshAbs := fsh.FileSystemAbstraction

		//Translate the path to realpath
		rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
		if err != nil {
			utils.SendErrorResponse(w, "Invalid path given")
			return
		}

		//Check if directory is readonly
		accmode := userinfo.GetPathAccessPermission(vsrc)
		if accmode == arozfs.FsReadOnly {
			utils.SendErrorResponse(w, "This directory is Read Only")
			return
		} else if accmode == arozfs.FsDenied {
			utils.SendErrorResponse(w, "Access Denied")
			return
		}

		//Check if the filename contains web-unsafe characters
		if !utils.FilenameIsWebSafe(filename) {
			utils.SendErrorResponse(w, "Filename contains illegal characters")
			return
		}

		//Check if the file already exists. If yes, fix its filename.
		newfilePath := filepath.ToSlash(filepath.Join(rpath, filename))

		switch fileType {
		case "file":
			for fshAbs.FileExists(newfilePath) {
				utils.SendErrorResponse(w, "Given filename already exists")
				return
			}
			ext := filepath.Ext(filename)
			defaultFileCotent := []byte("")
			if ext != "" {
				templateFile, _ := fshAbs.Glob("system/newitem/*" + ext)
				if len(templateFile) > 0 {
					//Copy file from templateFile[0] to current dir with the given name
					input, _ := os.ReadFile(templateFile[0])
					defaultFileCotent = input
				}
			}

			err = fshAbs.WriteFile(newfilePath, defaultFileCotent, 0775)
			if err != nil {
				systemWideLogger.PrintAndLog("File System", "Unable to create new file: "+err.Error(), err)
				utils.SendErrorResponse(w, err.Error())
				return
			}

		case "folder":
			if fshAbs.FileExists(newfilePath) {
				utils.SendErrorResponse(w, "Given folder already exists")
				return
			}
			//Create the folder at target location
			err := fshAbs.Mkdir(newfilePath, 0755)
			if err != nil {
				utils.SendErrorResponse(w, err.Error())
				return
			}
		}

		utils.SendOK(w)
	} else {
		utils.SendErrorResponse(w, "Missing paramter(s).")
		return
	}
}

/*

	Handle file operations via WebSocket

	This handler only handle zip, unzip, copy and move. Not other operations.
	For other operations, please use the legacy handleOpr endpoint

	The actual operation logic is shared with the asynchronous (background)
	file operation endpoint, see runFileOperationTask below.
*/

func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
	//Get and check user permission
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	operation, _ := utils.GetPara(r, "opr") //Accept copy and move
	vsrcFiles, _ := utils.GetPara(r, "src")
	vdestFile, _ := utils.GetPara(r, "dest")
	existsOpr, _ := utils.GetPara(r, "existsresp")

	sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Permission checking
	if !userinfo.CanWrite(vdestFile) {
		systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
		w.WriteHeader(http.StatusForbidden)
		w.Write([]byte("403 - Access Denied"))
		return
	}

	//Upgrade to websocket
	var upgrader = websocket.Upgrader{}
	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - " + err.Error()))
		systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
		return
	}

	//Create the file operation task and remember it
	task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)

	//Send over the oprId for this file operation for tracking
	time.Sleep(300 * time.Millisecond)
	c.WriteMessage(1, []byte("{\"oprid\":\""+task.ID+"\"}"))

	//Run the operation on this request goroutine and stream the progress back
	runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, func(update fileOprProgressUpdate) {
		js, _ := json.Marshal(update)
		c.WriteMessage(1, js)
	})

	//This endpoint do not keep the finished record. Remove it right away.
	wsConnectionStore.Delete(task.ID)

	//Close WebSocket connection after finished
	time.Sleep(1 * time.Second)
	c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
	c.Close()
}

/*
	Handle file operations as a background task

	Unlike the WebSocket endpoint above, this endpoint returns the operation id
	right away and let the operation run in the background. The caller can then
	watch the progress of all of its operations through a single status
	WebSocket, see system_fs_handleFileOprStatusWebSocket below.
*/

func system_fs_handleAsyncOpr(w http.ResponseWriter, r *http.Request) {
	//Get and check user permission
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Validate the token
	tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
	if !tokenValid {
		http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
		return
	}

	operation, _ := utils.PostPara(r, "opr")
	vsrcFiles, _ := utils.PostPara(r, "src")
	vdestFile, _ := utils.PostPara(r, "dest")
	existsOpr, _ := utils.PostPara(r, "existsresp")

	sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Permission checking
	if !userinfo.CanWrite(vdestFile) {
		systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
		utils.SendErrorResponse(w, "Access Denied")
		return
	}

	//Create the task record and start it in the background
	task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)
	go runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, nil)

	js, _ := json.Marshal(map[string]string{"oprid": task.ID})
	utils.SendJSONResponse(w, string(js))
}

/*
	Stream the status of all the file operations of the requesting user

	One connection is enough for all the file operations of a user. The client
	side file operation dialog opens this once and renders every task it reports.
	Control commands can also be sent over this socket in the following format
	{"cmd":"pause | continue | cancel | remove", "oprid":"<operation_id>"} or
	{"cmd":"clear"} for removing all the finished records of this user.
*/

func system_fs_handleFileOprStatusWebSocket(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	var upgrader = websocket.Upgrader{}
	upgrader.CheckOrigin = func(r *http.Request) bool { return true }
	c, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		w.Write([]byte("500 - " + err.Error()))
		systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
		return
	}
	defer c.Close()

	//Listen for control commands on this socket until it is closed by the client
	clientGone := make(chan bool, 1)
	go func() {
		type controlCommand struct {
			Cmd   string
			Oprid string
		}
		for {
			_, message, err := c.ReadMessage()
			if err != nil {
				clientGone <- true
				return
			}

			cmd := controlCommand{}
			if json.Unmarshal(message, &cmd) != nil {
				continue
			}
			ApplyFileOperationControl(userinfo.Username, cmd.Cmd, cmd.Oprid)
		}
	}()

	//Push the full task list of this user on every tick
	ticker := time.NewTicker(500 * time.Millisecond)
	defer ticker.Stop()
	for {
		js, err := MarshalFileOperationForUser(userinfo.Username, true)
		if err == nil {
			if c.WriteMessage(websocket.TextMessage, js) != nil {
				return
			}
		}

		select {
		case <-clientGone:
			return
		case <-ticker.C:
		}
	}
}

/*
	File operation task runner

	These are the shared internals used by both the legacy WebSocket endpoint
	and the background (async) file operation endpoint above.
*/

// fileOprProgressUpdate is the progress payload of the legacy WebSocket file operation endpoint
type fileOprProgressUpdate struct {
	LatestFile string
	Progress   int
	StatusFlag int
	Error      string
}

// parseFileOperationRequest validates a file operation request and returns the
// decoded source file list and the decoded destination virtual path
func parseFileOperationRequest(operation string, vsrcFiles string, vdestFile string) ([]string, string, error) {
	//Check if opr is supported
	if operation != "move" && operation != "copy" && operation != "zip" && operation != "unzip" {
		systemWideLogger.PrintAndLog("File System", "This file operation is not supported on the file operation endpoint. Received: "+operation, errors.New("operation not supported"))
		return nil, "", errors.New("operation not supported on this endpoint")
	}

	//Decode the source file list
	var sourceFiles []string
	decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
	err := json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "File operation source file JSON parse error", err)
		return nil, "", errors.New("Source file JSON parse error.")
	}

	if len(sourceFiles) == 0 {
		return nil, "", errors.New("No source file given")
	}

	//Bugged char filtering
	tmp := []string{}
	for _, src := range sourceFiles {
		tmp = append(tmp, strings.ReplaceAll(src, "{{plug_sign}}", "+"))
	}
	sourceFiles = tmp
	vdestFile = strings.ReplaceAll(vdestFile, "{{plug_sign}}", "+")

	//Decode the target position
	escapedVdest := system_fs_specialURIDecode(vdestFile)
	vdestFile = escapedVdest

	if vdestFile == "" {
		return nil, "", errors.New("Undefined dest location")
	}

	//Make sure the destination is resolvable before doing anything else
	_, _, err = GetFSHandlerSubpathFromVpath(vdestFile)
	if err != nil {
		return nil, "", err
	}

	return sourceFiles, vdestFile, nil
}

// runFileOperationTask executes the file operation described by the given task record.
// onUpdate is optional and, when given, is called on every progress update.
func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFiles []string, vdestFile string, existsOpr string, onUpdate func(fileOprProgressUpdate)) {
	if existsOpr == "" {
		existsOpr = "keep"
	}

	oprId := task.ID
	operation := task.Operation

	pushUpdate := func(update fileOprProgressUpdate) {
		if onUpdate != nil {
			onUpdate(update)
		}
	}

	failTask := func(filename string, errmsg string) {
		SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, errmsg)
		pushUpdate(fileOprProgressUpdate{
			LatestFile: filename,
			Progress:   -1,
			Error:      errmsg,
			StatusFlag: filesystem.FsOpr_Error,
		})
	}

	//Resolve the destination file system handler
	destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
	if err != nil {
		failTask(filepath.Base(vdestFile), err.Error())
		return
	}
	destFshAbs := destFsh.FileSystemAbstraction
	rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)

	if !userinfo.CanWrite(vdestFile) {
		failTask(filepath.Base(vdestFile), "Access Denied: No Write Permission")
		return
	}

	if operation == "zip" {
		//Zip files
		outputFilename := filepath.Join(rdestFile, strings.ReplaceAll(filepath.Base(filepath.Dir(sourceFiles[0])+".zip"), ":", ""))
		if len(sourceFiles) == 1 {
			//Use the basename of the source file as zip file name
			outputFilename = filepath.Join(rdestFile, filepath.Base(sourceFiles[0])) + ".zip"
		}

		//Translate source Files into real paths
		realSourceFiles := []string{}
		sourceFileFsh := []*filesystem.FileSystemHandler{}
		for _, vsrcs := range sourceFiles {
			thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
			if err != nil {
				failTask(filepath.Base(vsrcs), "Source file not exists")
				return
			}
			rsrc, err := thisSrcFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
			if err != nil {
				failTask(filepath.Base(rsrc), "Source file not exists")
				return
			}

			realSourceFiles = append(realSourceFiles, rsrc)
			sourceFileFsh = append(sourceFileFsh, thisSrcFsh)
		}

		zipDestPath := outputFilename
		zipDestFsh := destFsh
		if destFsh.RequireBuffer {
			zipDestPath = getFsBufferFilepath(outputFilename, false)
			zipDestFsh = nil
		}

		//Create the zip file
		err = filesystem.ArozZipFileWithProgress(sourceFileFsh, realSourceFiles, zipDestFsh, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) int {
			sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
			pushUpdate(fileOprProgressUpdate{
				LatestFile: currentFilename,
				Progress:   int(math.Ceil(progress)),
				Error:      "",
				StatusFlag: sig,
			})
			return sig
		})

		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
		}

		if destFsh.RequireBuffer {
			//Move the buffer result to remote
			f, _ := os.Open(zipDestPath)
			err = destFshAbs.WriteStream(outputFilename, f, 0775)
			if err != nil {
				systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver"+destFsh.Filesystem+" failed", err)
			}
			f.Close()

			//Clear local buffers
			os.Remove(zipDestPath)
			cleanFsBufferFileFromList(realSourceFiles)
		}
	} else if operation == "unzip" {
		//Create the destination folder
		destFshAbs.MkdirAll(rdestFile, 0755)

		//Convert the src files into realpaths
		realSourceFiles := []string{}
		for _, vsrcs := range sourceFiles {
			thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
			if err != nil {
				failTask(filepath.Base(vsrcs), "Source file not exists")
				return
			}
			thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
			rsrc, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
			if err != nil {
				failTask(filepath.Base(rsrc), "Source file not exists")
				return
			}
			if thisSrcFsh.RequireBuffer {
				localBufferFilepath, err := bufferRemoteFileToLocal(thisSrcFsh, rsrc, false)
				if err != nil {
					failTask(filepath.Base(rsrc), "Failed to buffer file to local disk")
					return
				}
				realSourceFiles = append(realSourceFiles, localBufferFilepath)
			} else {
				realSourceFiles = append(realSourceFiles, rsrc)
			}
		}

		unzipDest := rdestFile
		if destFsh.RequireBuffer {
			unzipDest = getFsBufferFilepath(rdestFile, true)
		}

		//Unzip the files
		filesystem.ArozUnzipFileWithProgress(realSourceFiles, unzipDest, func(currentFile string, filecount int, totalfile int, progress float64) int {
			//Generate the status update struct
			sig, _ := UpdateOngoingFileOperation(oprId, filepath.Base(currentFile), math.Ceil(progress))
			pushUpdate(fileOprProgressUpdate{
				LatestFile: filepath.Base(currentFile),
				Progress:   int(math.Ceil(progress)),
				Error:      "",
				StatusFlag: sig,
			})

			return sig
		})

		if destFsh.RequireBuffer {
			//Push the unzip results back to remote fs
			filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
				path = filepath.ToSlash(path)
				relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
				if info.IsDir() {
					destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
				} else {
					f, _ := os.Open(path)
					destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
					f.Close()
				}
				return nil
			})

			cleanFsBufferFileFromList([]string{unzipDest})
		}

		cleanFsBufferFileFromList(realSourceFiles)

	} else {
		//Other operations that allow multiple source files to handle one by one
		for i := 0; i < len(sourceFiles); i++ {
			vsrcFile := sourceFiles[i]
			thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
			if err != nil {
				MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
				failTask(filepath.Base(vsrcFile), "Source file not exists")
				return
			}
			thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
			rsrcFile, _ := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)

			if !thisSrcFshAbs.FileExists(rsrcFile) {
				//This source file not exists. Report Error and Stop
				MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
				failTask(filepath.Base(rsrcFile), "Source file not exists")
				return
			}

			//Progress handler shared by the move and copy operations. The overall
			//progress is worked out from the bytes of every source file, so a
			//small file no longer counts for as much of the bar as a large one.
			subtaskProgressHandler := func(currentFile string, bytesDone int64, bytesTotal int64) int {
				sig, overallProgress, _ := UpdateOngoingFileOperationSubtask(oprId, i, filepath.Base(currentFile), bytesDone, bytesTotal)
				pushUpdate(fileOprProgressUpdate{
					LatestFile: filepath.Base(currentFile),
					Progress:   int(math.Ceil(overallProgress)),
					Error:      "",
					StatusFlag: sig,
				})
				return sig
			}

			if operation == "move" {
				err := filesystem.FileMoveWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, subtaskProgressHandler)

				//Handle move starting error
				if err != nil {
					MarkFileOperationSubtaskEnded(oprId, i, err.Error())
					failTask(filepath.Base(rsrcFile), err.Error())
					return
				}

				//Remove the cache for the original file
				metadata.RemoveCache(thisSrcFsh, rsrcFile)

			} else if operation == "copy" {
				err := filesystem.FileCopyWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, subtaskProgressHandler)

				//Handle Copy starting error
				if err != nil {
					MarkFileOperationSubtaskEnded(oprId, i, err.Error())
					failTask(filepath.Base(rsrcFile), err.Error())
					return
				}
			}

			//This source file is done. Mark it as completed
			MarkFileOperationSubtaskEnded(oprId, i, "")
		}
	}

	//Check if the operation was cancelled by the user half way through
	endingSignal := filesystem.FsOpr_Continue
	if t, err := GetOngoingFileOperationByOprID(oprId); err == nil {
		fileOprTaskLock.RLock()
		if t.FileOperationSignal == filesystem.FsOpr_Cancel {
			endingSignal = filesystem.FsOpr_Cancel
		}
		fileOprTaskLock.RUnlock()
	}

	SetFileOperationTaskEnded(oprId, endingSignal, "")
}

/*
	Handle file operations

	Support {move, copy, delete, recycle, rename}
*/
//Handle file operations.
func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
	//Check if user logged in
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	//Validate the token
	tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
	if !tokenValid {
		http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
		return
	}

	operation, _ := utils.PostPara(r, "opr")
	vsrcFiles, _ := utils.PostPara(r, "src")
	vdestFile, _ := utils.PostPara(r, "dest")
	vnfilenames, _ := utils.PostPara(r, "new") //Only use when rename or create new file / folder

	//Check if operation valid.
	if operation == "" {
		//Undefined operations.
		utils.SendErrorResponse(w, "Undefined operations paramter: Missing 'opr' in request header.")
		return
	}

	//As the user can pass in multiple source files at the same time, parse sourceFiles from json string
	var sourceFiles []string
	//This line is required in order to allow passing of special charaters
	decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
	err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
	if err != nil {
		utils.SendErrorResponse(w, "Source file JSON parse error.")
		return
	}

	//Check if new filenames are also valid. If yes, translate it into string array
	var newFilenames []string
	if vnfilenames != "" {
		vnfilenames, _ := url.QueryUnescape(vnfilenames)
		err = json.Unmarshal([]byte(vnfilenames), &newFilenames)
		if err != nil {
			utils.SendErrorResponse(w, "Unable to parse JSON for new filenames")
			return
		}
	}

	if operation == "zip" {
		//Zip operation. Parse the real filepath list
		rsrcFiles := []string{}
		srcFshs := []*filesystem.FileSystemHandler{}
		destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
		if err != nil {
			utils.SendErrorResponse(w, "Unable to resolve zip destination path")
			return
		}
		destFshAbs := destFsh.FileSystemAbstraction
		rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
		for _, vsrcFile := range sourceFiles {
			vsrcFsh, vsrcSubpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
			if err != nil {
				continue
			}
			rsrcFile, _ := vsrcFsh.FileSystemAbstraction.VirtualPathToRealPath(vsrcSubpath, userinfo.Username)
			if vsrcFsh.FileSystemAbstraction.FileExists(rsrcFile) {
				//Push directly its local path to list
				rsrcFiles = append(rsrcFiles, rsrcFile)
				srcFshs = append(srcFshs, vsrcFsh)
			}
		}

		zipFilename := rdestFile
		if destFshAbs.IsDir(rdestFile) {
			//Append the filename to it
			if len(rsrcFiles) == 1 {
				zipFilename = filepath.Join(rdestFile, strings.TrimSuffix(filepath.Base(rsrcFiles[0]), filepath.Ext(filepath.Base(rsrcFiles[0])))+".zip")
			} else if len(rsrcFiles) > 1 {
				zipFilename = filepath.Join(rdestFile, filepath.Base(filepath.Dir(rsrcFiles[0]))+".zip")
			}
		}

		//Create a buffer if destination fsh request buffer
		zipFileTargetLocation := zipFilename
		zipDestFsh := destFsh
		if destFsh.RequireBuffer {
			zipFileTargetLocation = getFsBufferFilepath(zipFilename, false)
			zipDestFsh = nil
		}

		//Create a zip file at target location
		err = filesystem.ArozZipFile(srcFshs, rsrcFiles, zipDestFsh, zipFileTargetLocation, false)
		if err != nil {
			os.Remove(zipFileTargetLocation)
			utils.SendErrorResponse(w, err.Error())
			return
		}

		//Write it to final destination from buffer
		if destFsh.RequireBuffer {
			//Upload the finalized zip file
			f, _ := destFshAbs.Open(zipFileTargetLocation)
			destFshAbs.WriteStream(zipFilename, f, 0775)
			f.Close()

			//Remove all buff files
			os.Remove(zipFileTargetLocation)
			cleanFsBufferFileFromList(rsrcFiles)
		}

	} else {
		//For operations that is handled file by file
		for i, vsrcFile := range sourceFiles {
			//Convert the virtual path to realpath on disk
			srcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
			if err != nil {
				continue
			}
			srcFshAbs := srcFsh.FileSystemAbstraction
			rsrcFile, _ := srcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)

			destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
			var destFshAbs filesystem.FileSystemAbstraction = nil
			var rdestFile string = ""
			if err == nil {
				destFshAbs = destFsh.FileSystemAbstraction
				rdestFile, _ = destFshAbs.VirtualPathToRealPath(destSubpath, userinfo.Username)
			}

			//Check if the source file exists
			if operation == "rename" {
				//Check if the usage is correct.
				if vdestFile != "" {
					utils.SendErrorResponse(w, "Rename only accept 'src' and 'new'. Please use move if you want to move a file.")
					return
				}
				//Check if new name paramter is passed in.
				if len(newFilenames) == 0 {
					utils.SendErrorResponse(w, "Missing paramter (JSON string): 'new'")
					return
				}
				//Check if the source filenames and new filenanmes match
				if len(newFilenames) != len(sourceFiles) {
					utils.SendErrorResponse(w, "New filenames do not match with source filename's length.")
					return
				}

				//Check if the target dir is not readonly
				accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
				switch accmode {
				case arozfs.FsReadOnly:
					utils.SendErrorResponse(w, "This directory is Read Only")
					return
				case arozfs.FsDenied:
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				thisFilename := filepath.Base(newFilenames[i])

				//Check if the new filename contains web-unsafe characters
				if !utils.FilenameIsWebSafe(thisFilename) {
					utils.SendErrorResponse(w, "Filename contains illegal characters")
					return
				}
				//Check if the name already exists. If yes, return false
				if srcFshAbs.FileExists(filepath.Join(filepath.Dir(rsrcFile), thisFilename)) {
					utils.SendErrorResponse(w, "File already exists")
					return
				}

				//Everything is ok. Rename the file.
				targetNewName := filepath.Join(filepath.Dir(rsrcFile), thisFilename)
				err = srcFshAbs.Rename(rsrcFile, targetNewName)
				if err != nil {
					systemWideLogger.PrintAndLog("File System", "File rename failed", err)
					utils.SendErrorResponse(w, err.Error())
					return
				}

				//Remove the cache for the original file
				metadata.RemoveCache(srcFsh, rsrcFile)

			} else if operation == "move" {
				//File move operation. Check if the source file / dir and target directory exists
				/*
					Example usage from file explorer
					$.ajax({
						type: 'POST',
						url: `/system/file_system/fileOpr`,
						data: {opr: "move" ,src: JSON.stringify(fileList), dest: targetDir},
						success: function(data){
							if (data.error !== undefined){
								msgbox("remove",data.error);
							}else{
								//OK, do something
							}
						}
					});
				*/

				if !srcFshAbs.FileExists(rsrcFile) {
					utils.SendErrorResponse(w, "Source file not exists")
					return
				}

				//Check if the source file is read only.
				accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
				if accmode == arozfs.FsReadOnly {
					utils.SendErrorResponse(w, "This source file is Read Only")
					return
				} else if accmode == arozfs.FsDenied {
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				if rdestFile == "" {
					utils.SendErrorResponse(w, "Undefined dest location")
					return
				}

				//Get exists overwrite mode
				existsOpr, _ := utils.PostPara(r, "existsresp")

				//Check if use fast move instead
				//Check if the source and destination folder are under the same root. If yes, use os.Rename for faster move operations

				//Check if the two files are under the same user root path

				//srcAbs, _ := filepath.Abs(rsrcFile)
				//destAbs, _ := filepath.Abs(rdestFile)
				//underSameRoot, _ := filesystem.UnderTheSameRoot(srcAbs, destAbs)

				//Updates 19-10-2020: Added ownership management to file move and copy
				userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)

				err = filesystem.FileMove(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, nil)
				if err != nil {
					utils.SendErrorResponse(w, err.Error())
					//Restore the ownership if remove failed
					userinfo.SetOwnerOfFile(srcFsh, vsrcFile)
					return
				}

				//Set user to own the new file
				newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
				newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
				userinfo.SetOwnerOfFile(destFsh, newfileVpath)

				//Remove cache for the original file
				metadata.RemoveCache(srcFsh, rsrcFile)
			} else if operation == "copy" {
				//Copy file. See move example and change 'opr' to 'copy'
				if !srcFshAbs.FileExists(rsrcFile) {
					utils.SendErrorResponse(w, "Source file not exists")
					return
				}

				//Check if the desintation is read only.
				if !userinfo.CanWrite(vdestFile) {
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				if !destFshAbs.FileExists(rdestFile) {
					if destFshAbs.FileExists(filepath.Dir(rdestFile)) {
						//User pass in the whole path for the folder. Report error usecase.
						utils.SendErrorResponse(w, "Dest location should be an existing folder instead of the full path of the copied file")
						return
					}
					utils.SendErrorResponse(w, "Dest folder not found")
					return
				}

				existsOpr, _ := utils.PostPara(r, "existsresp")

				//Check if the user have space for the extra file
				if !userinfo.StorageQuota.HaveSpace(filesystem.GetFileSize(rdestFile)) {
					utils.SendErrorResponse(w, "Storage Quota Full")
					return
				}

				err = filesystem.FileCopy(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, nil)
				if err != nil {
					utils.SendErrorResponse(w, err.Error())
					return
				}

				//Set user to own this file
				newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
				newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
				userinfo.SetOwnerOfFile(destFsh, newfileVpath)

			} else if operation == "delete" {
				//Delete the file permanently
				if !srcFshAbs.FileExists(rsrcFile) {
					//Check if it is a non escapted file instead
					utils.SendErrorResponse(w, "Source file not exists")
					return

				}

				if !userinfo.CanWrite(vsrcFile) {
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				//Check if the user own this file
				isOwner := userinfo.IsOwnerOfFile(srcFsh, vsrcFile)
				if isOwner {
					//This user own this system. Remove this file from his quota
					userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
				}

				//Check if this file has any cached files. If yes, remove it
				metadata.RemoveCache(srcFsh, rsrcFile)

				//Clear the cache folder if there is no files inside
				fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
				if len(fc) == 0 && err == nil {
					srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
				}

				err = srcFshAbs.RemoveAll(rsrcFile)
				if err != nil {
					systemWideLogger.PrintAndLog("File System", "Unable to remove file from "+srcFsh.UUID, err)
					utils.SendErrorResponse(w, err.Error())
					return
				}

			} else if operation == "recycle" {
				//Put it into a subfolder named trash and allow it to to be removed later
				if !srcFshAbs.FileExists(rsrcFile) {
					//Check if it is a non escapted file instead
					utils.SendErrorResponse(w, "Source file not exists")
					return

				}

				//Check if the upload target is read only.
				if !userinfo.CanWrite(vsrcFile) {
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				//Check if this file has any cached files. If yes, remove it
				metadata.RemoveCache(srcFsh, rsrcFile)

				//Clear the cache folder if there is no files inside
				fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
				if len(fc) == 0 && err == nil {
					srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
				}

				/*
					Trash quota

					Checked before the move, not after: once the file is inside
					.trash it has already left the tree the user was looking at,
					and undoing that cleanly is harder than refusing up front.
					The client recognises this error code and offers to empty
					the bin, delete outright, or do nothing.
				*/
				userTrashQuota := system_fs_getTrashQuotaBytes(userinfo.Username)
				if userTrashQuota > 0 {
					incomingSize := system_fs_getTrashEntrySize(srcFsh, rsrcFile)
					if system_fs_getTrashUsage(userinfo.Username)+incomingSize > userTrashQuota {
						utils.SendErrorResponse(w, "TRASH_QUOTA_EXCEEDED")
						return
					}
				}

				//Create a trash directory for this folder
				trashDir := filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.trash/"
				srcFshAbs.MkdirAll(trashDir, 0755)
				hidden.HideFile(filepath.Dir(trashDir))
				hidden.HideFile(trashDir)
				err = srcFshAbs.Rename(rsrcFile, trashDir+filepath.Base(rsrcFile)+"."+utils.Int64ToString(time.Now().Unix()))
				if err != nil {
					if srcFsh.RequireBuffer {
						utils.SendErrorResponse(w, "Incompatible File System Type: Try SHIFT + DELETE to delete file permanently")
					} else {
						systemWideLogger.PrintAndLog("File System", "Failed to move file to trash. See log for more info.", err)
						utils.SendErrorResponse(w, "Failed to move file to trash")
					}
					return
				}
			} else if operation == "unzip" {
				//Unzip the file to destination

				//Check if the user can write to the target dest file
				if !userinfo.CanWrite(string(vdestFile)) {
					utils.SendErrorResponse(w, "Access Denied")
					return
				}

				//Make the rdest directory if not exists
				if !destFshAbs.FileExists(rdestFile) {
					err = destFshAbs.MkdirAll(rdestFile, 0755)
					if err != nil {
						utils.SendErrorResponse(w, err.Error())
						return
					}
				}

				unzipSource := rsrcFile
				unzipDest := rdestFile
				if srcFsh.RequireBuffer {
					localBufferedFile, _ := bufferRemoteFileToLocal(srcFsh, rsrcFile, false)
					unzipSource = localBufferedFile
				}

				if destFsh.RequireBuffer {
					localUnzipBuffer, _ := bufferRemoteFileToLocal(destFsh, rdestFile, true)
					unzipDest = localUnzipBuffer
				}

				//OK! Unzip to destination
				err := filesystem.Unzip(unzipSource, unzipDest)
				if err != nil {
					utils.SendErrorResponse(w, err.Error())
					return
				}

				if srcFsh.RequireBuffer {
					//Remove the local buffered file
					os.Remove(unzipSource)
				}

				if destFsh.RequireBuffer {
					//Push the buffer to target fs
					filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
						path = filepath.ToSlash(path)
						relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
						if info.IsDir() {
							destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
						} else {
							f, _ := os.Open(path)
							destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
							f.Close()
						}
						return nil
					})

					cleanFsBufferFileFromList([]string{unzipDest})
				}

			} else {
				utils.SendErrorResponse(w, "Unknown file opeartion given")
				return
			}
		}

	}
	utils.SendOK(w)
}

// Allow systems to store key value pairs in the database as preferences.
func system_fs_handleUserPreference(w http.ResponseWriter, r *http.Request) {
	username, err := authAgent.GetUserName(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	key, _ := utils.GetPara(r, "key")
	value, _ := utils.GetPara(r, "value")
	remove, _ := utils.GetPara(r, "remove")

	if key != "" && value == "" && remove == "" {
		//Get mode. Read the prefernece with given key
		result := ""
		err := sysdb.Read("fs", "pref/"+key+"/"+username, &result)
		if err != nil {
			utils.SendJSONResponse(w, `{"error":"Key not found."}`)
			return
		}
		utils.SendTextResponse(w, result)
	} else if key != "" && value == "" && remove == "true" {
		//Remove mode. Delete this key from sysdb
		err := sysdb.Delete("fs", "pref/"+key+"/"+username)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
		}

		utils.SendOK(w)
	} else if key != "" && value != "" {
		//Set mode. Set the preference with given key
		if len(value) > 1024*1024 { //1KB
			//Size too big. Reject storage
			utils.SendErrorResponse(w, "Preference value too long. Preference value can only store maximum 1KB.")
			return
		}
		sysdb.Write("fs", "pref/"+key+"/"+username, value)
		utils.SendOK(w)
	}
}

func system_fs_removeUserPreferences(username string) {
	entries, err := sysdb.ListTable("fs")
	if err != nil {
		return
	}

	for _, keypairs := range entries {
		if strings.Contains(string(keypairs[0]), "pref/") && strings.Contains(string(keypairs[0]), "/"+username) {
			//Remove this preference
			sysdb.Delete("fs", string(keypairs[0]))
		}
	}
}

func system_fs_listDrives(w http.ResponseWriter, r *http.Request) {
	if !authAgent.CheckAuth(r) {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}
	userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
	type driveInfo struct {
		Drivepath       string
		DriveFreeSpace  uint64
		DriveTotalSpace uint64
		DriveAvailSpace uint64
	}
	var drives []driveInfo
	if runtime.GOOS == "windows" {
		//Under windows
		for _, drive := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
			f, err := os.Open(string(drive) + ":\\")
			if err == nil {
				thisdrive := new(driveInfo)
				thisdrive.Drivepath = string(drive) + ":\\"
				free, total, avail := storage.GetDriveCapacity(string(drive) + ":\\")
				thisdrive.DriveFreeSpace = free
				thisdrive.DriveTotalSpace = total
				thisdrive.DriveAvailSpace = avail
				drives = append(drives, *thisdrive)
				f.Close()
			}
		}
	} else {
		//Under linux environment
		//Append all the virtual directories root as root instead
		storageDevices := []string{}
		for _, fshandler := range userinfo.GetAllFileSystemHandler() {
			storageDevices = append(storageDevices, fshandler.Path)
		}

		//List all storage information of each devices
		for _, dev := range storageDevices {
			thisdrive := new(driveInfo)
			thisdrive.Drivepath = filepath.Base(dev)
			free, total, avail := storage.GetDriveCapacity(string(dev))
			thisdrive.DriveFreeSpace = free
			thisdrive.DriveTotalSpace = total
			thisdrive.DriveAvailSpace = avail
			drives = append(drives, *thisdrive)
		}

	}

	jsonString, _ := json.Marshal(drives)
	utils.SendJSONResponse(w, string(jsonString))
}

func system_fs_listRoot(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	username := userinfo.Username
	userRoot, _ := utils.GetPara(r, "user")
	if userRoot == "true" {
		type fileObject struct {
			Filename string
			Filepath string
			IsDir    bool
		}
		//List the root media folders under user:/
		fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath("user:/")
		fshAbs := fsh.FileSystemAbstraction
		filesInUserRoot := []fileObject{}
		filesInRoot, _ := fshAbs.Glob(filepath.ToSlash(filepath.Clean(*root_directory)) + "/users/" + username + "/*")
		for _, file := range filesInRoot {
			//Check if this is a hidden file
			if len(filepath.Base(file)) > 0 && filepath.Base(file)[:1] == "." {
				continue
			}
			thisFile := new(fileObject)
			thisFile.Filename = filepath.Base(file)
			thisFile.Filepath, _ = fshAbs.RealPathToVirtualPath(file, userinfo.Username)
			thisFile.IsDir = fshAbs.IsDir(file)
			filesInUserRoot = append(filesInUserRoot, *thisFile)
		}
		jsonString, _ := json.Marshal(filesInUserRoot)
		utils.SendJSONResponse(w, string(jsonString))
	} else {
		type rootObject struct {
			rootID     string //The vroot id
			RootName   string //The name of this vroot
			RootPath   string //The path of this vroot
			BufferedFs bool   //If buffer typed FS
		}

		roots := []*rootObject{}
		for _, store := range userinfo.GetAllFileSystemHandler() {
			var thisDevice = new(rootObject)
			thisDevice.RootName = store.Name
			thisDevice.RootPath = store.UUID + ":/"
			thisDevice.rootID = store.UUID
			thisDevice.BufferedFs = store.RequireBuffer
			roots = append(roots, thisDevice)
		}

		jsonString, _ := json.Marshal(roots)
		utils.SendJSONResponse(w, string(jsonString))
	}

}

/*
	Special Glob for handling path with [ or ] inside.
	You can also pass in normal path for globing if you are not sure.
*/

func system_fs_specialURIDecode(inputPath string) string {
	inputPath = strings.ReplaceAll(inputPath, "+", "{{plus_sign}}")
	inputPath, _ = url.QueryUnescape(inputPath)
	inputPath = strings.ReplaceAll(inputPath, "{{plus_sign}}", "+")
	return inputPath
}

/*
func system_fs_specialURIEncode(inputPath string) string {
	inputPath = strings.ReplaceAll(inputPath, " ", "{{space_sign}}")
	inputPath, _ = url.QueryUnescape(inputPath)
	inputPath = strings.ReplaceAll(inputPath, "{{space_sign}}", "%20")
	return inputPath
}
*/

/*
The parent folder of a virtual path, as shown in the properties dialog.

A virtual path is always slash separated and carries a "vdID:" prefix, so
filepath.Dir must not be used here: on Windows it reads "user:" as a volume
name and Clean guards the result with a "./" prefix, turning
"user:/cluster/a.agi" into "./user:/cluster". GetIDFromVirtualPath strips
that prefix back off for the same reason.
*/
func virtualDirname(vpath string) string {
	return path.Dir(vpath)
}

// Handle file properties request
func system_fs_getFileProperties(w http.ResponseWriter, r *http.Request) {
	type fileProperties struct {
		VirtualPath    string
		StoragePath    string
		Basename       string
		VirtualDirname string
		StorageDirname string
		Ext            string
		MimeType       string
		Filesize       int64
		Permission     string
		LastModTime    string
		LastModUnix    int64
		CreationTime   string
		CreationUnix   int64
		IsDirectory    bool
		Owner          string
	}

	result := fileProperties{}

	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	vpath, err := utils.PostPara(r, "path")
	if err != nil {
		utils.SendErrorResponse(w, "path not defined")
		return
	}

	vrootID, subpath, _ := filesystem.GetIDFromVirtualPath(vpath)
	fsh, err := GetFsHandlerByUUID(vrootID)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	fshAbs := fsh.FileSystemAbstraction

	rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	fileStat, err := fshAbs.Stat(rpath)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	fileMime := "text/directory"
	if !fileStat.IsDir() {
		m, _, err := filesystem.GetMime(rpath)
		if err != nil {
			fileMime = mime.TypeByExtension(filepath.Ext(rpath))
		} else {
			fileMime = m
		}

	}

	filesize := fileStat.Size()
	//Get file overall size if this is folder
	if fileStat.IsDir() {
		if fsh.IsNetworkDrive() {
			filesize = -1
		} else {
			//Check if du exists
			usefallback := true //Use fallback

			if fsh.IsLocalDrive() {
				//Try using native syscall to grab directory size
				nativeSize, err := filesystem.GetDirectorySizeNative(rpath)
				if err == nil {
					usefallback = false
					filesize = nativeSize
				}
			}

			if usefallback {
				// invalid platform. walk the whole file system
				var size int64 = 0
				fshAbs.Walk(rpath, func(_ string, info os.FileInfo, err error) error {
					if err != nil {
						return err
					}
					if !info.IsDir() {
						size += info.Size()
					}
					return err
				})
				filesize = size
			}

		}
	}

	//Get the file creation time. Not every platform / file system records one,
	//in which case it is left empty for the client to grey out
	creationTimeString := ""
	var creationUnix int64 = 0
	if creationTime, ok := filesystem.GetFileCreationTime(fileStat); ok {
		creationTimeString = time.Unix(creationTime, 0).Format("2006-01-02 15:04:05")
		creationUnix = creationTime
	}

	//Get file owner
	owner := userinfo.GetFileOwner(fsh, vpath)

	if owner == "" {
		//Handle special virtual roots
		owner = "Unknown"
	}

	result = fileProperties{
		VirtualPath:    vpath,
		StoragePath:    filepath.ToSlash(filepath.Clean(rpath)),
		Basename:       filepath.Base(rpath),
		VirtualDirname: virtualDirname(vpath),
		StorageDirname: filepath.ToSlash(filepath.Dir(rpath)),
		Ext:            filepath.Ext(rpath),
		MimeType:       fileMime,
		Filesize:       filesize,
		Permission:     fileStat.Mode().Perm().String(),
		LastModTime:    fileStat.ModTime().Format("2006-01-02 15:04:05"),
		LastModUnix:    fileStat.ModTime().Unix(),
		CreationTime:   creationTimeString,
		CreationUnix:   creationUnix,
		IsDirectory:    fileStat.IsDir(),
		Owner:          owner,
	}

	jsonString, _ := json.Marshal(result)
	utils.SendJSONResponse(w, string(jsonString))

}

/*
	Storage info

	Where the file actually is. Most drives have nothing to add beyond the
	storage path already in the properties dialog, but a drive that spreads
	its files over several hosts does: cluster:/ answers with the nodes and
	volumes holding a copy of the file. The abstraction resolves it
	(arozfs.StorageInfoProvider), so this handler stays the same whatever
	kind of drive the file is on.
*/

func system_fs_getStorageInfo(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	vpath, err := utils.PostPara(r, "path")
	if err != nil {
		utils.SendErrorResponse(w, "path not defined")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Reading a file the user has no access to must not leak where it is kept
	if !userinfo.CanRead(vpath) {
		utils.SendErrorResponse(w, "Permission denied")
		return
	}

	provider, ok := fsh.FileSystemAbstraction.(arozfs.StorageInfoProvider)
	if !ok {
		//Nothing to add for this kind of drive
		utils.SendErrorResponse(w, "Storage info not supported by this drive")
		return
	}

	rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	info, err := provider.StorageInfo(rpath)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	js, _ := json.Marshal(info)
	utils.SendJSONResponse(w, string(js))
}

/*
	List directory in the given path

	Usage: Pass in dir like the following examples:
	AOR:/Desktop	<= Open /user/{username}/Desktop
	S1:/			<= Open {uuid=S1}/
*/

func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
	currentDir, err := utils.PostPara(r, "dir")
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	sortMode, _ := utils.PostPara(r, "sort")
	showHidden, _ := utils.PostPara(r, "showHidden")

	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		//user not logged in. Redirect to login page.
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	if currentDir == "" {
		utils.SendErrorResponse(w, "Invalid dir given.")
		return
	}

	// Pad a slash at the end of currentDir if not exists
	if !strings.HasSuffix(currentDir, "/") {
		currentDir = currentDir + "/"
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	fshAbs := fsh.FileSystemAbstraction

	realpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	if !fshAbs.FileExists(realpath) {
		//Path not exists
		userRoot, _ := fshAbs.VirtualPathToRealPath("/", userinfo.Username)
		if filepath.Clean(realpath) == filepath.Clean(userRoot) || realpath == "" {
			//Initiate user folder (Initiaed in user object)
			err = fshAbs.MkdirAll(userRoot, 0775)
			if err != nil {
				systemWideLogger.PrintAndLog("File System", "Unable to create user root on "+fsh.UUID+": "+err.Error(), nil)
				utils.SendErrorResponse(w, "Unable to create user root folder due to file system error")
				return
			}
		} else {
			//Folder not exists
			systemWideLogger.PrintAndLog("File System", "Requested path: "+realpath+" does not exists", nil)
			utils.SendErrorResponse(w, "Folder not exists")
			return
		}

	}

	if sortMode == "" {
		sortMode = "default"
	}

	files, err := fshAbs.ReadDir(realpath)
	if err != nil {
		utils.SendErrorResponse(w, "Readdir Failed: "+strings.ReplaceAll(err.Error(), "\\", "/"))
		systemWideLogger.PrintAndLog("File System", "Unable to read dir: "+err.Error(), err)
		return
	}

	//Remapping use parsed list
	parsedFilelist := map[string]filesystem.FileData{}

	//Sorting use list
	realpathList := []string{}
	fileInfoList := []fs.FileInfo{}
	for _, f := range files {
		//Check if it is hidden file
		isHidden, _ := hidden.IsHidden(f.Name(), false)
		if showHidden != "true" && isHidden {
			//Skipping hidden files
			continue
		}

		//Check if this file contains invalid characters
		if !utils.FilenameIsWebSafe(f.Name()) {
			continue
		}

		//Check if this is an aodb file
		if f.Name() == "aofs.db" || f.Name() == "aofs.db.lock" {
			//Database file (reserved)
			continue
		}

		//Check if it is shortcut file. If yes, render a shortcut data struct
		var shortCutInfo *arozfs.ShortcutData = nil
		if filepath.Ext(f.Name()) == ".shortcut" {
			//This is a shortcut file
			fcontent, err := fshAbs.ReadFile(arozfs.ToSlash(filepath.Join(realpath, f.Name())))
			if err != nil {
				shortCutInfo = nil
			} else {
				shorcutData, err := shortcut.ReadShortcut(fcontent)
				if err != nil {
					shortCutInfo = nil
				} else {
					shortCutInfo = shorcutData
				}
			}
		}

		statInfo, err := f.Info()
		if err != nil {
			continue
		}
		thisvPath, _ := fshAbs.RealPathToVirtualPath(filepath.Join(realpath, f.Name()), userinfo.Username)
		thisFile := filesystem.FileData{
			Filename:    f.Name(),
			Filepath:    currentDir + f.Name(),
			Realpath:    filepath.ToSlash(filepath.Join(realpath, f.Name())),
			IsDir:       f.IsDir(),
			Filesize:    statInfo.Size(),
			Displaysize: filesystem.GetFileDisplaySize(statInfo.Size(), 2),
			ModTime:     statInfo.ModTime().Unix(),
			IsShared:    shareManager.FileIsShared(userinfo, thisvPath),
			Shortcut:    shortCutInfo,
		}

		parsedFilelist[currentDir+f.Name()] = thisFile
		realpathList = append(realpathList, currentDir+f.Name())
		fileInfoList = append(fileInfoList, statInfo)
	}

	//Sort the filelist
	sortedRealpathList := fssort.SortFileList(realpathList, fileInfoList, sortMode)
	results := []filesystem.FileData{}

	for _, thisRpath := range sortedRealpathList {
		val, ok := parsedFilelist[thisRpath]
		if ok {
			results = append(results, val)
		}
	}

	jsonString, _ := json.Marshal(results)
	utils.SendJSONResponse(w, string(jsonString))
}

// Handle getting a hash from a given contents in the given path
func system_fs_handleDirHash(w http.ResponseWriter, r *http.Request) {
	currentDir, err := utils.GetPara(r, "dir")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid dir given")
		return
	}

	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
	if err != nil {
		utils.SendErrorResponse(w, "Unable to resolve target directory")
		return
	}
	fshAbs := fsh.FileSystemAbstraction

	rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, "Invalid dir given")
		return
	}

	//Get a list of files in this directory
	/*
		currentDir = filepath.ToSlash(filepath.Clean(rpath)) + "/"

			filesInDir, err := fshAbs.Glob(currentDir + "*")
			if err != nil {
				utils.SendErrorResponse(w, err.Error())
				return
			}


			filenames := []string{}
			for _, file := range filesInDir {
				if len(filepath.Base(file)) > 0 && string([]rune(filepath.Base(file))[0]) != "." {
					//Ignore hidden files
					filenames = append(filenames, filepath.Base(file))
				}

			}
	*/
	finfos, err := fshAbs.ReadDir(rpath)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	filenames := []string{}
	for _, fi := range finfos {
		isHiddenFile, _ := hidden.IsHidden(fi.Name(), false)
		if len(fi.Name()) > 0 && !isHiddenFile {
			//Ignore hidden files
			filenames = append(filenames, fi.Name())
		}
	}

	sort.Strings(filenames)

	//Build a hash base on the filelist
	h := sha256.New()
	h.Write([]byte(strings.Join(filenames, ",")))
	utils.SendTextResponse(w, hex.EncodeToString((h.Sum(nil))))
}

/*
	File zipping and unzipping functions
*/

// Handle all zip related API
func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	opr, err := utils.PostPara(r, "opr")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid opr or opr not defined")
		return
	}

	vsrc, _ := utils.PostPara(r, "src")
	if vsrc == "" {
		utils.SendErrorResponse(w, "Invalid src paramter")
		return
	}

	vdest, _ := utils.PostPara(r, "dest")
	rdest := ""

	//Convert source path from JSON string to object
	virtualSourcePaths := []string{}
	err = json.Unmarshal([]byte(vsrc), &virtualSourcePaths)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	//Check each of the path
	realSourcePaths := []string{}
	sourceFshs := []*filesystem.FileSystemHandler{}
	for _, vpath := range virtualSourcePaths {
		thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
		if err != nil {
			utils.SendErrorResponse(w, "Unable to resolve file: "+vpath)
			return
		}
		thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
		thisrpath, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
		if err != nil || !thisSrcFshAbs.FileExists(thisrpath) {
			utils.SendErrorResponse(w, "File not exists: "+vpath)
			return
		}

		realSourcePaths = append(realSourcePaths, thisrpath)
		sourceFshs = append(sourceFshs, thisSrcFsh)
	}

	///Convert dest to real if given
	var destFsh *filesystem.FileSystemHandler = nil
	var subpath string = ""
	var filename string = ""
	if vdest != "" {
		//Given target virtual dest
		destFsh, subpath, err = GetFSHandlerSubpathFromVpath(rdest)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
	} else {
		//Given no virtual dest. Zip to tmp:/
		filename = utils.Int64ToString(time.Now().Unix()) + ".zip"
		destFsh, subpath, err = GetFSHandlerSubpathFromVpath(filepath.Join("tmp:/", filename))
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
	}
	rdest, _ = destFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
	destFshAbs := destFsh.FileSystemAbstraction
	zipOutput := rdest
	zipDestFsh := destFsh
	if destFsh.RequireBuffer {
		zipOutput = getFsBufferFilepath(rdest, false)
		zipDestFsh = nil
	}

	if opr == "tmpzipAsync" {
		/*
			Same as tmpzip, but the zipping runs in the background and reports
			its progress into the file operation task list, so the caller can
			follow it on /system/file_system/ongoing instead of staring at a
			request that returns nothing until the archive is finished.
		*/
		if filename == "" {
			utils.SendErrorResponse(w, "tmpzipAsync does not support a custom destination")
			return
		}

		vzipPath := "tmp:/" + filename
		task := NewOngoingFileOperation(userinfo, "zip", virtualSourcePaths, vzipPath)
		go runTmpZipTask(task.ID, sourceFshs, realSourcePaths, zipDestFsh, zipOutput, destFsh, rdest)

		js, _ := json.Marshal(map[string]string{
			"oprid": task.ID,
			"dest":  vzipPath,
		})
		utils.SendJSONResponse(w, string(js))
		return
	}

	if opr == "zip" {
		//Check if destination location exists
		if rdest == "" || !destFshAbs.FileExists(filepath.Dir(zipOutput)) {
			utils.SendErrorResponse(w, "Invalid dest location")
			return
		}

		//OK. Create the zip at the desired location
		err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		utils.SendOK(w)
	} else if opr == "tmpzip" {
		//Zip to tmp folder
		err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		//Send the tmp filename to the user
		utils.SendTextResponse(w, "tmp:/"+filename)
	}

	if destFsh.RequireBuffer {
		//Write the buffer zip file to destination
		f, _ := os.Open(zipOutput)
		destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
		f.Close()
		os.Remove(zipOutput)
	}
	cleanFsBufferFileFromList(realSourcePaths)
}

/*
runTmpZipTask builds a temporary zip in the background, keeping the file
operation record of oprId up to date so the front end can poll its progress.

The progress handler returns the task's control signal, which is what lets a
user cancel the archive from the file operation list: ArozZipFileWithProgress
stops and reports the cancellation as an error, and finishTmpZipTask turns that
back into a cancelled record.
*/
func runTmpZipTask(oprId string, sourceFshs []*filesystem.FileSystemHandler, realSourcePaths []string, zipDestFsh *filesystem.FileSystemHandler, zipOutput string, destFsh *filesystem.FileSystemHandler, rdest string) {
	err := filesystem.ArozZipFileWithProgress(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false, func(currentFilename string, _ int, _ int, progress float64) int {
		sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
		return sig
	})

	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
		finishTmpZipTask(oprId, err)
		return
	}

	if destFsh.RequireBuffer {
		//Write the buffer zip file to destination
		f, _ := os.Open(zipOutput)
		err = destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
		f.Close()
		os.Remove(zipOutput)
		if err != nil {
			systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver "+destFsh.Filesystem+" failed", err)
			finishTmpZipTask(oprId, err)
			return
		}
	}

	cleanFsBufferFileFromList(realSourcePaths)
	finishTmpZipTask(oprId, nil)
}

// finishTmpZipTask closes off the task record of a background tmp zip.
// Pass in the error the zipping ended with, or nil when it completed.
func finishTmpZipTask(oprId string, err error) {
	if err != nil {
		SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, err.Error())
		return
	}
	SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Continue, "")
}

// Manage file version history
func system_fs_FileVersionHistory(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	path, err := utils.PostPara(r, "path")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid path given")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(path)
	if err != nil {
		if err != nil {
			utils.SendErrorResponse(w, "Invalid path given")
			return
		}
	}
	fshAbs := fsh.FileSystemAbstraction

	opr, _ := utils.PostPara(r, "opr")

	rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, "Unable to translate virtual path")
		return
	}

	if opr == "" {
		//List file history

		fileVersionData, err := localversion.GetFileVersionData(fsh, rpath)
		if err != nil {
			utils.SendErrorResponse(w, "Unable to load version information: "+err.Error())
			return
		}

		js, _ := json.Marshal(fileVersionData)
		utils.SendJSONResponse(w, string(js))

	} else if opr == "delete" {
		//Delete file history of given history ID
		historyID, err := utils.PostPara(r, "histid")
		if err != nil {
			utils.SendErrorResponse(w, "Invalid history id given")
			return
		}

		err = localversion.RemoveFileHistory(fsh, rpath, historyID)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		utils.SendOK(w)
	} else if opr == "deleteAll" {
		//Delete all file history of given vpath
		err = localversion.RemoveAllRelatedFileHistory(fsh, rpath)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		utils.SendOK(w)

	} else if opr == "restore" {
		//Restore file history of given history ID
		historyID, err := utils.PostPara(r, "histid")
		if err != nil {
			utils.SendErrorResponse(w, "Invalid history id given")
			return
		}
		err = localversion.RestoreFileHistory(fsh, rpath, historyID)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		utils.SendOK(w)
	} else if opr == "new" {
		//Create a new snapshot of this file
		err = localversion.CreateFileSnapshot(fsh, rpath)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		utils.SendOK(w)
	} else {
		utils.SendErrorResponse(w, "Unknown opr")
	}

}

func system_fs_clearVersionHistories() {
	allFsh := GetAllLoadedFsh()
	for _, fsh := range allFsh {
		if fsh.ReadOnly {
			continue
		}
		if !nightlyShouldMaintainFsh(fsh) {
			//A drive shared by the cluster is cleaned by the master node
			continue
		}
		localversion.CleanExpiredVersionBackups(fsh, fsh.Path, 30*86400)
	}
}

// Handle cache rendering with websocket pipeline
func system_fs_handleCacheRender(w http.ResponseWriter, r *http.Request) {
	userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
	vpath, err := utils.GetPara(r, "folder")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid folder paramter")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
	if err != nil {
		utils.SendErrorResponse(w, "Unable to resolve target directory")
		return
	}
	rpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)

	//Get folder sort mode
	sortMode := "default"
	folder := arozfs.Clean(vpath)
	if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
		sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
	}

	//Perform cache rendering
	thumbRenderHandler.HandleLoadCache(w, r, fsh, rpath, sortMode)
}

// Handle loading of one thumbnail
func system_fs_handleThumbnailLoad(w http.ResponseWriter, r *http.Request) {
	userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
	vpath, err := utils.GetPara(r, "vpath")
	if err != nil {
		utils.SendErrorResponse(w, "vpath not defined")
		return
	}

	byteMode, _ := utils.GetPara(r, "bytes")
	isByteMode := byteMode == "true"
	fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
	if err != nil {
		if isByteMode {
			http.NotFound(w, r)
			return
		}
		utils.SendErrorResponse(w, "Unable to resolve target directory")
		return
	}
	rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		if isByteMode {
			http.NotFound(w, r)
			return
		}
		utils.SendErrorResponse(w, err.Error())
		return
	}

	if isByteMode {
		thumbnailBytes, err := thumbRenderHandler.LoadCacheAsBytes(fsh, vpath, userinfo.Username, false)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		filetype := http.DetectContentType(thumbnailBytes)
		w.Header().Add("Content-Type", filetype)
		w.Write(thumbnailBytes)
	} else {
		thumbnailPath, err := thumbRenderHandler.LoadCache(fsh, rpath, false)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		js, _ := json.Marshal(thumbnailPath)
		utils.SendJSONResponse(w, string(js))
	}
}

// Handle file thumbnail caching
func system_fs_handleFolderCache(w http.ResponseWriter, r *http.Request) {
	userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
	vfolderpath, err := utils.GetPara(r, "folder")
	if err != nil {
		utils.SendErrorResponse(w, "folder not defined")
		return
	}

	fsh, _, err := GetFSHandlerSubpathFromVpath(vfolderpath)
	if err != nil {
		utils.SendErrorResponse(w, "unable to resolve path")
		return
	}

	thumbRenderHandler.BuildCacheForFolder(fsh, vfolderpath, userinfo.Username)
	utils.SendOK(w)
}

// Handle the get and set of sort mode of a particular folder
func system_fs_handleFolderSortModePreference(w http.ResponseWriter, r *http.Request) {
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}
	folder, err := utils.PostPara(r, "folder")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid folder given")
		return
	}

	opr, _ := utils.PostPara(r, "opr")

	folder = arozfs.Clean(folder)

	if opr == "" || opr == "get" {
		sortMode := "default"
		if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
			sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
		}

		js, err := json.Marshal(sortMode)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}
		utils.SendJSONResponse(w, string(js))
	} else if opr == "set" {
		sortMode, err := utils.PostPara(r, "mode")
		if err != nil {
			utils.SendErrorResponse(w, "Invalid sort mode given")
			return
		}

		if !utils.StringInArray(fssort.ValidSortModes, sortMode) {
			utils.SendErrorResponse(w, "Not supported sort mode: "+sortMode)
			return
		}

		sysdb.Write("fs-sortpref", userinfo.Username+"/"+folder, sortMode)
		utils.SendOK(w)
	} else {
		utils.SendErrorResponse(w, "Invalid opr mode")
		return
	}
}

// Handle setting and loading of file permission on Linux
func system_fs_handleFilePermission(w http.ResponseWriter, r *http.Request) {
	file, err := utils.PostPara(r, "file")
	if err != nil {
		utils.SendErrorResponse(w, "Invalid file")
		return
	}

	//Translate the file to real path
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	fsh, subpath, err := GetFSHandlerSubpathFromVpath(file)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	fshAbs := fsh.FileSystemAbstraction
	rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}
	newMode, _ := utils.PostPara(r, "mode")
	if newMode == "" {
		//Read the file mode

		//Check if the file exists
		if !fshAbs.FileExists(rpath) {
			utils.SendErrorResponse(w, "File not exists!")
			return
		}

		//Read the file permission
		filePermission, err := fsp.GetFilePermissions(fsh, rpath)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		}

		//Send the file permission to client
		js, _ := json.Marshal(filePermission)
		utils.SendJSONResponse(w, string(js))
	} else {
		//Set the file mode
		//Check if the file exists
		if !filesystem.FileExists(rpath) {
			utils.SendErrorResponse(w, "File not exists!")
			return
		}

		//Check if windows. If yes, ignore this request
		if runtime.GOOS == "windows" {
			utils.SendErrorResponse(w, "Windows host not supported")
			return
		}

		//Check if this user has permission to change the file permission
		//Aka user must be 1. This is his own folder or 2. Admin
		fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath(file)
		if fsh.Hierarchy == "user" {
			//Always ok as this is owned by the user
		} else if fsh.Hierarchy == "public" {
			//Require admin
			if !userinfo.IsAdmin() {
				utils.SendErrorResponse(w, "Permission Denied")
				return
			}
		} else {
			//Not implemeneted. Require admin
			if !userinfo.IsAdmin() {
				utils.SendErrorResponse(w, "Permission Denied")
				return
			}
		}

		//Be noted that if the system is not running in sudo mode,
		//File permission change might not works.

		err := fsp.SetFilePermisson(fsh, rpath, newMode)
		if err != nil {
			utils.SendErrorResponse(w, err.Error())
			return
		} else {
			utils.SendOK(w)
		}
	}
}

// Clear the old files inside the tmp file
func system_fs_clearOldTmpFiles() {
	filesToBeDelete := []string{}
	tmpAbs, _ := filepath.Abs(*tmp_directory)
	filepath.Walk(*tmp_directory, func(path string, info os.FileInfo, err error) error {
		if filepath.Base(path) != "aofs.db" && filepath.Base(path) != "aofs.db.lock" {
			//Check if root folders. Do not delete root folders
			parentAbs, _ := filepath.Abs(filepath.Dir(path))

			if tmpAbs == parentAbs {
				//Root folder. Do not remove
				return nil
			}
			//Get its modification time
			modTime, err := filesystem.GetModTime(path)
			if err != nil {
				return nil
			}

			//Check if mod time is more than 24 hours ago
			if time.Now().Unix()-modTime > int64(*maxTempFileKeepTime) {
				//Delete OK
				filesToBeDelete = append(filesToBeDelete, path)
			}
		}
		return nil
	})

	//Remove all files from the delete list
	for _, fileToBeDelete := range filesToBeDelete {
		os.RemoveAll(fileToBeDelete)
	}

}

/*
	File System Utilities for Buffered type FS

	These functions help create a local representation of file
	buffer from remote file systems like webdav or SMB
	**REMEMBER TO CLEAR THE BUFFER FILES YOURSELF**

	Example Usage
	//Replace a destination path (for file create) with local buffer filepath
	if destFsh.RequireBuffer {
		dest = getFsBufferFilepath(outputFilename)
	}

	//Buffer a remote file to local first before doing any advance file operations
	if thisSrcFsh.RequireBuffer {
		localBufferFilepath, err := bufferRemoteFileToLocal(fsh, remoteRealSrc)
		if err != nil{
			//Handle Error
		}
	}

	//Clean a list of source files that contains local buffer files
	clearnFsBufferFileFromList(realSourceFiles)

*/

// Generate a random buffer filepath. Remember to delete file after usage
func getFsBufferFilepath(originalFilename string, keepOriginalName bool) string {
	thisBuffFilename := uuid.NewV4().String()
	tmpDir := filepath.Join(*tmp_directory, "fsBuff")
	targetFile := filepath.Join(tmpDir, thisBuffFilename+filepath.Ext(originalFilename))
	if keepOriginalName {
		targetFile = filepath.Join(tmpDir, thisBuffFilename, filepath.Base(originalFilename))
	}
	os.MkdirAll(filepath.Dir(targetFile), 0775)

	return filepath.ToSlash(targetFile)
}

// Generate a buffer filepath and buffer the remote file to local. Remember to remove file after done.
func bufferRemoteFileToLocal(targetFsh *filesystem.FileSystemHandler, rpath string, keepOriginalName bool) (string, error) {
	newBufferFilename := getFsBufferFilepath(rpath, keepOriginalName)
	src, err := targetFsh.FileSystemAbstraction.ReadStream(rpath)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
		return "", err
	}
	defer src.Close()

	dest, err := os.OpenFile(newBufferFilename, os.O_CREATE|os.O_WRONLY, 0775)
	if err != nil {
		systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
		return "", err
	}
	io.Copy(dest, src)
	dest.Close()

	return newBufferFilename, nil
}

// Check if a file is buffer filepath
func isFsBufferFilepath(filename string) bool {
	tmpDir := filepath.Join(*tmp_directory, "fsBuff")
	filenameAbs, _ := filepath.Abs(filename)
	filenameAbs = filepath.ToSlash(filenameAbs)
	tmpDirAbs, _ := filepath.Abs(tmpDir)
	tmpDirAbs = filepath.ToSlash(tmpDirAbs)
	return strings.HasPrefix(filenameAbs, tmpDirAbs)
}

func cleanFsBufferFileFromList(filelist []string) {
	for _, thisFilepath := range filelist {
		if isFsBufferFilepath(thisFilepath) {
			os.RemoveAll(thisFilepath)
			folderContent, _ := os.ReadDir(filepath.Dir(thisFilepath))
			if len(folderContent) == 0 {
				//Nothing in this folder. Remove it
				os.Remove(filepath.Dir(thisFilepath))
			}
		}
	}
}

/*
	File operation task book keeping

	All file operations of all users are tracked in wsConnectionStore, keyed by
	the operation id. Finished records are kept around for a while so the file
	operation dialog can still show the completed entries to the user before
	they are cleared (either by the user or by the janitor below).
*/

// Handle all the on going task requests.
// Accept parameter: flag={continue / pause / cancel / remove / clear}
// When no flag is given, the ongoing task list of this user is returned.
// Pass in all=true to include the recently finished tasks in the listing.
func system_fs_HandleOnGoingTasks(w http.ResponseWriter, r *http.Request) {
	//Get the user information
	userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
	if err != nil {
		utils.SendErrorResponse(w, "User not logged in")
		return
	}

	statusFlag, _ := utils.PostPara(r, "flag")
	oprid, _ := utils.PostPara(r, "oprid")
	if statusFlag == "" {
		//Also accept the flag as a GET parameter for easier polling
		statusFlag, _ = utils.GetPara(r, "flag")
		oprid, _ = utils.GetPara(r, "oprid")
	}

	if statusFlag == "" {
		//No flag defined. Print all operations of this user
		includeFinished, _ := utils.GetPara(r, "all")
		js, _ := MarshalFileOperationForUser(userinfo.Username, includeFinished == "true")
		utils.SendJSONResponse(w, string(js))
		return
	}

	if statusFlag == "clear" {
		//Clear all the finished records of this user
		ClearFinishedFileOperationForUser(userinfo.Username)
		utils.SendOK(w)
		return
	}

	if oprid == "" {
		utils.SendErrorResponse(w, "oprid is empty or not set")
		return
	}

	err = ApplyFileOperationControl(userinfo.Username, statusFlag, oprid)
	if err != nil {
		utils.SendErrorResponse(w, err.Error())
		return
	}

	utils.SendOK(w)
}

// ApplyFileOperationControl applies a control signal to one of the file operations
// owned by the given user. Supported commands are continue, pause, cancel and remove.
func ApplyFileOperationControl(username string, command string, oprid string) error {
	if command == "clear" {
		ClearFinishedFileOperationForUser(username)
		return nil
	}

	//Get the operation record
	oprRecord, err := GetOngoingFileOperationByOprID(oprid)
	if err != nil {
		return err
	}

	//Only the owner of this operation can control it
	if oprRecord.Owner != username {
		return errors.New("permission denied")
	}

	fileOprTaskLock.Lock()
	defer fileOprTaskLock.Unlock()
	switch command {
	case "continue":
		//Continue the file operation
		oprRecord.FileOperationSignal = filesystem.FsOpr_Continue
	case "pause":
		//Pause the file operation until the flag is set to other status
		oprRecord.FileOperationSignal = filesystem.FsOpr_Pause
	case "cancel":
		//Cancel and stop the operation
		oprRecord.FileOperationSignal = filesystem.FsOpr_Cancel
		if taskIsFinished(oprRecord) {
			//Already finished. Nothing left to cancel, just drop the record
			wsConnectionStore.Delete(oprid)
		}
	case "remove":
		//Remove a finished record from the listing
		if !taskIsFinished(oprRecord) {
			return errors.New("task is still running")
		}
		wsConnectionStore.Delete(oprid)
	default:
		return errors.New("unsupported operation")
	}

	return nil
}

// NewOngoingFileOperation creates and registers a new file operation task record.
// The size of each source file is resolved here so the front end can show the
// transferred / total size of every file in the operation.
func NewOngoingFileOperation(userinfo *user.User, operation string, sourceFiles []string, vdestFile string) *fileOperationTask {
	/*
		Sizing a source folder means walking it. For a move that the storage can
		satisfy with a rename that walk costs far more than the move itself, and
		it would only feed a bar that jumps straight to done, so it is skipped.
		Should the rename fail after all and the files end up being streamed, the
		first progress report fills the real sizes in, see
		UpdateOngoingFileOperationSubtask.
	*/
	resolveSizes := !fileOperationCanBeRenamed(operation, sourceFiles, vdestFile)

	subtasks := []*fileOperationSubtask{}
	totalSize := int64(0)
	for _, vsrc := range sourceFiles {
		thisSize, thisIsDir := resolveFileOperationSourceInfo(userinfo, vsrc, resolveSizes)
		totalSize += thisSize
		subtasks = append(subtasks, &fileOperationSubtask{
			Filename: filepath.Base(strings.TrimSuffix(vsrc, "/")),
			Src:      arozfs.ToSlash(vsrc),
			IsDir:    thisIsDir,
			Size:     thisSize,
			Done:     0,
			Progress: 0,
			Status:   FsTask_Pending,
		})
	}

	thisTask := fileOperationTask{
		ID:                  strconv.Itoa(int(time.Now().Unix())) + "_" + uuid.NewV4().String(),
		Owner:               userinfo.Username,
		Operation:           operation,
		Src:                 arozfs.ToSlash(filepath.Dir(sourceFiles[0])),
		Dest:                arozfs.ToSlash(vdestFile),
		Progress:            0.0,
		LatestFile:          arozfs.ToSlash(filepath.Base(sourceFiles[0])),
		FileOperationSignal: filesystem.FsOpr_Continue,
		Files:               subtasks,
		TotalSize:           totalSize,
		DoneSize:            0,
		StartTime:           time.Now().Unix(),
		Status:              FsTask_Ongoing,
	}

	wsConnectionStore.Store(thisTask.ID, &thisTask)
	startFinishedFileOperationJanitor()
	return &thisTask
}

// resolveFileOperationSourceInfo returns the size of a source file in bytes and
// whether it is a folder. Set resolveSize to false to skip the folder walk when
// the size is not worth what it costs to work out.
// The size is 0 when it cannot be determined.
func resolveFileOperationSourceInfo(userinfo *user.User, vsrc string, resolveSize bool) (int64, bool) {
	fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
	if err != nil {
		return 0, false
	}
	rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
	if err != nil {
		return 0, false
	}

	isDir := fsh.FileSystemAbstraction.IsDir(rpath)
	if !resolveSize {
		return 0, isDir
	}
	if isDir {
		size, _ := fsh.GetDirctorySizeFromRealPath(rpath, false)
		return size, true
	}
	return fsh.FileSystemAbstraction.GetFileSize(rpath), false
}

// fileOperationCanBeRenamed reports whether an operation is a move that the
// storage holding both of its ends can carry out on its own, in which case no
// bytes travel through ArozOS and the operation finishes right away.
func fileOperationCanBeRenamed(operation string, sourceFiles []string, vdestFile string) bool {
	if operation != "move" {
		return false
	}

	destFsh, _, err := GetFSHandlerSubpathFromVpath(vdestFile)
	if err != nil || destFsh.ReadOnly {
		return false
	}

	for _, vsrc := range sourceFiles {
		srcFsh, _, err := GetFSHandlerSubpathFromVpath(vsrc)
		if err != nil || srcFsh.ReadOnly || !filesystem.SameFileSystem(srcFsh, destFsh) {
			return false
		}
	}
	return true
}

// GetAllFileOperationForUser returns all the file operation records of the given user.
// Set includeFinished to true to also return the recently finished records.
// The records are live objects that the running operations keep updating, so
// read them through MarshalFileOperationForUser unless you hold fileOprTaskLock.
func GetAllFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
	fileOprTaskLock.RLock()
	defer fileOprTaskLock.RUnlock()
	return collectFileOperationForUser(username, includeFinished)
}

// MarshalFileOperationForUser serializes the file operation records of a user
// while holding the task lock, so the running operations cannot update a record
// half way through the serialization.
func MarshalFileOperationForUser(username string, includeFinished bool) ([]byte, error) {
	fileOprTaskLock.RLock()
	defer fileOprTaskLock.RUnlock()
	return json.Marshal(collectFileOperationForUser(username, includeFinished))
}

// collectFileOperationForUser gathers the task records of a user in operation id
// order. The caller is expected to be holding fileOprTaskLock.
func collectFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
	results := []*fileOperationTask{}
	wsConnectionStore.Range(func(key, value interface{}) bool {
		taskInfo := value.(*fileOperationTask)
		if taskInfo.Owner == username && (includeFinished || !taskIsFinished(taskInfo)) {
			results = append(results, taskInfo)
		}
		return true
	})

	sort.Slice(results, func(i, j int) bool {
		return results[i].ID < results[j].ID
	})
	return results
}

// ClearFinishedFileOperationForUser removes all the finished records of a user
func ClearFinishedFileOperationForUser(username string) {
	fileOprTaskLock.Lock()
	defer fileOprTaskLock.Unlock()
	wsConnectionStore.Range(func(key, value interface{}) bool {
		taskInfo := value.(*fileOperationTask)
		if taskInfo.Owner == username && taskIsFinished(taskInfo) {
			wsConnectionStore.Delete(key)
		}
		return true
	})
}

// Get an ongoing task record
func GetOngoingFileOperationByOprID(oprid string) (*fileOperationTask, error) {
	object, ok := wsConnectionStore.Load(oprid)
	if !ok {
		return nil, errors.New("task not exists")
	}

	return object.(*fileOperationTask), nil
}

// Set or update an ongoing task record
func SetOngoingFileOperation(opr *fileOperationTask) {
	wsConnectionStore.Store(opr.ID, opr)
}

// Update the status of an ongoing task record, return latest status code and error if any
func UpdateOngoingFileOperation(oprid string, currentFile string, progress float64) (int, error) {
	t, err := GetOngoingFileOperationByOprID(oprid)
	if err != nil {
		return 0, err
	}

	fileOprTaskLock.Lock()
	t.LatestFile = currentFile
	t.Progress = progress
	t.DoneSize = int64(float64(t.TotalSize) * progress / 100)

	//Zip and unzip report one progress value for the whole operation. Spread it
	//over the source files in order so the dialog can still show them one by one.
	remaining := t.DoneSize
	for _, thisSubtask := range t.Files {
		if remaining >= thisSubtask.Size {
			thisSubtask.Done = thisSubtask.Size
			thisSubtask.Progress = 100
			thisSubtask.Status = FsTask_Completed
			remaining -= thisSubtask.Size
		} else {
			thisSubtask.Done = remaining
			if thisSubtask.Size > 0 {
				thisSubtask.Progress = float64(remaining) / float64(thisSubtask.Size) * 100
			} else {
				thisSubtask.Progress = progress
			}
			thisSubtask.Status = FsTask_Ongoing
			remaining = 0
		}
	}
	sig := t.FileOperationSignal
	fileOprTaskLock.Unlock()

	return sig, nil
}

/*
UpdateOngoingFileOperationSubtask records how many bytes of one source file are
already processed and recalculates the progress of the whole operation from it.
It returns the current control signal and the overall progress of the operation.

bytesTotal is the size the file operation itself reports for this source, which
wins over the size resolved when the task was created: a folder can grow or
shrink between the two, and the transfer knows better than the earlier walk.
*/
func UpdateOngoingFileOperationSubtask(oprid string, subtaskIndex int, currentFile string, bytesDone int64, bytesTotal int64) (int, float64, error) {
	t, err := GetOngoingFileOperationByOprID(oprid)
	if err != nil {
		return 0, 0, err
	}

	fileOprTaskLock.Lock()
	if currentFile != "" {
		t.LatestFile = currentFile
	}
	if subtaskIndex >= 0 && subtaskIndex < len(t.Files) {
		thisSubtask := t.Files[subtaskIndex]
		thisSubtask.Status = FsTask_Ongoing

		if bytesTotal > 0 && bytesTotal != thisSubtask.Size {
			//Keep the operation total in step with the corrected file size
			t.TotalSize += bytesTotal - thisSubtask.Size
			thisSubtask.Size = bytesTotal
		}

		if bytesDone > thisSubtask.Size {
			bytesDone = thisSubtask.Size
		}
		thisSubtask.Done = bytesDone
		if thisSubtask.Size > 0 {
			thisSubtask.Progress = float64(bytesDone) / float64(thisSubtask.Size) * 100
		}
	}

	t.DoneSize = sumFileOperationDoneSize(t)
	recalculateFileOperationProgress(t)
	sig := t.FileOperationSignal
	progress := t.Progress
	fileOprTaskLock.Unlock()

	return sig, progress, nil
}

/*
recalculateFileOperationProgress works out the overall progress of a task from
the bytes of the files inside it, so copying a 1 KB text file next to a 4 GB
image no longer moves the bar half way when the text file lands.

Operations whose sizes could not be resolved fall back to weighting every source
file equally. The caller is expected to be holding fileOprTaskLock.
*/
func recalculateFileOperationProgress(t *fileOperationTask) {
	if t.TotalSize > 0 {
		progress := float64(t.DoneSize) / float64(t.TotalSize) * 100
		if progress > 100 {
			progress = 100
		}
		t.Progress = progress
		return
	}

	if len(t.Files) == 0 {
		return
	}
	totalProgress := float64(0)
	for _, thisSubtask := range t.Files {
		totalProgress += thisSubtask.Progress
	}
	t.Progress = totalProgress / float64(len(t.Files))
}

// MarkFileOperationSubtaskEnded marks one source file inside a task as finished.
// Pass in an empty errmsg for a successful completion.
func MarkFileOperationSubtaskEnded(oprid string, subtaskIndex int, errmsg string) {
	t, err := GetOngoingFileOperationByOprID(oprid)
	if err != nil {
		return
	}

	fileOprTaskLock.Lock()
	defer fileOprTaskLock.Unlock()
	if subtaskIndex < 0 || subtaskIndex >= len(t.Files) {
		return
	}

	thisSubtask := t.Files[subtaskIndex]
	if errmsg != "" {
		thisSubtask.Status = FsTask_Error
		thisSubtask.Error = errmsg
	} else {
		thisSubtask.Status = FsTask_Completed
		thisSubtask.Progress = 100
		thisSubtask.Done = thisSubtask.Size
	}
	t.DoneSize = sumFileOperationDoneSize(t)
	recalculateFileOperationProgress(t)
}

// SetFileOperationTaskEnded closes off a task record with the given ending signal.
// The record is kept in the store so the front end can still render the result.
func SetFileOperationTaskEnded(oprid string, endingSignal int, errmsg string) {
	t, err := GetOngoingFileOperationByOprID(oprid)
	if err != nil {
		return
	}

	fileOprTaskLock.Lock()
	defer fileOprTaskLock.Unlock()
	t.EndTime = time.Now().Unix()
	t.Error = errmsg
	switch {
	case endingSignal == filesystem.FsOpr_Cancel || t.FileOperationSignal == filesystem.FsOpr_Cancel:
		//A file operation stopped by the user reports the abort as an error on
		//its way out. That is not a failure, so record it as a cancellation.
		t.Status = FsTask_Cancelled
		t.Error = ""
		for _, thisSubtask := range t.Files {
			if thisSubtask.Status == FsTask_Error {
				thisSubtask.Status = FsTask_Cancelled
				thisSubtask.Error = ""
			}
		}
	case errmsg != "" || endingSignal == filesystem.FsOpr_Error:
		t.Status = FsTask_Error
	default:
		t.Status = FsTask_Completed
		t.Progress = 100
		t.DoneSize = t.TotalSize
		for _, thisSubtask := range t.Files {
			if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
				thisSubtask.Status = FsTask_Completed
				thisSubtask.Progress = 100
				thisSubtask.Done = thisSubtask.Size
			}
		}
	}

	//Any file still marked as running at this point will never move again
	for _, thisSubtask := range t.Files {
		if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
			thisSubtask.Status = t.Status
		}
	}
}

// taskIsFinished checks if a task record has stopped running.
// The caller is expected to be holding fileOprTaskLock when required.
func taskIsFinished(t *fileOperationTask) bool {
	return t.Status != FsTask_Ongoing && t.Status != FsTask_Pending
}

// sumFileOperationDoneSize sums up the transferred bytes of all the files in a task.
// The caller is expected to be holding fileOprTaskLock.
func sumFileOperationDoneSize(t *fileOperationTask) int64 {
	total := int64(0)
	for _, thisSubtask := range t.Files {
		total += thisSubtask.Done
	}
	return total
}

// startFinishedFileOperationJanitor starts the background cleaner that tidies up
// the finished task records, see clearExpiredFileOperationRecords.
func startFinishedFileOperationJanitor() {
	if !fileOprJanitorStarted.CompareAndSwap(false, true) {
		return
	}

	go func() {
		for {
			time.Sleep(fileOprJanitorInterval)
			clearExpiredFileOperationRecords()
		}
	}()
}

/*
clearExpiredFileOperationRecords drops the finished task records that are no
longer worth keeping.

An operation that finished without an error is cleared on its own shortly
after it ended, whether or not anyone was watching it: closing the browser
tab half way through a copy must not leave the record behind forever.
A failed operation is kept instead, so the user can still find out what went
wrong after reopening the desktop from this or from another machine. Only the
newest fileOprErrorRecordLimit failures of each user are kept.
*/
func clearExpiredFileOperationRecords() {
	expireTime := time.Now().Unix() - fileOprFinishedRecordTTL
	errorRecords := map[string][]*fileOperationTask{}

	fileOprTaskLock.Lock()
	defer fileOprTaskLock.Unlock()
	wsConnectionStore.Range(func(key, value interface{}) bool {
		taskInfo := value.(*fileOperationTask)
		if !taskIsFinished(taskInfo) || taskInfo.EndTime == 0 {
			//Still running, leave it alone
			return true
		}

		if taskInfo.Status == FsTask_Error {
			errorRecords[taskInfo.Owner] = append(errorRecords[taskInfo.Owner], taskInfo)
			return true
		}

		if taskInfo.EndTime < expireTime {
			wsConnectionStore.Delete(key)
		}
		return true
	})

	//Trim the failure backlog of each user down to the newest few
	for _, records := range errorRecords {
		if len(records) <= fileOprErrorRecordLimit {
			continue
		}
		sort.Slice(records, func(i, j int) bool {
			return records[i].EndTime > records[j].EndTime
		})
		for _, expiredRecord := range records[fileOprErrorRecordLimit:] {
			wsConnectionStore.Delete(expiredRecord.ID)
		}
	}
}