file_system.go 147 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766
  1. package main
  2. import (
  3. "crypto/sha256"
  4. "encoding/hex"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "hash/crc32"
  9. "io"
  10. "io/fs"
  11. "math"
  12. "mime"
  13. "net/http"
  14. "net/url"
  15. "os"
  16. "sync"
  17. "sync/atomic"
  18. "path"
  19. "path/filepath"
  20. "runtime"
  21. "sort"
  22. "strconv"
  23. "strings"
  24. "time"
  25. "github.com/gorilla/websocket"
  26. uuid "github.com/satori/go.uuid"
  27. "imuslab.com/arozos/mod/compatibility"
  28. "imuslab.com/arozos/mod/filesystem"
  29. "imuslab.com/arozos/mod/filesystem/arozfs"
  30. fsp "imuslab.com/arozos/mod/filesystem/fspermission"
  31. "imuslab.com/arozos/mod/filesystem/fssort"
  32. "imuslab.com/arozos/mod/filesystem/fuzzy"
  33. hidden "imuslab.com/arozos/mod/filesystem/hidden"
  34. "imuslab.com/arozos/mod/filesystem/localversion"
  35. metadata "imuslab.com/arozos/mod/filesystem/metadata"
  36. "imuslab.com/arozos/mod/filesystem/shortcut"
  37. module "imuslab.com/arozos/mod/modules"
  38. prout "imuslab.com/arozos/mod/prouter"
  39. "imuslab.com/arozos/mod/share"
  40. "imuslab.com/arozos/mod/share/shareEntry"
  41. storage "imuslab.com/arozos/mod/storage"
  42. user "imuslab.com/arozos/mod/user"
  43. "imuslab.com/arozos/mod/utils"
  44. )
  45. var (
  46. thumbRenderHandler *metadata.RenderHandler
  47. shareEntryTable *shareEntry.ShareEntryTable
  48. shareManager *share.Manager
  49. wsConnectionStore sync.Map //Storage of all the file operation tasks, keyed by operation id
  50. fileOprTaskLock sync.RWMutex //Lock guarding the mutable fields of the task records above
  51. fileOprJanitorStarted atomic.Bool //Whether the finished task record janitor is already running
  52. )
  53. // Status of a file operation task or of one of the files inside it
  54. const (
  55. FsTask_Pending = "pending"
  56. FsTask_Ongoing = "ongoing"
  57. FsTask_Completed = "completed"
  58. FsTask_Error = "error"
  59. FsTask_Cancelled = "cancelled"
  60. //How long an operation that finished without error is kept before the
  61. //janitor drops it, in seconds. Long enough for a connected dialog to render
  62. //the result and fire its completion callback, short enough that the listing
  63. //does not fill up with finished transfers.
  64. fileOprFinishedRecordTTL = 3
  65. //How many failed operations are kept per user. Failures are never dropped on
  66. //a timer so they can still be reviewed after reopening the desktop from
  67. //another browser or machine, but the backlog is bounded.
  68. fileOprErrorRecordLimit = 32
  69. //How often the janitor sweeps the finished task records. Kept well under
  70. //fileOprFinishedRecordTTL so a record is actually dropped near its deadline
  71. //rather than up to a whole sweep later.
  72. fileOprJanitorInterval = 2 * time.Second
  73. )
  74. type trashedFile struct {
  75. Filename string
  76. Filepath string
  77. FileExt string
  78. IsDir bool
  79. Filesize int64
  80. RemoveTimestamp int64
  81. RemoveDate string
  82. OriginalPath string
  83. OriginalFilename string
  84. }
  85. // A single source file inside a file operation task
  86. type fileOperationSubtask struct {
  87. Filename string //Base name of this source file
  88. Src string //Virtual path of this source file
  89. IsDir bool //Whether this source is a folder
  90. Size int64 //Total size of this source file in bytes
  91. Done int64 //Bytes of this source file that are already processed
  92. Progress float64 //Progress of this source file, in percentage
  93. Status string //Status of this source file, see the FsTask_* constants
  94. Error string //Error message of this source file, if any
  95. }
  96. type fileOperationTask struct {
  97. ID string //Unique id for the task operation
  98. Owner string //Owner of the file opr
  99. Operation string //Type of the file opr: move / copy / zip / unzip
  100. Src string //Source folder for opr
  101. Dest string //Destination folder for opr
  102. Progress float64 //Progress for the operation
  103. LatestFile string //Latest file that is current transfering
  104. FileOperationSignal int //Current control signal of the file opr
  105. Files []*fileOperationSubtask //Per source file progress of this operation
  106. TotalSize int64 //Total size of all the source files in bytes
  107. DoneSize int64 //Total bytes processed so far
  108. StartTime int64 //Unix timestamp of when this operation started
  109. EndTime int64 //Unix timestamp of when this operation ended, 0 if still running
  110. Status string //Status of this operation, see the FsTask_* constants
  111. Error string //Error message of this operation, if any
  112. }
  113. func FileSystemInit() {
  114. router := prout.NewModuleRouter(prout.RouterOption{
  115. ModuleName: "File Manager",
  116. AdminOnly: false,
  117. UserHandler: userHandler,
  118. DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
  119. utils.SendErrorResponse(w, "Permission Denied")
  120. },
  121. })
  122. //Upload related functions
  123. router.HandleFunc("/system/file_system/upload", system_fs_handleUpload)
  124. router.HandleFunc("/system/file_system/lowmemUpload", system_fs_handleLowMemoryUpload)
  125. //Other file operations
  126. router.HandleFunc("/system/file_system/validateFileOpr", system_fs_validateFileOpr)
  127. router.HandleFunc("/system/file_system/fileOpr", system_fs_handleOpr)
  128. router.HandleFunc("/system/file_system/ws/fileOpr", system_fs_handleWebSocketOpr)
  129. router.HandleFunc("/system/file_system/fileOprAsync", system_fs_handleAsyncOpr)
  130. router.HandleFunc("/system/file_system/ws/fileOprStatus", system_fs_handleFileOprStatusWebSocket)
  131. router.HandleFunc("/system/file_system/listDir", system_fs_handleList)
  132. router.HandleFunc("/system/file_system/listDirHash", system_fs_handleDirHash)
  133. router.HandleFunc("/system/file_system/listRoots", system_fs_listRoot)
  134. router.HandleFunc("/system/file_system/listDrives", system_fs_listDrives)
  135. router.HandleFunc("/system/file_system/newItem", system_fs_handleNewObjects)
  136. router.HandleFunc("/system/file_system/preference", system_fs_handleUserPreference)
  137. router.HandleFunc("/system/file_system/listTrash", system_fs_scanTrashBin)
  138. router.HandleFunc("/system/file_system/ws/listTrash", system_fs_WebSocketScanTrashBin)
  139. router.HandleFunc("/system/file_system/clearTrash", system_fs_clearTrashBin)
  140. router.HandleFunc("/system/file_system/trashSettings", system_fs_handleTrashSettings)
  141. router.HandleFunc("/system/file_system/restoreTrash", system_fs_restoreFile)
  142. router.HandleFunc("/system/file_system/zipHandler", system_fs_zipHandler)
  143. router.HandleFunc("/system/file_system/getProperties", system_fs_getFileProperties)
  144. router.HandleFunc("/system/file_system/getStorageInfo", system_fs_getStorageInfo)
  145. router.HandleFunc("/system/file_system/versionHistory", system_fs_FileVersionHistory)
  146. router.HandleFunc("/system/file_system/handleFilePermission", system_fs_handleFilePermission)
  147. router.HandleFunc("/system/file_system/search", system_fs_handleFileSearch)
  148. //Thumbnail caching functions
  149. router.HandleFunc("/system/file_system/handleFolderCache", system_fs_handleFolderCache)
  150. router.HandleFunc("/system/file_system/handleCacheRender", system_fs_handleCacheRender)
  151. router.HandleFunc("/system/file_system/loadThumbnail", system_fs_handleThumbnailLoad)
  152. //Directory specific config
  153. router.HandleFunc("/system/file_system/sortMode", system_fs_handleFolderSortModePreference)
  154. //Register the module
  155. moduleHandler.RegisterModule(module.ModuleInfo{
  156. Name: "File Manager",
  157. Group: "System Tools",
  158. IconPath: "SystemAO/file_system/img/small_icon.png",
  159. Version: "1.0",
  160. StartDir: "SystemAO/file_system/file_explorer.html",
  161. SupportFW: true,
  162. InitFWSize: []int{1075, 610},
  163. LaunchFWDir: "SystemAO/file_system/file_explorer.html",
  164. SupportEmb: false,
  165. })
  166. //Register the Trashbin module
  167. moduleHandler.RegisterModule(module.ModuleInfo{
  168. Name: "Trash Bin",
  169. Group: "System Tools",
  170. IconPath: "SystemAO/file_system/trashbin_img/small_icon.png",
  171. Version: "1.0",
  172. StartDir: "SystemAO/file_system/trashbin.html",
  173. SupportFW: true,
  174. InitFWSize: []int{400, 200},
  175. LaunchFWDir: "SystemAO/file_system/trashbin.html",
  176. SupportEmb: false,
  177. SupportedExt: []string{"*"},
  178. })
  179. //Register the Zip Extractor module
  180. moduleHandler.RegisterModule(module.ModuleInfo{
  181. Name: "Zip Extractor",
  182. Group: "System Tools",
  183. IconPath: "SystemAO/file_system/img/zip_extractor.png",
  184. Version: "1.0",
  185. SupportFW: false,
  186. LaunchEmb: "SystemAO/file_system/zip_extractor.html",
  187. SupportEmb: true,
  188. InitEmbSize: []int{260, 120},
  189. SupportedExt: []string{".zip"},
  190. })
  191. //Create user root if not exists
  192. err := os.MkdirAll(filepath.Join(*root_directory, "users/"), 0755)
  193. if err != nil {
  194. systemWideLogger.PrintAndLog("File System", "Failed to create system storage root", err)
  195. panic(err)
  196. }
  197. //Create database table if not exists
  198. err = sysdb.NewTable("fs")
  199. if err != nil {
  200. systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
  201. panic(err)
  202. }
  203. //Create new table for sort preference
  204. err = sysdb.NewTable("fs-sortpref")
  205. if err != nil {
  206. systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
  207. panic(err)
  208. }
  209. //Create a RenderHandler for caching thumbnails
  210. thumbRenderHandler = metadata.NewRenderHandler()
  211. //Thumbnails of drives that render them where the file is stored (e.g.
  212. //cluster:/) are kept on this host, outside the drive and outside tmp:/
  213. //(whose files are cleared after a day)
  214. metadata.SetExternalCacheDir(filepath.Join("system", "cache", "thumbnails"))
  215. nightlyManager.RegisterNightlyTask(func() {
  216. metadata.PruneExternalCache(thumbnailCacheMaxAge)
  217. })
  218. /*
  219. Share Related Registering
  220. This section of functions create and register the file share service
  221. for the arozos
  222. */
  223. //Create a share manager to handle user file sharae
  224. shareEntryTable = shareEntry.NewShareEntryTable(sysdb)
  225. shareManager = share.NewShareManager(share.Options{
  226. AuthAgent: authAgent,
  227. ShareEntryTable: shareEntryTable,
  228. UserHandler: userHandler,
  229. HostName: *host_name,
  230. TmpFolder: *tmp_directory,
  231. //Resolved per call, so it works although notificationInit runs later
  232. NotificationSender: sendUserNotification,
  233. })
  234. //Share management API (/system/file_system/share/*), behind the File Manager permission
  235. shareManager.RegisterAPIEndpoints(router)
  236. //Handle the main share function
  237. //Share function is now routed by the main router
  238. //http.HandleFunc("/share", shareManager.HandleShareAccess)
  239. /*
  240. File Operation Resume Functions
  241. */
  242. //Create a sync map for file operation opened connections
  243. wsConnectionStore = sync.Map{}
  244. router.HandleFunc("/system/file_system/ongoing", system_fs_HandleOnGoingTasks)
  245. /*
  246. Nighly Tasks
  247. These functions allow file system to clear and maintain
  248. the arozos file system when no one is using the system
  249. */
  250. //Trash bin retention and size limit, per user
  251. registerSetting(settingModule{
  252. Name: "File Manager",
  253. Desc: "Trash Bin Retention and Size",
  254. IconPath: "SystemAO/file_system/trashbin_img/small_icon.png",
  255. Group: "Disk",
  256. StartDir: "SystemAO/disk/filemanager/trashsettings.html",
  257. })
  258. //Clear tmp folder if files is placed here too long
  259. nightlyManager.RegisterNightlyTask(system_fs_clearOldTmpFiles)
  260. //Clear shares that have expired or whose parent file no longer exists in the system
  261. shareManager.ValidateAndClearShares()
  262. nightlyManager.RegisterNightlyTask(shareManager.ValidateAndClearShares)
  263. //Clear file version history that is more than 30 days
  264. go func() {
  265. //Start version history cleaning in background
  266. system_fs_clearVersionHistories()
  267. systemWideLogger.PrintAndLog("File System", "Startup File Version History Cleaning Completed", nil)
  268. }()
  269. systemWideLogger.PrintAndLog("File System", "Started File Version History Cleaning in background", nil)
  270. nightlyManager.RegisterNightlyTask(system_fs_clearVersionHistories)
  271. //Purge trashed files older than each user's retention setting
  272. nightlyManager.RegisterNightlyTask(system_fs_clearExpiredTrash)
  273. }
  274. /*
  275. File Search
  276. Handle file search in wildcard and recursive search
  277. */
  278. func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
  279. //Get the user information
  280. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  281. if err != nil {
  282. utils.SendErrorResponse(w, "User not logged in")
  283. return
  284. }
  285. //Get the search target root path
  286. vpath, err := utils.PostPara(r, "path")
  287. if err != nil {
  288. utils.SendErrorResponse(w, "Invalid vpath given")
  289. return
  290. }
  291. keyword, err := utils.PostPara(r, "keyword")
  292. if err != nil {
  293. utils.SendErrorResponse(w, "Invalid keyword given")
  294. return
  295. }
  296. //Check if case sensitive is enabled
  297. casesensitve, _ := utils.PostPara(r, "casesensitive")
  298. vrootID, _, err := filesystem.GetIDFromVirtualPath(vpath)
  299. var targetFSH *filesystem.FileSystemHandler = nil
  300. if err != nil {
  301. utils.SendErrorResponse(w, "Invalid path given")
  302. return
  303. }
  304. targetFSH, err = GetFsHandlerByUUID(vrootID)
  305. if err != nil {
  306. utils.SendErrorResponse(w, err.Error())
  307. return
  308. }
  309. //Translate the vpath to realpath if this is an actual path on disk
  310. resolvedPath, err := targetFSH.FileSystemAbstraction.VirtualPathToRealPath(vpath, userinfo.Username)
  311. if err != nil {
  312. utils.SendErrorResponse(w, "Invalid path given")
  313. return
  314. }
  315. rpath := resolvedPath
  316. //Check if the search mode is recursive keyword or wildcard
  317. if len(keyword) > 1 && keyword[:1] == "/" {
  318. //Wildcard
  319. //Updates 31-12-2021: Do not allow wildcard search on virtual type's FSH
  320. if targetFSH == nil {
  321. utils.SendErrorResponse(w, "Invalid path given")
  322. return
  323. }
  324. targetFshAbs := targetFSH.FileSystemAbstraction
  325. wildcard := keyword[1:]
  326. //The pattern is joined onto the real path, so it must not climb out of it
  327. for _, segment := range strings.Split(filepath.ToSlash(wildcard), "/") {
  328. if segment == ".." {
  329. utils.SendErrorResponse(w, "Invalid wildcard given")
  330. return
  331. }
  332. }
  333. matchingFiles, err := targetFshAbs.Glob(filepath.Join(rpath, wildcard))
  334. if err != nil {
  335. utils.SendErrorResponse(w, err.Error())
  336. return
  337. }
  338. //Prepare result struct
  339. results := []filesystem.FileData{}
  340. escaped := false
  341. for _, matchedFile := range matchingFiles {
  342. thisVpath, _ := targetFSH.FileSystemAbstraction.RealPathToVirtualPath(matchedFile, userinfo.Username)
  343. isHidden, _ := hidden.IsHidden(thisVpath, true)
  344. if !isHidden {
  345. results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, matchedFile, 2))
  346. }
  347. }
  348. if escaped {
  349. utils.SendErrorResponse(w, "Search keywords contain escape character!")
  350. return
  351. }
  352. //OK. Tidy up the results
  353. js, _ := json.Marshal(results)
  354. utils.SendJSONResponse(w, string(js))
  355. } else {
  356. //Updates 2022-02-16: Build the fuzzy matcher if it is not a wildcard search
  357. matcher := fuzzy.NewFuzzyMatcher(keyword, casesensitve == "true")
  358. //Recursive keyword
  359. results := []filesystem.FileData{}
  360. var err error = nil
  361. fshAbs := targetFSH.FileSystemAbstraction
  362. err = fshAbs.Walk(rpath, func(path string, info os.FileInfo, err error) error {
  363. thisFilename := filepath.Base(path)
  364. if casesensitve != "true" {
  365. thisFilename = strings.ToLower(thisFilename)
  366. }
  367. if !filesystem.IsInsideHiddenFolder(path) {
  368. if matcher.Match(thisFilename) {
  369. //This is a matching file
  370. thisVpath, _ := fshAbs.RealPathToVirtualPath(path, userinfo.Username)
  371. results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, path, 2))
  372. }
  373. }
  374. return nil
  375. })
  376. if err != nil {
  377. utils.SendErrorResponse(w, err.Error())
  378. return
  379. }
  380. //OK. Tidy up the results
  381. js, _ := json.Marshal(results)
  382. utils.SendJSONResponse(w, string(js))
  383. }
  384. }
  385. /*
  386. Handle low-memory upload operations
  387. This function is specailly designed to work with low memory devices
  388. (e.g. ZeroPi / Orange Pi Zero with 512MB RAM)
  389. Two cases
  390. 1. Not Buffer FS + Huge File
  391. => Write chunks to fsa + merge to fsa
  392. 2. Else
  393. => write chunks to tmp (via os package) + merge to fsa
  394. */
  395. /*
  396. Low memory (websocket) upload tunables
  397. uploadIdleTimeout is how long the server waits for the next chunk before it
  398. assumes the client is gone. A paused upload is deliberately exempt from it -
  399. the client is still there, it is just not sending - so uploadPauseTimeout
  400. takes over instead, to cover an upload someone paused and forgot about.
  401. uploadPauseCloseCode is a private-use websocket close code, so the client can
  402. tell "your pause expired" apart from an ordinary disconnect and offer retry.
  403. */
  404. const (
  405. uploadIdleTimeout = 300 * time.Second
  406. uploadPauseTimeout = 2 * time.Hour
  407. uploadPauseCloseCode = 4001
  408. )
  409. func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
  410. //Get user info
  411. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  412. if err != nil {
  413. w.WriteHeader(http.StatusUnauthorized)
  414. w.Write([]byte("401 - Unauthorized"))
  415. return
  416. }
  417. //Get filename and upload path
  418. filename, err := utils.GetPara(r, "filename")
  419. if filename == "" || err != nil {
  420. w.WriteHeader(http.StatusInternalServerError)
  421. w.Write([]byte("500 - Invalid filename given"))
  422. return
  423. }
  424. //Get upload target directory
  425. uploadTarget, err := utils.GetPara(r, "path")
  426. if uploadTarget == "" || err != nil {
  427. w.WriteHeader(http.StatusInternalServerError)
  428. w.Write([]byte("500 - Invalid path given"))
  429. return
  430. }
  431. //Unescape the upload target path
  432. unescapedPath, err := url.PathUnescape(uploadTarget)
  433. if err != nil {
  434. unescapedPath = uploadTarget
  435. }
  436. //Check if the user can write to this folder
  437. if !userinfo.CanWrite(unescapedPath) {
  438. //No permission
  439. w.WriteHeader(http.StatusForbidden)
  440. w.Write([]byte("403 - Access Denied"))
  441. return
  442. }
  443. fsh, subpath, err := GetFSHandlerSubpathFromVpath(unescapedPath)
  444. if err != nil {
  445. w.WriteHeader(http.StatusInternalServerError)
  446. w.Write([]byte("500 - Path translation failed"))
  447. return
  448. }
  449. fshAbs := fsh.FileSystemAbstraction
  450. //Translate the upload target directory
  451. realUploadPath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  452. if err != nil {
  453. w.WriteHeader(http.StatusInternalServerError)
  454. w.Write([]byte("500 - Path translation failed"))
  455. return
  456. }
  457. //Check if it is huge file upload mode
  458. isHugeFile := false
  459. hugefile, _ := utils.GetPara(r, "hugefile")
  460. if hugefile == "true" && !fsh.RequireBuffer {
  461. //Huge file mode is only compatible with local file systems
  462. //For remote file system, use buffer to tmp then upload method
  463. isHugeFile = true
  464. }
  465. //Create destination folder if not exists
  466. targetUploadLocation := arozfs.ToSlash(filepath.Join(realUploadPath, filename))
  467. if !fshAbs.FileExists(realUploadPath) {
  468. fshAbs.MkdirAll(realUploadPath, 0755)
  469. }
  470. //Generate an UUID for this upload
  471. uploadUUID := uuid.NewV4().String()
  472. uploadFolder := filepath.Join(*tmp_directory, "uploads", uploadUUID)
  473. if isHugeFile {
  474. //Change to upload directly to target disk
  475. uploadFolder = filepath.Join(realUploadPath, ".metadata/.upload", uploadUUID)
  476. fshAbs.MkdirAll(uploadFolder, 0700)
  477. } else {
  478. os.MkdirAll(uploadFolder, 0700)
  479. }
  480. //Start websocket connection
  481. var upgrader = websocket.Upgrader{}
  482. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  483. c, err := upgrader.Upgrade(w, r, nil)
  484. if err != nil {
  485. systemWideLogger.PrintAndLog("File System", "Failed to upgrade websocket connection: ", err)
  486. w.WriteHeader(http.StatusInternalServerError)
  487. w.Write([]byte("500 WebSocket upgrade failed"))
  488. return
  489. }
  490. defer c.Close()
  491. //Handle WebSocket upload
  492. blockCounter := 0
  493. chunkName := []string{}
  494. /*
  495. Shared with the watchdog goroutine below, so these are read and written
  496. atomically. pausedSince is 0 while the upload is running, and holds the
  497. unix time the client paused at otherwise.
  498. */
  499. var lastChunkArrivalTime int64 = time.Now().Unix()
  500. var pausedSince int64 = 0
  501. //Setup a timeout listener, check if connection still active every 1 minute
  502. ticker := time.NewTicker(60 * time.Second)
  503. done := make(chan bool)
  504. go func() {
  505. for {
  506. select {
  507. case <-done:
  508. return
  509. case <-ticker.C:
  510. if pausedAt := atomic.LoadInt64(&pausedSince); pausedAt > 0 {
  511. //Paused: silence is expected here, so only the pause clock
  512. //applies. WriteControl is the one write that is safe to make
  513. //from this goroutine while the reader holds the connection.
  514. if time.Since(time.Unix(pausedAt, 0)) > uploadPauseTimeout {
  515. systemWideLogger.PrintAndLog("File System", "Upload paused for too long. Disconnecting.", errors.New("upload pause timeout"))
  516. c.WriteControl(websocket.CloseMessage,
  517. websocket.FormatCloseMessage(uploadPauseCloseCode, "upload pause timeout"),
  518. time.Now().Add(time.Second))
  519. time.Sleep(1 * time.Second)
  520. c.Close()
  521. return
  522. }
  523. continue
  524. }
  525. if time.Now().Unix()-atomic.LoadInt64(&lastChunkArrivalTime) > int64(uploadIdleTimeout.Seconds()) {
  526. //Already 5 minutes without new data arraival. Stop connection
  527. systemWideLogger.PrintAndLog("File System", "Upload WebSocket connection timeout. Disconnecting.", errors.New("websocket connection timeout"))
  528. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  529. time.Sleep(1 * time.Second)
  530. c.Close()
  531. return
  532. }
  533. }
  534. }
  535. }()
  536. totalFileSize := int64(0)
  537. // Full-file CRC32 hasher (IEEE polynomial, matches JS crc32Table implementation)
  538. fileCRC32Hasher := crc32.NewIEEE()
  539. // Per-chunk state machine:
  540. // The client sends a text metadata frame {"index":N,"checksum":"hex"} before each binary frame.
  541. var pendingChunkIndex int
  542. var pendingChunkChecksum string
  543. expectingBinary := false
  544. for {
  545. mt, message, err := c.ReadMessage()
  546. if err != nil {
  547. //Connection closed by client. Clear the tmp folder and exit
  548. systemWideLogger.PrintAndLog("File System", "Upload terminated by client. Cleaning tmp folder", err)
  549. //Clear the tmp folder
  550. time.Sleep(1 * time.Second)
  551. if isHugeFile {
  552. fshAbs.RemoveAll(uploadFolder)
  553. } else {
  554. os.RemoveAll(uploadFolder)
  555. }
  556. return
  557. }
  558. if mt == 1 {
  559. // Text frame – either a control message, chunk metadata or done signal
  560. textMsg := strings.TrimSpace(string(message))
  561. /*
  562. Control messages are checked before anything else, and regardless
  563. of expectingBinary: a pause can land after the metadata frame but
  564. before its binary payload, and the flow control below would
  565. otherwise mistake it for a malformed chunk header.
  566. A chunk header or done signal unmarshals into this struct with
  567. every field false, so this is safe to try first.
  568. */
  569. var ctrl struct {
  570. Pause bool `json:"pause"`
  571. Resume bool `json:"resume"`
  572. Ping bool `json:"ping"`
  573. }
  574. if jsonErr := json.Unmarshal([]byte(textMsg), &ctrl); jsonErr == nil &&
  575. (ctrl.Pause || ctrl.Resume || ctrl.Ping) {
  576. if ctrl.Pause {
  577. atomic.StoreInt64(&pausedSince, time.Now().Unix())
  578. } else if ctrl.Resume {
  579. atomic.StoreInt64(&pausedSince, 0)
  580. //Do not count the paused stretch as idle time
  581. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  582. }
  583. /*
  584. Any control message counts as the client still being there,
  585. so it also holds off the idle timeout. That matters when a
  586. pause is raised before this socket finished connecting: the
  587. pause frame is lost, but the heartbeat still arrives, and
  588. without this the upload would be reaped as idle anyway.
  589. Answering every control message is what puts traffic on the
  590. wire in both directions - a reverse proxy in front of us will
  591. drop a connection it sees nothing on.
  592. */
  593. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  594. c.WriteMessage(1, []byte(`{"pong":true}`))
  595. continue
  596. }
  597. if !expectingBinary {
  598. // Check if this is the done signal
  599. var doneSignal struct {
  600. Done bool `json:"done"`
  601. TotalChunks int `json:"totalChunks"`
  602. FileChecksum string `json:"fileChecksum"`
  603. }
  604. if jsonErr := json.Unmarshal([]byte(textMsg), &doneSignal); jsonErr == nil && doneSignal.Done {
  605. // Verify the full-file CRC32 before merging
  606. if doneSignal.FileChecksum != "" {
  607. computedSum := fileCRC32Hasher.Sum32()
  608. computedSumBytes := []byte{byte(computedSum >> 24), byte(computedSum >> 16), byte(computedSum >> 8), byte(computedSum)}
  609. computedHex := hex.EncodeToString(computedSumBytes)
  610. if doneSignal.FileChecksum != computedHex {
  611. systemWideLogger.PrintAndLog("File System", "Upload file checksum mismatch: client="+doneSignal.FileChecksum+" server="+computedHex, nil)
  612. c.WriteMessage(1, []byte(`{"error":"File integrity check failed: full-file checksum mismatch"}`))
  613. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  614. time.Sleep(1 * time.Second)
  615. c.Close()
  616. if isHugeFile {
  617. fshAbs.RemoveAll(uploadFolder)
  618. } else {
  619. os.RemoveAll(uploadFolder)
  620. }
  621. return
  622. }
  623. }
  624. // Checksum verified – proceed to merge
  625. break
  626. }
  627. // Parse as chunk metadata
  628. var meta struct {
  629. Index int `json:"index"`
  630. Checksum string `json:"checksum"`
  631. }
  632. if jsonErr := json.Unmarshal([]byte(textMsg), &meta); jsonErr != nil {
  633. systemWideLogger.PrintAndLog("File System", "Invalid chunk metadata received: "+textMsg, jsonErr)
  634. continue
  635. }
  636. pendingChunkIndex = meta.Index
  637. pendingChunkChecksum = meta.Checksum
  638. expectingBinary = true
  639. }
  640. } else if mt == 2 {
  641. // Binary frame – the chunk data that follows a metadata frame
  642. if !expectingBinary {
  643. systemWideLogger.PrintAndLog("File System", "Received binary chunk without preceding metadata, ignoring", nil)
  644. continue
  645. }
  646. expectingBinary = false
  647. /*
  648. Chunks are accepted strictly in index order, and "next" is sent
  649. once per accepted chunk - clients that keep several chunks in
  650. flight count those acknowledgements to know which chunks landed.
  651. Anything else is dropped without a reply:
  652. - index < blockCounter: a copy of a chunk already written, resent
  653. by the client after an acknowledgement timeout. Accepting it
  654. again would count its bytes twice in the size and full-file
  655. checksum.
  656. - index > blockCounter: sent ahead in a pipelined window before
  657. the chunk at blockCounter failed its CRC check. The client
  658. rewinds to the retry and sends it again.
  659. */
  660. if pendingChunkIndex != blockCounter {
  661. continue
  662. }
  663. // Verify chunk CRC32
  664. chunkSum := crc32.ChecksumIEEE(message)
  665. chunkSumBytes := []byte{byte(chunkSum >> 24), byte(chunkSum >> 16), byte(chunkSum >> 8), byte(chunkSum)}
  666. chunkHex := hex.EncodeToString(chunkSumBytes)
  667. if pendingChunkChecksum != "" && pendingChunkChecksum != chunkHex {
  668. // CRC32 mismatch – ask the client to re-send this chunk
  669. systemWideLogger.PrintAndLog("File System", "Chunk "+strconv.Itoa(pendingChunkIndex)+" CRC32 mismatch: expected "+pendingChunkChecksum+" got "+chunkHex, nil)
  670. retryMsg, _ := json.Marshal(map[string]int{"retryChunk": pendingChunkIndex})
  671. c.WriteMessage(1, retryMsg)
  672. // Reset state; client will re-send the metadata+binary for this chunk
  673. continue
  674. }
  675. // Chunk verified – write to tmp folder
  676. chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(pendingChunkIndex))
  677. chunkName = append(chunkName, chunkFilepath)
  678. blockCounter++
  679. var writeErr error
  680. if isHugeFile {
  681. writeErr = fshAbs.WriteFile(chunkFilepath, message, 0700)
  682. } else {
  683. writeErr = os.WriteFile(chunkFilepath, message, 0700)
  684. }
  685. if writeErr != nil {
  686. systemWideLogger.PrintAndLog("File System", "Upload chunk write failed: "+writeErr.Error(), writeErr)
  687. c.WriteMessage(1, []byte(`{"error":"Write file chunk to disk failed"}`))
  688. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  689. time.Sleep(1 * time.Second)
  690. c.Close()
  691. if isHugeFile {
  692. fshAbs.RemoveAll(uploadFolder)
  693. } else {
  694. os.RemoveAll(uploadFolder)
  695. }
  696. return
  697. }
  698. // Update running full-file CRC32 with the verified chunk data
  699. fileCRC32Hasher.Write(message)
  700. // Update timing and quota tracking
  701. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  702. totalFileSize += int64(len(message))
  703. if totalFileSize > max_upload_size {
  704. c.WriteMessage(1, []byte(`{"error":"File size too large"}`))
  705. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  706. time.Sleep(1 * time.Second)
  707. c.Close()
  708. if isHugeFile {
  709. fshAbs.RemoveAll(uploadFolder)
  710. } else {
  711. os.RemoveAll(uploadFolder)
  712. }
  713. return
  714. } else if !userinfo.StorageQuota.HaveSpace(totalFileSize) {
  715. c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
  716. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  717. time.Sleep(1 * time.Second)
  718. c.Close()
  719. if isHugeFile {
  720. fshAbs.RemoveAll(uploadFolder)
  721. } else {
  722. os.RemoveAll(uploadFolder)
  723. }
  724. return
  725. }
  726. // Acknowledge the chunk; client will send the next metadata+binary pair
  727. c.WriteMessage(1, []byte("next"))
  728. }
  729. }
  730. //Try to decode the location if possible
  731. decodedUploadLocation, err := url.PathUnescape(targetUploadLocation)
  732. if err != nil {
  733. decodedUploadLocation = targetUploadLocation
  734. }
  735. //Do not allow % sign in filename. Replace all with underscore
  736. decodedUploadLocation = strings.ReplaceAll(decodedUploadLocation, "%", "_")
  737. //Merge the file. Merge file location must be on local machine
  738. mergeFileLocation := decodedUploadLocation
  739. var out arozfs.File
  740. if fsh.RequireBuffer {
  741. //The merge file location must be local buffer
  742. mergeFileLocation = getFsBufferFilepath(decodedUploadLocation, false)
  743. out, err = os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
  744. } else {
  745. //The merge file location can be local or remote that support OpenFile.
  746. out, err = fshAbs.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
  747. }
  748. defer out.Close()
  749. if err != nil {
  750. systemWideLogger.PrintAndLog("File System", "Failed to open file:"+err.Error(), err)
  751. c.WriteMessage(1, []byte(`{"error":"Failed to open destination file"}`))
  752. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  753. time.Sleep(1 * time.Second)
  754. c.Close()
  755. return
  756. }
  757. for counter, filesrc := range chunkName {
  758. var srcChunkReader arozfs.File
  759. if isHugeFile {
  760. srcChunkReader, err = fshAbs.Open(filesrc)
  761. } else {
  762. srcChunkReader, err = os.Open(filesrc)
  763. }
  764. if err != nil {
  765. systemWideLogger.PrintAndLog("File System", "Failed to open Source Chunk"+filesrc+" with error "+err.Error(), err)
  766. c.WriteMessage(1, []byte(`{"error":"Failed to open Source Chunk"}`))
  767. return
  768. }
  769. io.Copy(out, srcChunkReader)
  770. srcChunkReader.Close()
  771. //Delete file immediately to save space
  772. if isHugeFile {
  773. fshAbs.Remove(filesrc)
  774. } else {
  775. os.Remove(filesrc)
  776. }
  777. //Write to websocket for the percentage of upload is written fro tmp to dest
  778. moveProg := strconv.Itoa(int(math.Round(float64(counter)/float64(len(chunkName))*100))) + "%"
  779. c.WriteMessage(1, []byte(`{"move":"`+moveProg+`"}`))
  780. }
  781. out.Close()
  782. //Check if the size fit in user quota
  783. var fi fs.FileInfo
  784. if fsh.RequireBuffer {
  785. fi, err = os.Stat(mergeFileLocation)
  786. } else {
  787. fi, err = fshAbs.Stat(mergeFileLocation)
  788. }
  789. if err != nil {
  790. // Could not obtain stat, handle error
  791. systemWideLogger.PrintAndLog("File System", "Failed to validate uploaded file: "+mergeFileLocation+". Error Message: "+err.Error(), err)
  792. c.WriteMessage(1, []byte(`{"error":"Failed to validate uploaded file"}`))
  793. return
  794. }
  795. if !userinfo.StorageQuota.HaveSpace(fi.Size()) {
  796. c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
  797. if fsh.RequireBuffer {
  798. os.RemoveAll(mergeFileLocation)
  799. } else {
  800. fshAbs.RemoveAll(mergeFileLocation)
  801. }
  802. return
  803. }
  804. //Upload it to remote side if it fits the user quota && is buffer file
  805. if fsh.RequireBuffer {
  806. //This is local buffer file. Upload to dest fsh
  807. f, err := os.Open(mergeFileLocation)
  808. if err != nil {
  809. systemWideLogger.PrintAndLog("File System", "Failed to open buffered file at "+mergeFileLocation+" with error "+err.Error(), err)
  810. c.WriteMessage(1, []byte(`{"error":"Failed to open buffered object"}`))
  811. f.Close()
  812. return
  813. }
  814. err = fsh.FileSystemAbstraction.WriteStream(decodedUploadLocation, f, 0775)
  815. if err != nil {
  816. systemWideLogger.PrintAndLog("File System", "Failed to write to file system: "+fsh.UUID+" with error "+err.Error(), err)
  817. c.WriteMessage(1, []byte(`{"error":"Failed to upload to remote file system"}`))
  818. f.Close()
  819. return
  820. }
  821. //Remove the buffered file
  822. f.Close()
  823. os.Remove(mergeFileLocation)
  824. }
  825. //Log the upload filename
  826. //systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+filepath.Base(decodedUploadLocation), nil)
  827. //Set owner of the new uploaded file
  828. userinfo.SetOwnerOfFile(fsh, unescapedPath)
  829. //Return complete signal
  830. c.WriteMessage(1, []byte("OK"))
  831. //Stop the timeout listner
  832. done <- true
  833. //Clear the tmp folder
  834. time.Sleep(300 * time.Millisecond)
  835. if isHugeFile {
  836. fshAbs.RemoveAll(uploadFolder)
  837. } else {
  838. os.RemoveAll(uploadFolder)
  839. }
  840. //Close WebSocket connection after finished
  841. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  842. time.Sleep(300 * time.Second)
  843. c.Close()
  844. }
  845. /*
  846. Handle FORM POST based upload
  847. This function is design for general SBCs or computers with more than 2GB of RAM
  848. (e.g. Raspberry Pi 4 / Linux Server)
  849. */
  850. func system_fs_handleUpload(w http.ResponseWriter, r *http.Request) {
  851. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  852. if err != nil {
  853. utils.SendErrorResponse(w, "User not logged in")
  854. return
  855. }
  856. //Limit the max upload size to the user defined size
  857. if max_upload_size != 0 {
  858. r.Body = http.MaxBytesReader(w, r.Body, max_upload_size)
  859. }
  860. err = r.ParseMultipartForm(int64(*upload_buf) << 20)
  861. if err != nil {
  862. //Filesize too big
  863. systemWideLogger.PrintAndLog("File System", "Upload file size too big", err)
  864. utils.SendErrorResponse(w, "File too large")
  865. return
  866. }
  867. file, handler, err := r.FormFile("file")
  868. if err != nil {
  869. systemWideLogger.PrintAndLog("File System", "Error Retrieving File from upload by user: "+userinfo.Username, err)
  870. utils.SendErrorResponse(w, "Unable to parse file from upload")
  871. return
  872. }
  873. //Get upload target directory
  874. uploadTarget, _ := utils.PostPara(r, "path")
  875. if uploadTarget == "" {
  876. utils.SendErrorResponse(w, "Upload target cannot be empty.")
  877. return
  878. }
  879. fsh, subpath, err := GetFSHandlerSubpathFromVpath(uploadTarget)
  880. if err != nil {
  881. utils.SendErrorResponse(w, "Invalid upload target")
  882. return
  883. }
  884. targetFs := fsh.FileSystemAbstraction
  885. //Translate the upload target directory
  886. realUploadPath, err := targetFs.VirtualPathToRealPath(subpath, userinfo.Username)
  887. if err != nil {
  888. utils.SendErrorResponse(w, "Upload target is invalid or permission denied.")
  889. return
  890. }
  891. storeFilename := handler.Filename //Filename of the uploaded file
  892. //Get request time
  893. uploadStartTime := time.Now().UnixNano() / int64(time.Millisecond)
  894. //Update for Firefox 94.0.2 (x64) -> Now firefox put its relative path inside Content-Disposition -> filename
  895. //Skip this handler logic if Firefox version is in between 84.0.2 to 94.0.2
  896. bypassMetaCheck := compatibility.FirefoxBrowserVersionForBypassUploadMetaHeaderCheck(r.UserAgent())
  897. if !bypassMetaCheck && strings.Contains(handler.Header["Content-Disposition"][0], "filename=") && strings.Contains(handler.Header["Content-Disposition"][0], "/") {
  898. //This is a firefox MIME Header for file inside folder. Look for the actual filename
  899. headerFields := strings.Split(handler.Header["Content-Disposition"][0], "; ")
  900. possibleRelativePathname := ""
  901. for _, hf := range headerFields {
  902. if strings.Contains(hf, "filename=") && len(hf) > 11 {
  903. //Found. Overwrite original filename with the latest one
  904. possibleRelativePathname = hf[10 : len(hf)-1]
  905. storeFilename = possibleRelativePathname
  906. break
  907. }
  908. }
  909. }
  910. destFilepath := arozfs.ToSlash(filepath.Join(realUploadPath, storeFilename))
  911. //fmt.Println(destFilepath, realUploadPath, storeFilename)
  912. if !targetFs.FileExists(filepath.Dir(destFilepath)) {
  913. targetFs.MkdirAll(filepath.Dir(destFilepath), 0775)
  914. }
  915. //Check if the upload target is read only.
  916. accmode := userinfo.GetPathAccessPermission(uploadTarget)
  917. if accmode == arozfs.FsReadOnly {
  918. utils.SendErrorResponse(w, "The upload target is Read Only.")
  919. return
  920. } else if accmode == arozfs.FsDenied {
  921. utils.SendErrorResponse(w, "Access Denied")
  922. return
  923. }
  924. //Check for storage quota
  925. uploadFileSize := handler.Size
  926. if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
  927. utils.SendErrorResponse(w, "User Storage Quota Exceeded")
  928. return
  929. }
  930. //Do not allow % sign in filename. Replace all with underscore
  931. destFilepath = strings.ReplaceAll(destFilepath, "%", "_")
  932. //Move the file to destination file location
  933. if *enable_asyncFileUpload {
  934. //Use Async upload method
  935. systemWideLogger.PrintAndLog("File System", "AsyncFileUpload flag has been deprecated. Falling back to blocking upload.", errors.New("call to deprecated flag: asyncFileUpload"))
  936. }
  937. err = targetFs.WriteStream(destFilepath, file, 0775)
  938. if err != nil {
  939. systemWideLogger.PrintAndLog("File System", "Write stream to destination file system abstraction from upload failed", err)
  940. utils.SendErrorResponse(w, "Write upload to destination disk failed")
  941. return
  942. }
  943. file.Close()
  944. //Clear up buffered files
  945. r.MultipartForm.RemoveAll()
  946. //Set the ownership of file
  947. userinfo.SetOwnerOfFile(fsh, uploadTarget)
  948. //Finish up the upload
  949. /*
  950. fmt.Printf("Uploaded File: %+v\n", handler.Filename)
  951. fmt.Printf("File Size: %+v\n", handler.Size)
  952. fmt.Printf("MIME Header: %+v\n", handler.Header)
  953. fmt.Println("Upload target: " + realUploadPath)
  954. */
  955. //Fnish upload. Fix the tmp filename
  956. //systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+handler.Filename, nil)
  957. //Do upload finishing stuff
  958. //Add a delay to the complete message to make sure browser catch the return value
  959. currentTimeMilli := time.Now().UnixNano() / int64(time.Millisecond)
  960. if currentTimeMilli-uploadStartTime < 100 {
  961. //Sleep until at least 300 ms
  962. time.Sleep(time.Duration(100 - (currentTimeMilli - uploadStartTime)))
  963. }
  964. //Completed
  965. utils.SendOK(w)
  966. }
  967. // Validate if the copy and target process will involve file overwriting problem.
  968. func system_fs_validateFileOpr(w http.ResponseWriter, r *http.Request) {
  969. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  970. if err != nil {
  971. utils.SendErrorResponse(w, err.Error())
  972. return
  973. }
  974. vsrcFiles, _ := utils.PostPara(r, "src")
  975. vdestFile, _ := utils.PostPara(r, "dest")
  976. var duplicateFiles []string = []string{}
  977. //Loop through all files are see if there are duplication during copy and paste
  978. sourceFiles := []string{}
  979. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  980. err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  981. if err != nil {
  982. utils.SendErrorResponse(w, "Source file JSON parse error.")
  983. return
  984. }
  985. destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  986. if err != nil {
  987. utils.SendErrorResponse(w, "Operation Valid Failed: "+err.Error())
  988. return
  989. }
  990. rdestFile, _ := destFsh.FileSystemAbstraction.VirtualPathToRealPath(destSubpath, userinfo.Username)
  991. for _, file := range sourceFiles {
  992. srcFsh, srcSubpath, _ := GetFSHandlerSubpathFromVpath(string(file))
  993. rsrcFile, _ := srcFsh.FileSystemAbstraction.VirtualPathToRealPath(srcSubpath, userinfo.Username)
  994. if destFsh.FileSystemAbstraction.FileExists(filepath.Join(rdestFile, filepath.Base(rsrcFile))) {
  995. //File exists already.
  996. vpath, _ := srcFsh.FileSystemAbstraction.RealPathToVirtualPath(rsrcFile, userinfo.Username)
  997. duplicateFiles = append(duplicateFiles, vpath)
  998. }
  999. }
  1000. jsonString, _ := json.Marshal(duplicateFiles)
  1001. utils.SendJSONResponse(w, string(jsonString))
  1002. }
  1003. /*
  1004. Trash bin settings
  1005. Stored per user in the same preference table the File Manager already uses,
  1006. so they follow the account rather than the browser.
  1007. Both settings use zero as "no limit", which is also what an account that has
  1008. never opened the settings page reads back:
  1009. trash/retentionDays 0 = never auto remove, otherwise 1..365
  1010. trash/quotaBytes 0 = unlimited
  1011. */
  1012. const (
  1013. trashRetentionPrefKey = "trash/retentionDays"
  1014. trashQuotaPrefKey = "trash/quotaBytes"
  1015. trashMaxRetentionDays = 365
  1016. //What an account that has never opened the settings page holds trash for.
  1017. //An explicitly stored zero still means "never remove" - only the absence
  1018. //of a stored value falls back to this.
  1019. trashDefaultRetentionDays = 30
  1020. )
  1021. func system_fs_getTrashRetentionDays(username string) int {
  1022. result := ""
  1023. err := sysdb.Read("fs", "pref/"+trashRetentionPrefKey+"/"+username, &result)
  1024. if err != nil {
  1025. //Never set for this account
  1026. return trashDefaultRetentionDays
  1027. }
  1028. days, err := strconv.Atoi(result)
  1029. if err != nil || days < 0 {
  1030. //Stored but unreadable - the default is a safer answer than "never"
  1031. return trashDefaultRetentionDays
  1032. }
  1033. if days > trashMaxRetentionDays {
  1034. days = trashMaxRetentionDays
  1035. }
  1036. return days
  1037. }
  1038. func system_fs_getTrashQuotaBytes(username string) int64 {
  1039. result := ""
  1040. err := sysdb.Read("fs", "pref/"+trashQuotaPrefKey+"/"+username, &result)
  1041. if err != nil {
  1042. return 0
  1043. }
  1044. quota, err := utils.StringToInt64(result)
  1045. if err != nil || quota < 0 {
  1046. return 0
  1047. }
  1048. return quota
  1049. }
  1050. /*
  1051. Space a single trashed entry takes up
  1052. A folder counts for everything inside it, not the zero bytes its own entry
  1053. reports - otherwise a user could fill the bin with folders and never reach
  1054. the quota. Hidden files are included because they were moved along with the
  1055. rest, and the trash itself is a hidden folder.
  1056. */
  1057. func system_fs_getTrashEntrySize(fsh *filesystem.FileSystemHandler, rpath string) int64 {
  1058. if fsh.FileSystemAbstraction.IsDir(rpath) {
  1059. size, _ := fsh.GetDirctorySizeFromRealPath(rpath, true)
  1060. return size
  1061. }
  1062. return int64(fsh.FileSystemAbstraction.GetFileSize(rpath))
  1063. }
  1064. // Total bytes currently sitting in a user's trash across every file system
  1065. func system_fs_getTrashUsage(username string) int64 {
  1066. total := int64(0)
  1067. for _, item := range system_fs_listTrash(username) {
  1068. total += system_fs_getTrashEntrySize(item.Fsh, item.Realpath)
  1069. }
  1070. return total
  1071. }
  1072. /*
  1073. Read or write the trash settings for the logged in user.
  1074. GET with no value -> {"RetentionDays":n,"QuotaBytes":n,"MaxRetentionDays":365,"UsedBytes":n}
  1075. POST retentionDays / quotaBytes -> stores them
  1076. */
  1077. func system_fs_handleTrashSettings(w http.ResponseWriter, r *http.Request) {
  1078. username, err := authAgent.GetUserName(w, r)
  1079. if err != nil {
  1080. utils.SendErrorResponse(w, "User not logged in")
  1081. return
  1082. }
  1083. retention, retentionSet := utils.PostPara(r, "retentionDays")
  1084. quota, quotaSet := utils.PostPara(r, "quotaBytes")
  1085. if retentionSet == nil && retention != "" {
  1086. days, err := strconv.Atoi(retention)
  1087. if err != nil || days < 0 || days > trashMaxRetentionDays {
  1088. utils.SendErrorResponse(w, "Invalid retention day given")
  1089. return
  1090. }
  1091. sysdb.Write("fs", "pref/"+trashRetentionPrefKey+"/"+username, strconv.Itoa(days))
  1092. }
  1093. if quotaSet == nil && quota != "" {
  1094. bytes, err := utils.StringToInt64(quota)
  1095. if err != nil || bytes < 0 {
  1096. utils.SendErrorResponse(w, "Invalid quota given")
  1097. return
  1098. }
  1099. sysdb.Write("fs", "pref/"+trashQuotaPrefKey+"/"+username, utils.Int64ToString(bytes))
  1100. }
  1101. type trashSettings struct {
  1102. RetentionDays int
  1103. QuotaBytes int64
  1104. MaxRetentionDays int
  1105. UsedBytes int64
  1106. }
  1107. js, _ := json.Marshal(trashSettings{
  1108. RetentionDays: system_fs_getTrashRetentionDays(username),
  1109. QuotaBytes: system_fs_getTrashQuotaBytes(username),
  1110. MaxRetentionDays: trashMaxRetentionDays,
  1111. UsedBytes: system_fs_getTrashUsage(username),
  1112. })
  1113. utils.SendJSONResponse(w, string(js))
  1114. }
  1115. /*
  1116. Trash layout
  1117. Every file system root keeps a single .trash folder. A removed item keeps the
  1118. folders it was removed from, and the unix time it was removed at is appended
  1119. to its name:
  1120. user:/foo/bar/deleteme.txt -> user:/.trash/foo/bar/deleteme.txt.1712345678
  1121. so scanning the trash only reads that one folder instead of the whole drive,
  1122. and never looks inside a trashed folder. A folder on the way whose own name
  1123. looks like a trashed item ("backup.2024") would be taken for one, so such a
  1124. name gets a "~" appended (plus one for every "~" it already ends with), which
  1125. keeps the mapping reversible.
  1126. */
  1127. const trashFolderSubpath = "/.trash"
  1128. // A trashed item found in the .trash folder of a file system root
  1129. type trashItem struct {
  1130. Fsh *filesystem.FileSystemHandler
  1131. Subpath string //Where it sits, e.g. /.trash/foo/bar/deleteme.txt.1712345678
  1132. Realpath string
  1133. OriginalSubpath string //Where it was removed from, e.g. /foo/bar/deleteme.txt
  1134. Timestamp int64 //Unix time it was removed at
  1135. }
  1136. // Check if a name is a trashed item name: <name>.<unix time>
  1137. func system_fs_isTrashName(name string) bool {
  1138. ext := path.Ext(name)
  1139. if len(ext) < 2 || len(ext) == len(name) {
  1140. return false
  1141. }
  1142. for _, c := range ext[1:] {
  1143. if c < '0' || c > '9' {
  1144. return false
  1145. }
  1146. }
  1147. _, err := strconv.ParseInt(ext[1:], 10, 64)
  1148. return err == nil
  1149. }
  1150. // Escape a folder name on the way to a trashed item so it is never taken for one
  1151. func system_fs_escapeTrashFolder(name string) string {
  1152. if system_fs_isTrashName(strings.TrimRight(name, "~")) {
  1153. return name + "~"
  1154. }
  1155. return name
  1156. }
  1157. func system_fs_unescapeTrashFolder(name string) string {
  1158. if strings.HasSuffix(name, "~") && system_fs_isTrashName(strings.TrimRight(name, "~")) {
  1159. return strings.TrimSuffix(name, "~")
  1160. }
  1161. return name
  1162. }
  1163. // Check if a subpath is the .trash folder or anything inside it
  1164. func system_fs_isInTrash(subpath string) bool {
  1165. cleaned := path.Clean("/" + filepath.ToSlash(subpath))
  1166. return cleaned == trashFolderSubpath || strings.HasPrefix(cleaned, trashFolderSubpath+"/")
  1167. }
  1168. /*
  1169. Where an item removed at the given time goes inside the trash
  1170. /foo/bar/deleteme.txt -> /.trash/foo/bar/deleteme.txt.<timestamp>
  1171. Returns an empty string for the root folder, which cannot be trashed.
  1172. */
  1173. func system_fs_trashSubpathOf(originalSubpath string, timestamp int64) string {
  1174. cleaned := strings.Trim(path.Clean("/"+filepath.ToSlash(originalSubpath)), "/")
  1175. if cleaned == "" {
  1176. return ""
  1177. }
  1178. segments := strings.Split(cleaned, "/")
  1179. last := len(segments) - 1
  1180. for i := 0; i < last; i++ {
  1181. segments[i] = system_fs_escapeTrashFolder(segments[i])
  1182. }
  1183. segments[last] += "." + strconv.FormatInt(timestamp, 10)
  1184. return trashFolderSubpath + "/" + strings.Join(segments, "/")
  1185. }
  1186. /*
  1187. Where a trashed item came from and when it was removed
  1188. /.trash/foo/bar/deleteme.txt.<timestamp> -> /foo/bar/deleteme.txt, <timestamp>
  1189. ok is false for anything that is not a trashed item: a folder on the way, a
  1190. file inside a trashed folder or a path outside the trash.
  1191. */
  1192. func system_fs_originalSubpathOf(trashSubpath string) (string, int64, bool) {
  1193. cleaned := path.Clean("/" + filepath.ToSlash(trashSubpath))
  1194. if !strings.HasPrefix(cleaned, trashFolderSubpath+"/") {
  1195. return "", 0, false
  1196. }
  1197. segments := strings.Split(strings.TrimPrefix(cleaned, trashFolderSubpath+"/"), "/")
  1198. last := len(segments) - 1
  1199. for i := 0; i < last; i++ {
  1200. if system_fs_isTrashName(segments[i]) {
  1201. //Inside a trashed folder
  1202. return "", 0, false
  1203. }
  1204. segments[i] = system_fs_unescapeTrashFolder(segments[i])
  1205. }
  1206. if !system_fs_isTrashName(segments[last]) {
  1207. return "", 0, false
  1208. }
  1209. ext := path.Ext(segments[last])
  1210. timestamp, _ := strconv.ParseInt(ext[1:], 10, 64)
  1211. segments[last] = strings.TrimSuffix(segments[last], ext)
  1212. return "/" + strings.Join(segments, "/"), timestamp, true
  1213. }
  1214. /*
  1215. Scan the .trash folder of one file system root and call fn for every trashed
  1216. item in it. Only the folders on the way are read, never the inside of a
  1217. trashed folder. An error returned by fn stops the scan and is returned.
  1218. */
  1219. func system_fs_scanTrashOf(fsh *filesystem.FileSystemHandler, username string, fn func(trashItem) error) error {
  1220. fshAbs := fsh.FileSystemAbstraction
  1221. trashRoot, err := fshAbs.VirtualPathToRealPath(trashFolderSubpath, username)
  1222. if err != nil || !fshAbs.IsDir(trashRoot) {
  1223. return nil
  1224. }
  1225. var visit func(rdir string, subdir string) error
  1226. visit = func(rdir string, subdir string) error {
  1227. entries, err := fshAbs.ReadDir(rdir)
  1228. if err != nil {
  1229. return nil
  1230. }
  1231. for _, entry := range entries {
  1232. rpath := rdir + "/" + entry.Name()
  1233. subpath := subdir + "/" + entry.Name()
  1234. if system_fs_isTrashName(entry.Name()) {
  1235. originalSubpath, timestamp, ok := system_fs_originalSubpathOf(subpath)
  1236. if !ok {
  1237. continue
  1238. }
  1239. err := fn(trashItem{
  1240. Fsh: fsh,
  1241. Subpath: subpath,
  1242. Realpath: rpath,
  1243. OriginalSubpath: originalSubpath,
  1244. Timestamp: timestamp,
  1245. })
  1246. if err != nil {
  1247. return err
  1248. }
  1249. } else if entry.IsDir() {
  1250. //A folder on the way, or leftovers like a thumbnail .metadata folder
  1251. if err := visit(rpath, subpath); err != nil {
  1252. return err
  1253. }
  1254. }
  1255. }
  1256. return nil
  1257. }
  1258. return visit(filepath.ToSlash(trashRoot), trashFolderSubpath)
  1259. }
  1260. // The file system roots whose trash a user sees
  1261. func system_fs_trashRootsOf(userinfo *user.User) []*filesystem.FileSystemHandler {
  1262. roots := []*filesystem.FileSystemHandler{}
  1263. for _, storage := range userinfo.GetAllFileSystemHandler() {
  1264. if storage.Hierarchy == "backup" || share.IsSharedDrive(storage) {
  1265. //Skip this fsh. Shared files are in their owner's trash, not ours
  1266. continue
  1267. }
  1268. roots = append(roots, storage)
  1269. }
  1270. return roots
  1271. }
  1272. // Get every trashed item of a user across all their file system roots
  1273. func system_fs_listTrash(username string) []trashItem {
  1274. userinfo, err := userHandler.GetUserInfoFromUsername(username)
  1275. if err != nil {
  1276. return []trashItem{}
  1277. }
  1278. items := []trashItem{}
  1279. for _, fsh := range system_fs_trashRootsOf(userinfo) {
  1280. system_fs_scanTrashOf(fsh, username, func(item trashItem) error {
  1281. items = append(items, item)
  1282. return nil
  1283. })
  1284. }
  1285. return items
  1286. }
  1287. // Describe a trashed item to the client
  1288. func system_fs_getTrashedFileInfo(item trashItem) trashedFile {
  1289. fshAbs := item.Fsh.FileSystemAbstraction
  1290. originalName := path.Base(item.OriginalSubpath)
  1291. isDir := fshAbs.IsDir(item.Realpath)
  1292. originalExt := ""
  1293. if !isDir {
  1294. originalExt = path.Ext(originalName)
  1295. }
  1296. return trashedFile{
  1297. Filename: path.Base(item.Subpath),
  1298. Filepath: item.Fsh.UUID + ":" + item.Subpath,
  1299. FileExt: originalExt,
  1300. IsDir: isDir,
  1301. Filesize: int64(fshAbs.GetFileSize(item.Realpath)),
  1302. RemoveTimestamp: item.Timestamp,
  1303. RemoveDate: time.Unix(item.Timestamp, 0).Format("2006-01-02 15:04:05"),
  1304. OriginalPath: item.Fsh.UUID + ":" + path.Dir(item.OriginalSubpath),
  1305. OriginalFilename: originalName,
  1306. }
  1307. }
  1308. // Remove a trashed item for good, along with the owner's quota record of it
  1309. func system_fs_removeTrashItem(userinfo *user.User, item trashItem) error {
  1310. vpath := item.Fsh.UUID + ":" + item.Subpath
  1311. if userinfo.IsOwnerOfFile(item.Fsh, vpath) {
  1312. userinfo.RemoveOwnershipFromFile(item.Fsh, vpath)
  1313. }
  1314. return item.Fsh.FileSystemAbstraction.RemoveAll(item.Realpath)
  1315. }
  1316. /*
  1317. Remove every folder below rdir (rdir included) that holds no trashed item,
  1318. and report whether rdir still holds one. Leftovers such as the .metadata
  1319. folder a thumbnail of a trashed file leaves behind, or a .DS_Store, go too.
  1320. */
  1321. func system_fs_pruneTrashTree(fshAbs filesystem.FileSystemAbstraction, rdir string) bool {
  1322. entries, err := fshAbs.ReadDir(rdir)
  1323. if err != nil {
  1324. //Cannot tell what is in there; leave it alone
  1325. return true
  1326. }
  1327. kept := false
  1328. for _, entry := range entries {
  1329. if system_fs_isTrashName(entry.Name()) {
  1330. kept = true
  1331. } else if entry.IsDir() && system_fs_pruneTrashTree(fshAbs, rdir+"/"+entry.Name()) {
  1332. kept = true
  1333. }
  1334. }
  1335. if !kept {
  1336. fshAbs.RemoveAll(rdir)
  1337. }
  1338. return kept
  1339. }
  1340. // After an item left the trash, remove the folders it left empty, up to the .trash folder itself
  1341. func system_fs_pruneTrashParents(fsh *filesystem.FileSystemHandler, username string, rdir string) {
  1342. fshAbs := fsh.FileSystemAbstraction
  1343. trashRoot, err := fshAbs.VirtualPathToRealPath(trashFolderSubpath, username)
  1344. if err != nil {
  1345. return
  1346. }
  1347. trashRoot = path.Clean(filepath.ToSlash(trashRoot))
  1348. rdir = path.Clean(filepath.ToSlash(rdir))
  1349. for rdir == trashRoot || strings.HasPrefix(rdir, trashRoot+"/") {
  1350. if !fshAbs.FileExists(rdir) {
  1351. //Already gone, try its parent
  1352. } else if system_fs_pruneTrashTree(fshAbs, rdir) {
  1353. return
  1354. }
  1355. if rdir == trashRoot {
  1356. return
  1357. }
  1358. rdir = path.Dir(rdir)
  1359. }
  1360. }
  1361. /*
  1362. Nightly cleanup
  1363. Removes trashed files past their owner's retention window. Users who have
  1364. set the retention to zero keep their trash indefinitely.
  1365. */
  1366. func system_fs_clearExpiredTrash() {
  1367. for _, username := range authAgent.ListUsers() {
  1368. retentionDays := system_fs_getTrashRetentionDays(username)
  1369. if retentionDays <= 0 {
  1370. //Auto removal disabled for this user
  1371. continue
  1372. }
  1373. userinfo, err := userHandler.GetUserInfoFromUsername(username)
  1374. if err != nil {
  1375. continue
  1376. }
  1377. cutoff := time.Now().Unix() - int64(retentionDays)*86400
  1378. removed := 0
  1379. for _, fsh := range system_fs_trashRootsOf(userinfo) {
  1380. if !nightlyShouldMaintainFsh(fsh) {
  1381. //Trash inside a drive shared by the cluster is emptied by the
  1382. //master node, so it is not purged once per member
  1383. continue
  1384. }
  1385. expired := []trashItem{}
  1386. system_fs_scanTrashOf(fsh, username, func(item trashItem) error {
  1387. if item.Timestamp <= cutoff {
  1388. expired = append(expired, item)
  1389. }
  1390. return nil
  1391. })
  1392. if len(expired) == 0 {
  1393. continue
  1394. }
  1395. for _, item := range expired {
  1396. if system_fs_removeTrashItem(userinfo, item) == nil {
  1397. removed++
  1398. }
  1399. }
  1400. //Drop the folders the removed items left empty in one pass
  1401. trashRoot, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(trashFolderSubpath, username)
  1402. if err == nil {
  1403. system_fs_pruneTrashTree(fsh.FileSystemAbstraction, filepath.ToSlash(trashRoot))
  1404. }
  1405. }
  1406. if removed > 0 {
  1407. systemWideLogger.PrintAndLog("File System", "Removed "+strconv.Itoa(removed)+
  1408. " expired trash item(s) for user "+username, nil)
  1409. }
  1410. }
  1411. }
  1412. // Scan the trash of every root and send back the results with WebSocket
  1413. func system_fs_WebSocketScanTrashBin(w http.ResponseWriter, r *http.Request) {
  1414. //Get and check user permission
  1415. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1416. if err != nil {
  1417. utils.SendErrorResponse(w, "User not logged in")
  1418. return
  1419. }
  1420. //Upgrade to websocket
  1421. var upgrader = websocket.Upgrader{}
  1422. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1423. c, err := upgrader.Upgrade(w, r, nil)
  1424. if err != nil {
  1425. w.WriteHeader(http.StatusInternalServerError)
  1426. w.Write([]byte("500 - " + err.Error()))
  1427. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1428. return
  1429. }
  1430. for _, fsh := range system_fs_trashRootsOf(userinfo) {
  1431. err = system_fs_scanTrashOf(fsh, userinfo.Username, func(item trashItem) error {
  1432. //Send out the result as JSON string
  1433. js, _ := json.Marshal(system_fs_getTrashedFileInfo(item))
  1434. return c.WriteMessage(1, js)
  1435. })
  1436. if err != nil {
  1437. //Client connection error (Connection closed?)
  1438. return
  1439. }
  1440. }
  1441. //Close connection after finished
  1442. c.Close()
  1443. }
  1444. // Scan the trash of every root and return the trashed files within the system
  1445. func system_fs_scanTrashBin(w http.ResponseWriter, r *http.Request) {
  1446. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1447. if err != nil {
  1448. utils.SendErrorResponse(w, err.Error())
  1449. return
  1450. }
  1451. results := []trashedFile{}
  1452. for _, item := range system_fs_listTrash(userinfo.Username) {
  1453. results = append(results, system_fs_getTrashedFileInfo(item))
  1454. }
  1455. //Sort the results by date, latest on top
  1456. sort.Slice(results[:], func(i, j int) bool {
  1457. return results[i].RemoveTimestamp > results[j].RemoveTimestamp
  1458. })
  1459. //Format and return the json results
  1460. jsonString, _ := json.Marshal(results)
  1461. utils.SendJSONResponse(w, string(jsonString))
  1462. }
  1463. // Restore a trashed file to the folder it was removed from
  1464. func system_fs_restoreFile(w http.ResponseWriter, r *http.Request) {
  1465. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1466. if err != nil {
  1467. utils.SendErrorResponse(w, err.Error())
  1468. return
  1469. }
  1470. targetTrashedFile, err := utils.PostPara(r, "src")
  1471. if err != nil {
  1472. utils.SendErrorResponse(w, "Invalid src given")
  1473. return
  1474. }
  1475. fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetTrashedFile)
  1476. if err != nil {
  1477. utils.SendErrorResponse(w, err.Error())
  1478. return
  1479. }
  1480. fshAbs := fsh.FileSystemAbstraction
  1481. //Check if this is really a trashed file
  1482. originalSubpath, _, ok := system_fs_originalSubpathOf(subpath)
  1483. if !ok {
  1484. utils.SendErrorResponse(w, "File not in trashbin")
  1485. return
  1486. }
  1487. realpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1488. if err != nil || !fshAbs.FileExists(realpath) {
  1489. utils.SendErrorResponse(w, "File not exists")
  1490. return
  1491. }
  1492. if !userinfo.CanWrite(fsh.UUID + ":" + originalSubpath) {
  1493. utils.SendErrorResponse(w, "Access Denied")
  1494. return
  1495. }
  1496. targetPath, err := fshAbs.VirtualPathToRealPath(originalSubpath, userinfo.Username)
  1497. if err != nil {
  1498. utils.SendErrorResponse(w, err.Error())
  1499. return
  1500. }
  1501. if fshAbs.FileExists(targetPath) {
  1502. utils.SendErrorResponse(w, "An item with the same name already exists in the original location")
  1503. return
  1504. }
  1505. //The folder it came from may have been removed since
  1506. fshAbs.MkdirAll(path.Dir(filepath.ToSlash(targetPath)), 0755)
  1507. err = fshAbs.Rename(realpath, targetPath)
  1508. if err != nil {
  1509. systemWideLogger.PrintAndLog("File System", "Failed to restore file from trash", err)
  1510. utils.SendErrorResponse(w, "Failed to restore file from trash")
  1511. return
  1512. }
  1513. //Drop the folders in the trash it leaves empty
  1514. system_fs_pruneTrashParents(fsh, userinfo.Username, path.Dir(filepath.ToSlash(realpath)))
  1515. utils.SendOK(w)
  1516. }
  1517. // Clear all trashed file in the system
  1518. func system_fs_clearTrashBin(w http.ResponseWriter, r *http.Request) {
  1519. u, err := userHandler.GetUserInfoFromRequest(w, r)
  1520. if err != nil {
  1521. utils.SendErrorResponse(w, "User not logged in")
  1522. return
  1523. }
  1524. for _, fsh := range system_fs_trashRootsOf(u) {
  1525. items := []trashItem{}
  1526. system_fs_scanTrashOf(fsh, u.Username, func(item trashItem) error {
  1527. items = append(items, item)
  1528. return nil
  1529. })
  1530. for _, item := range items {
  1531. system_fs_removeTrashItem(u, item)
  1532. }
  1533. //Everything left in there is a leftover. Remove the whole .trash folder
  1534. fshAbs := fsh.FileSystemAbstraction
  1535. trashRoot, err := fshAbs.VirtualPathToRealPath(trashFolderSubpath, u.Username)
  1536. if err == nil && fshAbs.FileExists(trashRoot) {
  1537. fshAbs.RemoveAll(trashRoot)
  1538. }
  1539. }
  1540. utils.SendOK(w)
  1541. }
  1542. /*
  1543. Handle new file or folder functions
  1544. Required information
  1545. @type {folder / file}
  1546. @ext {any that is listed in the template folder}
  1547. if no paramter is passed in, default listing all the supported template file
  1548. */
  1549. func system_fs_handleNewObjects(w http.ResponseWriter, r *http.Request) {
  1550. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1551. if err != nil {
  1552. utils.SendErrorResponse(w, "User not logged in")
  1553. return
  1554. }
  1555. //Validate the token
  1556. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  1557. if !tokenValid {
  1558. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  1559. return
  1560. }
  1561. fileType, _ := utils.PostPara(r, "type") //File creation type, {file, folder}
  1562. vsrc, _ := utils.PostPara(r, "src") //Virtual file source folder, do not include filename
  1563. filename, _ := utils.PostPara(r, "filename") //Filename for the new file
  1564. if fileType == "" && filename == "" {
  1565. //List all the supported new filetype
  1566. if !filesystem.FileExists("system/newitem/") {
  1567. os.MkdirAll("system/newitem/", 0755)
  1568. }
  1569. type newItemObject struct {
  1570. Desc string
  1571. Ext string
  1572. }
  1573. var newItemList []newItemObject
  1574. newItemTemplate, _ := filepath.Glob("system/newitem/*")
  1575. for _, file := range newItemTemplate {
  1576. thisItem := new(newItemObject)
  1577. thisItem.Desc = strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
  1578. thisItem.Ext = filepath.Ext(file)[1:]
  1579. newItemList = append(newItemList, *thisItem)
  1580. }
  1581. jsonString, err := json.Marshal(newItemList)
  1582. if err != nil {
  1583. systemWideLogger.PrintAndLog("File System", "Unable to parse JSON string for new item list", err)
  1584. utils.SendErrorResponse(w, "Unable to parse new item list. See server log for more information.")
  1585. return
  1586. }
  1587. utils.SendJSONResponse(w, string(jsonString))
  1588. return
  1589. } else if fileType != "" && filename != "" {
  1590. if vsrc == "" {
  1591. utils.SendErrorResponse(w, "Missing paramter: 'src'")
  1592. return
  1593. }
  1594. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
  1595. if err != nil {
  1596. utils.SendErrorResponse(w, err.Error())
  1597. return
  1598. }
  1599. fshAbs := fsh.FileSystemAbstraction
  1600. //Translate the path to realpath
  1601. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1602. if err != nil {
  1603. utils.SendErrorResponse(w, "Invalid path given")
  1604. return
  1605. }
  1606. //Check if directory is readonly
  1607. accmode := userinfo.GetPathAccessPermission(vsrc)
  1608. if accmode == arozfs.FsReadOnly {
  1609. utils.SendErrorResponse(w, "This directory is Read Only")
  1610. return
  1611. } else if accmode == arozfs.FsDenied {
  1612. utils.SendErrorResponse(w, "Access Denied")
  1613. return
  1614. }
  1615. //Check if the filename contains web-unsafe characters
  1616. if !utils.FilenameIsWebSafe(filename) {
  1617. utils.SendErrorResponse(w, "Filename contains illegal characters")
  1618. return
  1619. }
  1620. //Check if the file already exists. If yes, fix its filename.
  1621. newfilePath := filepath.ToSlash(filepath.Join(rpath, filename))
  1622. switch fileType {
  1623. case "file":
  1624. for fshAbs.FileExists(newfilePath) {
  1625. utils.SendErrorResponse(w, "Given filename already exists")
  1626. return
  1627. }
  1628. ext := filepath.Ext(filename)
  1629. defaultFileCotent := []byte("")
  1630. if ext != "" {
  1631. templateFile, _ := fshAbs.Glob("system/newitem/*" + ext)
  1632. if len(templateFile) > 0 {
  1633. //Copy file from templateFile[0] to current dir with the given name
  1634. input, _ := os.ReadFile(templateFile[0])
  1635. defaultFileCotent = input
  1636. }
  1637. }
  1638. err = fshAbs.WriteFile(newfilePath, defaultFileCotent, 0775)
  1639. if err != nil {
  1640. systemWideLogger.PrintAndLog("File System", "Unable to create new file: "+err.Error(), err)
  1641. utils.SendErrorResponse(w, err.Error())
  1642. return
  1643. }
  1644. case "folder":
  1645. if fshAbs.FileExists(newfilePath) {
  1646. utils.SendErrorResponse(w, "Given folder already exists")
  1647. return
  1648. }
  1649. //Create the folder at target location
  1650. err := fshAbs.Mkdir(newfilePath, 0755)
  1651. if err != nil {
  1652. utils.SendErrorResponse(w, err.Error())
  1653. return
  1654. }
  1655. }
  1656. utils.SendOK(w)
  1657. } else {
  1658. utils.SendErrorResponse(w, "Missing paramter(s).")
  1659. return
  1660. }
  1661. }
  1662. /*
  1663. Handle file operations via WebSocket
  1664. This handler only handle zip, unzip, copy and move. Not other operations.
  1665. For other operations, please use the legacy handleOpr endpoint
  1666. The actual operation logic is shared with the asynchronous (background)
  1667. file operation endpoint, see runFileOperationTask below.
  1668. */
  1669. func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
  1670. //Get and check user permission
  1671. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1672. if err != nil {
  1673. utils.SendErrorResponse(w, "User not logged in")
  1674. return
  1675. }
  1676. operation, _ := utils.GetPara(r, "opr") //Accept copy and move
  1677. vsrcFiles, _ := utils.GetPara(r, "src")
  1678. vdestFile, _ := utils.GetPara(r, "dest")
  1679. existsOpr, _ := utils.GetPara(r, "existsresp")
  1680. sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
  1681. if err != nil {
  1682. utils.SendErrorResponse(w, err.Error())
  1683. return
  1684. }
  1685. //Permission checking
  1686. if !userinfo.CanWrite(vdestFile) {
  1687. systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
  1688. w.WriteHeader(http.StatusForbidden)
  1689. w.Write([]byte("403 - Access Denied"))
  1690. return
  1691. }
  1692. //Upgrade to websocket
  1693. var upgrader = websocket.Upgrader{}
  1694. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1695. c, err := upgrader.Upgrade(w, r, nil)
  1696. if err != nil {
  1697. w.WriteHeader(http.StatusInternalServerError)
  1698. w.Write([]byte("500 - " + err.Error()))
  1699. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1700. return
  1701. }
  1702. //Create the file operation task and remember it
  1703. task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)
  1704. //Send over the oprId for this file operation for tracking
  1705. time.Sleep(300 * time.Millisecond)
  1706. c.WriteMessage(1, []byte("{\"oprid\":\""+task.ID+"\"}"))
  1707. //Run the operation on this request goroutine and stream the progress back
  1708. runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, func(update fileOprProgressUpdate) {
  1709. js, _ := json.Marshal(update)
  1710. c.WriteMessage(1, js)
  1711. })
  1712. //This endpoint do not keep the finished record. Remove it right away.
  1713. wsConnectionStore.Delete(task.ID)
  1714. //Close WebSocket connection after finished
  1715. time.Sleep(1 * time.Second)
  1716. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  1717. c.Close()
  1718. }
  1719. /*
  1720. Handle file operations as a background task
  1721. Unlike the WebSocket endpoint above, this endpoint returns the operation id
  1722. right away and let the operation run in the background. The caller can then
  1723. watch the progress of all of its operations through a single status
  1724. WebSocket, see system_fs_handleFileOprStatusWebSocket below.
  1725. */
  1726. func system_fs_handleAsyncOpr(w http.ResponseWriter, r *http.Request) {
  1727. //Get and check user permission
  1728. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1729. if err != nil {
  1730. utils.SendErrorResponse(w, "User not logged in")
  1731. return
  1732. }
  1733. //Validate the token
  1734. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  1735. if !tokenValid {
  1736. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  1737. return
  1738. }
  1739. operation, _ := utils.PostPara(r, "opr")
  1740. vsrcFiles, _ := utils.PostPara(r, "src")
  1741. vdestFile, _ := utils.PostPara(r, "dest")
  1742. existsOpr, _ := utils.PostPara(r, "existsresp")
  1743. sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
  1744. if err != nil {
  1745. utils.SendErrorResponse(w, err.Error())
  1746. return
  1747. }
  1748. //Permission checking
  1749. if !userinfo.CanWrite(vdestFile) {
  1750. systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
  1751. utils.SendErrorResponse(w, "Access Denied")
  1752. return
  1753. }
  1754. //Create the task record and start it in the background
  1755. task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)
  1756. go runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, nil)
  1757. js, _ := json.Marshal(map[string]string{"oprid": task.ID})
  1758. utils.SendJSONResponse(w, string(js))
  1759. }
  1760. /*
  1761. Stream the status of all the file operations of the requesting user
  1762. One connection is enough for all the file operations of a user. The client
  1763. side file operation dialog opens this once and renders every task it reports.
  1764. Control commands can also be sent over this socket in the following format
  1765. {"cmd":"pause | continue | cancel | remove", "oprid":"<operation_id>"} or
  1766. {"cmd":"clear"} for removing all the finished records of this user.
  1767. */
  1768. func system_fs_handleFileOprStatusWebSocket(w http.ResponseWriter, r *http.Request) {
  1769. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1770. if err != nil {
  1771. utils.SendErrorResponse(w, "User not logged in")
  1772. return
  1773. }
  1774. var upgrader = websocket.Upgrader{}
  1775. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1776. c, err := upgrader.Upgrade(w, r, nil)
  1777. if err != nil {
  1778. w.WriteHeader(http.StatusInternalServerError)
  1779. w.Write([]byte("500 - " + err.Error()))
  1780. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1781. return
  1782. }
  1783. defer c.Close()
  1784. //Listen for control commands on this socket until it is closed by the client
  1785. clientGone := make(chan bool, 1)
  1786. go func() {
  1787. type controlCommand struct {
  1788. Cmd string
  1789. Oprid string
  1790. }
  1791. for {
  1792. _, message, err := c.ReadMessage()
  1793. if err != nil {
  1794. clientGone <- true
  1795. return
  1796. }
  1797. cmd := controlCommand{}
  1798. if json.Unmarshal(message, &cmd) != nil {
  1799. continue
  1800. }
  1801. ApplyFileOperationControl(userinfo.Username, cmd.Cmd, cmd.Oprid)
  1802. }
  1803. }()
  1804. //Push the full task list of this user on every tick
  1805. ticker := time.NewTicker(500 * time.Millisecond)
  1806. defer ticker.Stop()
  1807. for {
  1808. js, err := MarshalFileOperationForUser(userinfo.Username, true)
  1809. if err == nil {
  1810. if c.WriteMessage(websocket.TextMessage, js) != nil {
  1811. return
  1812. }
  1813. }
  1814. select {
  1815. case <-clientGone:
  1816. return
  1817. case <-ticker.C:
  1818. }
  1819. }
  1820. }
  1821. /*
  1822. File operation task runner
  1823. These are the shared internals used by both the legacy WebSocket endpoint
  1824. and the background (async) file operation endpoint above.
  1825. */
  1826. // fileOprProgressUpdate is the progress payload of the legacy WebSocket file operation endpoint
  1827. type fileOprProgressUpdate struct {
  1828. LatestFile string
  1829. Progress int
  1830. StatusFlag int
  1831. Error string
  1832. }
  1833. // parseFileOperationRequest validates a file operation request and returns the
  1834. // decoded source file list and the decoded destination virtual path
  1835. func parseFileOperationRequest(operation string, vsrcFiles string, vdestFile string) ([]string, string, error) {
  1836. //Check if opr is supported
  1837. if operation != "move" && operation != "copy" && operation != "zip" && operation != "unzip" {
  1838. systemWideLogger.PrintAndLog("File System", "This file operation is not supported on the file operation endpoint. Received: "+operation, errors.New("operation not supported"))
  1839. return nil, "", errors.New("operation not supported on this endpoint")
  1840. }
  1841. //Decode the source file list
  1842. var sourceFiles []string
  1843. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  1844. err := json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  1845. if err != nil {
  1846. systemWideLogger.PrintAndLog("File System", "File operation source file JSON parse error", err)
  1847. return nil, "", errors.New("Source file JSON parse error.")
  1848. }
  1849. if len(sourceFiles) == 0 {
  1850. return nil, "", errors.New("No source file given")
  1851. }
  1852. //Bugged char filtering
  1853. tmp := []string{}
  1854. for _, src := range sourceFiles {
  1855. tmp = append(tmp, strings.ReplaceAll(src, "{{plug_sign}}", "+"))
  1856. }
  1857. sourceFiles = tmp
  1858. vdestFile = strings.ReplaceAll(vdestFile, "{{plug_sign}}", "+")
  1859. //Decode the target position
  1860. escapedVdest := system_fs_specialURIDecode(vdestFile)
  1861. vdestFile = escapedVdest
  1862. if vdestFile == "" {
  1863. return nil, "", errors.New("Undefined dest location")
  1864. }
  1865. //Make sure the destination is resolvable before doing anything else
  1866. _, _, err = GetFSHandlerSubpathFromVpath(vdestFile)
  1867. if err != nil {
  1868. return nil, "", err
  1869. }
  1870. return sourceFiles, vdestFile, nil
  1871. }
  1872. // runFileOperationTask executes the file operation described by the given task record.
  1873. // onUpdate is optional and, when given, is called on every progress update.
  1874. func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFiles []string, vdestFile string, existsOpr string, onUpdate func(fileOprProgressUpdate)) {
  1875. if existsOpr == "" {
  1876. existsOpr = "keep"
  1877. }
  1878. oprId := task.ID
  1879. operation := task.Operation
  1880. pushUpdate := func(update fileOprProgressUpdate) {
  1881. if onUpdate != nil {
  1882. onUpdate(update)
  1883. }
  1884. }
  1885. failTask := func(filename string, errmsg string) {
  1886. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, errmsg)
  1887. pushUpdate(fileOprProgressUpdate{
  1888. LatestFile: filename,
  1889. Progress: -1,
  1890. Error: errmsg,
  1891. StatusFlag: filesystem.FsOpr_Error,
  1892. })
  1893. }
  1894. //Resolve the destination file system handler
  1895. destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  1896. if err != nil {
  1897. failTask(filepath.Base(vdestFile), err.Error())
  1898. return
  1899. }
  1900. destFshAbs := destFsh.FileSystemAbstraction
  1901. rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1902. if !userinfo.CanWrite(vdestFile) {
  1903. failTask(filepath.Base(vdestFile), "Access Denied: No Write Permission")
  1904. return
  1905. }
  1906. //A copy of a file shared with the user needs its owner to allow downloads
  1907. if operation == "copy" || operation == "zip" {
  1908. if err := sharedDriveCopyDenied(userinfo, sourceFiles); err != nil {
  1909. failTask(filepath.Base(sourceFiles[0]), err.Error())
  1910. return
  1911. }
  1912. }
  1913. if operation == "zip" {
  1914. //Zip files
  1915. outputFilename := filepath.Join(rdestFile, strings.ReplaceAll(filepath.Base(filepath.Dir(sourceFiles[0])+".zip"), ":", ""))
  1916. if len(sourceFiles) == 1 {
  1917. //Use the basename of the source file as zip file name
  1918. outputFilename = filepath.Join(rdestFile, filepath.Base(sourceFiles[0])) + ".zip"
  1919. }
  1920. //Translate source Files into real paths
  1921. realSourceFiles := []string{}
  1922. sourceFileFsh := []*filesystem.FileSystemHandler{}
  1923. for _, vsrcs := range sourceFiles {
  1924. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
  1925. if err != nil {
  1926. failTask(filepath.Base(vsrcs), "Source file not exists")
  1927. return
  1928. }
  1929. rsrc, err := thisSrcFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  1930. if err != nil {
  1931. failTask(filepath.Base(rsrc), "Source file not exists")
  1932. return
  1933. }
  1934. realSourceFiles = append(realSourceFiles, rsrc)
  1935. sourceFileFsh = append(sourceFileFsh, thisSrcFsh)
  1936. }
  1937. zipDestPath := outputFilename
  1938. zipDestFsh := destFsh
  1939. if destFsh.RequireBuffer {
  1940. zipDestPath = getFsBufferFilepath(outputFilename, false)
  1941. zipDestFsh = nil
  1942. }
  1943. //Create the zip file
  1944. err = filesystem.ArozZipFileWithProgress(sourceFileFsh, realSourceFiles, zipDestFsh, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) int {
  1945. sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
  1946. pushUpdate(fileOprProgressUpdate{
  1947. LatestFile: currentFilename,
  1948. Progress: int(math.Ceil(progress)),
  1949. Error: "",
  1950. StatusFlag: sig,
  1951. })
  1952. return sig
  1953. })
  1954. if err != nil {
  1955. //The half written archive is already removed by the zipper
  1956. if destFsh.RequireBuffer {
  1957. cleanFsBufferFileFromList(realSourceFiles)
  1958. }
  1959. if !fileOperationCancelled(oprId) {
  1960. systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
  1961. failTask(filepath.Base(outputFilename), err.Error())
  1962. return
  1963. }
  1964. } else if destFsh.RequireBuffer {
  1965. //Move the buffer result to remote
  1966. f, _ := os.Open(zipDestPath)
  1967. err = destFshAbs.WriteStream(outputFilename, f, 0775)
  1968. if err != nil {
  1969. systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver"+destFsh.Filesystem+" failed", err)
  1970. }
  1971. f.Close()
  1972. //Clear local buffers
  1973. os.Remove(zipDestPath)
  1974. cleanFsBufferFileFromList(realSourceFiles)
  1975. }
  1976. } else if operation == "unzip" {
  1977. //Create the destination folder
  1978. destFshAbs.MkdirAll(rdestFile, 0755)
  1979. //Convert the src files into realpaths
  1980. realSourceFiles := []string{}
  1981. for _, vsrcs := range sourceFiles {
  1982. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
  1983. if err != nil {
  1984. failTask(filepath.Base(vsrcs), "Source file not exists")
  1985. return
  1986. }
  1987. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  1988. rsrc, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1989. if err != nil {
  1990. failTask(filepath.Base(rsrc), "Source file not exists")
  1991. return
  1992. }
  1993. if thisSrcFsh.RequireBuffer {
  1994. localBufferFilepath, err := bufferRemoteFileToLocal(thisSrcFsh, rsrc, false)
  1995. if err != nil {
  1996. failTask(filepath.Base(rsrc), "Failed to buffer file to local disk")
  1997. return
  1998. }
  1999. realSourceFiles = append(realSourceFiles, localBufferFilepath)
  2000. } else {
  2001. realSourceFiles = append(realSourceFiles, rsrc)
  2002. }
  2003. }
  2004. unzipDest := rdestFile
  2005. if destFsh.RequireBuffer {
  2006. unzipDest = getFsBufferFilepath(rdestFile, true)
  2007. }
  2008. //Unzip the files
  2009. filesystem.ArozUnzipFileWithProgress(realSourceFiles, unzipDest, func(currentFile string, filecount int, totalfile int, progress float64) int {
  2010. //Generate the status update struct
  2011. sig, _ := UpdateOngoingFileOperation(oprId, filepath.Base(currentFile), math.Ceil(progress))
  2012. pushUpdate(fileOprProgressUpdate{
  2013. LatestFile: filepath.Base(currentFile),
  2014. Progress: int(math.Ceil(progress)),
  2015. Error: "",
  2016. StatusFlag: sig,
  2017. })
  2018. return sig
  2019. })
  2020. if destFsh.RequireBuffer {
  2021. //Push the unzip results back to remote fs
  2022. filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
  2023. path = filepath.ToSlash(path)
  2024. relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
  2025. if info.IsDir() {
  2026. destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
  2027. } else {
  2028. f, _ := os.Open(path)
  2029. destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
  2030. f.Close()
  2031. }
  2032. return nil
  2033. })
  2034. cleanFsBufferFileFromList([]string{unzipDest})
  2035. }
  2036. cleanFsBufferFileFromList(realSourceFiles)
  2037. } else {
  2038. //Other operations that allow multiple source files to handle one by one
  2039. for i := 0; i < len(sourceFiles); i++ {
  2040. vsrcFile := sourceFiles[i]
  2041. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  2042. if err != nil {
  2043. MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
  2044. failTask(filepath.Base(vsrcFile), "Source file not exists")
  2045. return
  2046. }
  2047. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  2048. rsrcFile, _ := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2049. if !thisSrcFshAbs.FileExists(rsrcFile) {
  2050. //This source file not exists. Report Error and Stop
  2051. MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
  2052. failTask(filepath.Base(rsrcFile), "Source file not exists")
  2053. return
  2054. }
  2055. //Progress handler shared by the move and copy operations. The overall
  2056. //progress is worked out from the bytes of every source file, so a
  2057. //small file no longer counts for as much of the bar as a large one.
  2058. subtaskProgressHandler := func(currentFile string, bytesDone int64, bytesTotal int64) int {
  2059. sig, overallProgress, _ := UpdateOngoingFileOperationSubtask(oprId, i, filepath.Base(currentFile), bytesDone, bytesTotal)
  2060. pushUpdate(fileOprProgressUpdate{
  2061. LatestFile: filepath.Base(currentFile),
  2062. Progress: int(math.Ceil(overallProgress)),
  2063. Error: "",
  2064. StatusFlag: sig,
  2065. })
  2066. return sig
  2067. }
  2068. if operation == "move" {
  2069. err := filesystem.FileMoveWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, subtaskProgressHandler)
  2070. //Handle move starting error
  2071. if err != nil {
  2072. MarkFileOperationSubtaskEnded(oprId, i, err.Error())
  2073. failTask(filepath.Base(rsrcFile), err.Error())
  2074. return
  2075. }
  2076. //Remove the cache for the original file
  2077. metadata.RemoveCache(thisSrcFsh, rsrcFile)
  2078. } else if operation == "copy" {
  2079. err := filesystem.FileCopyWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, subtaskProgressHandler)
  2080. //Handle Copy starting error
  2081. if err != nil {
  2082. MarkFileOperationSubtaskEnded(oprId, i, err.Error())
  2083. failTask(filepath.Base(rsrcFile), err.Error())
  2084. return
  2085. }
  2086. }
  2087. //This source file is done. Mark it as completed
  2088. MarkFileOperationSubtaskEnded(oprId, i, "")
  2089. }
  2090. }
  2091. //Check if the operation was cancelled by the user half way through
  2092. endingSignal := filesystem.FsOpr_Continue
  2093. if fileOperationCancelled(oprId) {
  2094. endingSignal = filesystem.FsOpr_Cancel
  2095. }
  2096. SetFileOperationTaskEnded(oprId, endingSignal, "")
  2097. }
  2098. // fileOperationCancelled reports whether the user asked to cancel the given operation
  2099. func fileOperationCancelled(oprId string) bool {
  2100. t, err := GetOngoingFileOperationByOprID(oprId)
  2101. if err != nil {
  2102. return false
  2103. }
  2104. fileOprTaskLock.RLock()
  2105. defer fileOprTaskLock.RUnlock()
  2106. return t.FileOperationSignal == filesystem.FsOpr_Cancel
  2107. }
  2108. /*
  2109. Handle file operations
  2110. Support {move, copy, delete, recycle, rename}
  2111. */
  2112. //Handle file operations.
  2113. func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
  2114. //Check if user logged in
  2115. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2116. if err != nil {
  2117. utils.SendErrorResponse(w, "User not logged in")
  2118. return
  2119. }
  2120. //Validate the token
  2121. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  2122. if !tokenValid {
  2123. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  2124. return
  2125. }
  2126. operation, _ := utils.PostPara(r, "opr")
  2127. vsrcFiles, _ := utils.PostPara(r, "src")
  2128. vdestFile, _ := utils.PostPara(r, "dest")
  2129. vnfilenames, _ := utils.PostPara(r, "new") //Only use when rename or create new file / folder
  2130. //Check if operation valid.
  2131. if operation == "" {
  2132. //Undefined operations.
  2133. utils.SendErrorResponse(w, "Undefined operations paramter: Missing 'opr' in request header.")
  2134. return
  2135. }
  2136. //As the user can pass in multiple source files at the same time, parse sourceFiles from json string
  2137. var sourceFiles []string
  2138. //This line is required in order to allow passing of special charaters
  2139. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  2140. err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  2141. if err != nil {
  2142. utils.SendErrorResponse(w, "Source file JSON parse error.")
  2143. return
  2144. }
  2145. //Check if new filenames are also valid. If yes, translate it into string array
  2146. var newFilenames []string
  2147. if vnfilenames != "" {
  2148. vnfilenames, _ := url.QueryUnescape(vnfilenames)
  2149. err = json.Unmarshal([]byte(vnfilenames), &newFilenames)
  2150. if err != nil {
  2151. utils.SendErrorResponse(w, "Unable to parse JSON for new filenames")
  2152. return
  2153. }
  2154. }
  2155. //A copy of a file shared with the user needs its owner to allow downloads
  2156. if operation == "copy" || operation == "zip" {
  2157. if err := sharedDriveCopyDenied(userinfo, sourceFiles); err != nil {
  2158. utils.SendErrorResponse(w, err.Error())
  2159. return
  2160. }
  2161. }
  2162. if operation == "zip" {
  2163. //Zip operation. Parse the real filepath list
  2164. rsrcFiles := []string{}
  2165. srcFshs := []*filesystem.FileSystemHandler{}
  2166. destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  2167. if err != nil {
  2168. utils.SendErrorResponse(w, "Unable to resolve zip destination path")
  2169. return
  2170. }
  2171. destFshAbs := destFsh.FileSystemAbstraction
  2172. rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2173. for _, vsrcFile := range sourceFiles {
  2174. vsrcFsh, vsrcSubpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  2175. if err != nil {
  2176. continue
  2177. }
  2178. rsrcFile, _ := vsrcFsh.FileSystemAbstraction.VirtualPathToRealPath(vsrcSubpath, userinfo.Username)
  2179. if vsrcFsh.FileSystemAbstraction.FileExists(rsrcFile) {
  2180. //Push directly its local path to list
  2181. rsrcFiles = append(rsrcFiles, rsrcFile)
  2182. srcFshs = append(srcFshs, vsrcFsh)
  2183. }
  2184. }
  2185. zipFilename := rdestFile
  2186. if destFshAbs.IsDir(rdestFile) {
  2187. //Append the filename to it
  2188. if len(rsrcFiles) == 1 {
  2189. zipFilename = filepath.Join(rdestFile, strings.TrimSuffix(filepath.Base(rsrcFiles[0]), filepath.Ext(filepath.Base(rsrcFiles[0])))+".zip")
  2190. } else if len(rsrcFiles) > 1 {
  2191. zipFilename = filepath.Join(rdestFile, filepath.Base(filepath.Dir(rsrcFiles[0]))+".zip")
  2192. }
  2193. }
  2194. //Create a buffer if destination fsh request buffer
  2195. zipFileTargetLocation := zipFilename
  2196. zipDestFsh := destFsh
  2197. if destFsh.RequireBuffer {
  2198. zipFileTargetLocation = getFsBufferFilepath(zipFilename, false)
  2199. zipDestFsh = nil
  2200. }
  2201. //Create a zip file at target location
  2202. err = filesystem.ArozZipFile(srcFshs, rsrcFiles, zipDestFsh, zipFileTargetLocation, false)
  2203. if err != nil {
  2204. os.Remove(zipFileTargetLocation)
  2205. utils.SendErrorResponse(w, err.Error())
  2206. return
  2207. }
  2208. //Write it to final destination from buffer
  2209. if destFsh.RequireBuffer {
  2210. //Upload the finalized zip file
  2211. f, _ := destFshAbs.Open(zipFileTargetLocation)
  2212. destFshAbs.WriteStream(zipFilename, f, 0775)
  2213. f.Close()
  2214. //Remove all buff files
  2215. os.Remove(zipFileTargetLocation)
  2216. cleanFsBufferFileFromList(rsrcFiles)
  2217. }
  2218. } else {
  2219. //For operations that is handled file by file
  2220. for i, vsrcFile := range sourceFiles {
  2221. //Convert the virtual path to realpath on disk
  2222. srcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  2223. if err != nil {
  2224. continue
  2225. }
  2226. srcFshAbs := srcFsh.FileSystemAbstraction
  2227. rsrcFile, _ := srcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2228. destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  2229. var destFshAbs filesystem.FileSystemAbstraction = nil
  2230. var rdestFile string = ""
  2231. if err == nil {
  2232. destFshAbs = destFsh.FileSystemAbstraction
  2233. rdestFile, _ = destFshAbs.VirtualPathToRealPath(destSubpath, userinfo.Username)
  2234. }
  2235. //Check if the source file exists
  2236. if operation == "rename" {
  2237. //Check if the usage is correct.
  2238. if vdestFile != "" {
  2239. utils.SendErrorResponse(w, "Rename only accept 'src' and 'new'. Please use move if you want to move a file.")
  2240. return
  2241. }
  2242. //Check if new name paramter is passed in.
  2243. if len(newFilenames) == 0 {
  2244. utils.SendErrorResponse(w, "Missing paramter (JSON string): 'new'")
  2245. return
  2246. }
  2247. //Check if the source filenames and new filenanmes match
  2248. if len(newFilenames) != len(sourceFiles) {
  2249. utils.SendErrorResponse(w, "New filenames do not match with source filename's length.")
  2250. return
  2251. }
  2252. //Check if the target dir is not readonly
  2253. accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
  2254. switch accmode {
  2255. case arozfs.FsReadOnly:
  2256. utils.SendErrorResponse(w, "This directory is Read Only")
  2257. return
  2258. case arozfs.FsDenied:
  2259. utils.SendErrorResponse(w, "Access Denied")
  2260. return
  2261. }
  2262. thisFilename := filepath.Base(newFilenames[i])
  2263. //Check if the new filename contains web-unsafe characters
  2264. if !utils.FilenameIsWebSafe(thisFilename) {
  2265. utils.SendErrorResponse(w, "Filename contains illegal characters")
  2266. return
  2267. }
  2268. //Check if the name already exists. If yes, return false
  2269. if srcFshAbs.FileExists(filepath.Join(filepath.Dir(rsrcFile), thisFilename)) {
  2270. utils.SendErrorResponse(w, "File already exists")
  2271. return
  2272. }
  2273. //Everything is ok. Rename the file.
  2274. targetNewName := filepath.Join(filepath.Dir(rsrcFile), thisFilename)
  2275. err = srcFshAbs.Rename(rsrcFile, targetNewName)
  2276. if err != nil {
  2277. systemWideLogger.PrintAndLog("File System", "File rename failed", err)
  2278. utils.SendErrorResponse(w, err.Error())
  2279. return
  2280. }
  2281. //Remove the cache for the original file
  2282. metadata.RemoveCache(srcFsh, rsrcFile)
  2283. } else if operation == "move" {
  2284. //File move operation. Check if the source file / dir and target directory exists
  2285. /*
  2286. Example usage from file explorer
  2287. $.ajax({
  2288. type: 'POST',
  2289. url: `/system/file_system/fileOpr`,
  2290. data: {opr: "move" ,src: JSON.stringify(fileList), dest: targetDir},
  2291. success: function(data){
  2292. if (data.error !== undefined){
  2293. msgbox("remove",data.error);
  2294. }else{
  2295. //OK, do something
  2296. }
  2297. }
  2298. });
  2299. */
  2300. if !srcFshAbs.FileExists(rsrcFile) {
  2301. utils.SendErrorResponse(w, "Source file not exists")
  2302. return
  2303. }
  2304. //Check if the source file is read only.
  2305. accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
  2306. if accmode == arozfs.FsReadOnly {
  2307. utils.SendErrorResponse(w, "This source file is Read Only")
  2308. return
  2309. } else if accmode == arozfs.FsDenied {
  2310. utils.SendErrorResponse(w, "Access Denied")
  2311. return
  2312. }
  2313. if rdestFile == "" {
  2314. utils.SendErrorResponse(w, "Undefined dest location")
  2315. return
  2316. }
  2317. //Get exists overwrite mode
  2318. existsOpr, _ := utils.PostPara(r, "existsresp")
  2319. //Check if use fast move instead
  2320. //Check if the source and destination folder are under the same root. If yes, use os.Rename for faster move operations
  2321. //Check if the two files are under the same user root path
  2322. //srcAbs, _ := filepath.Abs(rsrcFile)
  2323. //destAbs, _ := filepath.Abs(rdestFile)
  2324. //underSameRoot, _ := filesystem.UnderTheSameRoot(srcAbs, destAbs)
  2325. //Updates 19-10-2020: Added ownership management to file move and copy
  2326. userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
  2327. err = filesystem.FileMove(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, nil)
  2328. if err != nil {
  2329. utils.SendErrorResponse(w, err.Error())
  2330. //Restore the ownership if remove failed
  2331. userinfo.SetOwnerOfFile(srcFsh, vsrcFile)
  2332. return
  2333. }
  2334. //Set user to own the new file
  2335. newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
  2336. newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
  2337. userinfo.SetOwnerOfFile(destFsh, newfileVpath)
  2338. //Remove cache for the original file
  2339. metadata.RemoveCache(srcFsh, rsrcFile)
  2340. } else if operation == "copy" {
  2341. //Copy file. See move example and change 'opr' to 'copy'
  2342. if !srcFshAbs.FileExists(rsrcFile) {
  2343. utils.SendErrorResponse(w, "Source file not exists")
  2344. return
  2345. }
  2346. //Check if the desintation is read only.
  2347. if !userinfo.CanWrite(vdestFile) {
  2348. utils.SendErrorResponse(w, "Access Denied")
  2349. return
  2350. }
  2351. if !destFshAbs.FileExists(rdestFile) {
  2352. if destFshAbs.FileExists(filepath.Dir(rdestFile)) {
  2353. //User pass in the whole path for the folder. Report error usecase.
  2354. utils.SendErrorResponse(w, "Dest location should be an existing folder instead of the full path of the copied file")
  2355. return
  2356. }
  2357. utils.SendErrorResponse(w, "Dest folder not found")
  2358. return
  2359. }
  2360. existsOpr, _ := utils.PostPara(r, "existsresp")
  2361. //Check if the user have space for the extra file
  2362. if !userinfo.StorageQuota.HaveSpace(filesystem.GetFileSize(rdestFile)) {
  2363. utils.SendErrorResponse(w, "Storage Quota Full")
  2364. return
  2365. }
  2366. err = filesystem.FileCopy(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, nil)
  2367. if err != nil {
  2368. utils.SendErrorResponse(w, err.Error())
  2369. return
  2370. }
  2371. //Set user to own this file
  2372. newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
  2373. newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
  2374. userinfo.SetOwnerOfFile(destFsh, newfileVpath)
  2375. } else if operation == "delete" {
  2376. //Delete the file permanently
  2377. if !srcFshAbs.FileExists(rsrcFile) {
  2378. //Check if it is a non escapted file instead
  2379. utils.SendErrorResponse(w, "Source file not exists")
  2380. return
  2381. }
  2382. if !userinfo.CanWrite(vsrcFile) {
  2383. utils.SendErrorResponse(w, "Access Denied")
  2384. return
  2385. }
  2386. //Check if the user own this file
  2387. isOwner := userinfo.IsOwnerOfFile(srcFsh, vsrcFile)
  2388. if isOwner {
  2389. //This user own this system. Remove this file from his quota
  2390. userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
  2391. }
  2392. //Check if this file has any cached files. If yes, remove it
  2393. metadata.RemoveCache(srcFsh, rsrcFile)
  2394. //Clear the cache folder if there is no files inside
  2395. fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
  2396. if len(fc) == 0 && err == nil {
  2397. srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
  2398. }
  2399. err = srcFshAbs.RemoveAll(rsrcFile)
  2400. if err != nil {
  2401. systemWideLogger.PrintAndLog("File System", "Unable to remove file from "+srcFsh.UUID, err)
  2402. utils.SendErrorResponse(w, err.Error())
  2403. return
  2404. }
  2405. //Deleted from the trash bin. Drop the folders it left empty there
  2406. if system_fs_isInTrash(subpath) {
  2407. system_fs_pruneTrashParents(srcFsh, userinfo.Username, path.Dir(filepath.ToSlash(rsrcFile)))
  2408. }
  2409. } else if operation == "recycle" {
  2410. //Move it into the .trash folder of its root and allow it to to be removed later
  2411. if !srcFshAbs.FileExists(rsrcFile) {
  2412. //Check if it is a non escapted file instead
  2413. utils.SendErrorResponse(w, "Source file not exists")
  2414. return
  2415. }
  2416. //Check if the upload target is read only.
  2417. if !userinfo.CanWrite(vsrcFile) {
  2418. utils.SendErrorResponse(w, "Access Denied")
  2419. return
  2420. }
  2421. if system_fs_isInTrash(subpath) {
  2422. utils.SendErrorResponse(w, "This item is already in the trash bin")
  2423. return
  2424. }
  2425. if system_fs_trashSubpathOf(subpath, 0) == "" {
  2426. utils.SendErrorResponse(w, "The root folder cannot be moved to the trash bin")
  2427. return
  2428. }
  2429. //Check if this file has any cached files. If yes, remove it
  2430. metadata.RemoveCache(srcFsh, rsrcFile)
  2431. //Clear the cache folder if there is no files inside
  2432. fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
  2433. if len(fc) == 0 && err == nil {
  2434. srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
  2435. }
  2436. /*
  2437. Trash quota
  2438. Checked before the move, not after: once the file is inside
  2439. .trash it has already left the tree the user was looking at,
  2440. and undoing that cleanly is harder than refusing up front.
  2441. The client recognises this error code and offers to empty
  2442. the bin, delete outright, or do nothing.
  2443. */
  2444. userTrashQuota := system_fs_getTrashQuotaBytes(userinfo.Username)
  2445. if userTrashQuota > 0 {
  2446. incomingSize := system_fs_getTrashEntrySize(srcFsh, rsrcFile)
  2447. if system_fs_getTrashUsage(userinfo.Username)+incomingSize > userTrashQuota {
  2448. utils.SendErrorResponse(w, "TRASH_QUOTA_EXCEEDED")
  2449. return
  2450. }
  2451. }
  2452. //Mirror the folders it was removed from inside the .trash folder of this root
  2453. removeTime := time.Now().Unix()
  2454. trashPath, err := srcFshAbs.VirtualPathToRealPath(system_fs_trashSubpathOf(subpath, removeTime), userinfo.Username)
  2455. for err == nil && srcFshAbs.FileExists(trashPath) {
  2456. //Another item of the same name was removed from there within this second
  2457. removeTime++
  2458. trashPath, err = srcFshAbs.VirtualPathToRealPath(system_fs_trashSubpathOf(subpath, removeTime), userinfo.Username)
  2459. }
  2460. trashRoot, rootErr := srcFshAbs.VirtualPathToRealPath(trashFolderSubpath, userinfo.Username)
  2461. if err != nil || rootErr != nil {
  2462. utils.SendErrorResponse(w, "Failed to move file to trash")
  2463. return
  2464. }
  2465. srcFshAbs.MkdirAll(path.Dir(filepath.ToSlash(trashPath)), 0755)
  2466. hidden.HideFile(trashRoot)
  2467. err = srcFshAbs.Rename(rsrcFile, trashPath)
  2468. if err != nil {
  2469. //Do not leave the folders made for it behind
  2470. system_fs_pruneTrashParents(srcFsh, userinfo.Username, path.Dir(filepath.ToSlash(trashPath)))
  2471. if srcFsh.RequireBuffer {
  2472. utils.SendErrorResponse(w, "Incompatible File System Type: Try SHIFT + DELETE to delete file permanently")
  2473. } else {
  2474. systemWideLogger.PrintAndLog("File System", "Failed to move file to trash. See log for more info.", err)
  2475. utils.SendErrorResponse(w, "Failed to move file to trash")
  2476. }
  2477. return
  2478. }
  2479. } else if operation == "unzip" {
  2480. //Unzip the file to destination
  2481. //Check if the user can write to the target dest file
  2482. if !userinfo.CanWrite(string(vdestFile)) {
  2483. utils.SendErrorResponse(w, "Access Denied")
  2484. return
  2485. }
  2486. //Make the rdest directory if not exists
  2487. if !destFshAbs.FileExists(rdestFile) {
  2488. err = destFshAbs.MkdirAll(rdestFile, 0755)
  2489. if err != nil {
  2490. utils.SendErrorResponse(w, err.Error())
  2491. return
  2492. }
  2493. }
  2494. unzipSource := rsrcFile
  2495. unzipDest := rdestFile
  2496. if srcFsh.RequireBuffer {
  2497. localBufferedFile, _ := bufferRemoteFileToLocal(srcFsh, rsrcFile, false)
  2498. unzipSource = localBufferedFile
  2499. }
  2500. if destFsh.RequireBuffer {
  2501. localUnzipBuffer, _ := bufferRemoteFileToLocal(destFsh, rdestFile, true)
  2502. unzipDest = localUnzipBuffer
  2503. }
  2504. //OK! Unzip to destination
  2505. err := filesystem.Unzip(unzipSource, unzipDest)
  2506. if err != nil {
  2507. utils.SendErrorResponse(w, err.Error())
  2508. return
  2509. }
  2510. if srcFsh.RequireBuffer {
  2511. //Remove the local buffered file
  2512. os.Remove(unzipSource)
  2513. }
  2514. if destFsh.RequireBuffer {
  2515. //Push the buffer to target fs
  2516. filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
  2517. path = filepath.ToSlash(path)
  2518. relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
  2519. if info.IsDir() {
  2520. destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
  2521. } else {
  2522. f, _ := os.Open(path)
  2523. destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
  2524. f.Close()
  2525. }
  2526. return nil
  2527. })
  2528. cleanFsBufferFileFromList([]string{unzipDest})
  2529. }
  2530. } else {
  2531. utils.SendErrorResponse(w, "Unknown file opeartion given")
  2532. return
  2533. }
  2534. }
  2535. }
  2536. utils.SendOK(w)
  2537. }
  2538. // Allow systems to store key value pairs in the database as preferences.
  2539. func system_fs_handleUserPreference(w http.ResponseWriter, r *http.Request) {
  2540. username, err := authAgent.GetUserName(w, r)
  2541. if err != nil {
  2542. utils.SendErrorResponse(w, "User not logged in")
  2543. return
  2544. }
  2545. key, _ := utils.GetPara(r, "key")
  2546. value, _ := utils.GetPara(r, "value")
  2547. remove, _ := utils.GetPara(r, "remove")
  2548. if key != "" && value == "" && remove == "" {
  2549. //Get mode. Read the prefernece with given key
  2550. result := ""
  2551. err := sysdb.Read("fs", "pref/"+key+"/"+username, &result)
  2552. if err != nil {
  2553. utils.SendJSONResponse(w, `{"error":"Key not found."}`)
  2554. return
  2555. }
  2556. utils.SendTextResponse(w, result)
  2557. } else if key != "" && value == "" && remove == "true" {
  2558. //Remove mode. Delete this key from sysdb
  2559. err := sysdb.Delete("fs", "pref/"+key+"/"+username)
  2560. if err != nil {
  2561. utils.SendErrorResponse(w, err.Error())
  2562. }
  2563. utils.SendOK(w)
  2564. } else if key != "" && value != "" {
  2565. //Set mode. Set the preference with given key
  2566. if len(value) > 1024*1024 { //1KB
  2567. //Size too big. Reject storage
  2568. utils.SendErrorResponse(w, "Preference value too long. Preference value can only store maximum 1KB.")
  2569. return
  2570. }
  2571. sysdb.Write("fs", "pref/"+key+"/"+username, value)
  2572. utils.SendOK(w)
  2573. }
  2574. }
  2575. func system_fs_removeUserPreferences(username string) {
  2576. entries, err := sysdb.ListTable("fs")
  2577. if err != nil {
  2578. return
  2579. }
  2580. for _, keypairs := range entries {
  2581. if strings.Contains(string(keypairs[0]), "pref/") && strings.Contains(string(keypairs[0]), "/"+username) {
  2582. //Remove this preference
  2583. sysdb.Delete("fs", string(keypairs[0]))
  2584. }
  2585. }
  2586. }
  2587. func system_fs_listDrives(w http.ResponseWriter, r *http.Request) {
  2588. if !authAgent.CheckAuth(r) {
  2589. utils.SendErrorResponse(w, "User not logged in")
  2590. return
  2591. }
  2592. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  2593. type driveInfo struct {
  2594. Drivepath string
  2595. DriveFreeSpace uint64
  2596. DriveTotalSpace uint64
  2597. DriveAvailSpace uint64
  2598. }
  2599. var drives []driveInfo
  2600. if runtime.GOOS == "windows" {
  2601. //Under windows
  2602. for _, drive := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
  2603. f, err := os.Open(string(drive) + ":\\")
  2604. if err == nil {
  2605. thisdrive := new(driveInfo)
  2606. thisdrive.Drivepath = string(drive) + ":\\"
  2607. free, total, avail := storage.GetDriveCapacity(string(drive) + ":\\")
  2608. thisdrive.DriveFreeSpace = free
  2609. thisdrive.DriveTotalSpace = total
  2610. thisdrive.DriveAvailSpace = avail
  2611. drives = append(drives, *thisdrive)
  2612. f.Close()
  2613. }
  2614. }
  2615. } else {
  2616. //Under linux environment
  2617. //Append all the virtual directories root as root instead
  2618. storageDevices := []string{}
  2619. for _, fshandler := range userinfo.GetAllFileSystemHandler() {
  2620. storageDevices = append(storageDevices, fshandler.Path)
  2621. }
  2622. //List all storage information of each devices
  2623. for _, dev := range storageDevices {
  2624. thisdrive := new(driveInfo)
  2625. thisdrive.Drivepath = filepath.Base(dev)
  2626. free, total, avail := storage.GetDriveCapacity(string(dev))
  2627. thisdrive.DriveFreeSpace = free
  2628. thisdrive.DriveTotalSpace = total
  2629. thisdrive.DriveAvailSpace = avail
  2630. drives = append(drives, *thisdrive)
  2631. }
  2632. }
  2633. jsonString, _ := json.Marshal(drives)
  2634. utils.SendJSONResponse(w, string(jsonString))
  2635. }
  2636. func system_fs_listRoot(w http.ResponseWriter, r *http.Request) {
  2637. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2638. if err != nil {
  2639. utils.SendErrorResponse(w, err.Error())
  2640. return
  2641. }
  2642. username := userinfo.Username
  2643. userRoot, _ := utils.GetPara(r, "user")
  2644. if userRoot == "true" {
  2645. type fileObject struct {
  2646. Filename string
  2647. Filepath string
  2648. IsDir bool
  2649. }
  2650. //List the root media folders under user:/
  2651. fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath("user:/")
  2652. fshAbs := fsh.FileSystemAbstraction
  2653. filesInUserRoot := []fileObject{}
  2654. filesInRoot, _ := fshAbs.Glob(filepath.ToSlash(filepath.Clean(*root_directory)) + "/users/" + username + "/*")
  2655. for _, file := range filesInRoot {
  2656. //Check if this is a hidden file
  2657. if len(filepath.Base(file)) > 0 && filepath.Base(file)[:1] == "." {
  2658. continue
  2659. }
  2660. thisFile := new(fileObject)
  2661. thisFile.Filename = filepath.Base(file)
  2662. thisFile.Filepath, _ = fshAbs.RealPathToVirtualPath(file, userinfo.Username)
  2663. thisFile.IsDir = fshAbs.IsDir(file)
  2664. filesInUserRoot = append(filesInUserRoot, *thisFile)
  2665. }
  2666. jsonString, _ := json.Marshal(filesInUserRoot)
  2667. utils.SendJSONResponse(w, string(jsonString))
  2668. } else {
  2669. type rootObject struct {
  2670. rootID string //The vroot id
  2671. RootName string //The name of this vroot
  2672. RootPath string //The path of this vroot
  2673. BufferedFs bool //If buffer typed FS
  2674. }
  2675. roots := []*rootObject{}
  2676. for _, store := range userinfo.GetAllFileSystemHandler() {
  2677. if share.IsSharedDrive(store) {
  2678. //Not a storage device; the File Manager lists it as Shared with me
  2679. continue
  2680. }
  2681. var thisDevice = new(rootObject)
  2682. thisDevice.RootName = store.Name
  2683. thisDevice.RootPath = store.UUID + ":/"
  2684. thisDevice.rootID = store.UUID
  2685. thisDevice.BufferedFs = store.RequireBuffer
  2686. roots = append(roots, thisDevice)
  2687. }
  2688. jsonString, _ := json.Marshal(roots)
  2689. utils.SendJSONResponse(w, string(jsonString))
  2690. }
  2691. }
  2692. /*
  2693. Special Glob for handling path with [ or ] inside.
  2694. You can also pass in normal path for globing if you are not sure.
  2695. */
  2696. func system_fs_specialURIDecode(inputPath string) string {
  2697. inputPath = strings.ReplaceAll(inputPath, "+", "{{plus_sign}}")
  2698. inputPath, _ = url.QueryUnescape(inputPath)
  2699. inputPath = strings.ReplaceAll(inputPath, "{{plus_sign}}", "+")
  2700. return inputPath
  2701. }
  2702. /*
  2703. func system_fs_specialURIEncode(inputPath string) string {
  2704. inputPath = strings.ReplaceAll(inputPath, " ", "{{space_sign}}")
  2705. inputPath, _ = url.QueryUnescape(inputPath)
  2706. inputPath = strings.ReplaceAll(inputPath, "{{space_sign}}", "%20")
  2707. return inputPath
  2708. }
  2709. */
  2710. /*
  2711. The parent folder of a virtual path, as shown in the properties dialog.
  2712. A virtual path is always slash separated and carries a "vdID:" prefix, so
  2713. filepath.Dir must not be used here: on Windows it reads "user:" as a volume
  2714. name and Clean guards the result with a "./" prefix, turning
  2715. "user:/cluster/a.agi" into "./user:/cluster". GetIDFromVirtualPath strips
  2716. that prefix back off for the same reason.
  2717. */
  2718. func virtualDirname(vpath string) string {
  2719. return path.Dir(vpath)
  2720. }
  2721. // Handle file properties request
  2722. func system_fs_getFileProperties(w http.ResponseWriter, r *http.Request) {
  2723. type fileProperties struct {
  2724. VirtualPath string
  2725. StoragePath string
  2726. Basename string
  2727. VirtualDirname string
  2728. StorageDirname string
  2729. Ext string
  2730. MimeType string
  2731. Filesize int64
  2732. Permission string
  2733. LastModTime string
  2734. LastModUnix int64
  2735. CreationTime string
  2736. CreationUnix int64
  2737. IsDirectory bool
  2738. Owner string
  2739. }
  2740. result := fileProperties{}
  2741. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2742. if err != nil {
  2743. utils.SendErrorResponse(w, err.Error())
  2744. return
  2745. }
  2746. vpath, err := utils.PostPara(r, "path")
  2747. if err != nil {
  2748. utils.SendErrorResponse(w, "path not defined")
  2749. return
  2750. }
  2751. vrootID, subpath, _ := filesystem.GetIDFromVirtualPath(vpath)
  2752. fsh, err := GetFsHandlerByUUID(vrootID)
  2753. if err != nil {
  2754. utils.SendErrorResponse(w, err.Error())
  2755. return
  2756. }
  2757. fshAbs := fsh.FileSystemAbstraction
  2758. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2759. if err != nil {
  2760. utils.SendErrorResponse(w, err.Error())
  2761. return
  2762. }
  2763. fileStat, err := fshAbs.Stat(rpath)
  2764. if err != nil {
  2765. utils.SendErrorResponse(w, err.Error())
  2766. return
  2767. }
  2768. fileMime := "text/directory"
  2769. if !fileStat.IsDir() {
  2770. m, _, err := filesystem.GetMime(rpath)
  2771. if err != nil {
  2772. fileMime = mime.TypeByExtension(filepath.Ext(rpath))
  2773. } else {
  2774. fileMime = m
  2775. }
  2776. }
  2777. filesize := fileStat.Size()
  2778. //Get file overall size if this is folder
  2779. if fileStat.IsDir() {
  2780. if fsh.IsNetworkDrive() {
  2781. filesize = -1
  2782. } else {
  2783. //Check if du exists
  2784. usefallback := true //Use fallback
  2785. if fsh.IsLocalDrive() {
  2786. //Try using native syscall to grab directory size
  2787. nativeSize, err := filesystem.GetDirectorySizeNative(rpath)
  2788. if err == nil {
  2789. usefallback = false
  2790. filesize = nativeSize
  2791. }
  2792. }
  2793. if usefallback {
  2794. // invalid platform. walk the whole file system
  2795. var size int64 = 0
  2796. fshAbs.Walk(rpath, func(_ string, info os.FileInfo, err error) error {
  2797. if err != nil {
  2798. return err
  2799. }
  2800. if !info.IsDir() {
  2801. size += info.Size()
  2802. }
  2803. return err
  2804. })
  2805. filesize = size
  2806. }
  2807. }
  2808. }
  2809. //Get the file creation time. Not every platform / file system records one,
  2810. //in which case it is left empty for the client to grey out
  2811. creationTimeString := ""
  2812. var creationUnix int64 = 0
  2813. if creationTime, ok := filesystem.GetFileCreationTime(fileStat); ok {
  2814. creationTimeString = time.Unix(creationTime, 0).Format("2006-01-02 15:04:05")
  2815. creationUnix = creationTime
  2816. }
  2817. //Get file owner
  2818. owner := userinfo.GetFileOwner(fsh, vpath)
  2819. if owner == "" {
  2820. //A file someone shared with this user belongs to whoever shared it
  2821. owner = sharedDriveOwnerOf(userinfo, fsh, vpath)
  2822. }
  2823. if owner == "" {
  2824. //Handle special virtual roots
  2825. owner = "Unknown"
  2826. }
  2827. result = fileProperties{
  2828. VirtualPath: vpath,
  2829. StoragePath: filepath.ToSlash(filepath.Clean(rpath)),
  2830. Basename: filepath.Base(rpath),
  2831. VirtualDirname: virtualDirname(vpath),
  2832. StorageDirname: filepath.ToSlash(filepath.Dir(rpath)),
  2833. Ext: filepath.Ext(rpath),
  2834. MimeType: fileMime,
  2835. Filesize: filesize,
  2836. Permission: fileStat.Mode().Perm().String(),
  2837. LastModTime: fileStat.ModTime().Format("2006-01-02 15:04:05"),
  2838. LastModUnix: fileStat.ModTime().Unix(),
  2839. CreationTime: creationTimeString,
  2840. CreationUnix: creationUnix,
  2841. IsDirectory: fileStat.IsDir(),
  2842. Owner: owner,
  2843. }
  2844. jsonString, _ := json.Marshal(result)
  2845. utils.SendJSONResponse(w, string(jsonString))
  2846. }
  2847. /*
  2848. Storage info
  2849. Where the file actually is. Most drives have nothing to add beyond the
  2850. storage path already in the properties dialog, but a drive that spreads
  2851. its files over several hosts does: cluster:/ answers with the nodes and
  2852. volumes holding a copy of the file. The abstraction resolves it
  2853. (arozfs.StorageInfoProvider), so this handler stays the same whatever
  2854. kind of drive the file is on.
  2855. */
  2856. func system_fs_getStorageInfo(w http.ResponseWriter, r *http.Request) {
  2857. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2858. if err != nil {
  2859. utils.SendErrorResponse(w, "User not logged in")
  2860. return
  2861. }
  2862. vpath, err := utils.PostPara(r, "path")
  2863. if err != nil {
  2864. utils.SendErrorResponse(w, "path not defined")
  2865. return
  2866. }
  2867. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  2868. if err != nil {
  2869. utils.SendErrorResponse(w, err.Error())
  2870. return
  2871. }
  2872. //Reading a file the user has no access to must not leak where it is kept
  2873. if !userinfo.CanRead(vpath) {
  2874. utils.SendErrorResponse(w, "Permission denied")
  2875. return
  2876. }
  2877. provider, ok := fsh.FileSystemAbstraction.(arozfs.StorageInfoProvider)
  2878. if !ok {
  2879. //Nothing to add for this kind of drive
  2880. utils.SendErrorResponse(w, "Storage info not supported by this drive")
  2881. return
  2882. }
  2883. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  2884. if err != nil {
  2885. utils.SendErrorResponse(w, err.Error())
  2886. return
  2887. }
  2888. info, err := provider.StorageInfo(rpath)
  2889. if err != nil {
  2890. utils.SendErrorResponse(w, err.Error())
  2891. return
  2892. }
  2893. js, _ := json.Marshal(info)
  2894. utils.SendJSONResponse(w, string(js))
  2895. }
  2896. /*
  2897. List directory in the given path
  2898. Usage: Pass in dir like the following examples:
  2899. AOR:/Desktop <= Open /user/{username}/Desktop
  2900. S1:/ <= Open {uuid=S1}/
  2901. */
  2902. func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
  2903. currentDir, err := utils.PostPara(r, "dir")
  2904. if err != nil {
  2905. utils.SendErrorResponse(w, err.Error())
  2906. return
  2907. }
  2908. sortMode, _ := utils.PostPara(r, "sort")
  2909. showHidden, _ := utils.PostPara(r, "showHidden")
  2910. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2911. if err != nil {
  2912. //user not logged in. Redirect to login page.
  2913. utils.SendErrorResponse(w, "User not logged in")
  2914. return
  2915. }
  2916. if currentDir == "" {
  2917. utils.SendErrorResponse(w, "Invalid dir given.")
  2918. return
  2919. }
  2920. // Pad a slash at the end of currentDir if not exists
  2921. if !strings.HasSuffix(currentDir, "/") {
  2922. currentDir = currentDir + "/"
  2923. }
  2924. fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
  2925. if err != nil {
  2926. utils.SendErrorResponse(w, err.Error())
  2927. return
  2928. }
  2929. fshAbs := fsh.FileSystemAbstraction
  2930. realpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2931. if err != nil {
  2932. utils.SendErrorResponse(w, err.Error())
  2933. return
  2934. }
  2935. if !fshAbs.FileExists(realpath) {
  2936. //Path not exists
  2937. userRoot, _ := fshAbs.VirtualPathToRealPath("/", userinfo.Username)
  2938. if filepath.Clean(realpath) == filepath.Clean(userRoot) || realpath == "" {
  2939. //Initiate user folder (Initiaed in user object)
  2940. err = fshAbs.MkdirAll(userRoot, 0775)
  2941. if err != nil {
  2942. systemWideLogger.PrintAndLog("File System", "Unable to create user root on "+fsh.UUID+": "+err.Error(), nil)
  2943. utils.SendErrorResponse(w, "Unable to create user root folder due to file system error")
  2944. return
  2945. }
  2946. } else {
  2947. //Folder not exists
  2948. systemWideLogger.PrintAndLog("File System", "Requested path: "+realpath+" does not exists", nil)
  2949. utils.SendErrorResponse(w, "Folder not exists")
  2950. return
  2951. }
  2952. }
  2953. if sortMode == "" {
  2954. sortMode = "default"
  2955. }
  2956. files, err := fshAbs.ReadDir(realpath)
  2957. if err != nil {
  2958. utils.SendErrorResponse(w, "Readdir Failed: "+strings.ReplaceAll(err.Error(), "\\", "/"))
  2959. systemWideLogger.PrintAndLog("File System", "Unable to read dir: "+err.Error(), err)
  2960. return
  2961. }
  2962. //Remapping use parsed list
  2963. parsedFilelist := map[string]filesystem.FileData{}
  2964. //Sorting use list
  2965. realpathList := []string{}
  2966. fileInfoList := []fs.FileInfo{}
  2967. for _, f := range files {
  2968. //Check if it is hidden file
  2969. isHidden, _ := hidden.IsHidden(f.Name(), false)
  2970. if showHidden != "true" && isHidden {
  2971. //Skipping hidden files
  2972. continue
  2973. }
  2974. //Check if this file contains invalid characters
  2975. if !utils.FilenameIsWebSafe(f.Name()) {
  2976. continue
  2977. }
  2978. //Check if this is an aodb file
  2979. if f.Name() == "aofs.db" || f.Name() == "aofs.db.lock" {
  2980. //Database file (reserved)
  2981. continue
  2982. }
  2983. //Check if it is shortcut file. If yes, render a shortcut data struct
  2984. var shortCutInfo *arozfs.ShortcutData = nil
  2985. if filepath.Ext(f.Name()) == ".shortcut" {
  2986. //This is a shortcut file
  2987. fcontent, err := fshAbs.ReadFile(arozfs.ToSlash(filepath.Join(realpath, f.Name())))
  2988. if err != nil {
  2989. shortCutInfo = nil
  2990. } else {
  2991. shorcutData, err := shortcut.ReadShortcut(fcontent)
  2992. if err != nil {
  2993. shortCutInfo = nil
  2994. } else {
  2995. shortCutInfo = shorcutData
  2996. }
  2997. }
  2998. }
  2999. statInfo, err := f.Info()
  3000. if err != nil {
  3001. continue
  3002. }
  3003. thisvPath, _ := fshAbs.RealPathToVirtualPath(filepath.Join(realpath, f.Name()), userinfo.Username)
  3004. thisFile := filesystem.FileData{
  3005. Filename: f.Name(),
  3006. Filepath: currentDir + f.Name(),
  3007. Realpath: filepath.ToSlash(filepath.Join(realpath, f.Name())),
  3008. IsDir: f.IsDir(),
  3009. Filesize: statInfo.Size(),
  3010. Displaysize: filesystem.GetFileDisplaySize(statInfo.Size(), 2),
  3011. ModTime: statInfo.ModTime().Unix(),
  3012. IsShared: shareManager.FileIsShared(userinfo, thisvPath),
  3013. Shortcut: shortCutInfo,
  3014. }
  3015. parsedFilelist[currentDir+f.Name()] = thisFile
  3016. realpathList = append(realpathList, currentDir+f.Name())
  3017. fileInfoList = append(fileInfoList, statInfo)
  3018. }
  3019. //Sort the filelist
  3020. sortedRealpathList := fssort.SortFileList(realpathList, fileInfoList, sortMode)
  3021. results := []filesystem.FileData{}
  3022. for _, thisRpath := range sortedRealpathList {
  3023. val, ok := parsedFilelist[thisRpath]
  3024. if ok {
  3025. results = append(results, val)
  3026. }
  3027. }
  3028. jsonString, _ := json.Marshal(results)
  3029. utils.SendJSONResponse(w, string(jsonString))
  3030. }
  3031. // Handle getting a hash from a given contents in the given path
  3032. func system_fs_handleDirHash(w http.ResponseWriter, r *http.Request) {
  3033. currentDir, err := utils.GetPara(r, "dir")
  3034. if err != nil {
  3035. utils.SendErrorResponse(w, "Invalid dir given")
  3036. return
  3037. }
  3038. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3039. if err != nil {
  3040. utils.SendErrorResponse(w, "User not logged in")
  3041. return
  3042. }
  3043. fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
  3044. if err != nil {
  3045. utils.SendErrorResponse(w, "Unable to resolve target directory")
  3046. return
  3047. }
  3048. fshAbs := fsh.FileSystemAbstraction
  3049. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3050. if err != nil {
  3051. utils.SendErrorResponse(w, "Invalid dir given")
  3052. return
  3053. }
  3054. //The folder might have been removed by another window or client
  3055. if !fshAbs.FileExists(rpath) || !fshAbs.IsDir(rpath) {
  3056. utils.SendErrorResponse(w, "Directory not found")
  3057. return
  3058. }
  3059. //Get a list of files in this directory
  3060. /*
  3061. currentDir = filepath.ToSlash(filepath.Clean(rpath)) + "/"
  3062. filesInDir, err := fshAbs.Glob(currentDir + "*")
  3063. if err != nil {
  3064. utils.SendErrorResponse(w, err.Error())
  3065. return
  3066. }
  3067. filenames := []string{}
  3068. for _, file := range filesInDir {
  3069. if len(filepath.Base(file)) > 0 && string([]rune(filepath.Base(file))[0]) != "." {
  3070. //Ignore hidden files
  3071. filenames = append(filenames, filepath.Base(file))
  3072. }
  3073. }
  3074. */
  3075. finfos, err := fshAbs.ReadDir(rpath)
  3076. if err != nil {
  3077. utils.SendErrorResponse(w, err.Error())
  3078. return
  3079. }
  3080. filenames := []string{}
  3081. for _, fi := range finfos {
  3082. isHiddenFile, _ := hidden.IsHidden(fi.Name(), false)
  3083. if len(fi.Name()) > 0 && !isHiddenFile {
  3084. //Ignore hidden files
  3085. filenames = append(filenames, fi.Name())
  3086. }
  3087. }
  3088. sort.Strings(filenames)
  3089. //Build a hash base on the filelist
  3090. h := sha256.New()
  3091. h.Write([]byte(strings.Join(filenames, ",")))
  3092. utils.SendTextResponse(w, hex.EncodeToString((h.Sum(nil))))
  3093. }
  3094. /*
  3095. File zipping and unzipping functions
  3096. */
  3097. // Handle all zip related API
  3098. func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
  3099. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3100. if err != nil {
  3101. utils.SendErrorResponse(w, err.Error())
  3102. return
  3103. }
  3104. opr, err := utils.PostPara(r, "opr")
  3105. if err != nil {
  3106. utils.SendErrorResponse(w, "Invalid opr or opr not defined")
  3107. return
  3108. }
  3109. vsrc, _ := utils.PostPara(r, "src")
  3110. if vsrc == "" {
  3111. utils.SendErrorResponse(w, "Invalid src paramter")
  3112. return
  3113. }
  3114. vdest, _ := utils.PostPara(r, "dest")
  3115. rdest := ""
  3116. //Convert source path from JSON string to object
  3117. virtualSourcePaths := []string{}
  3118. err = json.Unmarshal([]byte(vsrc), &virtualSourcePaths)
  3119. if err != nil {
  3120. utils.SendErrorResponse(w, err.Error())
  3121. return
  3122. }
  3123. //A zip of a file shared with the user needs its owner to allow downloads
  3124. if err := sharedDriveCopyDenied(userinfo, virtualSourcePaths); err != nil {
  3125. utils.SendErrorResponse(w, err.Error())
  3126. return
  3127. }
  3128. //Check each of the path
  3129. realSourcePaths := []string{}
  3130. sourceFshs := []*filesystem.FileSystemHandler{}
  3131. for _, vpath := range virtualSourcePaths {
  3132. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  3133. if err != nil {
  3134. utils.SendErrorResponse(w, "Unable to resolve file: "+vpath)
  3135. return
  3136. }
  3137. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  3138. thisrpath, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3139. if err != nil || !thisSrcFshAbs.FileExists(thisrpath) {
  3140. utils.SendErrorResponse(w, "File not exists: "+vpath)
  3141. return
  3142. }
  3143. realSourcePaths = append(realSourcePaths, thisrpath)
  3144. sourceFshs = append(sourceFshs, thisSrcFsh)
  3145. }
  3146. ///Convert dest to real if given
  3147. var destFsh *filesystem.FileSystemHandler = nil
  3148. var subpath string = ""
  3149. var filename string = ""
  3150. if vdest != "" {
  3151. //Given target virtual dest
  3152. destFsh, subpath, err = GetFSHandlerSubpathFromVpath(rdest)
  3153. if err != nil {
  3154. utils.SendErrorResponse(w, err.Error())
  3155. return
  3156. }
  3157. } else {
  3158. //Given no virtual dest. Zip to tmp:/
  3159. filename = utils.Int64ToString(time.Now().Unix()) + ".zip"
  3160. destFsh, subpath, err = GetFSHandlerSubpathFromVpath(filepath.Join("tmp:/", filename))
  3161. if err != nil {
  3162. utils.SendErrorResponse(w, err.Error())
  3163. return
  3164. }
  3165. }
  3166. rdest, _ = destFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3167. destFshAbs := destFsh.FileSystemAbstraction
  3168. zipOutput := rdest
  3169. zipDestFsh := destFsh
  3170. if destFsh.RequireBuffer {
  3171. zipOutput = getFsBufferFilepath(rdest, false)
  3172. zipDestFsh = nil
  3173. }
  3174. if opr == "tmpzipAsync" {
  3175. /*
  3176. Same as tmpzip, but the zipping runs in the background and reports
  3177. its progress into the file operation task list, so the caller can
  3178. follow it on /system/file_system/ongoing instead of staring at a
  3179. request that returns nothing until the archive is finished.
  3180. */
  3181. if filename == "" {
  3182. utils.SendErrorResponse(w, "tmpzipAsync does not support a custom destination")
  3183. return
  3184. }
  3185. vzipPath := "tmp:/" + filename
  3186. task := NewOngoingFileOperation(userinfo, "zip", virtualSourcePaths, vzipPath)
  3187. go runTmpZipTask(task.ID, sourceFshs, realSourcePaths, zipDestFsh, zipOutput, destFsh, rdest)
  3188. js, _ := json.Marshal(map[string]string{
  3189. "oprid": task.ID,
  3190. "dest": vzipPath,
  3191. })
  3192. utils.SendJSONResponse(w, string(js))
  3193. return
  3194. }
  3195. if opr == "zip" {
  3196. //Check if destination location exists
  3197. if rdest == "" || !destFshAbs.FileExists(filepath.Dir(zipOutput)) {
  3198. utils.SendErrorResponse(w, "Invalid dest location")
  3199. return
  3200. }
  3201. //OK. Create the zip at the desired location
  3202. err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
  3203. if err != nil {
  3204. utils.SendErrorResponse(w, err.Error())
  3205. return
  3206. }
  3207. utils.SendOK(w)
  3208. } else if opr == "tmpzip" {
  3209. //Zip to tmp folder
  3210. err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
  3211. if err != nil {
  3212. utils.SendErrorResponse(w, err.Error())
  3213. return
  3214. }
  3215. //Send the tmp filename to the user
  3216. utils.SendTextResponse(w, "tmp:/"+filename)
  3217. }
  3218. if destFsh.RequireBuffer {
  3219. //Write the buffer zip file to destination
  3220. f, _ := os.Open(zipOutput)
  3221. destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
  3222. f.Close()
  3223. os.Remove(zipOutput)
  3224. }
  3225. cleanFsBufferFileFromList(realSourcePaths)
  3226. }
  3227. /*
  3228. runTmpZipTask builds a temporary zip in the background, keeping the file
  3229. operation record of oprId up to date so the front end can poll its progress.
  3230. The progress handler returns the task's control signal, which is what lets a
  3231. user cancel the archive from the file operation list: ArozZipFileWithProgress
  3232. stops and reports the cancellation as an error, and finishTmpZipTask turns that
  3233. back into a cancelled record.
  3234. */
  3235. func runTmpZipTask(oprId string, sourceFshs []*filesystem.FileSystemHandler, realSourcePaths []string, zipDestFsh *filesystem.FileSystemHandler, zipOutput string, destFsh *filesystem.FileSystemHandler, rdest string) {
  3236. err := filesystem.ArozZipFileWithProgress(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false, func(currentFilename string, _ int, _ int, progress float64) int {
  3237. sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
  3238. return sig
  3239. })
  3240. if err != nil {
  3241. systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
  3242. finishTmpZipTask(oprId, err)
  3243. return
  3244. }
  3245. if destFsh.RequireBuffer {
  3246. //Write the buffer zip file to destination
  3247. f, _ := os.Open(zipOutput)
  3248. err = destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
  3249. f.Close()
  3250. os.Remove(zipOutput)
  3251. if err != nil {
  3252. systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver "+destFsh.Filesystem+" failed", err)
  3253. finishTmpZipTask(oprId, err)
  3254. return
  3255. }
  3256. }
  3257. cleanFsBufferFileFromList(realSourcePaths)
  3258. finishTmpZipTask(oprId, nil)
  3259. }
  3260. // finishTmpZipTask closes off the task record of a background tmp zip.
  3261. // Pass in the error the zipping ended with, or nil when it completed.
  3262. func finishTmpZipTask(oprId string, err error) {
  3263. if err != nil {
  3264. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, err.Error())
  3265. return
  3266. }
  3267. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Continue, "")
  3268. }
  3269. // Manage file version history
  3270. func system_fs_FileVersionHistory(w http.ResponseWriter, r *http.Request) {
  3271. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3272. if err != nil {
  3273. utils.SendErrorResponse(w, err.Error())
  3274. return
  3275. }
  3276. path, err := utils.PostPara(r, "path")
  3277. if err != nil {
  3278. utils.SendErrorResponse(w, "Invalid path given")
  3279. return
  3280. }
  3281. fsh, subpath, err := GetFSHandlerSubpathFromVpath(path)
  3282. if err != nil {
  3283. if err != nil {
  3284. utils.SendErrorResponse(w, "Invalid path given")
  3285. return
  3286. }
  3287. }
  3288. fshAbs := fsh.FileSystemAbstraction
  3289. opr, _ := utils.PostPara(r, "opr")
  3290. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3291. if err != nil {
  3292. utils.SendErrorResponse(w, "Unable to translate virtual path")
  3293. return
  3294. }
  3295. if opr == "" {
  3296. //List file history
  3297. fileVersionData, err := localversion.GetFileVersionData(fsh, rpath)
  3298. if err != nil {
  3299. utils.SendErrorResponse(w, "Unable to load version information: "+err.Error())
  3300. return
  3301. }
  3302. js, _ := json.Marshal(fileVersionData)
  3303. utils.SendJSONResponse(w, string(js))
  3304. } else if opr == "delete" {
  3305. //Delete file history of given history ID
  3306. historyID, err := utils.PostPara(r, "histid")
  3307. if err != nil {
  3308. utils.SendErrorResponse(w, "Invalid history id given")
  3309. return
  3310. }
  3311. err = localversion.RemoveFileHistory(fsh, rpath, historyID)
  3312. if err != nil {
  3313. utils.SendErrorResponse(w, err.Error())
  3314. return
  3315. }
  3316. utils.SendOK(w)
  3317. } else if opr == "deleteAll" {
  3318. //Delete all file history of given vpath
  3319. err = localversion.RemoveAllRelatedFileHistory(fsh, rpath)
  3320. if err != nil {
  3321. utils.SendErrorResponse(w, err.Error())
  3322. return
  3323. }
  3324. utils.SendOK(w)
  3325. } else if opr == "restore" {
  3326. //Restore file history of given history ID
  3327. historyID, err := utils.PostPara(r, "histid")
  3328. if err != nil {
  3329. utils.SendErrorResponse(w, "Invalid history id given")
  3330. return
  3331. }
  3332. err = localversion.RestoreFileHistory(fsh, rpath, historyID)
  3333. if err != nil {
  3334. utils.SendErrorResponse(w, err.Error())
  3335. return
  3336. }
  3337. utils.SendOK(w)
  3338. } else if opr == "new" {
  3339. //Create a new snapshot of this file
  3340. err = localversion.CreateFileSnapshot(fsh, rpath)
  3341. if err != nil {
  3342. utils.SendErrorResponse(w, err.Error())
  3343. return
  3344. }
  3345. utils.SendOK(w)
  3346. } else {
  3347. utils.SendErrorResponse(w, "Unknown opr")
  3348. }
  3349. }
  3350. func system_fs_clearVersionHistories() {
  3351. allFsh := GetAllLoadedFsh()
  3352. for _, fsh := range allFsh {
  3353. if fsh.ReadOnly {
  3354. continue
  3355. }
  3356. if !nightlyShouldMaintainFsh(fsh) {
  3357. //A drive shared by the cluster is cleaned by the master node
  3358. continue
  3359. }
  3360. localversion.CleanExpiredVersionBackups(fsh, fsh.Path, 30*86400)
  3361. }
  3362. }
  3363. // Handle cache rendering with websocket pipeline
  3364. func system_fs_handleCacheRender(w http.ResponseWriter, r *http.Request) {
  3365. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3366. vpath, err := utils.GetPara(r, "folder")
  3367. if err != nil {
  3368. utils.SendErrorResponse(w, "Invalid folder paramter")
  3369. return
  3370. }
  3371. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  3372. if err != nil {
  3373. utils.SendErrorResponse(w, "Unable to resolve target directory")
  3374. return
  3375. }
  3376. rpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3377. //Get folder sort mode
  3378. sortMode := "default"
  3379. folder := arozfs.Clean(vpath)
  3380. if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
  3381. sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
  3382. }
  3383. //Perform cache rendering
  3384. thumbRenderHandler.HandleLoadCache(w, r, fsh, rpath, sortMode)
  3385. }
  3386. // Handle loading of one thumbnail
  3387. func system_fs_handleThumbnailLoad(w http.ResponseWriter, r *http.Request) {
  3388. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3389. vpath, err := utils.GetPara(r, "vpath")
  3390. if err != nil {
  3391. utils.SendErrorResponse(w, "vpath not defined")
  3392. return
  3393. }
  3394. byteMode, _ := utils.GetPara(r, "bytes")
  3395. isByteMode := byteMode == "true"
  3396. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  3397. if err != nil {
  3398. if isByteMode {
  3399. http.NotFound(w, r)
  3400. return
  3401. }
  3402. utils.SendErrorResponse(w, "Unable to resolve target directory")
  3403. return
  3404. }
  3405. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3406. if err != nil {
  3407. if isByteMode {
  3408. http.NotFound(w, r)
  3409. return
  3410. }
  3411. utils.SendErrorResponse(w, err.Error())
  3412. return
  3413. }
  3414. if isByteMode {
  3415. thumbnailBytes, err := thumbRenderHandler.LoadCacheAsBytes(fsh, vpath, userinfo.Username, false)
  3416. if err != nil {
  3417. http.NotFound(w, r)
  3418. return
  3419. }
  3420. filetype := http.DetectContentType(thumbnailBytes)
  3421. w.Header().Add("Content-Type", filetype)
  3422. w.Write(thumbnailBytes)
  3423. } else {
  3424. thumbnailPath, err := thumbRenderHandler.LoadCache(fsh, rpath, false)
  3425. if err != nil {
  3426. utils.SendErrorResponse(w, err.Error())
  3427. return
  3428. }
  3429. js, _ := json.Marshal(thumbnailPath)
  3430. utils.SendJSONResponse(w, string(js))
  3431. }
  3432. }
  3433. // Handle file thumbnail caching
  3434. func system_fs_handleFolderCache(w http.ResponseWriter, r *http.Request) {
  3435. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3436. vfolderpath, err := utils.GetPara(r, "folder")
  3437. if err != nil {
  3438. utils.SendErrorResponse(w, "folder not defined")
  3439. return
  3440. }
  3441. fsh, _, err := GetFSHandlerSubpathFromVpath(vfolderpath)
  3442. if err != nil {
  3443. utils.SendErrorResponse(w, "unable to resolve path")
  3444. return
  3445. }
  3446. thumbRenderHandler.BuildCacheForFolder(fsh, vfolderpath, userinfo.Username)
  3447. utils.SendOK(w)
  3448. }
  3449. // Handle the get and set of sort mode of a particular folder
  3450. func system_fs_handleFolderSortModePreference(w http.ResponseWriter, r *http.Request) {
  3451. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3452. if err != nil {
  3453. utils.SendErrorResponse(w, "User not logged in")
  3454. return
  3455. }
  3456. folder, err := utils.PostPara(r, "folder")
  3457. if err != nil {
  3458. utils.SendErrorResponse(w, "Invalid folder given")
  3459. return
  3460. }
  3461. opr, _ := utils.PostPara(r, "opr")
  3462. folder = arozfs.Clean(folder)
  3463. if opr == "" || opr == "get" {
  3464. sortMode := "default"
  3465. if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
  3466. sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
  3467. }
  3468. js, err := json.Marshal(sortMode)
  3469. if err != nil {
  3470. utils.SendErrorResponse(w, err.Error())
  3471. return
  3472. }
  3473. utils.SendJSONResponse(w, string(js))
  3474. } else if opr == "set" {
  3475. sortMode, err := utils.PostPara(r, "mode")
  3476. if err != nil {
  3477. utils.SendErrorResponse(w, "Invalid sort mode given")
  3478. return
  3479. }
  3480. if !utils.StringInArray(fssort.ValidSortModes, sortMode) {
  3481. utils.SendErrorResponse(w, "Not supported sort mode: "+sortMode)
  3482. return
  3483. }
  3484. sysdb.Write("fs-sortpref", userinfo.Username+"/"+folder, sortMode)
  3485. utils.SendOK(w)
  3486. } else {
  3487. utils.SendErrorResponse(w, "Invalid opr mode")
  3488. return
  3489. }
  3490. }
  3491. // Handle setting and loading of file permission on Linux
  3492. func system_fs_handleFilePermission(w http.ResponseWriter, r *http.Request) {
  3493. file, err := utils.PostPara(r, "file")
  3494. if err != nil {
  3495. utils.SendErrorResponse(w, "Invalid file")
  3496. return
  3497. }
  3498. //Translate the file to real path
  3499. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3500. if err != nil {
  3501. utils.SendErrorResponse(w, "User not logged in")
  3502. return
  3503. }
  3504. fsh, subpath, err := GetFSHandlerSubpathFromVpath(file)
  3505. if err != nil {
  3506. utils.SendErrorResponse(w, err.Error())
  3507. return
  3508. }
  3509. fshAbs := fsh.FileSystemAbstraction
  3510. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3511. if err != nil {
  3512. utils.SendErrorResponse(w, err.Error())
  3513. return
  3514. }
  3515. newMode, _ := utils.PostPara(r, "mode")
  3516. if newMode == "" {
  3517. //Read the file mode
  3518. //Check if the file exists
  3519. if !fshAbs.FileExists(rpath) {
  3520. utils.SendErrorResponse(w, "File not exists!")
  3521. return
  3522. }
  3523. //Read the file permission
  3524. filePermission, err := fsp.GetFilePermissions(fsh, rpath)
  3525. if err != nil {
  3526. utils.SendErrorResponse(w, err.Error())
  3527. return
  3528. }
  3529. //Send the file permission to client
  3530. js, _ := json.Marshal(filePermission)
  3531. utils.SendJSONResponse(w, string(js))
  3532. } else {
  3533. //Set the file mode
  3534. //Check if the file exists
  3535. if !filesystem.FileExists(rpath) {
  3536. utils.SendErrorResponse(w, "File not exists!")
  3537. return
  3538. }
  3539. //Check if windows. If yes, ignore this request
  3540. if runtime.GOOS == "windows" {
  3541. utils.SendErrorResponse(w, "Windows host not supported")
  3542. return
  3543. }
  3544. //Check if this user has permission to change the file permission
  3545. //Aka user must be 1. This is his own folder or 2. Admin
  3546. fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath(file)
  3547. if fsh.Hierarchy == "user" {
  3548. //Always ok as this is owned by the user
  3549. } else if fsh.Hierarchy == "public" {
  3550. //Require admin
  3551. if !userinfo.IsAdmin() {
  3552. utils.SendErrorResponse(w, "Permission Denied")
  3553. return
  3554. }
  3555. } else {
  3556. //Not implemeneted. Require admin
  3557. if !userinfo.IsAdmin() {
  3558. utils.SendErrorResponse(w, "Permission Denied")
  3559. return
  3560. }
  3561. }
  3562. //Be noted that if the system is not running in sudo mode,
  3563. //File permission change might not works.
  3564. err := fsp.SetFilePermisson(fsh, rpath, newMode)
  3565. if err != nil {
  3566. utils.SendErrorResponse(w, err.Error())
  3567. return
  3568. } else {
  3569. utils.SendOK(w)
  3570. }
  3571. }
  3572. }
  3573. // Clear the old files inside the tmp file
  3574. func system_fs_clearOldTmpFiles() {
  3575. filesToBeDelete := []string{}
  3576. tmpAbs, _ := filepath.Abs(*tmp_directory)
  3577. filepath.Walk(*tmp_directory, func(path string, info os.FileInfo, err error) error {
  3578. if filepath.Base(path) != "aofs.db" && filepath.Base(path) != "aofs.db.lock" {
  3579. //Check if root folders. Do not delete root folders
  3580. parentAbs, _ := filepath.Abs(filepath.Dir(path))
  3581. if tmpAbs == parentAbs {
  3582. //Root folder. Do not remove
  3583. return nil
  3584. }
  3585. //Get its modification time
  3586. modTime, err := filesystem.GetModTime(path)
  3587. if err != nil {
  3588. return nil
  3589. }
  3590. //Check if mod time is more than 24 hours ago
  3591. if time.Now().Unix()-modTime > int64(*maxTempFileKeepTime) {
  3592. //Delete OK
  3593. filesToBeDelete = append(filesToBeDelete, path)
  3594. }
  3595. }
  3596. return nil
  3597. })
  3598. //Remove all files from the delete list
  3599. for _, fileToBeDelete := range filesToBeDelete {
  3600. os.RemoveAll(fileToBeDelete)
  3601. }
  3602. }
  3603. /*
  3604. File System Utilities for Buffered type FS
  3605. These functions help create a local representation of file
  3606. buffer from remote file systems like webdav or SMB
  3607. **REMEMBER TO CLEAR THE BUFFER FILES YOURSELF**
  3608. Example Usage
  3609. //Replace a destination path (for file create) with local buffer filepath
  3610. if destFsh.RequireBuffer {
  3611. dest = getFsBufferFilepath(outputFilename)
  3612. }
  3613. //Buffer a remote file to local first before doing any advance file operations
  3614. if thisSrcFsh.RequireBuffer {
  3615. localBufferFilepath, err := bufferRemoteFileToLocal(fsh, remoteRealSrc)
  3616. if err != nil{
  3617. //Handle Error
  3618. }
  3619. }
  3620. //Clean a list of source files that contains local buffer files
  3621. clearnFsBufferFileFromList(realSourceFiles)
  3622. */
  3623. // Generate a random buffer filepath. Remember to delete file after usage
  3624. func getFsBufferFilepath(originalFilename string, keepOriginalName bool) string {
  3625. thisBuffFilename := uuid.NewV4().String()
  3626. tmpDir := filepath.Join(*tmp_directory, "fsBuff")
  3627. targetFile := filepath.Join(tmpDir, thisBuffFilename+filepath.Ext(originalFilename))
  3628. if keepOriginalName {
  3629. targetFile = filepath.Join(tmpDir, thisBuffFilename, filepath.Base(originalFilename))
  3630. }
  3631. os.MkdirAll(filepath.Dir(targetFile), 0775)
  3632. return filepath.ToSlash(targetFile)
  3633. }
  3634. // Generate a buffer filepath and buffer the remote file to local. Remember to remove file after done.
  3635. func bufferRemoteFileToLocal(targetFsh *filesystem.FileSystemHandler, rpath string, keepOriginalName bool) (string, error) {
  3636. newBufferFilename := getFsBufferFilepath(rpath, keepOriginalName)
  3637. src, err := targetFsh.FileSystemAbstraction.ReadStream(rpath)
  3638. if err != nil {
  3639. systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
  3640. return "", err
  3641. }
  3642. defer src.Close()
  3643. dest, err := os.OpenFile(newBufferFilename, os.O_CREATE|os.O_WRONLY, 0775)
  3644. if err != nil {
  3645. systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
  3646. return "", err
  3647. }
  3648. io.Copy(dest, src)
  3649. dest.Close()
  3650. return newBufferFilename, nil
  3651. }
  3652. // Check if a file is buffer filepath
  3653. func isFsBufferFilepath(filename string) bool {
  3654. tmpDir := filepath.Join(*tmp_directory, "fsBuff")
  3655. filenameAbs, _ := filepath.Abs(filename)
  3656. filenameAbs = filepath.ToSlash(filenameAbs)
  3657. tmpDirAbs, _ := filepath.Abs(tmpDir)
  3658. tmpDirAbs = filepath.ToSlash(tmpDirAbs)
  3659. return strings.HasPrefix(filenameAbs, tmpDirAbs)
  3660. }
  3661. func cleanFsBufferFileFromList(filelist []string) {
  3662. for _, thisFilepath := range filelist {
  3663. if isFsBufferFilepath(thisFilepath) {
  3664. os.RemoveAll(thisFilepath)
  3665. folderContent, _ := os.ReadDir(filepath.Dir(thisFilepath))
  3666. if len(folderContent) == 0 {
  3667. //Nothing in this folder. Remove it
  3668. os.Remove(filepath.Dir(thisFilepath))
  3669. }
  3670. }
  3671. }
  3672. }
  3673. /*
  3674. File operation task book keeping
  3675. All file operations of all users are tracked in wsConnectionStore, keyed by
  3676. the operation id. Finished records are kept around for a while so the file
  3677. operation dialog can still show the completed entries to the user before
  3678. they are cleared (either by the user or by the janitor below).
  3679. */
  3680. // Handle all the on going task requests.
  3681. // Accept parameter: flag={continue / pause / cancel / remove / clear}
  3682. // When no flag is given, the ongoing task list of this user is returned.
  3683. // Pass in all=true to include the recently finished tasks in the listing.
  3684. func system_fs_HandleOnGoingTasks(w http.ResponseWriter, r *http.Request) {
  3685. //Get the user information
  3686. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3687. if err != nil {
  3688. utils.SendErrorResponse(w, "User not logged in")
  3689. return
  3690. }
  3691. statusFlag, _ := utils.PostPara(r, "flag")
  3692. oprid, _ := utils.PostPara(r, "oprid")
  3693. if statusFlag == "" {
  3694. //Also accept the flag as a GET parameter for easier polling
  3695. statusFlag, _ = utils.GetPara(r, "flag")
  3696. oprid, _ = utils.GetPara(r, "oprid")
  3697. }
  3698. if statusFlag == "" {
  3699. //No flag defined. Print all operations of this user
  3700. includeFinished, _ := utils.GetPara(r, "all")
  3701. js, _ := MarshalFileOperationForUser(userinfo.Username, includeFinished == "true")
  3702. utils.SendJSONResponse(w, string(js))
  3703. return
  3704. }
  3705. if statusFlag == "clear" {
  3706. //Clear all the finished records of this user
  3707. ClearFinishedFileOperationForUser(userinfo.Username)
  3708. utils.SendOK(w)
  3709. return
  3710. }
  3711. if oprid == "" {
  3712. utils.SendErrorResponse(w, "oprid is empty or not set")
  3713. return
  3714. }
  3715. err = ApplyFileOperationControl(userinfo.Username, statusFlag, oprid)
  3716. if err != nil {
  3717. utils.SendErrorResponse(w, err.Error())
  3718. return
  3719. }
  3720. utils.SendOK(w)
  3721. }
  3722. // ApplyFileOperationControl applies a control signal to one of the file operations
  3723. // owned by the given user. Supported commands are continue, pause, cancel and remove.
  3724. func ApplyFileOperationControl(username string, command string, oprid string) error {
  3725. if command == "clear" {
  3726. ClearFinishedFileOperationForUser(username)
  3727. return nil
  3728. }
  3729. //Get the operation record
  3730. oprRecord, err := GetOngoingFileOperationByOprID(oprid)
  3731. if err != nil {
  3732. return err
  3733. }
  3734. //Only the owner of this operation can control it
  3735. if oprRecord.Owner != username {
  3736. return errors.New("permission denied")
  3737. }
  3738. fileOprTaskLock.Lock()
  3739. defer fileOprTaskLock.Unlock()
  3740. switch command {
  3741. case "continue":
  3742. //Continue the file operation
  3743. oprRecord.FileOperationSignal = filesystem.FsOpr_Continue
  3744. case "pause":
  3745. //Pause the file operation until the flag is set to other status
  3746. oprRecord.FileOperationSignal = filesystem.FsOpr_Pause
  3747. case "cancel":
  3748. //Cancel and stop the operation
  3749. oprRecord.FileOperationSignal = filesystem.FsOpr_Cancel
  3750. if taskIsFinished(oprRecord) {
  3751. //Already finished. Nothing left to cancel, just drop the record
  3752. wsConnectionStore.Delete(oprid)
  3753. }
  3754. case "remove":
  3755. //Remove a finished record from the listing
  3756. if !taskIsFinished(oprRecord) {
  3757. return errors.New("task is still running")
  3758. }
  3759. wsConnectionStore.Delete(oprid)
  3760. default:
  3761. return errors.New("unsupported operation")
  3762. }
  3763. return nil
  3764. }
  3765. // NewOngoingFileOperation creates and registers a new file operation task record.
  3766. // The size of each source file is resolved here so the front end can show the
  3767. // transferred / total size of every file in the operation.
  3768. func NewOngoingFileOperation(userinfo *user.User, operation string, sourceFiles []string, vdestFile string) *fileOperationTask {
  3769. /*
  3770. Sizing a source folder means walking it. For a move that the storage can
  3771. satisfy with a rename that walk costs far more than the move itself, and
  3772. it would only feed a bar that jumps straight to done, so it is skipped.
  3773. Should the rename fail after all and the files end up being streamed, the
  3774. first progress report fills the real sizes in, see
  3775. UpdateOngoingFileOperationSubtask.
  3776. */
  3777. resolveSizes := !fileOperationCanBeRenamed(operation, sourceFiles, vdestFile)
  3778. subtasks := []*fileOperationSubtask{}
  3779. totalSize := int64(0)
  3780. for _, vsrc := range sourceFiles {
  3781. thisSize, thisIsDir := resolveFileOperationSourceInfo(userinfo, vsrc, resolveSizes)
  3782. totalSize += thisSize
  3783. subtasks = append(subtasks, &fileOperationSubtask{
  3784. Filename: filepath.Base(strings.TrimSuffix(vsrc, "/")),
  3785. Src: arozfs.ToSlash(vsrc),
  3786. IsDir: thisIsDir,
  3787. Size: thisSize,
  3788. Done: 0,
  3789. Progress: 0,
  3790. Status: FsTask_Pending,
  3791. })
  3792. }
  3793. thisTask := fileOperationTask{
  3794. ID: strconv.Itoa(int(time.Now().Unix())) + "_" + uuid.NewV4().String(),
  3795. Owner: userinfo.Username,
  3796. Operation: operation,
  3797. Src: arozfs.ToSlash(filepath.Dir(sourceFiles[0])),
  3798. Dest: arozfs.ToSlash(vdestFile),
  3799. Progress: 0.0,
  3800. LatestFile: arozfs.ToSlash(filepath.Base(sourceFiles[0])),
  3801. FileOperationSignal: filesystem.FsOpr_Continue,
  3802. Files: subtasks,
  3803. TotalSize: totalSize,
  3804. DoneSize: 0,
  3805. StartTime: time.Now().Unix(),
  3806. Status: FsTask_Ongoing,
  3807. }
  3808. wsConnectionStore.Store(thisTask.ID, &thisTask)
  3809. startFinishedFileOperationJanitor()
  3810. return &thisTask
  3811. }
  3812. // resolveFileOperationSourceInfo returns the size of a source file in bytes and
  3813. // whether it is a folder. Set resolveSize to false to skip the folder walk when
  3814. // the size is not worth what it costs to work out.
  3815. // The size is 0 when it cannot be determined.
  3816. func resolveFileOperationSourceInfo(userinfo *user.User, vsrc string, resolveSize bool) (int64, bool) {
  3817. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
  3818. if err != nil {
  3819. return 0, false
  3820. }
  3821. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3822. if err != nil {
  3823. return 0, false
  3824. }
  3825. isDir := fsh.FileSystemAbstraction.IsDir(rpath)
  3826. if !resolveSize {
  3827. return 0, isDir
  3828. }
  3829. if isDir {
  3830. size, _ := fsh.GetDirctorySizeFromRealPath(rpath, false)
  3831. return size, true
  3832. }
  3833. return fsh.FileSystemAbstraction.GetFileSize(rpath), false
  3834. }
  3835. // fileOperationCanBeRenamed reports whether an operation is a move that the
  3836. // storage holding both of its ends can carry out on its own, in which case no
  3837. // bytes travel through ArozOS and the operation finishes right away.
  3838. func fileOperationCanBeRenamed(operation string, sourceFiles []string, vdestFile string) bool {
  3839. if operation != "move" {
  3840. return false
  3841. }
  3842. destFsh, _, err := GetFSHandlerSubpathFromVpath(vdestFile)
  3843. if err != nil || destFsh.ReadOnly {
  3844. return false
  3845. }
  3846. for _, vsrc := range sourceFiles {
  3847. srcFsh, _, err := GetFSHandlerSubpathFromVpath(vsrc)
  3848. if err != nil || srcFsh.ReadOnly || !filesystem.SameFileSystem(srcFsh, destFsh) {
  3849. return false
  3850. }
  3851. }
  3852. return true
  3853. }
  3854. // GetAllFileOperationForUser returns all the file operation records of the given user.
  3855. // Set includeFinished to true to also return the recently finished records.
  3856. // The records are live objects that the running operations keep updating, so
  3857. // read them through MarshalFileOperationForUser unless you hold fileOprTaskLock.
  3858. func GetAllFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
  3859. fileOprTaskLock.RLock()
  3860. defer fileOprTaskLock.RUnlock()
  3861. return collectFileOperationForUser(username, includeFinished)
  3862. }
  3863. // MarshalFileOperationForUser serializes the file operation records of a user
  3864. // while holding the task lock, so the running operations cannot update a record
  3865. // half way through the serialization.
  3866. func MarshalFileOperationForUser(username string, includeFinished bool) ([]byte, error) {
  3867. fileOprTaskLock.RLock()
  3868. defer fileOprTaskLock.RUnlock()
  3869. return json.Marshal(collectFileOperationForUser(username, includeFinished))
  3870. }
  3871. // collectFileOperationForUser gathers the task records of a user in operation id
  3872. // order. The caller is expected to be holding fileOprTaskLock.
  3873. func collectFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
  3874. results := []*fileOperationTask{}
  3875. wsConnectionStore.Range(func(key, value interface{}) bool {
  3876. taskInfo := value.(*fileOperationTask)
  3877. if taskInfo.Owner == username && (includeFinished || !taskIsFinished(taskInfo)) {
  3878. results = append(results, taskInfo)
  3879. }
  3880. return true
  3881. })
  3882. sort.Slice(results, func(i, j int) bool {
  3883. return results[i].ID < results[j].ID
  3884. })
  3885. return results
  3886. }
  3887. // ClearFinishedFileOperationForUser removes all the finished records of a user
  3888. func ClearFinishedFileOperationForUser(username string) {
  3889. fileOprTaskLock.Lock()
  3890. defer fileOprTaskLock.Unlock()
  3891. wsConnectionStore.Range(func(key, value interface{}) bool {
  3892. taskInfo := value.(*fileOperationTask)
  3893. if taskInfo.Owner == username && taskIsFinished(taskInfo) {
  3894. wsConnectionStore.Delete(key)
  3895. }
  3896. return true
  3897. })
  3898. }
  3899. // Get an ongoing task record
  3900. func GetOngoingFileOperationByOprID(oprid string) (*fileOperationTask, error) {
  3901. object, ok := wsConnectionStore.Load(oprid)
  3902. if !ok {
  3903. return nil, errors.New("task not exists")
  3904. }
  3905. return object.(*fileOperationTask), nil
  3906. }
  3907. // Set or update an ongoing task record
  3908. func SetOngoingFileOperation(opr *fileOperationTask) {
  3909. wsConnectionStore.Store(opr.ID, opr)
  3910. }
  3911. // Update the status of an ongoing task record, return latest status code and error if any
  3912. func UpdateOngoingFileOperation(oprid string, currentFile string, progress float64) (int, error) {
  3913. t, err := GetOngoingFileOperationByOprID(oprid)
  3914. if err != nil {
  3915. return 0, err
  3916. }
  3917. fileOprTaskLock.Lock()
  3918. t.LatestFile = currentFile
  3919. t.Progress = progress
  3920. t.DoneSize = int64(float64(t.TotalSize) * progress / 100)
  3921. //Zip and unzip report one progress value for the whole operation. Spread it
  3922. //over the source files in order so the dialog can still show them one by one.
  3923. remaining := t.DoneSize
  3924. for _, thisSubtask := range t.Files {
  3925. if remaining >= thisSubtask.Size {
  3926. thisSubtask.Done = thisSubtask.Size
  3927. thisSubtask.Progress = 100
  3928. thisSubtask.Status = FsTask_Completed
  3929. remaining -= thisSubtask.Size
  3930. } else {
  3931. thisSubtask.Done = remaining
  3932. if thisSubtask.Size > 0 {
  3933. thisSubtask.Progress = float64(remaining) / float64(thisSubtask.Size) * 100
  3934. } else {
  3935. thisSubtask.Progress = progress
  3936. }
  3937. thisSubtask.Status = FsTask_Ongoing
  3938. remaining = 0
  3939. }
  3940. }
  3941. sig := t.FileOperationSignal
  3942. fileOprTaskLock.Unlock()
  3943. return sig, nil
  3944. }
  3945. /*
  3946. UpdateOngoingFileOperationSubtask records how many bytes of one source file are
  3947. already processed and recalculates the progress of the whole operation from it.
  3948. It returns the current control signal and the overall progress of the operation.
  3949. bytesTotal is the size the file operation itself reports for this source, which
  3950. wins over the size resolved when the task was created: a folder can grow or
  3951. shrink between the two, and the transfer knows better than the earlier walk.
  3952. */
  3953. func UpdateOngoingFileOperationSubtask(oprid string, subtaskIndex int, currentFile string, bytesDone int64, bytesTotal int64) (int, float64, error) {
  3954. t, err := GetOngoingFileOperationByOprID(oprid)
  3955. if err != nil {
  3956. return 0, 0, err
  3957. }
  3958. fileOprTaskLock.Lock()
  3959. if currentFile != "" {
  3960. t.LatestFile = currentFile
  3961. }
  3962. if subtaskIndex >= 0 && subtaskIndex < len(t.Files) {
  3963. thisSubtask := t.Files[subtaskIndex]
  3964. thisSubtask.Status = FsTask_Ongoing
  3965. if bytesTotal > 0 && bytesTotal != thisSubtask.Size {
  3966. //Keep the operation total in step with the corrected file size
  3967. t.TotalSize += bytesTotal - thisSubtask.Size
  3968. thisSubtask.Size = bytesTotal
  3969. }
  3970. if bytesDone > thisSubtask.Size {
  3971. bytesDone = thisSubtask.Size
  3972. }
  3973. thisSubtask.Done = bytesDone
  3974. if thisSubtask.Size > 0 {
  3975. thisSubtask.Progress = float64(bytesDone) / float64(thisSubtask.Size) * 100
  3976. }
  3977. }
  3978. t.DoneSize = sumFileOperationDoneSize(t)
  3979. recalculateFileOperationProgress(t)
  3980. sig := t.FileOperationSignal
  3981. progress := t.Progress
  3982. fileOprTaskLock.Unlock()
  3983. return sig, progress, nil
  3984. }
  3985. /*
  3986. recalculateFileOperationProgress works out the overall progress of a task from
  3987. the bytes of the files inside it, so copying a 1 KB text file next to a 4 GB
  3988. image no longer moves the bar half way when the text file lands.
  3989. Operations whose sizes could not be resolved fall back to weighting every source
  3990. file equally. The caller is expected to be holding fileOprTaskLock.
  3991. */
  3992. func recalculateFileOperationProgress(t *fileOperationTask) {
  3993. if t.TotalSize > 0 {
  3994. progress := float64(t.DoneSize) / float64(t.TotalSize) * 100
  3995. if progress > 100 {
  3996. progress = 100
  3997. }
  3998. t.Progress = progress
  3999. return
  4000. }
  4001. if len(t.Files) == 0 {
  4002. return
  4003. }
  4004. totalProgress := float64(0)
  4005. for _, thisSubtask := range t.Files {
  4006. totalProgress += thisSubtask.Progress
  4007. }
  4008. t.Progress = totalProgress / float64(len(t.Files))
  4009. }
  4010. // MarkFileOperationSubtaskEnded marks one source file inside a task as finished.
  4011. // Pass in an empty errmsg for a successful completion.
  4012. func MarkFileOperationSubtaskEnded(oprid string, subtaskIndex int, errmsg string) {
  4013. t, err := GetOngoingFileOperationByOprID(oprid)
  4014. if err != nil {
  4015. return
  4016. }
  4017. fileOprTaskLock.Lock()
  4018. defer fileOprTaskLock.Unlock()
  4019. if subtaskIndex < 0 || subtaskIndex >= len(t.Files) {
  4020. return
  4021. }
  4022. thisSubtask := t.Files[subtaskIndex]
  4023. if errmsg != "" {
  4024. thisSubtask.Status = FsTask_Error
  4025. thisSubtask.Error = errmsg
  4026. } else {
  4027. thisSubtask.Status = FsTask_Completed
  4028. thisSubtask.Progress = 100
  4029. thisSubtask.Done = thisSubtask.Size
  4030. }
  4031. t.DoneSize = sumFileOperationDoneSize(t)
  4032. recalculateFileOperationProgress(t)
  4033. }
  4034. // SetFileOperationTaskEnded closes off a task record with the given ending signal.
  4035. // The record is kept in the store so the front end can still render the result.
  4036. func SetFileOperationTaskEnded(oprid string, endingSignal int, errmsg string) {
  4037. t, err := GetOngoingFileOperationByOprID(oprid)
  4038. if err != nil {
  4039. return
  4040. }
  4041. fileOprTaskLock.Lock()
  4042. defer fileOprTaskLock.Unlock()
  4043. t.EndTime = time.Now().Unix()
  4044. t.Error = errmsg
  4045. switch {
  4046. case endingSignal == filesystem.FsOpr_Cancel || t.FileOperationSignal == filesystem.FsOpr_Cancel:
  4047. //A file operation stopped by the user reports the abort as an error on
  4048. //its way out. That is not a failure, so record it as a cancellation.
  4049. t.Status = FsTask_Cancelled
  4050. t.Error = ""
  4051. for _, thisSubtask := range t.Files {
  4052. if thisSubtask.Status == FsTask_Error {
  4053. thisSubtask.Status = FsTask_Cancelled
  4054. thisSubtask.Error = ""
  4055. }
  4056. }
  4057. case errmsg != "" || endingSignal == filesystem.FsOpr_Error:
  4058. t.Status = FsTask_Error
  4059. default:
  4060. t.Status = FsTask_Completed
  4061. t.Progress = 100
  4062. t.DoneSize = t.TotalSize
  4063. for _, thisSubtask := range t.Files {
  4064. if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
  4065. thisSubtask.Status = FsTask_Completed
  4066. thisSubtask.Progress = 100
  4067. thisSubtask.Done = thisSubtask.Size
  4068. }
  4069. }
  4070. }
  4071. //Any file still marked as running at this point will never move again
  4072. for _, thisSubtask := range t.Files {
  4073. if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
  4074. thisSubtask.Status = t.Status
  4075. }
  4076. }
  4077. }
  4078. // taskIsFinished checks if a task record has stopped running.
  4079. // The caller is expected to be holding fileOprTaskLock when required.
  4080. func taskIsFinished(t *fileOperationTask) bool {
  4081. return t.Status != FsTask_Ongoing && t.Status != FsTask_Pending
  4082. }
  4083. // sumFileOperationDoneSize sums up the transferred bytes of all the files in a task.
  4084. // The caller is expected to be holding fileOprTaskLock.
  4085. func sumFileOperationDoneSize(t *fileOperationTask) int64 {
  4086. total := int64(0)
  4087. for _, thisSubtask := range t.Files {
  4088. total += thisSubtask.Done
  4089. }
  4090. return total
  4091. }
  4092. // startFinishedFileOperationJanitor starts the background cleaner that tidies up
  4093. // the finished task records, see clearExpiredFileOperationRecords.
  4094. func startFinishedFileOperationJanitor() {
  4095. if !fileOprJanitorStarted.CompareAndSwap(false, true) {
  4096. return
  4097. }
  4098. go func() {
  4099. for {
  4100. time.Sleep(fileOprJanitorInterval)
  4101. clearExpiredFileOperationRecords()
  4102. }
  4103. }()
  4104. }
  4105. /*
  4106. clearExpiredFileOperationRecords drops the finished task records that are no
  4107. longer worth keeping.
  4108. An operation that finished without an error is cleared on its own shortly
  4109. after it ended, whether or not anyone was watching it: closing the browser
  4110. tab half way through a copy must not leave the record behind forever.
  4111. A failed operation is kept instead, so the user can still find out what went
  4112. wrong after reopening the desktop from this or from another machine. Only the
  4113. newest fileOprErrorRecordLimit failures of each user are kept.
  4114. */
  4115. func clearExpiredFileOperationRecords() {
  4116. expireTime := time.Now().Unix() - fileOprFinishedRecordTTL
  4117. errorRecords := map[string][]*fileOperationTask{}
  4118. fileOprTaskLock.Lock()
  4119. defer fileOprTaskLock.Unlock()
  4120. wsConnectionStore.Range(func(key, value interface{}) bool {
  4121. taskInfo := value.(*fileOperationTask)
  4122. if !taskIsFinished(taskInfo) || taskInfo.EndTime == 0 {
  4123. //Still running, leave it alone
  4124. return true
  4125. }
  4126. if taskInfo.Status == FsTask_Error {
  4127. errorRecords[taskInfo.Owner] = append(errorRecords[taskInfo.Owner], taskInfo)
  4128. return true
  4129. }
  4130. if taskInfo.EndTime < expireTime {
  4131. wsConnectionStore.Delete(key)
  4132. }
  4133. return true
  4134. })
  4135. //Trim the failure backlog of each user down to the newest few
  4136. for _, records := range errorRecords {
  4137. if len(records) <= fileOprErrorRecordLimit {
  4138. continue
  4139. }
  4140. sort.Slice(records, func(i, j int) bool {
  4141. return records[i].EndTime > records[j].EndTime
  4142. })
  4143. for _, expiredRecord := range records[fileOprErrorRecordLimit:] {
  4144. wsConnectionStore.Delete(expiredRecord.ID)
  4145. }
  4146. }
  4147. }