file_system.go 141 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096309730983099310031013102310331043105310631073108310931103111311231133114311531163117311831193120312131223123312431253126312731283129313031313132313331343135313631373138313931403141314231433144314531463147314831493150315131523153315431553156315731583159316031613162316331643165316631673168316931703171317231733174317531763177317831793180318131823183318431853186318731883189319031913192319331943195319631973198319932003201320232033204320532063207320832093210321132123213321432153216321732183219322032213222322332243225322632273228322932303231323232333234323532363237323832393240324132423243324432453246324732483249325032513252325332543255325632573258325932603261326232633264326532663267326832693270327132723273327432753276327732783279328032813282328332843285328632873288328932903291329232933294329532963297329832993300330133023303330433053306330733083309331033113312331333143315331633173318331933203321332233233324332533263327332833293330333133323333333433353336333733383339334033413342334333443345334633473348334933503351335233533354335533563357335833593360336133623363336433653366336733683369337033713372337333743375337633773378337933803381338233833384338533863387338833893390339133923393339433953396339733983399340034013402340334043405340634073408340934103411341234133414341534163417341834193420342134223423342434253426342734283429343034313432343334343435343634373438343934403441344234433444344534463447344834493450345134523453345434553456345734583459346034613462346334643465346634673468346934703471347234733474347534763477347834793480348134823483348434853486348734883489349034913492349334943495349634973498349935003501350235033504350535063507350835093510351135123513351435153516351735183519352035213522352335243525352635273528352935303531353235333534353535363537353835393540354135423543354435453546354735483549355035513552355335543555355635573558355935603561356235633564356535663567356835693570357135723573357435753576357735783579358035813582358335843585358635873588358935903591359235933594359535963597359835993600360136023603360436053606360736083609361036113612361336143615361636173618361936203621362236233624362536263627362836293630363136323633363436353636363736383639364036413642364336443645364636473648364936503651365236533654365536563657365836593660366136623663366436653666366736683669367036713672367336743675367636773678367936803681368236833684368536863687368836893690369136923693369436953696369736983699370037013702370337043705370637073708370937103711371237133714371537163717371837193720372137223723372437253726372737283729373037313732373337343735373637373738373937403741374237433744374537463747374837493750375137523753375437553756375737583759376037613762376337643765376637673768376937703771377237733774377537763777377837793780378137823783378437853786378737883789379037913792379337943795379637973798379938003801380238033804380538063807380838093810381138123813381438153816381738183819382038213822382338243825382638273828382938303831383238333834383538363837383838393840384138423843384438453846384738483849385038513852385338543855385638573858385938603861386238633864386538663867386838693870387138723873387438753876387738783879388038813882388338843885388638873888388938903891389238933894389538963897389838993900390139023903390439053906390739083909391039113912391339143915391639173918391939203921392239233924392539263927392839293930393139323933393439353936393739383939394039413942394339443945394639473948394939503951395239533954395539563957395839593960396139623963396439653966396739683969397039713972397339743975397639773978397939803981398239833984398539863987398839893990399139923993399439953996399739983999400040014002400340044005400640074008400940104011401240134014401540164017401840194020402140224023402440254026402740284029403040314032403340344035403640374038403940404041404240434044404540464047404840494050405140524053405440554056405740584059406040614062406340644065406640674068406940704071407240734074407540764077407840794080408140824083408440854086408740884089409040914092409340944095409640974098409941004101410241034104410541064107410841094110411141124113411441154116411741184119412041214122412341244125412641274128412941304131413241334134413541364137413841394140414141424143414441454146414741484149415041514152415341544155415641574158415941604161416241634164416541664167416841694170417141724173417441754176417741784179418041814182418341844185418641874188418941904191419241934194419541964197419841994200420142024203420442054206420742084209421042114212421342144215421642174218421942204221422242234224422542264227422842294230423142324233423442354236423742384239424042414242424342444245424642474248424942504251425242534254425542564257425842594260426142624263426442654266426742684269427042714272427342744275427642774278427942804281428242834284428542864287428842894290429142924293429442954296429742984299430043014302430343044305430643074308430943104311431243134314431543164317431843194320432143224323432443254326432743284329433043314332433343344335433643374338433943404341434243434344434543464347434843494350435143524353435443554356435743584359436043614362436343644365436643674368436943704371437243734374437543764377437843794380438143824383438443854386438743884389439043914392439343944395439643974398439944004401440244034404440544064407440844094410441144124413441444154416441744184419442044214422442344244425442644274428442944304431443244334434443544364437443844394440444144424443444444454446444744484449445044514452445344544455445644574458445944604461446244634464446544664467446844694470447144724473447444754476447744784479448044814482448344844485448644874488448944904491449244934494449544964497449844994500450145024503450445054506450745084509451045114512451345144515451645174518451945204521452245234524452545264527452845294530453145324533453445354536453745384539454045414542454345444545454645474548454945504551455245534554455545564557455845594560456145624563456445654566456745684569457045714572457345744575457645774578457945804581458245834584458545864587
  1. package main
  2. import (
  3. "crypto/sha256"
  4. "encoding/hex"
  5. "encoding/json"
  6. "errors"
  7. "fmt"
  8. "hash/crc32"
  9. "io"
  10. "io/fs"
  11. "math"
  12. "mime"
  13. "net/http"
  14. "net/url"
  15. "os"
  16. "sync"
  17. "sync/atomic"
  18. "path"
  19. "path/filepath"
  20. "runtime"
  21. "sort"
  22. "strconv"
  23. "strings"
  24. "time"
  25. "github.com/gorilla/websocket"
  26. uuid "github.com/satori/go.uuid"
  27. "imuslab.com/arozos/mod/compatibility"
  28. "imuslab.com/arozos/mod/filesystem"
  29. "imuslab.com/arozos/mod/filesystem/arozfs"
  30. fsp "imuslab.com/arozos/mod/filesystem/fspermission"
  31. "imuslab.com/arozos/mod/filesystem/fssort"
  32. "imuslab.com/arozos/mod/filesystem/fuzzy"
  33. hidden "imuslab.com/arozos/mod/filesystem/hidden"
  34. "imuslab.com/arozos/mod/filesystem/localversion"
  35. metadata "imuslab.com/arozos/mod/filesystem/metadata"
  36. "imuslab.com/arozos/mod/filesystem/shortcut"
  37. module "imuslab.com/arozos/mod/modules"
  38. prout "imuslab.com/arozos/mod/prouter"
  39. "imuslab.com/arozos/mod/share"
  40. "imuslab.com/arozos/mod/share/shareEntry"
  41. storage "imuslab.com/arozos/mod/storage"
  42. user "imuslab.com/arozos/mod/user"
  43. "imuslab.com/arozos/mod/utils"
  44. )
  45. var (
  46. thumbRenderHandler *metadata.RenderHandler
  47. shareEntryTable *shareEntry.ShareEntryTable
  48. shareManager *share.Manager
  49. wsConnectionStore sync.Map //Storage of all the file operation tasks, keyed by operation id
  50. fileOprTaskLock sync.RWMutex //Lock guarding the mutable fields of the task records above
  51. fileOprJanitorStarted atomic.Bool //Whether the finished task record janitor is already running
  52. )
  53. // Status of a file operation task or of one of the files inside it
  54. const (
  55. FsTask_Pending = "pending"
  56. FsTask_Ongoing = "ongoing"
  57. FsTask_Completed = "completed"
  58. FsTask_Error = "error"
  59. FsTask_Cancelled = "cancelled"
  60. //How long an operation that finished without error is kept before the
  61. //janitor drops it, in seconds. Long enough for a connected dialog to render
  62. //the result and fire its completion callback, short enough that the listing
  63. //does not fill up with finished transfers.
  64. fileOprFinishedRecordTTL = 3
  65. //How many failed operations are kept per user. Failures are never dropped on
  66. //a timer so they can still be reviewed after reopening the desktop from
  67. //another browser or machine, but the backlog is bounded.
  68. fileOprErrorRecordLimit = 32
  69. //How often the janitor sweeps the finished task records. Kept well under
  70. //fileOprFinishedRecordTTL so a record is actually dropped near its deadline
  71. //rather than up to a whole sweep later.
  72. fileOprJanitorInterval = 2 * time.Second
  73. )
  74. type trashedFile struct {
  75. Filename string
  76. Filepath string
  77. FileExt string
  78. IsDir bool
  79. Filesize int64
  80. RemoveTimestamp int64
  81. RemoveDate string
  82. OriginalPath string
  83. OriginalFilename string
  84. }
  85. // A single source file inside a file operation task
  86. type fileOperationSubtask struct {
  87. Filename string //Base name of this source file
  88. Src string //Virtual path of this source file
  89. IsDir bool //Whether this source is a folder
  90. Size int64 //Total size of this source file in bytes
  91. Done int64 //Bytes of this source file that are already processed
  92. Progress float64 //Progress of this source file, in percentage
  93. Status string //Status of this source file, see the FsTask_* constants
  94. Error string //Error message of this source file, if any
  95. }
  96. type fileOperationTask struct {
  97. ID string //Unique id for the task operation
  98. Owner string //Owner of the file opr
  99. Operation string //Type of the file opr: move / copy / zip / unzip
  100. Src string //Source folder for opr
  101. Dest string //Destination folder for opr
  102. Progress float64 //Progress for the operation
  103. LatestFile string //Latest file that is current transfering
  104. FileOperationSignal int //Current control signal of the file opr
  105. Files []*fileOperationSubtask //Per source file progress of this operation
  106. TotalSize int64 //Total size of all the source files in bytes
  107. DoneSize int64 //Total bytes processed so far
  108. StartTime int64 //Unix timestamp of when this operation started
  109. EndTime int64 //Unix timestamp of when this operation ended, 0 if still running
  110. Status string //Status of this operation, see the FsTask_* constants
  111. Error string //Error message of this operation, if any
  112. }
  113. func FileSystemInit() {
  114. router := prout.NewModuleRouter(prout.RouterOption{
  115. ModuleName: "File Manager",
  116. AdminOnly: false,
  117. UserHandler: userHandler,
  118. DeniedHandler: func(w http.ResponseWriter, r *http.Request) {
  119. utils.SendErrorResponse(w, "Permission Denied")
  120. },
  121. })
  122. //Upload related functions
  123. router.HandleFunc("/system/file_system/upload", system_fs_handleUpload)
  124. router.HandleFunc("/system/file_system/lowmemUpload", system_fs_handleLowMemoryUpload)
  125. //Other file operations
  126. router.HandleFunc("/system/file_system/validateFileOpr", system_fs_validateFileOpr)
  127. router.HandleFunc("/system/file_system/fileOpr", system_fs_handleOpr)
  128. router.HandleFunc("/system/file_system/ws/fileOpr", system_fs_handleWebSocketOpr)
  129. router.HandleFunc("/system/file_system/fileOprAsync", system_fs_handleAsyncOpr)
  130. router.HandleFunc("/system/file_system/ws/fileOprStatus", system_fs_handleFileOprStatusWebSocket)
  131. router.HandleFunc("/system/file_system/listDir", system_fs_handleList)
  132. router.HandleFunc("/system/file_system/listDirHash", system_fs_handleDirHash)
  133. router.HandleFunc("/system/file_system/listRoots", system_fs_listRoot)
  134. router.HandleFunc("/system/file_system/listDrives", system_fs_listDrives)
  135. router.HandleFunc("/system/file_system/newItem", system_fs_handleNewObjects)
  136. router.HandleFunc("/system/file_system/preference", system_fs_handleUserPreference)
  137. router.HandleFunc("/system/file_system/listTrash", system_fs_scanTrashBin)
  138. router.HandleFunc("/system/file_system/ws/listTrash", system_fs_WebSocketScanTrashBin)
  139. router.HandleFunc("/system/file_system/clearTrash", system_fs_clearTrashBin)
  140. router.HandleFunc("/system/file_system/trashSettings", system_fs_handleTrashSettings)
  141. router.HandleFunc("/system/file_system/restoreTrash", system_fs_restoreFile)
  142. router.HandleFunc("/system/file_system/zipHandler", system_fs_zipHandler)
  143. router.HandleFunc("/system/file_system/getProperties", system_fs_getFileProperties)
  144. router.HandleFunc("/system/file_system/getStorageInfo", system_fs_getStorageInfo)
  145. router.HandleFunc("/system/file_system/versionHistory", system_fs_FileVersionHistory)
  146. router.HandleFunc("/system/file_system/handleFilePermission", system_fs_handleFilePermission)
  147. router.HandleFunc("/system/file_system/search", system_fs_handleFileSearch)
  148. //Thumbnail caching functions
  149. router.HandleFunc("/system/file_system/handleFolderCache", system_fs_handleFolderCache)
  150. router.HandleFunc("/system/file_system/handleCacheRender", system_fs_handleCacheRender)
  151. router.HandleFunc("/system/file_system/loadThumbnail", system_fs_handleThumbnailLoad)
  152. //Directory specific config
  153. router.HandleFunc("/system/file_system/sortMode", system_fs_handleFolderSortModePreference)
  154. //Register the module
  155. moduleHandler.RegisterModule(module.ModuleInfo{
  156. Name: "File Manager",
  157. Group: "System Tools",
  158. IconPath: "SystemAO/file_system/img/small_icon.png",
  159. Version: "1.0",
  160. StartDir: "SystemAO/file_system/file_explorer.html",
  161. SupportFW: true,
  162. InitFWSize: []int{1075, 610},
  163. LaunchFWDir: "SystemAO/file_system/file_explorer.html",
  164. SupportEmb: false,
  165. })
  166. //Register the Trashbin module
  167. moduleHandler.RegisterModule(module.ModuleInfo{
  168. Name: "Trash Bin",
  169. Group: "System Tools",
  170. IconPath: "SystemAO/file_system/trashbin_img/small_icon.png",
  171. Version: "1.0",
  172. StartDir: "SystemAO/file_system/trashbin.html",
  173. SupportFW: true,
  174. InitFWSize: []int{400, 200},
  175. LaunchFWDir: "SystemAO/file_system/trashbin.html",
  176. SupportEmb: false,
  177. SupportedExt: []string{"*"},
  178. })
  179. //Register the Zip Extractor module
  180. moduleHandler.RegisterModule(module.ModuleInfo{
  181. Name: "Zip Extractor",
  182. Group: "System Tools",
  183. IconPath: "SystemAO/file_system/img/zip_extractor.png",
  184. Version: "1.0",
  185. SupportFW: false,
  186. LaunchEmb: "SystemAO/file_system/zip_extractor.html",
  187. SupportEmb: true,
  188. InitEmbSize: []int{260, 120},
  189. SupportedExt: []string{".zip"},
  190. })
  191. //Create user root if not exists
  192. err := os.MkdirAll(filepath.Join(*root_directory, "users/"), 0755)
  193. if err != nil {
  194. systemWideLogger.PrintAndLog("File System", "Failed to create system storage root", err)
  195. panic(err)
  196. }
  197. //Create database table if not exists
  198. err = sysdb.NewTable("fs")
  199. if err != nil {
  200. systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
  201. panic(err)
  202. }
  203. //Create new table for sort preference
  204. err = sysdb.NewTable("fs-sortpref")
  205. if err != nil {
  206. systemWideLogger.PrintAndLog("File System", "Failed to create table for file system", err)
  207. panic(err)
  208. }
  209. //Create a RenderHandler for caching thumbnails
  210. thumbRenderHandler = metadata.NewRenderHandler()
  211. //Thumbnails of drives that render them where the file is stored (e.g.
  212. //cluster:/) are kept on this host, outside the drive and outside tmp:/
  213. //(whose files are cleared after a day)
  214. metadata.SetExternalCacheDir(filepath.Join("system", "cache", "thumbnails"))
  215. nightlyManager.RegisterNightlyTask(func() {
  216. metadata.PruneExternalCache(thumbnailCacheMaxAge)
  217. })
  218. /*
  219. Share Related Registering
  220. This section of functions create and register the file share service
  221. for the arozos
  222. */
  223. //Create a share manager to handle user file sharae
  224. shareEntryTable = shareEntry.NewShareEntryTable(sysdb)
  225. shareManager = share.NewShareManager(share.Options{
  226. AuthAgent: authAgent,
  227. ShareEntryTable: shareEntryTable,
  228. UserHandler: userHandler,
  229. HostName: *host_name,
  230. TmpFolder: *tmp_directory,
  231. //Resolved per call, so it works although notificationInit runs later
  232. NotificationSender: sendUserNotification,
  233. })
  234. //Share management API (/system/file_system/share/*), behind the File Manager permission
  235. shareManager.RegisterAPIEndpoints(router)
  236. //Handle the main share function
  237. //Share function is now routed by the main router
  238. //http.HandleFunc("/share", shareManager.HandleShareAccess)
  239. /*
  240. File Operation Resume Functions
  241. */
  242. //Create a sync map for file operation opened connections
  243. wsConnectionStore = sync.Map{}
  244. router.HandleFunc("/system/file_system/ongoing", system_fs_HandleOnGoingTasks)
  245. /*
  246. Nighly Tasks
  247. These functions allow file system to clear and maintain
  248. the arozos file system when no one is using the system
  249. */
  250. //Trash bin retention and size limit, per user
  251. registerSetting(settingModule{
  252. Name: "File Manager",
  253. Desc: "Trash Bin Retention and Size",
  254. IconPath: "SystemAO/file_system/trashbin_img/small_icon.png",
  255. Group: "Disk",
  256. StartDir: "SystemAO/disk/filemanager/trashsettings.html",
  257. })
  258. //Clear tmp folder if files is placed here too long
  259. nightlyManager.RegisterNightlyTask(system_fs_clearOldTmpFiles)
  260. //Clear shares that have expired or whose parent file no longer exists in the system
  261. shareManager.ValidateAndClearShares()
  262. nightlyManager.RegisterNightlyTask(shareManager.ValidateAndClearShares)
  263. //Clear file version history that is more than 30 days
  264. go func() {
  265. //Start version history cleaning in background
  266. system_fs_clearVersionHistories()
  267. systemWideLogger.PrintAndLog("File System", "Startup File Version History Cleaning Completed", nil)
  268. }()
  269. systemWideLogger.PrintAndLog("File System", "Started File Version History Cleaning in background", nil)
  270. nightlyManager.RegisterNightlyTask(system_fs_clearVersionHistories)
  271. //Purge trashed files older than each user's retention setting
  272. nightlyManager.RegisterNightlyTask(system_fs_clearExpiredTrash)
  273. }
  274. /*
  275. File Search
  276. Handle file search in wildcard and recursive search
  277. */
  278. func system_fs_handleFileSearch(w http.ResponseWriter, r *http.Request) {
  279. //Get the user information
  280. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  281. if err != nil {
  282. utils.SendErrorResponse(w, "User not logged in")
  283. return
  284. }
  285. //Get the search target root path
  286. vpath, err := utils.PostPara(r, "path")
  287. if err != nil {
  288. utils.SendErrorResponse(w, "Invalid vpath given")
  289. return
  290. }
  291. keyword, err := utils.PostPara(r, "keyword")
  292. if err != nil {
  293. utils.SendErrorResponse(w, "Invalid keyword given")
  294. return
  295. }
  296. //Check if case sensitive is enabled
  297. casesensitve, _ := utils.PostPara(r, "casesensitive")
  298. vrootID, _, err := filesystem.GetIDFromVirtualPath(vpath)
  299. var targetFSH *filesystem.FileSystemHandler = nil
  300. if err != nil {
  301. utils.SendErrorResponse(w, "Invalid path given")
  302. return
  303. }
  304. targetFSH, err = GetFsHandlerByUUID(vrootID)
  305. if err != nil {
  306. utils.SendErrorResponse(w, err.Error())
  307. return
  308. }
  309. //Translate the vpath to realpath if this is an actual path on disk
  310. resolvedPath, err := targetFSH.FileSystemAbstraction.VirtualPathToRealPath(vpath, userinfo.Username)
  311. if err != nil {
  312. utils.SendErrorResponse(w, "Invalid path given")
  313. return
  314. }
  315. rpath := resolvedPath
  316. //Check if the search mode is recursive keyword or wildcard
  317. if len(keyword) > 1 && keyword[:1] == "/" {
  318. //Wildcard
  319. //Updates 31-12-2021: Do not allow wildcard search on virtual type's FSH
  320. if targetFSH == nil {
  321. utils.SendErrorResponse(w, "Invalid path given")
  322. return
  323. }
  324. targetFshAbs := targetFSH.FileSystemAbstraction
  325. wildcard := keyword[1:]
  326. matchingFiles, err := targetFshAbs.Glob(filepath.Join(rpath, wildcard))
  327. if err != nil {
  328. utils.SendErrorResponse(w, err.Error())
  329. return
  330. }
  331. //Prepare result struct
  332. results := []filesystem.FileData{}
  333. escaped := false
  334. for _, matchedFile := range matchingFiles {
  335. thisVpath, _ := targetFSH.FileSystemAbstraction.RealPathToVirtualPath(matchedFile, userinfo.Username)
  336. isHidden, _ := hidden.IsHidden(thisVpath, true)
  337. if !isHidden {
  338. results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, matchedFile, 2))
  339. }
  340. }
  341. if escaped {
  342. utils.SendErrorResponse(w, "Search keywords contain escape character!")
  343. return
  344. }
  345. //OK. Tidy up the results
  346. js, _ := json.Marshal(results)
  347. utils.SendJSONResponse(w, string(js))
  348. } else {
  349. //Updates 2022-02-16: Build the fuzzy matcher if it is not a wildcard search
  350. matcher := fuzzy.NewFuzzyMatcher(keyword, casesensitve == "true")
  351. //Recursive keyword
  352. results := []filesystem.FileData{}
  353. var err error = nil
  354. fshAbs := targetFSH.FileSystemAbstraction
  355. err = fshAbs.Walk(rpath, func(path string, info os.FileInfo, err error) error {
  356. thisFilename := filepath.Base(path)
  357. if casesensitve != "true" {
  358. thisFilename = strings.ToLower(thisFilename)
  359. }
  360. if !filesystem.IsInsideHiddenFolder(path) {
  361. if matcher.Match(thisFilename) {
  362. //This is a matching file
  363. thisVpath, _ := fshAbs.RealPathToVirtualPath(path, userinfo.Username)
  364. results = append(results, filesystem.GetFileDataFromPath(targetFSH, thisVpath, path, 2))
  365. }
  366. }
  367. return nil
  368. })
  369. if err != nil {
  370. utils.SendErrorResponse(w, err.Error())
  371. return
  372. }
  373. //OK. Tidy up the results
  374. js, _ := json.Marshal(results)
  375. utils.SendJSONResponse(w, string(js))
  376. }
  377. }
  378. /*
  379. Handle low-memory upload operations
  380. This function is specailly designed to work with low memory devices
  381. (e.g. ZeroPi / Orange Pi Zero with 512MB RAM)
  382. Two cases
  383. 1. Not Buffer FS + Huge File
  384. => Write chunks to fsa + merge to fsa
  385. 2. Else
  386. => write chunks to tmp (via os package) + merge to fsa
  387. */
  388. /*
  389. Low memory (websocket) upload tunables
  390. uploadIdleTimeout is how long the server waits for the next chunk before it
  391. assumes the client is gone. A paused upload is deliberately exempt from it -
  392. the client is still there, it is just not sending - so uploadPauseTimeout
  393. takes over instead, to cover an upload someone paused and forgot about.
  394. uploadPauseCloseCode is a private-use websocket close code, so the client can
  395. tell "your pause expired" apart from an ordinary disconnect and offer retry.
  396. */
  397. const (
  398. uploadIdleTimeout = 300 * time.Second
  399. uploadPauseTimeout = 2 * time.Hour
  400. uploadPauseCloseCode = 4001
  401. )
  402. func system_fs_handleLowMemoryUpload(w http.ResponseWriter, r *http.Request) {
  403. //Get user info
  404. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  405. if err != nil {
  406. w.WriteHeader(http.StatusUnauthorized)
  407. w.Write([]byte("401 - Unauthorized"))
  408. return
  409. }
  410. //Get filename and upload path
  411. filename, err := utils.GetPara(r, "filename")
  412. if filename == "" || err != nil {
  413. w.WriteHeader(http.StatusInternalServerError)
  414. w.Write([]byte("500 - Invalid filename given"))
  415. return
  416. }
  417. //Get upload target directory
  418. uploadTarget, err := utils.GetPara(r, "path")
  419. if uploadTarget == "" || err != nil {
  420. w.WriteHeader(http.StatusInternalServerError)
  421. w.Write([]byte("500 - Invalid path given"))
  422. return
  423. }
  424. //Unescape the upload target path
  425. unescapedPath, err := url.PathUnescape(uploadTarget)
  426. if err != nil {
  427. unescapedPath = uploadTarget
  428. }
  429. //Check if the user can write to this folder
  430. if !userinfo.CanWrite(unescapedPath) {
  431. //No permission
  432. w.WriteHeader(http.StatusForbidden)
  433. w.Write([]byte("403 - Access Denied"))
  434. return
  435. }
  436. fsh, subpath, err := GetFSHandlerSubpathFromVpath(unescapedPath)
  437. if err != nil {
  438. w.WriteHeader(http.StatusInternalServerError)
  439. w.Write([]byte("500 - Path translation failed"))
  440. return
  441. }
  442. fshAbs := fsh.FileSystemAbstraction
  443. //Translate the upload target directory
  444. realUploadPath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  445. if err != nil {
  446. w.WriteHeader(http.StatusInternalServerError)
  447. w.Write([]byte("500 - Path translation failed"))
  448. return
  449. }
  450. //Check if it is huge file upload mode
  451. isHugeFile := false
  452. hugefile, _ := utils.GetPara(r, "hugefile")
  453. if hugefile == "true" && !fsh.RequireBuffer {
  454. //Huge file mode is only compatible with local file systems
  455. //For remote file system, use buffer to tmp then upload method
  456. isHugeFile = true
  457. }
  458. //Create destination folder if not exists
  459. targetUploadLocation := arozfs.ToSlash(filepath.Join(realUploadPath, filename))
  460. if !fshAbs.FileExists(realUploadPath) {
  461. fshAbs.MkdirAll(realUploadPath, 0755)
  462. }
  463. //Generate an UUID for this upload
  464. uploadUUID := uuid.NewV4().String()
  465. uploadFolder := filepath.Join(*tmp_directory, "uploads", uploadUUID)
  466. if isHugeFile {
  467. //Change to upload directly to target disk
  468. uploadFolder = filepath.Join(realUploadPath, ".metadata/.upload", uploadUUID)
  469. fshAbs.MkdirAll(uploadFolder, 0700)
  470. } else {
  471. os.MkdirAll(uploadFolder, 0700)
  472. }
  473. //Start websocket connection
  474. var upgrader = websocket.Upgrader{}
  475. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  476. c, err := upgrader.Upgrade(w, r, nil)
  477. if err != nil {
  478. systemWideLogger.PrintAndLog("File System", "Failed to upgrade websocket connection: ", err)
  479. w.WriteHeader(http.StatusInternalServerError)
  480. w.Write([]byte("500 WebSocket upgrade failed"))
  481. return
  482. }
  483. defer c.Close()
  484. //Handle WebSocket upload
  485. blockCounter := 0
  486. chunkName := []string{}
  487. /*
  488. Shared with the watchdog goroutine below, so these are read and written
  489. atomically. pausedSince is 0 while the upload is running, and holds the
  490. unix time the client paused at otherwise.
  491. */
  492. var lastChunkArrivalTime int64 = time.Now().Unix()
  493. var pausedSince int64 = 0
  494. //Setup a timeout listener, check if connection still active every 1 minute
  495. ticker := time.NewTicker(60 * time.Second)
  496. done := make(chan bool)
  497. go func() {
  498. for {
  499. select {
  500. case <-done:
  501. return
  502. case <-ticker.C:
  503. if pausedAt := atomic.LoadInt64(&pausedSince); pausedAt > 0 {
  504. //Paused: silence is expected here, so only the pause clock
  505. //applies. WriteControl is the one write that is safe to make
  506. //from this goroutine while the reader holds the connection.
  507. if time.Since(time.Unix(pausedAt, 0)) > uploadPauseTimeout {
  508. systemWideLogger.PrintAndLog("File System", "Upload paused for too long. Disconnecting.", errors.New("upload pause timeout"))
  509. c.WriteControl(websocket.CloseMessage,
  510. websocket.FormatCloseMessage(uploadPauseCloseCode, "upload pause timeout"),
  511. time.Now().Add(time.Second))
  512. time.Sleep(1 * time.Second)
  513. c.Close()
  514. return
  515. }
  516. continue
  517. }
  518. if time.Now().Unix()-atomic.LoadInt64(&lastChunkArrivalTime) > int64(uploadIdleTimeout.Seconds()) {
  519. //Already 5 minutes without new data arraival. Stop connection
  520. systemWideLogger.PrintAndLog("File System", "Upload WebSocket connection timeout. Disconnecting.", errors.New("websocket connection timeout"))
  521. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  522. time.Sleep(1 * time.Second)
  523. c.Close()
  524. return
  525. }
  526. }
  527. }
  528. }()
  529. totalFileSize := int64(0)
  530. // Full-file CRC32 hasher (IEEE polynomial, matches JS crc32Table implementation)
  531. fileCRC32Hasher := crc32.NewIEEE()
  532. // Per-chunk state machine:
  533. // The client sends a text metadata frame {"index":N,"checksum":"hex"} before each binary frame.
  534. var pendingChunkIndex int
  535. var pendingChunkChecksum string
  536. expectingBinary := false
  537. for {
  538. mt, message, err := c.ReadMessage()
  539. if err != nil {
  540. //Connection closed by client. Clear the tmp folder and exit
  541. systemWideLogger.PrintAndLog("File System", "Upload terminated by client. Cleaning tmp folder", err)
  542. //Clear the tmp folder
  543. time.Sleep(1 * time.Second)
  544. if isHugeFile {
  545. fshAbs.RemoveAll(uploadFolder)
  546. } else {
  547. os.RemoveAll(uploadFolder)
  548. }
  549. return
  550. }
  551. if mt == 1 {
  552. // Text frame – either a control message, chunk metadata or done signal
  553. textMsg := strings.TrimSpace(string(message))
  554. /*
  555. Control messages are checked before anything else, and regardless
  556. of expectingBinary: a pause can land after the metadata frame but
  557. before its binary payload, and the flow control below would
  558. otherwise mistake it for a malformed chunk header.
  559. A chunk header or done signal unmarshals into this struct with
  560. every field false, so this is safe to try first.
  561. */
  562. var ctrl struct {
  563. Pause bool `json:"pause"`
  564. Resume bool `json:"resume"`
  565. Ping bool `json:"ping"`
  566. }
  567. if jsonErr := json.Unmarshal([]byte(textMsg), &ctrl); jsonErr == nil &&
  568. (ctrl.Pause || ctrl.Resume || ctrl.Ping) {
  569. if ctrl.Pause {
  570. atomic.StoreInt64(&pausedSince, time.Now().Unix())
  571. } else if ctrl.Resume {
  572. atomic.StoreInt64(&pausedSince, 0)
  573. //Do not count the paused stretch as idle time
  574. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  575. }
  576. /*
  577. Any control message counts as the client still being there,
  578. so it also holds off the idle timeout. That matters when a
  579. pause is raised before this socket finished connecting: the
  580. pause frame is lost, but the heartbeat still arrives, and
  581. without this the upload would be reaped as idle anyway.
  582. Answering every control message is what puts traffic on the
  583. wire in both directions - a reverse proxy in front of us will
  584. drop a connection it sees nothing on.
  585. */
  586. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  587. c.WriteMessage(1, []byte(`{"pong":true}`))
  588. continue
  589. }
  590. if !expectingBinary {
  591. // Check if this is the done signal
  592. var doneSignal struct {
  593. Done bool `json:"done"`
  594. TotalChunks int `json:"totalChunks"`
  595. FileChecksum string `json:"fileChecksum"`
  596. }
  597. if jsonErr := json.Unmarshal([]byte(textMsg), &doneSignal); jsonErr == nil && doneSignal.Done {
  598. // Verify the full-file CRC32 before merging
  599. if doneSignal.FileChecksum != "" {
  600. computedSum := fileCRC32Hasher.Sum32()
  601. computedSumBytes := []byte{byte(computedSum >> 24), byte(computedSum >> 16), byte(computedSum >> 8), byte(computedSum)}
  602. computedHex := hex.EncodeToString(computedSumBytes)
  603. if doneSignal.FileChecksum != computedHex {
  604. systemWideLogger.PrintAndLog("File System", "Upload file checksum mismatch: client="+doneSignal.FileChecksum+" server="+computedHex, nil)
  605. c.WriteMessage(1, []byte(`{"error":"File integrity check failed: full-file checksum mismatch"}`))
  606. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  607. time.Sleep(1 * time.Second)
  608. c.Close()
  609. if isHugeFile {
  610. fshAbs.RemoveAll(uploadFolder)
  611. } else {
  612. os.RemoveAll(uploadFolder)
  613. }
  614. return
  615. }
  616. }
  617. // Checksum verified – proceed to merge
  618. break
  619. }
  620. // Parse as chunk metadata
  621. var meta struct {
  622. Index int `json:"index"`
  623. Checksum string `json:"checksum"`
  624. }
  625. if jsonErr := json.Unmarshal([]byte(textMsg), &meta); jsonErr != nil {
  626. systemWideLogger.PrintAndLog("File System", "Invalid chunk metadata received: "+textMsg, jsonErr)
  627. continue
  628. }
  629. pendingChunkIndex = meta.Index
  630. pendingChunkChecksum = meta.Checksum
  631. expectingBinary = true
  632. }
  633. } else if mt == 2 {
  634. // Binary frame – the chunk data that follows a metadata frame
  635. if !expectingBinary {
  636. systemWideLogger.PrintAndLog("File System", "Received binary chunk without preceding metadata, ignoring", nil)
  637. continue
  638. }
  639. expectingBinary = false
  640. /*
  641. Chunks are accepted strictly in index order, and "next" is sent
  642. once per accepted chunk - clients that keep several chunks in
  643. flight count those acknowledgements to know which chunks landed.
  644. Anything else is dropped without a reply:
  645. - index < blockCounter: a copy of a chunk already written, resent
  646. by the client after an acknowledgement timeout. Accepting it
  647. again would count its bytes twice in the size and full-file
  648. checksum.
  649. - index > blockCounter: sent ahead in a pipelined window before
  650. the chunk at blockCounter failed its CRC check. The client
  651. rewinds to the retry and sends it again.
  652. */
  653. if pendingChunkIndex != blockCounter {
  654. continue
  655. }
  656. // Verify chunk CRC32
  657. chunkSum := crc32.ChecksumIEEE(message)
  658. chunkSumBytes := []byte{byte(chunkSum >> 24), byte(chunkSum >> 16), byte(chunkSum >> 8), byte(chunkSum)}
  659. chunkHex := hex.EncodeToString(chunkSumBytes)
  660. if pendingChunkChecksum != "" && pendingChunkChecksum != chunkHex {
  661. // CRC32 mismatch – ask the client to re-send this chunk
  662. systemWideLogger.PrintAndLog("File System", "Chunk "+strconv.Itoa(pendingChunkIndex)+" CRC32 mismatch: expected "+pendingChunkChecksum+" got "+chunkHex, nil)
  663. retryMsg, _ := json.Marshal(map[string]int{"retryChunk": pendingChunkIndex})
  664. c.WriteMessage(1, retryMsg)
  665. // Reset state; client will re-send the metadata+binary for this chunk
  666. continue
  667. }
  668. // Chunk verified – write to tmp folder
  669. chunkFilepath := filepath.Join(uploadFolder, "upld_"+strconv.Itoa(pendingChunkIndex))
  670. chunkName = append(chunkName, chunkFilepath)
  671. blockCounter++
  672. var writeErr error
  673. if isHugeFile {
  674. writeErr = fshAbs.WriteFile(chunkFilepath, message, 0700)
  675. } else {
  676. writeErr = os.WriteFile(chunkFilepath, message, 0700)
  677. }
  678. if writeErr != nil {
  679. systemWideLogger.PrintAndLog("File System", "Upload chunk write failed: "+writeErr.Error(), writeErr)
  680. c.WriteMessage(1, []byte(`{"error":"Write file chunk to disk failed"}`))
  681. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  682. time.Sleep(1 * time.Second)
  683. c.Close()
  684. if isHugeFile {
  685. fshAbs.RemoveAll(uploadFolder)
  686. } else {
  687. os.RemoveAll(uploadFolder)
  688. }
  689. return
  690. }
  691. // Update running full-file CRC32 with the verified chunk data
  692. fileCRC32Hasher.Write(message)
  693. // Update timing and quota tracking
  694. atomic.StoreInt64(&lastChunkArrivalTime, time.Now().Unix())
  695. totalFileSize += int64(len(message))
  696. if totalFileSize > max_upload_size {
  697. c.WriteMessage(1, []byte(`{"error":"File size too large"}`))
  698. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  699. time.Sleep(1 * time.Second)
  700. c.Close()
  701. if isHugeFile {
  702. fshAbs.RemoveAll(uploadFolder)
  703. } else {
  704. os.RemoveAll(uploadFolder)
  705. }
  706. return
  707. } else if !userinfo.StorageQuota.HaveSpace(totalFileSize) {
  708. c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
  709. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  710. time.Sleep(1 * time.Second)
  711. c.Close()
  712. if isHugeFile {
  713. fshAbs.RemoveAll(uploadFolder)
  714. } else {
  715. os.RemoveAll(uploadFolder)
  716. }
  717. return
  718. }
  719. // Acknowledge the chunk; client will send the next metadata+binary pair
  720. c.WriteMessage(1, []byte("next"))
  721. }
  722. }
  723. //Try to decode the location if possible
  724. decodedUploadLocation, err := url.PathUnescape(targetUploadLocation)
  725. if err != nil {
  726. decodedUploadLocation = targetUploadLocation
  727. }
  728. //Do not allow % sign in filename. Replace all with underscore
  729. decodedUploadLocation = strings.ReplaceAll(decodedUploadLocation, "%", "_")
  730. //Merge the file. Merge file location must be on local machine
  731. mergeFileLocation := decodedUploadLocation
  732. var out arozfs.File
  733. if fsh.RequireBuffer {
  734. //The merge file location must be local buffer
  735. mergeFileLocation = getFsBufferFilepath(decodedUploadLocation, false)
  736. out, err = os.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
  737. } else {
  738. //The merge file location can be local or remote that support OpenFile.
  739. out, err = fshAbs.OpenFile(mergeFileLocation, os.O_CREATE|os.O_WRONLY, 0755)
  740. }
  741. defer out.Close()
  742. if err != nil {
  743. systemWideLogger.PrintAndLog("File System", "Failed to open file:"+err.Error(), err)
  744. c.WriteMessage(1, []byte(`{"error":"Failed to open destination file"}`))
  745. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  746. time.Sleep(1 * time.Second)
  747. c.Close()
  748. return
  749. }
  750. for counter, filesrc := range chunkName {
  751. var srcChunkReader arozfs.File
  752. if isHugeFile {
  753. srcChunkReader, err = fshAbs.Open(filesrc)
  754. } else {
  755. srcChunkReader, err = os.Open(filesrc)
  756. }
  757. if err != nil {
  758. systemWideLogger.PrintAndLog("File System", "Failed to open Source Chunk"+filesrc+" with error "+err.Error(), err)
  759. c.WriteMessage(1, []byte(`{"error":"Failed to open Source Chunk"}`))
  760. return
  761. }
  762. io.Copy(out, srcChunkReader)
  763. srcChunkReader.Close()
  764. //Delete file immediately to save space
  765. if isHugeFile {
  766. fshAbs.Remove(filesrc)
  767. } else {
  768. os.Remove(filesrc)
  769. }
  770. //Write to websocket for the percentage of upload is written fro tmp to dest
  771. moveProg := strconv.Itoa(int(math.Round(float64(counter)/float64(len(chunkName))*100))) + "%"
  772. c.WriteMessage(1, []byte(`{"move":"`+moveProg+`"}`))
  773. }
  774. out.Close()
  775. //Check if the size fit in user quota
  776. var fi fs.FileInfo
  777. if fsh.RequireBuffer {
  778. fi, err = os.Stat(mergeFileLocation)
  779. } else {
  780. fi, err = fshAbs.Stat(mergeFileLocation)
  781. }
  782. if err != nil {
  783. // Could not obtain stat, handle error
  784. systemWideLogger.PrintAndLog("File System", "Failed to validate uploaded file: "+mergeFileLocation+". Error Message: "+err.Error(), err)
  785. c.WriteMessage(1, []byte(`{"error":"Failed to validate uploaded file"}`))
  786. return
  787. }
  788. if !userinfo.StorageQuota.HaveSpace(fi.Size()) {
  789. c.WriteMessage(1, []byte(`{"error":"User Storage Quota Exceeded"}`))
  790. if fsh.RequireBuffer {
  791. os.RemoveAll(mergeFileLocation)
  792. } else {
  793. fshAbs.RemoveAll(mergeFileLocation)
  794. }
  795. return
  796. }
  797. //Upload it to remote side if it fits the user quota && is buffer file
  798. if fsh.RequireBuffer {
  799. //This is local buffer file. Upload to dest fsh
  800. f, err := os.Open(mergeFileLocation)
  801. if err != nil {
  802. systemWideLogger.PrintAndLog("File System", "Failed to open buffered file at "+mergeFileLocation+" with error "+err.Error(), err)
  803. c.WriteMessage(1, []byte(`{"error":"Failed to open buffered object"}`))
  804. f.Close()
  805. return
  806. }
  807. err = fsh.FileSystemAbstraction.WriteStream(decodedUploadLocation, f, 0775)
  808. if err != nil {
  809. systemWideLogger.PrintAndLog("File System", "Failed to write to file system: "+fsh.UUID+" with error "+err.Error(), err)
  810. c.WriteMessage(1, []byte(`{"error":"Failed to upload to remote file system"}`))
  811. f.Close()
  812. return
  813. }
  814. //Remove the buffered file
  815. f.Close()
  816. os.Remove(mergeFileLocation)
  817. }
  818. //Log the upload filename
  819. //systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+filepath.Base(decodedUploadLocation), nil)
  820. //Set owner of the new uploaded file
  821. userinfo.SetOwnerOfFile(fsh, unescapedPath)
  822. //Return complete signal
  823. c.WriteMessage(1, []byte("OK"))
  824. //Stop the timeout listner
  825. done <- true
  826. //Clear the tmp folder
  827. time.Sleep(300 * time.Millisecond)
  828. if isHugeFile {
  829. fshAbs.RemoveAll(uploadFolder)
  830. } else {
  831. os.RemoveAll(uploadFolder)
  832. }
  833. //Close WebSocket connection after finished
  834. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  835. time.Sleep(300 * time.Second)
  836. c.Close()
  837. }
  838. /*
  839. Handle FORM POST based upload
  840. This function is design for general SBCs or computers with more than 2GB of RAM
  841. (e.g. Raspberry Pi 4 / Linux Server)
  842. */
  843. func system_fs_handleUpload(w http.ResponseWriter, r *http.Request) {
  844. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  845. if err != nil {
  846. utils.SendErrorResponse(w, "User not logged in")
  847. return
  848. }
  849. //Limit the max upload size to the user defined size
  850. if max_upload_size != 0 {
  851. r.Body = http.MaxBytesReader(w, r.Body, max_upload_size)
  852. }
  853. err = r.ParseMultipartForm(int64(*upload_buf) << 20)
  854. if err != nil {
  855. //Filesize too big
  856. systemWideLogger.PrintAndLog("File System", "Upload file size too big", err)
  857. utils.SendErrorResponse(w, "File too large")
  858. return
  859. }
  860. file, handler, err := r.FormFile("file")
  861. if err != nil {
  862. systemWideLogger.PrintAndLog("File System", "Error Retrieving File from upload by user: "+userinfo.Username, err)
  863. utils.SendErrorResponse(w, "Unable to parse file from upload")
  864. return
  865. }
  866. //Get upload target directory
  867. uploadTarget, _ := utils.PostPara(r, "path")
  868. if uploadTarget == "" {
  869. utils.SendErrorResponse(w, "Upload target cannot be empty.")
  870. return
  871. }
  872. fsh, subpath, err := GetFSHandlerSubpathFromVpath(uploadTarget)
  873. if err != nil {
  874. utils.SendErrorResponse(w, "Invalid upload target")
  875. return
  876. }
  877. targetFs := fsh.FileSystemAbstraction
  878. //Translate the upload target directory
  879. realUploadPath, err := targetFs.VirtualPathToRealPath(subpath, userinfo.Username)
  880. if err != nil {
  881. utils.SendErrorResponse(w, "Upload target is invalid or permission denied.")
  882. return
  883. }
  884. storeFilename := handler.Filename //Filename of the uploaded file
  885. //Get request time
  886. uploadStartTime := time.Now().UnixNano() / int64(time.Millisecond)
  887. //Update for Firefox 94.0.2 (x64) -> Now firefox put its relative path inside Content-Disposition -> filename
  888. //Skip this handler logic if Firefox version is in between 84.0.2 to 94.0.2
  889. bypassMetaCheck := compatibility.FirefoxBrowserVersionForBypassUploadMetaHeaderCheck(r.UserAgent())
  890. if !bypassMetaCheck && strings.Contains(handler.Header["Content-Disposition"][0], "filename=") && strings.Contains(handler.Header["Content-Disposition"][0], "/") {
  891. //This is a firefox MIME Header for file inside folder. Look for the actual filename
  892. headerFields := strings.Split(handler.Header["Content-Disposition"][0], "; ")
  893. possibleRelativePathname := ""
  894. for _, hf := range headerFields {
  895. if strings.Contains(hf, "filename=") && len(hf) > 11 {
  896. //Found. Overwrite original filename with the latest one
  897. possibleRelativePathname = hf[10 : len(hf)-1]
  898. storeFilename = possibleRelativePathname
  899. break
  900. }
  901. }
  902. }
  903. destFilepath := arozfs.ToSlash(filepath.Join(realUploadPath, storeFilename))
  904. //fmt.Println(destFilepath, realUploadPath, storeFilename)
  905. if !targetFs.FileExists(filepath.Dir(destFilepath)) {
  906. targetFs.MkdirAll(filepath.Dir(destFilepath), 0775)
  907. }
  908. //Check if the upload target is read only.
  909. accmode := userinfo.GetPathAccessPermission(uploadTarget)
  910. if accmode == arozfs.FsReadOnly {
  911. utils.SendErrorResponse(w, "The upload target is Read Only.")
  912. return
  913. } else if accmode == arozfs.FsDenied {
  914. utils.SendErrorResponse(w, "Access Denied")
  915. return
  916. }
  917. //Check for storage quota
  918. uploadFileSize := handler.Size
  919. if !userinfo.StorageQuota.HaveSpace(uploadFileSize) {
  920. utils.SendErrorResponse(w, "User Storage Quota Exceeded")
  921. return
  922. }
  923. //Do not allow % sign in filename. Replace all with underscore
  924. destFilepath = strings.ReplaceAll(destFilepath, "%", "_")
  925. //Move the file to destination file location
  926. if *enable_asyncFileUpload {
  927. //Use Async upload method
  928. systemWideLogger.PrintAndLog("File System", "AsyncFileUpload flag has been deprecated. Falling back to blocking upload.", errors.New("call to deprecated flag: asyncFileUpload"))
  929. }
  930. err = targetFs.WriteStream(destFilepath, file, 0775)
  931. if err != nil {
  932. systemWideLogger.PrintAndLog("File System", "Write stream to destination file system abstraction from upload failed", err)
  933. utils.SendErrorResponse(w, "Write upload to destination disk failed")
  934. return
  935. }
  936. file.Close()
  937. //Clear up buffered files
  938. r.MultipartForm.RemoveAll()
  939. //Set the ownership of file
  940. userinfo.SetOwnerOfFile(fsh, uploadTarget)
  941. //Finish up the upload
  942. /*
  943. fmt.Printf("Uploaded File: %+v\n", handler.Filename)
  944. fmt.Printf("File Size: %+v\n", handler.Size)
  945. fmt.Printf("MIME Header: %+v\n", handler.Header)
  946. fmt.Println("Upload target: " + realUploadPath)
  947. */
  948. //Fnish upload. Fix the tmp filename
  949. //systemWideLogger.PrintAndLog("File System", userinfo.Username+" uploaded a file: "+handler.Filename, nil)
  950. //Do upload finishing stuff
  951. //Add a delay to the complete message to make sure browser catch the return value
  952. currentTimeMilli := time.Now().UnixNano() / int64(time.Millisecond)
  953. if currentTimeMilli-uploadStartTime < 100 {
  954. //Sleep until at least 300 ms
  955. time.Sleep(time.Duration(100 - (currentTimeMilli - uploadStartTime)))
  956. }
  957. //Completed
  958. utils.SendOK(w)
  959. }
  960. // Validate if the copy and target process will involve file overwriting problem.
  961. func system_fs_validateFileOpr(w http.ResponseWriter, r *http.Request) {
  962. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  963. if err != nil {
  964. utils.SendErrorResponse(w, err.Error())
  965. return
  966. }
  967. vsrcFiles, _ := utils.PostPara(r, "src")
  968. vdestFile, _ := utils.PostPara(r, "dest")
  969. var duplicateFiles []string = []string{}
  970. //Loop through all files are see if there are duplication during copy and paste
  971. sourceFiles := []string{}
  972. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  973. err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  974. if err != nil {
  975. utils.SendErrorResponse(w, "Source file JSON parse error.")
  976. return
  977. }
  978. destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  979. if err != nil {
  980. utils.SendErrorResponse(w, "Operation Valid Failed: "+err.Error())
  981. return
  982. }
  983. rdestFile, _ := destFsh.FileSystemAbstraction.VirtualPathToRealPath(destSubpath, userinfo.Username)
  984. for _, file := range sourceFiles {
  985. srcFsh, srcSubpath, _ := GetFSHandlerSubpathFromVpath(string(file))
  986. rsrcFile, _ := srcFsh.FileSystemAbstraction.VirtualPathToRealPath(srcSubpath, userinfo.Username)
  987. if destFsh.FileSystemAbstraction.FileExists(filepath.Join(rdestFile, filepath.Base(rsrcFile))) {
  988. //File exists already.
  989. vpath, _ := srcFsh.FileSystemAbstraction.RealPathToVirtualPath(rsrcFile, userinfo.Username)
  990. duplicateFiles = append(duplicateFiles, vpath)
  991. }
  992. }
  993. jsonString, _ := json.Marshal(duplicateFiles)
  994. utils.SendJSONResponse(w, string(jsonString))
  995. }
  996. /*
  997. Trash bin settings
  998. Stored per user in the same preference table the File Manager already uses,
  999. so they follow the account rather than the browser.
  1000. Both settings use zero as "no limit", which is also what an account that has
  1001. never opened the settings page reads back:
  1002. trash/retentionDays 0 = never auto remove, otherwise 1..365
  1003. trash/quotaBytes 0 = unlimited
  1004. */
  1005. const (
  1006. trashRetentionPrefKey = "trash/retentionDays"
  1007. trashQuotaPrefKey = "trash/quotaBytes"
  1008. trashMaxRetentionDays = 365
  1009. //What an account that has never opened the settings page holds trash for.
  1010. //An explicitly stored zero still means "never remove" - only the absence
  1011. //of a stored value falls back to this.
  1012. trashDefaultRetentionDays = 30
  1013. )
  1014. func system_fs_getTrashRetentionDays(username string) int {
  1015. result := ""
  1016. err := sysdb.Read("fs", "pref/"+trashRetentionPrefKey+"/"+username, &result)
  1017. if err != nil {
  1018. //Never set for this account
  1019. return trashDefaultRetentionDays
  1020. }
  1021. days, err := strconv.Atoi(result)
  1022. if err != nil || days < 0 {
  1023. //Stored but unreadable - the default is a safer answer than "never"
  1024. return trashDefaultRetentionDays
  1025. }
  1026. if days > trashMaxRetentionDays {
  1027. days = trashMaxRetentionDays
  1028. }
  1029. return days
  1030. }
  1031. func system_fs_getTrashQuotaBytes(username string) int64 {
  1032. result := ""
  1033. err := sysdb.Read("fs", "pref/"+trashQuotaPrefKey+"/"+username, &result)
  1034. if err != nil {
  1035. return 0
  1036. }
  1037. quota, err := utils.StringToInt64(result)
  1038. if err != nil || quota < 0 {
  1039. return 0
  1040. }
  1041. return quota
  1042. }
  1043. /*
  1044. Space a single trashed entry takes up
  1045. A folder counts for everything inside it, not the zero bytes its own entry
  1046. reports - otherwise a user could fill the bin with folders and never reach
  1047. the quota. Hidden files are included because they were moved along with the
  1048. rest, and the trash itself lives inside a hidden folder.
  1049. */
  1050. func system_fs_getTrashEntrySize(fsh *filesystem.FileSystemHandler, rpath string) int64 {
  1051. if fsh.FileSystemAbstraction.IsDir(rpath) {
  1052. size, _ := fsh.GetDirctorySizeFromRealPath(rpath, true)
  1053. return size
  1054. }
  1055. return int64(fsh.FileSystemAbstraction.GetFileSize(rpath))
  1056. }
  1057. // Total bytes currently sitting in a user's trash across every file system
  1058. func system_fs_getTrashUsage(username string) int64 {
  1059. files, fshs, err := system_fs_listTrash(username)
  1060. if err != nil {
  1061. return 0
  1062. }
  1063. total := int64(0)
  1064. for c, file := range files {
  1065. total += system_fs_getTrashEntrySize(fshs[c], file)
  1066. }
  1067. return total
  1068. }
  1069. /*
  1070. Read or write the trash settings for the logged in user.
  1071. GET with no value -> {"RetentionDays":n,"QuotaBytes":n,"MaxRetentionDays":365,"UsedBytes":n}
  1072. POST retentionDays / quotaBytes -> stores them
  1073. */
  1074. func system_fs_handleTrashSettings(w http.ResponseWriter, r *http.Request) {
  1075. username, err := authAgent.GetUserName(w, r)
  1076. if err != nil {
  1077. utils.SendErrorResponse(w, "User not logged in")
  1078. return
  1079. }
  1080. retention, retentionSet := utils.PostPara(r, "retentionDays")
  1081. quota, quotaSet := utils.PostPara(r, "quotaBytes")
  1082. if retentionSet == nil && retention != "" {
  1083. days, err := strconv.Atoi(retention)
  1084. if err != nil || days < 0 || days > trashMaxRetentionDays {
  1085. utils.SendErrorResponse(w, "Invalid retention day given")
  1086. return
  1087. }
  1088. sysdb.Write("fs", "pref/"+trashRetentionPrefKey+"/"+username, strconv.Itoa(days))
  1089. }
  1090. if quotaSet == nil && quota != "" {
  1091. bytes, err := utils.StringToInt64(quota)
  1092. if err != nil || bytes < 0 {
  1093. utils.SendErrorResponse(w, "Invalid quota given")
  1094. return
  1095. }
  1096. sysdb.Write("fs", "pref/"+trashQuotaPrefKey+"/"+username, utils.Int64ToString(bytes))
  1097. }
  1098. type trashSettings struct {
  1099. RetentionDays int
  1100. QuotaBytes int64
  1101. MaxRetentionDays int
  1102. UsedBytes int64
  1103. }
  1104. js, _ := json.Marshal(trashSettings{
  1105. RetentionDays: system_fs_getTrashRetentionDays(username),
  1106. QuotaBytes: system_fs_getTrashQuotaBytes(username),
  1107. MaxRetentionDays: trashMaxRetentionDays,
  1108. UsedBytes: system_fs_getTrashUsage(username),
  1109. })
  1110. utils.SendJSONResponse(w, string(js))
  1111. }
  1112. /*
  1113. Nightly cleanup
  1114. Removes trashed files past their owner's retention window. Users who have
  1115. not set a retention (or set it to zero) keep their trash indefinitely, which
  1116. is the behaviour every account had before this setting existed.
  1117. */
  1118. func system_fs_clearExpiredTrash() {
  1119. for _, username := range authAgent.ListUsers() {
  1120. retentionDays := system_fs_getTrashRetentionDays(username)
  1121. if retentionDays <= 0 {
  1122. //Auto removal disabled for this user
  1123. continue
  1124. }
  1125. userinfo, err := userHandler.GetUserInfoFromUsername(username)
  1126. if err != nil {
  1127. continue
  1128. }
  1129. cutoff := time.Now().Unix() - int64(retentionDays)*86400
  1130. files, fshs, err := system_fs_listTrash(username)
  1131. if err != nil {
  1132. continue
  1133. }
  1134. removed := 0
  1135. for c, file := range files {
  1136. if !nightlyShouldMaintainFsh(fshs[c]) {
  1137. //Trash inside a drive shared by the cluster is emptied by the
  1138. //master node, so it is not purged once per member
  1139. continue
  1140. }
  1141. /*
  1142. The removal time is stored as the file extension when the file
  1143. was recycled. Anything without a parsable timestamp is left
  1144. alone rather than guessed at.
  1145. */
  1146. ext := filepath.Ext(file)
  1147. if len(ext) < 2 {
  1148. continue
  1149. }
  1150. timestamp, err := utils.StringToInt64(ext[1:])
  1151. if err != nil || timestamp > cutoff {
  1152. continue
  1153. }
  1154. fshAbs := fshs[c].FileSystemAbstraction
  1155. fileVpath, err := fshAbs.RealPathToVirtualPath(file, username)
  1156. if err == nil && userinfo.IsOwnerOfFile(fshs[c], fileVpath) {
  1157. userinfo.RemoveOwnershipFromFile(fshs[c], fileVpath)
  1158. }
  1159. fshAbs.RemoveAll(file)
  1160. //Drop the .trash folder too once nothing is left in it
  1161. system_fs_removeTrashDirIfEmpty(fshAbs, filepath.Dir(file))
  1162. removed++
  1163. }
  1164. if removed > 0 {
  1165. systemWideLogger.PrintAndLog("File System", "Removed "+strconv.Itoa(removed)+
  1166. " expired trash item(s) for user "+username, nil)
  1167. }
  1168. }
  1169. }
  1170. // Scan all directory and get trash file and send back results with WebSocket
  1171. func system_fs_WebSocketScanTrashBin(w http.ResponseWriter, r *http.Request) {
  1172. //Get and check user permission
  1173. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1174. if err != nil {
  1175. utils.SendErrorResponse(w, "User not logged in")
  1176. return
  1177. }
  1178. //Upgrade to websocket
  1179. var upgrader = websocket.Upgrader{}
  1180. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1181. c, err := upgrader.Upgrade(w, r, nil)
  1182. if err != nil {
  1183. w.WriteHeader(http.StatusInternalServerError)
  1184. w.Write([]byte("500 - " + err.Error()))
  1185. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1186. return
  1187. }
  1188. //Start Scanning
  1189. scanningRoots := []*filesystem.FileSystemHandler{}
  1190. //Get all roots to scan
  1191. for _, storage := range userinfo.GetAllFileSystemHandler() {
  1192. if storage.Hierarchy == "backup" {
  1193. //Skip this fsh
  1194. continue
  1195. }
  1196. scanningRoots = append(scanningRoots, storage)
  1197. }
  1198. for _, fsh := range scanningRoots {
  1199. thisFshAbs := fsh.FileSystemAbstraction
  1200. rootPath, err := thisFshAbs.VirtualPathToRealPath("", userinfo.Username)
  1201. if err != nil {
  1202. continue
  1203. }
  1204. err = thisFshAbs.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
  1205. if system_fs_isTrashEntry(path) {
  1206. //This is a trashbin dir.
  1207. file := path
  1208. //Parse the trashFile struct
  1209. timestamp := filepath.Ext(file)[1:]
  1210. originalName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(filepath.Base(file)))
  1211. originalExt := filepath.Ext(filepath.Base(originalName))
  1212. virtualFilepath, _ := thisFshAbs.RealPathToVirtualPath(file, userinfo.Username)
  1213. virtualOrgPath, _ := thisFshAbs.RealPathToVirtualPath(filepath.Dir(filepath.Dir(filepath.Dir(file))), userinfo.Username)
  1214. rawsize := thisFshAbs.GetFileSize(file)
  1215. timestampInt64, _ := utils.StringToInt64(timestamp)
  1216. removeTimeDate := time.Unix(timestampInt64, 0)
  1217. if thisFshAbs.IsDir(file) {
  1218. originalExt = ""
  1219. }
  1220. thisTrashFileObject := trashedFile{
  1221. Filename: filepath.Base(file),
  1222. Filepath: virtualFilepath,
  1223. FileExt: originalExt,
  1224. IsDir: thisFshAbs.IsDir(file),
  1225. Filesize: int64(rawsize),
  1226. RemoveTimestamp: timestampInt64,
  1227. RemoveDate: removeTimeDate.Format("2006-01-02 15:04:05"),
  1228. OriginalPath: virtualOrgPath,
  1229. OriginalFilename: originalName,
  1230. }
  1231. //Send out the result as JSON string
  1232. js, _ := json.Marshal(thisTrashFileObject)
  1233. err := c.WriteMessage(1, js)
  1234. if err != nil {
  1235. //Connection already closed
  1236. return err
  1237. }
  1238. }
  1239. return nil
  1240. })
  1241. if err != nil {
  1242. //Scan or client connection error (Connection closed?)
  1243. return
  1244. }
  1245. }
  1246. //Close connection after finished
  1247. c.Close()
  1248. }
  1249. // Scan all the directory and get trash files within the system
  1250. func system_fs_scanTrashBin(w http.ResponseWriter, r *http.Request) {
  1251. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1252. if err != nil {
  1253. utils.SendErrorResponse(w, err.Error())
  1254. return
  1255. }
  1256. username := userinfo.Username
  1257. results := []trashedFile{}
  1258. files, fshs, err := system_fs_listTrash(username)
  1259. if err != nil {
  1260. utils.SendErrorResponse(w, err.Error())
  1261. return
  1262. }
  1263. //Get information of each files and process it into results
  1264. for c, file := range files {
  1265. fsAbs := fshs[c].FileSystemAbstraction
  1266. timestamp := filepath.Ext(file)[1:]
  1267. originalName := strings.TrimSuffix(filepath.Base(file), filepath.Ext(filepath.Base(file)))
  1268. originalExt := filepath.Ext(filepath.Base(originalName))
  1269. virtualFilepath, _ := fsAbs.RealPathToVirtualPath(file, userinfo.Username)
  1270. virtualOrgPath, _ := fsAbs.RealPathToVirtualPath(filepath.Dir(filepath.Dir(filepath.Dir(file))), userinfo.Username)
  1271. rawsize := fsAbs.GetFileSize(file)
  1272. timestampInt64, _ := utils.StringToInt64(timestamp)
  1273. removeTimeDate := time.Unix(timestampInt64, 0)
  1274. if fsAbs.IsDir(file) {
  1275. originalExt = ""
  1276. }
  1277. results = append(results, trashedFile{
  1278. Filename: filepath.Base(file),
  1279. Filepath: virtualFilepath,
  1280. FileExt: originalExt,
  1281. IsDir: fsAbs.IsDir(file),
  1282. Filesize: int64(rawsize),
  1283. RemoveTimestamp: timestampInt64,
  1284. RemoveDate: removeTimeDate.Format("2006-01-02 15:04:05"),
  1285. OriginalPath: virtualOrgPath,
  1286. OriginalFilename: originalName,
  1287. })
  1288. }
  1289. //Sort the results by date, latest on top
  1290. sort.Slice(results[:], func(i, j int) bool {
  1291. return results[i].RemoveTimestamp > results[j].RemoveTimestamp
  1292. })
  1293. //Format and return the json results
  1294. jsonString, _ := json.Marshal(results)
  1295. utils.SendJSONResponse(w, string(jsonString))
  1296. }
  1297. // Restore a trashed file to its parent dir
  1298. func system_fs_restoreFile(w http.ResponseWriter, r *http.Request) {
  1299. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1300. if err != nil {
  1301. utils.SendErrorResponse(w, err.Error())
  1302. return
  1303. }
  1304. targetTrashedFile, err := utils.PostPara(r, "src")
  1305. if err != nil {
  1306. utils.SendErrorResponse(w, "Invalid src given")
  1307. return
  1308. }
  1309. fsh, subpath, err := GetFSHandlerSubpathFromVpath(targetTrashedFile)
  1310. if err != nil {
  1311. utils.SendErrorResponse(w, err.Error())
  1312. return
  1313. }
  1314. fshAbs := fsh.FileSystemAbstraction
  1315. //Translate it to realpath
  1316. realpath, _ := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1317. if !fshAbs.FileExists(realpath) {
  1318. utils.SendErrorResponse(w, "File not exists")
  1319. return
  1320. }
  1321. //Check if this is really a trashed file
  1322. if filepath.Base(filepath.Dir(realpath)) != ".trash" {
  1323. utils.SendErrorResponse(w, "File not in trashbin")
  1324. return
  1325. }
  1326. //OK to proceed.
  1327. originalFilename := strings.TrimSuffix(filepath.Base(realpath), filepath.Ext(filepath.Base(realpath)))
  1328. restoreFolderRoot := filepath.Dir(filepath.Dir(filepath.Dir(realpath)))
  1329. targetPath := filepath.ToSlash(filepath.Join(restoreFolderRoot, originalFilename))
  1330. //systemWideLogger.PrintAndLog("File System", (targetPath)
  1331. fshAbs.Rename(realpath, targetPath)
  1332. //Check if the parent dir has no more fileds. If yes, remove it
  1333. system_fs_removeTrashDirIfEmpty(fshAbs, filepath.Dir(realpath))
  1334. utils.SendOK(w)
  1335. }
  1336. // Clear all trashed file in the system
  1337. func system_fs_clearTrashBin(w http.ResponseWriter, r *http.Request) {
  1338. u, err := userHandler.GetUserInfoFromRequest(w, r)
  1339. if err != nil {
  1340. utils.SendErrorResponse(w, "User not logged in")
  1341. return
  1342. }
  1343. fileList, fshs, err := system_fs_listTrash(u.Username)
  1344. if err != nil {
  1345. utils.SendErrorResponse(w, "Unable to clear trash: "+err.Error())
  1346. return
  1347. }
  1348. //Get list success. Remove each of them.
  1349. for c, file := range fileList {
  1350. fileVpath, _ := fshs[c].FileSystemAbstraction.RealPathToVirtualPath(file, u.Username)
  1351. isOwner := u.IsOwnerOfFile(fshs[c], fileVpath)
  1352. if isOwner {
  1353. //This user own this system. Remove this file from his quota
  1354. u.RemoveOwnershipFromFile(fshs[c], fileVpath)
  1355. }
  1356. fshAbs := fshs[c].FileSystemAbstraction
  1357. fshAbs.RemoveAll(file)
  1358. //Check if its parent directory have no files. If yes, remove the dir itself as well.
  1359. system_fs_removeTrashDirIfEmpty(fshAbs, filepath.Dir(file))
  1360. }
  1361. utils.SendOK(w)
  1362. }
  1363. /*
  1364. Check if a path is a trashed item: an entry right inside a .trash folder whose
  1365. name ends with the .<unix time> it was removed at. Anything else in there,
  1366. like the .metadata folder a thumbnail of a trashed file leaves behind or a
  1367. .DS_Store, is not something the user put in the trash.
  1368. */
  1369. func system_fs_isTrashEntry(path string) bool {
  1370. if filepath.Base(filepath.Dir(path)) != ".trash" {
  1371. return false
  1372. }
  1373. ext := filepath.Ext(path)
  1374. if len(ext) < 2 {
  1375. return false
  1376. }
  1377. _, err := utils.StringToInt64(ext[1:])
  1378. return err == nil
  1379. }
  1380. // Remove a .trash folder (with the leftovers in it) once no trashed item is left inside
  1381. func system_fs_removeTrashDirIfEmpty(fshAbs filesystem.FileSystemAbstraction, trashDir string) {
  1382. entries, err := fshAbs.Glob(filepath.ToSlash(trashDir) + "/*")
  1383. if err != nil {
  1384. return
  1385. }
  1386. for _, entry := range entries {
  1387. if system_fs_isTrashEntry(entry) {
  1388. return
  1389. }
  1390. }
  1391. fshAbs.RemoveAll(trashDir)
  1392. }
  1393. // Get all trash in a string list
  1394. func system_fs_listTrash(username string) ([]string, []*filesystem.FileSystemHandler, error) {
  1395. userinfo, _ := userHandler.GetUserInfoFromUsername(username)
  1396. scanningRoots := []*filesystem.FileSystemHandler{}
  1397. //Get all roots to scan
  1398. for _, storage := range userinfo.GetAllFileSystemHandler() {
  1399. if storage.Hierarchy == "backup" {
  1400. //Skip this fsh
  1401. continue
  1402. }
  1403. scanningRoots = append(scanningRoots, storage)
  1404. }
  1405. files := []string{}
  1406. fshs := []*filesystem.FileSystemHandler{}
  1407. for _, thisFsh := range scanningRoots {
  1408. thisFshAbs := thisFsh.FileSystemAbstraction
  1409. rootPath, _ := thisFshAbs.VirtualPathToRealPath("", userinfo.Username)
  1410. err := thisFshAbs.Walk(rootPath, func(path string, info os.FileInfo, err error) error {
  1411. if system_fs_isTrashEntry(path) {
  1412. //This is a trashbin dir.
  1413. files = append(files, path)
  1414. fshs = append(fshs, thisFsh)
  1415. }
  1416. return nil
  1417. })
  1418. if err != nil {
  1419. continue
  1420. }
  1421. }
  1422. return files, fshs, nil
  1423. }
  1424. /*
  1425. Handle new file or folder functions
  1426. Required information
  1427. @type {folder / file}
  1428. @ext {any that is listed in the template folder}
  1429. if no paramter is passed in, default listing all the supported template file
  1430. */
  1431. func system_fs_handleNewObjects(w http.ResponseWriter, r *http.Request) {
  1432. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1433. if err != nil {
  1434. utils.SendErrorResponse(w, "User not logged in")
  1435. return
  1436. }
  1437. //Validate the token
  1438. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  1439. if !tokenValid {
  1440. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  1441. return
  1442. }
  1443. fileType, _ := utils.PostPara(r, "type") //File creation type, {file, folder}
  1444. vsrc, _ := utils.PostPara(r, "src") //Virtual file source folder, do not include filename
  1445. filename, _ := utils.PostPara(r, "filename") //Filename for the new file
  1446. if fileType == "" && filename == "" {
  1447. //List all the supported new filetype
  1448. if !filesystem.FileExists("system/newitem/") {
  1449. os.MkdirAll("system/newitem/", 0755)
  1450. }
  1451. type newItemObject struct {
  1452. Desc string
  1453. Ext string
  1454. }
  1455. var newItemList []newItemObject
  1456. newItemTemplate, _ := filepath.Glob("system/newitem/*")
  1457. for _, file := range newItemTemplate {
  1458. thisItem := new(newItemObject)
  1459. thisItem.Desc = strings.TrimSuffix(filepath.Base(file), filepath.Ext(file))
  1460. thisItem.Ext = filepath.Ext(file)[1:]
  1461. newItemList = append(newItemList, *thisItem)
  1462. }
  1463. jsonString, err := json.Marshal(newItemList)
  1464. if err != nil {
  1465. systemWideLogger.PrintAndLog("File System", "Unable to parse JSON string for new item list", err)
  1466. utils.SendErrorResponse(w, "Unable to parse new item list. See server log for more information.")
  1467. return
  1468. }
  1469. utils.SendJSONResponse(w, string(jsonString))
  1470. return
  1471. } else if fileType != "" && filename != "" {
  1472. if vsrc == "" {
  1473. utils.SendErrorResponse(w, "Missing paramter: 'src'")
  1474. return
  1475. }
  1476. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
  1477. if err != nil {
  1478. utils.SendErrorResponse(w, err.Error())
  1479. return
  1480. }
  1481. fshAbs := fsh.FileSystemAbstraction
  1482. //Translate the path to realpath
  1483. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1484. if err != nil {
  1485. utils.SendErrorResponse(w, "Invalid path given")
  1486. return
  1487. }
  1488. //Check if directory is readonly
  1489. accmode := userinfo.GetPathAccessPermission(vsrc)
  1490. if accmode == arozfs.FsReadOnly {
  1491. utils.SendErrorResponse(w, "This directory is Read Only")
  1492. return
  1493. } else if accmode == arozfs.FsDenied {
  1494. utils.SendErrorResponse(w, "Access Denied")
  1495. return
  1496. }
  1497. //Check if the filename contains web-unsafe characters
  1498. if !utils.FilenameIsWebSafe(filename) {
  1499. utils.SendErrorResponse(w, "Filename contains illegal characters")
  1500. return
  1501. }
  1502. //Check if the file already exists. If yes, fix its filename.
  1503. newfilePath := filepath.ToSlash(filepath.Join(rpath, filename))
  1504. switch fileType {
  1505. case "file":
  1506. for fshAbs.FileExists(newfilePath) {
  1507. utils.SendErrorResponse(w, "Given filename already exists")
  1508. return
  1509. }
  1510. ext := filepath.Ext(filename)
  1511. defaultFileCotent := []byte("")
  1512. if ext != "" {
  1513. templateFile, _ := fshAbs.Glob("system/newitem/*" + ext)
  1514. if len(templateFile) > 0 {
  1515. //Copy file from templateFile[0] to current dir with the given name
  1516. input, _ := os.ReadFile(templateFile[0])
  1517. defaultFileCotent = input
  1518. }
  1519. }
  1520. err = fshAbs.WriteFile(newfilePath, defaultFileCotent, 0775)
  1521. if err != nil {
  1522. systemWideLogger.PrintAndLog("File System", "Unable to create new file: "+err.Error(), err)
  1523. utils.SendErrorResponse(w, err.Error())
  1524. return
  1525. }
  1526. case "folder":
  1527. if fshAbs.FileExists(newfilePath) {
  1528. utils.SendErrorResponse(w, "Given folder already exists")
  1529. return
  1530. }
  1531. //Create the folder at target location
  1532. err := fshAbs.Mkdir(newfilePath, 0755)
  1533. if err != nil {
  1534. utils.SendErrorResponse(w, err.Error())
  1535. return
  1536. }
  1537. }
  1538. utils.SendOK(w)
  1539. } else {
  1540. utils.SendErrorResponse(w, "Missing paramter(s).")
  1541. return
  1542. }
  1543. }
  1544. /*
  1545. Handle file operations via WebSocket
  1546. This handler only handle zip, unzip, copy and move. Not other operations.
  1547. For other operations, please use the legacy handleOpr endpoint
  1548. The actual operation logic is shared with the asynchronous (background)
  1549. file operation endpoint, see runFileOperationTask below.
  1550. */
  1551. func system_fs_handleWebSocketOpr(w http.ResponseWriter, r *http.Request) {
  1552. //Get and check user permission
  1553. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1554. if err != nil {
  1555. utils.SendErrorResponse(w, "User not logged in")
  1556. return
  1557. }
  1558. operation, _ := utils.GetPara(r, "opr") //Accept copy and move
  1559. vsrcFiles, _ := utils.GetPara(r, "src")
  1560. vdestFile, _ := utils.GetPara(r, "dest")
  1561. existsOpr, _ := utils.GetPara(r, "existsresp")
  1562. sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
  1563. if err != nil {
  1564. utils.SendErrorResponse(w, err.Error())
  1565. return
  1566. }
  1567. //Permission checking
  1568. if !userinfo.CanWrite(vdestFile) {
  1569. systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
  1570. w.WriteHeader(http.StatusForbidden)
  1571. w.Write([]byte("403 - Access Denied"))
  1572. return
  1573. }
  1574. //Upgrade to websocket
  1575. var upgrader = websocket.Upgrader{}
  1576. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1577. c, err := upgrader.Upgrade(w, r, nil)
  1578. if err != nil {
  1579. w.WriteHeader(http.StatusInternalServerError)
  1580. w.Write([]byte("500 - " + err.Error()))
  1581. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1582. return
  1583. }
  1584. //Create the file operation task and remember it
  1585. task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)
  1586. //Send over the oprId for this file operation for tracking
  1587. time.Sleep(300 * time.Millisecond)
  1588. c.WriteMessage(1, []byte("{\"oprid\":\""+task.ID+"\"}"))
  1589. //Run the operation on this request goroutine and stream the progress back
  1590. runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, func(update fileOprProgressUpdate) {
  1591. js, _ := json.Marshal(update)
  1592. c.WriteMessage(1, js)
  1593. })
  1594. //This endpoint do not keep the finished record. Remove it right away.
  1595. wsConnectionStore.Delete(task.ID)
  1596. //Close WebSocket connection after finished
  1597. time.Sleep(1 * time.Second)
  1598. c.WriteControl(8, []byte{}, time.Now().Add(time.Second))
  1599. c.Close()
  1600. }
  1601. /*
  1602. Handle file operations as a background task
  1603. Unlike the WebSocket endpoint above, this endpoint returns the operation id
  1604. right away and let the operation run in the background. The caller can then
  1605. watch the progress of all of its operations through a single status
  1606. WebSocket, see system_fs_handleFileOprStatusWebSocket below.
  1607. */
  1608. func system_fs_handleAsyncOpr(w http.ResponseWriter, r *http.Request) {
  1609. //Get and check user permission
  1610. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1611. if err != nil {
  1612. utils.SendErrorResponse(w, "User not logged in")
  1613. return
  1614. }
  1615. //Validate the token
  1616. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  1617. if !tokenValid {
  1618. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  1619. return
  1620. }
  1621. operation, _ := utils.PostPara(r, "opr")
  1622. vsrcFiles, _ := utils.PostPara(r, "src")
  1623. vdestFile, _ := utils.PostPara(r, "dest")
  1624. existsOpr, _ := utils.PostPara(r, "existsresp")
  1625. sourceFiles, vdestFile, err := parseFileOperationRequest(operation, vsrcFiles, vdestFile)
  1626. if err != nil {
  1627. utils.SendErrorResponse(w, err.Error())
  1628. return
  1629. }
  1630. //Permission checking
  1631. if !userinfo.CanWrite(vdestFile) {
  1632. systemWideLogger.PrintAndLog("File System", "Access denied for "+userinfo.Username+" try to access "+vdestFile, nil)
  1633. utils.SendErrorResponse(w, "Access Denied")
  1634. return
  1635. }
  1636. //Create the task record and start it in the background
  1637. task := NewOngoingFileOperation(userinfo, operation, sourceFiles, vdestFile)
  1638. go runFileOperationTask(task, userinfo, sourceFiles, vdestFile, existsOpr, nil)
  1639. js, _ := json.Marshal(map[string]string{"oprid": task.ID})
  1640. utils.SendJSONResponse(w, string(js))
  1641. }
  1642. /*
  1643. Stream the status of all the file operations of the requesting user
  1644. One connection is enough for all the file operations of a user. The client
  1645. side file operation dialog opens this once and renders every task it reports.
  1646. Control commands can also be sent over this socket in the following format
  1647. {"cmd":"pause | continue | cancel | remove", "oprid":"<operation_id>"} or
  1648. {"cmd":"clear"} for removing all the finished records of this user.
  1649. */
  1650. func system_fs_handleFileOprStatusWebSocket(w http.ResponseWriter, r *http.Request) {
  1651. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1652. if err != nil {
  1653. utils.SendErrorResponse(w, "User not logged in")
  1654. return
  1655. }
  1656. var upgrader = websocket.Upgrader{}
  1657. upgrader.CheckOrigin = func(r *http.Request) bool { return true }
  1658. c, err := upgrader.Upgrade(w, r, nil)
  1659. if err != nil {
  1660. w.WriteHeader(http.StatusInternalServerError)
  1661. w.Write([]byte("500 - " + err.Error()))
  1662. systemWideLogger.PrintAndLog("System", fmt.Sprint("Websocket Upgrade Error:", err.Error()), nil)
  1663. return
  1664. }
  1665. defer c.Close()
  1666. //Listen for control commands on this socket until it is closed by the client
  1667. clientGone := make(chan bool, 1)
  1668. go func() {
  1669. type controlCommand struct {
  1670. Cmd string
  1671. Oprid string
  1672. }
  1673. for {
  1674. _, message, err := c.ReadMessage()
  1675. if err != nil {
  1676. clientGone <- true
  1677. return
  1678. }
  1679. cmd := controlCommand{}
  1680. if json.Unmarshal(message, &cmd) != nil {
  1681. continue
  1682. }
  1683. ApplyFileOperationControl(userinfo.Username, cmd.Cmd, cmd.Oprid)
  1684. }
  1685. }()
  1686. //Push the full task list of this user on every tick
  1687. ticker := time.NewTicker(500 * time.Millisecond)
  1688. defer ticker.Stop()
  1689. for {
  1690. js, err := MarshalFileOperationForUser(userinfo.Username, true)
  1691. if err == nil {
  1692. if c.WriteMessage(websocket.TextMessage, js) != nil {
  1693. return
  1694. }
  1695. }
  1696. select {
  1697. case <-clientGone:
  1698. return
  1699. case <-ticker.C:
  1700. }
  1701. }
  1702. }
  1703. /*
  1704. File operation task runner
  1705. These are the shared internals used by both the legacy WebSocket endpoint
  1706. and the background (async) file operation endpoint above.
  1707. */
  1708. // fileOprProgressUpdate is the progress payload of the legacy WebSocket file operation endpoint
  1709. type fileOprProgressUpdate struct {
  1710. LatestFile string
  1711. Progress int
  1712. StatusFlag int
  1713. Error string
  1714. }
  1715. // parseFileOperationRequest validates a file operation request and returns the
  1716. // decoded source file list and the decoded destination virtual path
  1717. func parseFileOperationRequest(operation string, vsrcFiles string, vdestFile string) ([]string, string, error) {
  1718. //Check if opr is supported
  1719. if operation != "move" && operation != "copy" && operation != "zip" && operation != "unzip" {
  1720. systemWideLogger.PrintAndLog("File System", "This file operation is not supported on the file operation endpoint. Received: "+operation, errors.New("operation not supported"))
  1721. return nil, "", errors.New("operation not supported on this endpoint")
  1722. }
  1723. //Decode the source file list
  1724. var sourceFiles []string
  1725. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  1726. err := json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  1727. if err != nil {
  1728. systemWideLogger.PrintAndLog("File System", "File operation source file JSON parse error", err)
  1729. return nil, "", errors.New("Source file JSON parse error.")
  1730. }
  1731. if len(sourceFiles) == 0 {
  1732. return nil, "", errors.New("No source file given")
  1733. }
  1734. //Bugged char filtering
  1735. tmp := []string{}
  1736. for _, src := range sourceFiles {
  1737. tmp = append(tmp, strings.ReplaceAll(src, "{{plug_sign}}", "+"))
  1738. }
  1739. sourceFiles = tmp
  1740. vdestFile = strings.ReplaceAll(vdestFile, "{{plug_sign}}", "+")
  1741. //Decode the target position
  1742. escapedVdest := system_fs_specialURIDecode(vdestFile)
  1743. vdestFile = escapedVdest
  1744. if vdestFile == "" {
  1745. return nil, "", errors.New("Undefined dest location")
  1746. }
  1747. //Make sure the destination is resolvable before doing anything else
  1748. _, _, err = GetFSHandlerSubpathFromVpath(vdestFile)
  1749. if err != nil {
  1750. return nil, "", err
  1751. }
  1752. return sourceFiles, vdestFile, nil
  1753. }
  1754. // runFileOperationTask executes the file operation described by the given task record.
  1755. // onUpdate is optional and, when given, is called on every progress update.
  1756. func runFileOperationTask(task *fileOperationTask, userinfo *user.User, sourceFiles []string, vdestFile string, existsOpr string, onUpdate func(fileOprProgressUpdate)) {
  1757. if existsOpr == "" {
  1758. existsOpr = "keep"
  1759. }
  1760. oprId := task.ID
  1761. operation := task.Operation
  1762. pushUpdate := func(update fileOprProgressUpdate) {
  1763. if onUpdate != nil {
  1764. onUpdate(update)
  1765. }
  1766. }
  1767. failTask := func(filename string, errmsg string) {
  1768. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, errmsg)
  1769. pushUpdate(fileOprProgressUpdate{
  1770. LatestFile: filename,
  1771. Progress: -1,
  1772. Error: errmsg,
  1773. StatusFlag: filesystem.FsOpr_Error,
  1774. })
  1775. }
  1776. //Resolve the destination file system handler
  1777. destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  1778. if err != nil {
  1779. failTask(filepath.Base(vdestFile), err.Error())
  1780. return
  1781. }
  1782. destFshAbs := destFsh.FileSystemAbstraction
  1783. rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1784. if !userinfo.CanWrite(vdestFile) {
  1785. failTask(filepath.Base(vdestFile), "Access Denied: No Write Permission")
  1786. return
  1787. }
  1788. if operation == "zip" {
  1789. //Zip files
  1790. outputFilename := filepath.Join(rdestFile, strings.ReplaceAll(filepath.Base(filepath.Dir(sourceFiles[0])+".zip"), ":", ""))
  1791. if len(sourceFiles) == 1 {
  1792. //Use the basename of the source file as zip file name
  1793. outputFilename = filepath.Join(rdestFile, filepath.Base(sourceFiles[0])) + ".zip"
  1794. }
  1795. //Translate source Files into real paths
  1796. realSourceFiles := []string{}
  1797. sourceFileFsh := []*filesystem.FileSystemHandler{}
  1798. for _, vsrcs := range sourceFiles {
  1799. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
  1800. if err != nil {
  1801. failTask(filepath.Base(vsrcs), "Source file not exists")
  1802. return
  1803. }
  1804. rsrc, err := thisSrcFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  1805. if err != nil {
  1806. failTask(filepath.Base(rsrc), "Source file not exists")
  1807. return
  1808. }
  1809. realSourceFiles = append(realSourceFiles, rsrc)
  1810. sourceFileFsh = append(sourceFileFsh, thisSrcFsh)
  1811. }
  1812. zipDestPath := outputFilename
  1813. zipDestFsh := destFsh
  1814. if destFsh.RequireBuffer {
  1815. zipDestPath = getFsBufferFilepath(outputFilename, false)
  1816. zipDestFsh = nil
  1817. }
  1818. //Create the zip file
  1819. err = filesystem.ArozZipFileWithProgress(sourceFileFsh, realSourceFiles, zipDestFsh, zipDestPath, false, func(currentFilename string, _ int, _ int, progress float64) int {
  1820. sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
  1821. pushUpdate(fileOprProgressUpdate{
  1822. LatestFile: currentFilename,
  1823. Progress: int(math.Ceil(progress)),
  1824. Error: "",
  1825. StatusFlag: sig,
  1826. })
  1827. return sig
  1828. })
  1829. if err != nil {
  1830. //The half written archive is already removed by the zipper
  1831. if destFsh.RequireBuffer {
  1832. cleanFsBufferFileFromList(realSourceFiles)
  1833. }
  1834. if !fileOperationCancelled(oprId) {
  1835. systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
  1836. failTask(filepath.Base(outputFilename), err.Error())
  1837. return
  1838. }
  1839. } else if destFsh.RequireBuffer {
  1840. //Move the buffer result to remote
  1841. f, _ := os.Open(zipDestPath)
  1842. err = destFshAbs.WriteStream(outputFilename, f, 0775)
  1843. if err != nil {
  1844. systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver"+destFsh.Filesystem+" failed", err)
  1845. }
  1846. f.Close()
  1847. //Clear local buffers
  1848. os.Remove(zipDestPath)
  1849. cleanFsBufferFileFromList(realSourceFiles)
  1850. }
  1851. } else if operation == "unzip" {
  1852. //Create the destination folder
  1853. destFshAbs.MkdirAll(rdestFile, 0755)
  1854. //Convert the src files into realpaths
  1855. realSourceFiles := []string{}
  1856. for _, vsrcs := range sourceFiles {
  1857. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcs)
  1858. if err != nil {
  1859. failTask(filepath.Base(vsrcs), "Source file not exists")
  1860. return
  1861. }
  1862. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  1863. rsrc, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1864. if err != nil {
  1865. failTask(filepath.Base(rsrc), "Source file not exists")
  1866. return
  1867. }
  1868. if thisSrcFsh.RequireBuffer {
  1869. localBufferFilepath, err := bufferRemoteFileToLocal(thisSrcFsh, rsrc, false)
  1870. if err != nil {
  1871. failTask(filepath.Base(rsrc), "Failed to buffer file to local disk")
  1872. return
  1873. }
  1874. realSourceFiles = append(realSourceFiles, localBufferFilepath)
  1875. } else {
  1876. realSourceFiles = append(realSourceFiles, rsrc)
  1877. }
  1878. }
  1879. unzipDest := rdestFile
  1880. if destFsh.RequireBuffer {
  1881. unzipDest = getFsBufferFilepath(rdestFile, true)
  1882. }
  1883. //Unzip the files
  1884. filesystem.ArozUnzipFileWithProgress(realSourceFiles, unzipDest, func(currentFile string, filecount int, totalfile int, progress float64) int {
  1885. //Generate the status update struct
  1886. sig, _ := UpdateOngoingFileOperation(oprId, filepath.Base(currentFile), math.Ceil(progress))
  1887. pushUpdate(fileOprProgressUpdate{
  1888. LatestFile: filepath.Base(currentFile),
  1889. Progress: int(math.Ceil(progress)),
  1890. Error: "",
  1891. StatusFlag: sig,
  1892. })
  1893. return sig
  1894. })
  1895. if destFsh.RequireBuffer {
  1896. //Push the unzip results back to remote fs
  1897. filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
  1898. path = filepath.ToSlash(path)
  1899. relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
  1900. if info.IsDir() {
  1901. destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
  1902. } else {
  1903. f, _ := os.Open(path)
  1904. destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
  1905. f.Close()
  1906. }
  1907. return nil
  1908. })
  1909. cleanFsBufferFileFromList([]string{unzipDest})
  1910. }
  1911. cleanFsBufferFileFromList(realSourceFiles)
  1912. } else {
  1913. //Other operations that allow multiple source files to handle one by one
  1914. for i := 0; i < len(sourceFiles); i++ {
  1915. vsrcFile := sourceFiles[i]
  1916. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  1917. if err != nil {
  1918. MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
  1919. failTask(filepath.Base(vsrcFile), "Source file not exists")
  1920. return
  1921. }
  1922. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  1923. rsrcFile, _ := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  1924. if !thisSrcFshAbs.FileExists(rsrcFile) {
  1925. //This source file not exists. Report Error and Stop
  1926. MarkFileOperationSubtaskEnded(oprId, i, "Source file not exists")
  1927. failTask(filepath.Base(rsrcFile), "Source file not exists")
  1928. return
  1929. }
  1930. //Progress handler shared by the move and copy operations. The overall
  1931. //progress is worked out from the bytes of every source file, so a
  1932. //small file no longer counts for as much of the bar as a large one.
  1933. subtaskProgressHandler := func(currentFile string, bytesDone int64, bytesTotal int64) int {
  1934. sig, overallProgress, _ := UpdateOngoingFileOperationSubtask(oprId, i, filepath.Base(currentFile), bytesDone, bytesTotal)
  1935. pushUpdate(fileOprProgressUpdate{
  1936. LatestFile: filepath.Base(currentFile),
  1937. Progress: int(math.Ceil(overallProgress)),
  1938. Error: "",
  1939. StatusFlag: sig,
  1940. })
  1941. return sig
  1942. }
  1943. if operation == "move" {
  1944. err := filesystem.FileMoveWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, subtaskProgressHandler)
  1945. //Handle move starting error
  1946. if err != nil {
  1947. MarkFileOperationSubtaskEnded(oprId, i, err.Error())
  1948. failTask(filepath.Base(rsrcFile), err.Error())
  1949. return
  1950. }
  1951. //Remove the cache for the original file
  1952. metadata.RemoveCache(thisSrcFsh, rsrcFile)
  1953. } else if operation == "copy" {
  1954. err := filesystem.FileCopyWithProgress(thisSrcFsh, rsrcFile, destFsh, rdestFile, existsOpr, subtaskProgressHandler)
  1955. //Handle Copy starting error
  1956. if err != nil {
  1957. MarkFileOperationSubtaskEnded(oprId, i, err.Error())
  1958. failTask(filepath.Base(rsrcFile), err.Error())
  1959. return
  1960. }
  1961. }
  1962. //This source file is done. Mark it as completed
  1963. MarkFileOperationSubtaskEnded(oprId, i, "")
  1964. }
  1965. }
  1966. //Check if the operation was cancelled by the user half way through
  1967. endingSignal := filesystem.FsOpr_Continue
  1968. if fileOperationCancelled(oprId) {
  1969. endingSignal = filesystem.FsOpr_Cancel
  1970. }
  1971. SetFileOperationTaskEnded(oprId, endingSignal, "")
  1972. }
  1973. // fileOperationCancelled reports whether the user asked to cancel the given operation
  1974. func fileOperationCancelled(oprId string) bool {
  1975. t, err := GetOngoingFileOperationByOprID(oprId)
  1976. if err != nil {
  1977. return false
  1978. }
  1979. fileOprTaskLock.RLock()
  1980. defer fileOprTaskLock.RUnlock()
  1981. return t.FileOperationSignal == filesystem.FsOpr_Cancel
  1982. }
  1983. /*
  1984. Handle file operations
  1985. Support {move, copy, delete, recycle, rename}
  1986. */
  1987. //Handle file operations.
  1988. func system_fs_handleOpr(w http.ResponseWriter, r *http.Request) {
  1989. //Check if user logged in
  1990. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  1991. if err != nil {
  1992. utils.SendErrorResponse(w, "User not logged in")
  1993. return
  1994. }
  1995. //Validate the token
  1996. tokenValid := CSRFTokenManager.HandleTokenValidation(w, r)
  1997. if !tokenValid {
  1998. http.Error(w, "Invalid CSRF token", http.StatusUnauthorized)
  1999. return
  2000. }
  2001. operation, _ := utils.PostPara(r, "opr")
  2002. vsrcFiles, _ := utils.PostPara(r, "src")
  2003. vdestFile, _ := utils.PostPara(r, "dest")
  2004. vnfilenames, _ := utils.PostPara(r, "new") //Only use when rename or create new file / folder
  2005. //Check if operation valid.
  2006. if operation == "" {
  2007. //Undefined operations.
  2008. utils.SendErrorResponse(w, "Undefined operations paramter: Missing 'opr' in request header.")
  2009. return
  2010. }
  2011. //As the user can pass in multiple source files at the same time, parse sourceFiles from json string
  2012. var sourceFiles []string
  2013. //This line is required in order to allow passing of special charaters
  2014. decodedSourceFiles := system_fs_specialURIDecode(vsrcFiles)
  2015. err = json.Unmarshal([]byte(decodedSourceFiles), &sourceFiles)
  2016. if err != nil {
  2017. utils.SendErrorResponse(w, "Source file JSON parse error.")
  2018. return
  2019. }
  2020. //Check if new filenames are also valid. If yes, translate it into string array
  2021. var newFilenames []string
  2022. if vnfilenames != "" {
  2023. vnfilenames, _ := url.QueryUnescape(vnfilenames)
  2024. err = json.Unmarshal([]byte(vnfilenames), &newFilenames)
  2025. if err != nil {
  2026. utils.SendErrorResponse(w, "Unable to parse JSON for new filenames")
  2027. return
  2028. }
  2029. }
  2030. if operation == "zip" {
  2031. //Zip operation. Parse the real filepath list
  2032. rsrcFiles := []string{}
  2033. srcFshs := []*filesystem.FileSystemHandler{}
  2034. destFsh, subpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  2035. if err != nil {
  2036. utils.SendErrorResponse(w, "Unable to resolve zip destination path")
  2037. return
  2038. }
  2039. destFshAbs := destFsh.FileSystemAbstraction
  2040. rdestFile, _ := destFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2041. for _, vsrcFile := range sourceFiles {
  2042. vsrcFsh, vsrcSubpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  2043. if err != nil {
  2044. continue
  2045. }
  2046. rsrcFile, _ := vsrcFsh.FileSystemAbstraction.VirtualPathToRealPath(vsrcSubpath, userinfo.Username)
  2047. if vsrcFsh.FileSystemAbstraction.FileExists(rsrcFile) {
  2048. //Push directly its local path to list
  2049. rsrcFiles = append(rsrcFiles, rsrcFile)
  2050. srcFshs = append(srcFshs, vsrcFsh)
  2051. }
  2052. }
  2053. zipFilename := rdestFile
  2054. if destFshAbs.IsDir(rdestFile) {
  2055. //Append the filename to it
  2056. if len(rsrcFiles) == 1 {
  2057. zipFilename = filepath.Join(rdestFile, strings.TrimSuffix(filepath.Base(rsrcFiles[0]), filepath.Ext(filepath.Base(rsrcFiles[0])))+".zip")
  2058. } else if len(rsrcFiles) > 1 {
  2059. zipFilename = filepath.Join(rdestFile, filepath.Base(filepath.Dir(rsrcFiles[0]))+".zip")
  2060. }
  2061. }
  2062. //Create a buffer if destination fsh request buffer
  2063. zipFileTargetLocation := zipFilename
  2064. zipDestFsh := destFsh
  2065. if destFsh.RequireBuffer {
  2066. zipFileTargetLocation = getFsBufferFilepath(zipFilename, false)
  2067. zipDestFsh = nil
  2068. }
  2069. //Create a zip file at target location
  2070. err = filesystem.ArozZipFile(srcFshs, rsrcFiles, zipDestFsh, zipFileTargetLocation, false)
  2071. if err != nil {
  2072. os.Remove(zipFileTargetLocation)
  2073. utils.SendErrorResponse(w, err.Error())
  2074. return
  2075. }
  2076. //Write it to final destination from buffer
  2077. if destFsh.RequireBuffer {
  2078. //Upload the finalized zip file
  2079. f, _ := destFshAbs.Open(zipFileTargetLocation)
  2080. destFshAbs.WriteStream(zipFilename, f, 0775)
  2081. f.Close()
  2082. //Remove all buff files
  2083. os.Remove(zipFileTargetLocation)
  2084. cleanFsBufferFileFromList(rsrcFiles)
  2085. }
  2086. } else {
  2087. //For operations that is handled file by file
  2088. for i, vsrcFile := range sourceFiles {
  2089. //Convert the virtual path to realpath on disk
  2090. srcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrcFile)
  2091. if err != nil {
  2092. continue
  2093. }
  2094. srcFshAbs := srcFsh.FileSystemAbstraction
  2095. rsrcFile, _ := srcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2096. destFsh, destSubpath, err := GetFSHandlerSubpathFromVpath(vdestFile)
  2097. var destFshAbs filesystem.FileSystemAbstraction = nil
  2098. var rdestFile string = ""
  2099. if err == nil {
  2100. destFshAbs = destFsh.FileSystemAbstraction
  2101. rdestFile, _ = destFshAbs.VirtualPathToRealPath(destSubpath, userinfo.Username)
  2102. }
  2103. //Check if the source file exists
  2104. if operation == "rename" {
  2105. //Check if the usage is correct.
  2106. if vdestFile != "" {
  2107. utils.SendErrorResponse(w, "Rename only accept 'src' and 'new'. Please use move if you want to move a file.")
  2108. return
  2109. }
  2110. //Check if new name paramter is passed in.
  2111. if len(newFilenames) == 0 {
  2112. utils.SendErrorResponse(w, "Missing paramter (JSON string): 'new'")
  2113. return
  2114. }
  2115. //Check if the source filenames and new filenanmes match
  2116. if len(newFilenames) != len(sourceFiles) {
  2117. utils.SendErrorResponse(w, "New filenames do not match with source filename's length.")
  2118. return
  2119. }
  2120. //Check if the target dir is not readonly
  2121. accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
  2122. switch accmode {
  2123. case arozfs.FsReadOnly:
  2124. utils.SendErrorResponse(w, "This directory is Read Only")
  2125. return
  2126. case arozfs.FsDenied:
  2127. utils.SendErrorResponse(w, "Access Denied")
  2128. return
  2129. }
  2130. thisFilename := filepath.Base(newFilenames[i])
  2131. //Check if the new filename contains web-unsafe characters
  2132. if !utils.FilenameIsWebSafe(thisFilename) {
  2133. utils.SendErrorResponse(w, "Filename contains illegal characters")
  2134. return
  2135. }
  2136. //Check if the name already exists. If yes, return false
  2137. if srcFshAbs.FileExists(filepath.Join(filepath.Dir(rsrcFile), thisFilename)) {
  2138. utils.SendErrorResponse(w, "File already exists")
  2139. return
  2140. }
  2141. //Everything is ok. Rename the file.
  2142. targetNewName := filepath.Join(filepath.Dir(rsrcFile), thisFilename)
  2143. err = srcFshAbs.Rename(rsrcFile, targetNewName)
  2144. if err != nil {
  2145. systemWideLogger.PrintAndLog("File System", "File rename failed", err)
  2146. utils.SendErrorResponse(w, err.Error())
  2147. return
  2148. }
  2149. //Remove the cache for the original file
  2150. metadata.RemoveCache(srcFsh, rsrcFile)
  2151. } else if operation == "move" {
  2152. //File move operation. Check if the source file / dir and target directory exists
  2153. /*
  2154. Example usage from file explorer
  2155. $.ajax({
  2156. type: 'POST',
  2157. url: `/system/file_system/fileOpr`,
  2158. data: {opr: "move" ,src: JSON.stringify(fileList), dest: targetDir},
  2159. success: function(data){
  2160. if (data.error !== undefined){
  2161. msgbox("remove",data.error);
  2162. }else{
  2163. //OK, do something
  2164. }
  2165. }
  2166. });
  2167. */
  2168. if !srcFshAbs.FileExists(rsrcFile) {
  2169. utils.SendErrorResponse(w, "Source file not exists")
  2170. return
  2171. }
  2172. //Check if the source file is read only.
  2173. accmode := userinfo.GetPathAccessPermission(string(vsrcFile))
  2174. if accmode == arozfs.FsReadOnly {
  2175. utils.SendErrorResponse(w, "This source file is Read Only")
  2176. return
  2177. } else if accmode == arozfs.FsDenied {
  2178. utils.SendErrorResponse(w, "Access Denied")
  2179. return
  2180. }
  2181. if rdestFile == "" {
  2182. utils.SendErrorResponse(w, "Undefined dest location")
  2183. return
  2184. }
  2185. //Get exists overwrite mode
  2186. existsOpr, _ := utils.PostPara(r, "existsresp")
  2187. //Check if use fast move instead
  2188. //Check if the source and destination folder are under the same root. If yes, use os.Rename for faster move operations
  2189. //Check if the two files are under the same user root path
  2190. //srcAbs, _ := filepath.Abs(rsrcFile)
  2191. //destAbs, _ := filepath.Abs(rdestFile)
  2192. //underSameRoot, _ := filesystem.UnderTheSameRoot(srcAbs, destAbs)
  2193. //Updates 19-10-2020: Added ownership management to file move and copy
  2194. userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
  2195. err = filesystem.FileMove(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, true, nil)
  2196. if err != nil {
  2197. utils.SendErrorResponse(w, err.Error())
  2198. //Restore the ownership if remove failed
  2199. userinfo.SetOwnerOfFile(srcFsh, vsrcFile)
  2200. return
  2201. }
  2202. //Set user to own the new file
  2203. newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
  2204. newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
  2205. userinfo.SetOwnerOfFile(destFsh, newfileVpath)
  2206. //Remove cache for the original file
  2207. metadata.RemoveCache(srcFsh, rsrcFile)
  2208. } else if operation == "copy" {
  2209. //Copy file. See move example and change 'opr' to 'copy'
  2210. if !srcFshAbs.FileExists(rsrcFile) {
  2211. utils.SendErrorResponse(w, "Source file not exists")
  2212. return
  2213. }
  2214. //Check if the desintation is read only.
  2215. if !userinfo.CanWrite(vdestFile) {
  2216. utils.SendErrorResponse(w, "Access Denied")
  2217. return
  2218. }
  2219. if !destFshAbs.FileExists(rdestFile) {
  2220. if destFshAbs.FileExists(filepath.Dir(rdestFile)) {
  2221. //User pass in the whole path for the folder. Report error usecase.
  2222. utils.SendErrorResponse(w, "Dest location should be an existing folder instead of the full path of the copied file")
  2223. return
  2224. }
  2225. utils.SendErrorResponse(w, "Dest folder not found")
  2226. return
  2227. }
  2228. existsOpr, _ := utils.PostPara(r, "existsresp")
  2229. //Check if the user have space for the extra file
  2230. if !userinfo.StorageQuota.HaveSpace(filesystem.GetFileSize(rdestFile)) {
  2231. utils.SendErrorResponse(w, "Storage Quota Full")
  2232. return
  2233. }
  2234. err = filesystem.FileCopy(srcFsh, rsrcFile, destFsh, rdestFile, existsOpr, nil)
  2235. if err != nil {
  2236. utils.SendErrorResponse(w, err.Error())
  2237. return
  2238. }
  2239. //Set user to own this file
  2240. newfileRpath := filepath.ToSlash(filepath.Clean(rdestFile)) + "/" + filepath.Base(rsrcFile)
  2241. newfileVpath, _ := destFsh.FileSystemAbstraction.RealPathToVirtualPath(newfileRpath, userinfo.Username)
  2242. userinfo.SetOwnerOfFile(destFsh, newfileVpath)
  2243. } else if operation == "delete" {
  2244. //Delete the file permanently
  2245. if !srcFshAbs.FileExists(rsrcFile) {
  2246. //Check if it is a non escapted file instead
  2247. utils.SendErrorResponse(w, "Source file not exists")
  2248. return
  2249. }
  2250. if !userinfo.CanWrite(vsrcFile) {
  2251. utils.SendErrorResponse(w, "Access Denied")
  2252. return
  2253. }
  2254. //Check if the user own this file
  2255. isOwner := userinfo.IsOwnerOfFile(srcFsh, vsrcFile)
  2256. if isOwner {
  2257. //This user own this system. Remove this file from his quota
  2258. userinfo.RemoveOwnershipFromFile(srcFsh, vsrcFile)
  2259. }
  2260. //Check if this file has any cached files. If yes, remove it
  2261. metadata.RemoveCache(srcFsh, rsrcFile)
  2262. //Clear the cache folder if there is no files inside
  2263. fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
  2264. if len(fc) == 0 && err == nil {
  2265. srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
  2266. }
  2267. err = srcFshAbs.RemoveAll(rsrcFile)
  2268. if err != nil {
  2269. systemWideLogger.PrintAndLog("File System", "Unable to remove file from "+srcFsh.UUID, err)
  2270. utils.SendErrorResponse(w, err.Error())
  2271. return
  2272. }
  2273. //Deleted from the trash bin. Drop the .trash folder once it is empty
  2274. if filepath.Base(filepath.Dir(rsrcFile)) == ".trash" {
  2275. system_fs_removeTrashDirIfEmpty(srcFshAbs, filepath.Dir(rsrcFile))
  2276. }
  2277. } else if operation == "recycle" {
  2278. //Put it into a subfolder named trash and allow it to to be removed later
  2279. if !srcFshAbs.FileExists(rsrcFile) {
  2280. //Check if it is a non escapted file instead
  2281. utils.SendErrorResponse(w, "Source file not exists")
  2282. return
  2283. }
  2284. //Check if the upload target is read only.
  2285. if !userinfo.CanWrite(vsrcFile) {
  2286. utils.SendErrorResponse(w, "Access Denied")
  2287. return
  2288. }
  2289. //Check if this file has any cached files. If yes, remove it
  2290. metadata.RemoveCache(srcFsh, rsrcFile)
  2291. //Clear the cache folder if there is no files inside
  2292. fc, err := srcFshAbs.Glob(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/*")
  2293. if len(fc) == 0 && err == nil {
  2294. srcFshAbs.Remove(filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.cache/")
  2295. }
  2296. /*
  2297. Trash quota
  2298. Checked before the move, not after: once the file is inside
  2299. .trash it has already left the tree the user was looking at,
  2300. and undoing that cleanly is harder than refusing up front.
  2301. The client recognises this error code and offers to empty
  2302. the bin, delete outright, or do nothing.
  2303. */
  2304. userTrashQuota := system_fs_getTrashQuotaBytes(userinfo.Username)
  2305. if userTrashQuota > 0 {
  2306. incomingSize := system_fs_getTrashEntrySize(srcFsh, rsrcFile)
  2307. if system_fs_getTrashUsage(userinfo.Username)+incomingSize > userTrashQuota {
  2308. utils.SendErrorResponse(w, "TRASH_QUOTA_EXCEEDED")
  2309. return
  2310. }
  2311. }
  2312. //Create a trash directory for this folder
  2313. trashDir := filepath.ToSlash(filepath.Dir(rsrcFile)) + "/.metadata/.trash/"
  2314. srcFshAbs.MkdirAll(trashDir, 0755)
  2315. hidden.HideFile(filepath.Dir(trashDir))
  2316. hidden.HideFile(trashDir)
  2317. err = srcFshAbs.Rename(rsrcFile, trashDir+filepath.Base(rsrcFile)+"."+utils.Int64ToString(time.Now().Unix()))
  2318. if err != nil {
  2319. if srcFsh.RequireBuffer {
  2320. utils.SendErrorResponse(w, "Incompatible File System Type: Try SHIFT + DELETE to delete file permanently")
  2321. } else {
  2322. systemWideLogger.PrintAndLog("File System", "Failed to move file to trash. See log for more info.", err)
  2323. utils.SendErrorResponse(w, "Failed to move file to trash")
  2324. }
  2325. return
  2326. }
  2327. } else if operation == "unzip" {
  2328. //Unzip the file to destination
  2329. //Check if the user can write to the target dest file
  2330. if !userinfo.CanWrite(string(vdestFile)) {
  2331. utils.SendErrorResponse(w, "Access Denied")
  2332. return
  2333. }
  2334. //Make the rdest directory if not exists
  2335. if !destFshAbs.FileExists(rdestFile) {
  2336. err = destFshAbs.MkdirAll(rdestFile, 0755)
  2337. if err != nil {
  2338. utils.SendErrorResponse(w, err.Error())
  2339. return
  2340. }
  2341. }
  2342. unzipSource := rsrcFile
  2343. unzipDest := rdestFile
  2344. if srcFsh.RequireBuffer {
  2345. localBufferedFile, _ := bufferRemoteFileToLocal(srcFsh, rsrcFile, false)
  2346. unzipSource = localBufferedFile
  2347. }
  2348. if destFsh.RequireBuffer {
  2349. localUnzipBuffer, _ := bufferRemoteFileToLocal(destFsh, rdestFile, true)
  2350. unzipDest = localUnzipBuffer
  2351. }
  2352. //OK! Unzip to destination
  2353. err := filesystem.Unzip(unzipSource, unzipDest)
  2354. if err != nil {
  2355. utils.SendErrorResponse(w, err.Error())
  2356. return
  2357. }
  2358. if srcFsh.RequireBuffer {
  2359. //Remove the local buffered file
  2360. os.Remove(unzipSource)
  2361. }
  2362. if destFsh.RequireBuffer {
  2363. //Push the buffer to target fs
  2364. filepath.Walk(unzipDest, func(path string, info os.FileInfo, err error) error {
  2365. path = filepath.ToSlash(path)
  2366. relpath := strings.TrimPrefix(path, filepath.ToSlash(unzipDest))
  2367. if info.IsDir() {
  2368. destFshAbs.MkdirAll(filepath.Join(rdestFile, relpath), 0775)
  2369. } else {
  2370. f, _ := os.Open(path)
  2371. destFshAbs.WriteStream(filepath.Join(rdestFile, relpath), f, 0775)
  2372. f.Close()
  2373. }
  2374. return nil
  2375. })
  2376. cleanFsBufferFileFromList([]string{unzipDest})
  2377. }
  2378. } else {
  2379. utils.SendErrorResponse(w, "Unknown file opeartion given")
  2380. return
  2381. }
  2382. }
  2383. }
  2384. utils.SendOK(w)
  2385. }
  2386. // Allow systems to store key value pairs in the database as preferences.
  2387. func system_fs_handleUserPreference(w http.ResponseWriter, r *http.Request) {
  2388. username, err := authAgent.GetUserName(w, r)
  2389. if err != nil {
  2390. utils.SendErrorResponse(w, "User not logged in")
  2391. return
  2392. }
  2393. key, _ := utils.GetPara(r, "key")
  2394. value, _ := utils.GetPara(r, "value")
  2395. remove, _ := utils.GetPara(r, "remove")
  2396. if key != "" && value == "" && remove == "" {
  2397. //Get mode. Read the prefernece with given key
  2398. result := ""
  2399. err := sysdb.Read("fs", "pref/"+key+"/"+username, &result)
  2400. if err != nil {
  2401. utils.SendJSONResponse(w, `{"error":"Key not found."}`)
  2402. return
  2403. }
  2404. utils.SendTextResponse(w, result)
  2405. } else if key != "" && value == "" && remove == "true" {
  2406. //Remove mode. Delete this key from sysdb
  2407. err := sysdb.Delete("fs", "pref/"+key+"/"+username)
  2408. if err != nil {
  2409. utils.SendErrorResponse(w, err.Error())
  2410. }
  2411. utils.SendOK(w)
  2412. } else if key != "" && value != "" {
  2413. //Set mode. Set the preference with given key
  2414. if len(value) > 1024*1024 { //1KB
  2415. //Size too big. Reject storage
  2416. utils.SendErrorResponse(w, "Preference value too long. Preference value can only store maximum 1KB.")
  2417. return
  2418. }
  2419. sysdb.Write("fs", "pref/"+key+"/"+username, value)
  2420. utils.SendOK(w)
  2421. }
  2422. }
  2423. func system_fs_removeUserPreferences(username string) {
  2424. entries, err := sysdb.ListTable("fs")
  2425. if err != nil {
  2426. return
  2427. }
  2428. for _, keypairs := range entries {
  2429. if strings.Contains(string(keypairs[0]), "pref/") && strings.Contains(string(keypairs[0]), "/"+username) {
  2430. //Remove this preference
  2431. sysdb.Delete("fs", string(keypairs[0]))
  2432. }
  2433. }
  2434. }
  2435. func system_fs_listDrives(w http.ResponseWriter, r *http.Request) {
  2436. if !authAgent.CheckAuth(r) {
  2437. utils.SendErrorResponse(w, "User not logged in")
  2438. return
  2439. }
  2440. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  2441. type driveInfo struct {
  2442. Drivepath string
  2443. DriveFreeSpace uint64
  2444. DriveTotalSpace uint64
  2445. DriveAvailSpace uint64
  2446. }
  2447. var drives []driveInfo
  2448. if runtime.GOOS == "windows" {
  2449. //Under windows
  2450. for _, drive := range "ABCDEFGHIJKLMNOPQRSTUVWXYZ" {
  2451. f, err := os.Open(string(drive) + ":\\")
  2452. if err == nil {
  2453. thisdrive := new(driveInfo)
  2454. thisdrive.Drivepath = string(drive) + ":\\"
  2455. free, total, avail := storage.GetDriveCapacity(string(drive) + ":\\")
  2456. thisdrive.DriveFreeSpace = free
  2457. thisdrive.DriveTotalSpace = total
  2458. thisdrive.DriveAvailSpace = avail
  2459. drives = append(drives, *thisdrive)
  2460. f.Close()
  2461. }
  2462. }
  2463. } else {
  2464. //Under linux environment
  2465. //Append all the virtual directories root as root instead
  2466. storageDevices := []string{}
  2467. for _, fshandler := range userinfo.GetAllFileSystemHandler() {
  2468. storageDevices = append(storageDevices, fshandler.Path)
  2469. }
  2470. //List all storage information of each devices
  2471. for _, dev := range storageDevices {
  2472. thisdrive := new(driveInfo)
  2473. thisdrive.Drivepath = filepath.Base(dev)
  2474. free, total, avail := storage.GetDriveCapacity(string(dev))
  2475. thisdrive.DriveFreeSpace = free
  2476. thisdrive.DriveTotalSpace = total
  2477. thisdrive.DriveAvailSpace = avail
  2478. drives = append(drives, *thisdrive)
  2479. }
  2480. }
  2481. jsonString, _ := json.Marshal(drives)
  2482. utils.SendJSONResponse(w, string(jsonString))
  2483. }
  2484. func system_fs_listRoot(w http.ResponseWriter, r *http.Request) {
  2485. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2486. if err != nil {
  2487. utils.SendErrorResponse(w, err.Error())
  2488. return
  2489. }
  2490. username := userinfo.Username
  2491. userRoot, _ := utils.GetPara(r, "user")
  2492. if userRoot == "true" {
  2493. type fileObject struct {
  2494. Filename string
  2495. Filepath string
  2496. IsDir bool
  2497. }
  2498. //List the root media folders under user:/
  2499. fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath("user:/")
  2500. fshAbs := fsh.FileSystemAbstraction
  2501. filesInUserRoot := []fileObject{}
  2502. filesInRoot, _ := fshAbs.Glob(filepath.ToSlash(filepath.Clean(*root_directory)) + "/users/" + username + "/*")
  2503. for _, file := range filesInRoot {
  2504. //Check if this is a hidden file
  2505. if len(filepath.Base(file)) > 0 && filepath.Base(file)[:1] == "." {
  2506. continue
  2507. }
  2508. thisFile := new(fileObject)
  2509. thisFile.Filename = filepath.Base(file)
  2510. thisFile.Filepath, _ = fshAbs.RealPathToVirtualPath(file, userinfo.Username)
  2511. thisFile.IsDir = fshAbs.IsDir(file)
  2512. filesInUserRoot = append(filesInUserRoot, *thisFile)
  2513. }
  2514. jsonString, _ := json.Marshal(filesInUserRoot)
  2515. utils.SendJSONResponse(w, string(jsonString))
  2516. } else {
  2517. type rootObject struct {
  2518. rootID string //The vroot id
  2519. RootName string //The name of this vroot
  2520. RootPath string //The path of this vroot
  2521. BufferedFs bool //If buffer typed FS
  2522. }
  2523. roots := []*rootObject{}
  2524. for _, store := range userinfo.GetAllFileSystemHandler() {
  2525. var thisDevice = new(rootObject)
  2526. thisDevice.RootName = store.Name
  2527. thisDevice.RootPath = store.UUID + ":/"
  2528. thisDevice.rootID = store.UUID
  2529. thisDevice.BufferedFs = store.RequireBuffer
  2530. roots = append(roots, thisDevice)
  2531. }
  2532. jsonString, _ := json.Marshal(roots)
  2533. utils.SendJSONResponse(w, string(jsonString))
  2534. }
  2535. }
  2536. /*
  2537. Special Glob for handling path with [ or ] inside.
  2538. You can also pass in normal path for globing if you are not sure.
  2539. */
  2540. func system_fs_specialURIDecode(inputPath string) string {
  2541. inputPath = strings.ReplaceAll(inputPath, "+", "{{plus_sign}}")
  2542. inputPath, _ = url.QueryUnescape(inputPath)
  2543. inputPath = strings.ReplaceAll(inputPath, "{{plus_sign}}", "+")
  2544. return inputPath
  2545. }
  2546. /*
  2547. func system_fs_specialURIEncode(inputPath string) string {
  2548. inputPath = strings.ReplaceAll(inputPath, " ", "{{space_sign}}")
  2549. inputPath, _ = url.QueryUnescape(inputPath)
  2550. inputPath = strings.ReplaceAll(inputPath, "{{space_sign}}", "%20")
  2551. return inputPath
  2552. }
  2553. */
  2554. /*
  2555. The parent folder of a virtual path, as shown in the properties dialog.
  2556. A virtual path is always slash separated and carries a "vdID:" prefix, so
  2557. filepath.Dir must not be used here: on Windows it reads "user:" as a volume
  2558. name and Clean guards the result with a "./" prefix, turning
  2559. "user:/cluster/a.agi" into "./user:/cluster". GetIDFromVirtualPath strips
  2560. that prefix back off for the same reason.
  2561. */
  2562. func virtualDirname(vpath string) string {
  2563. return path.Dir(vpath)
  2564. }
  2565. // Handle file properties request
  2566. func system_fs_getFileProperties(w http.ResponseWriter, r *http.Request) {
  2567. type fileProperties struct {
  2568. VirtualPath string
  2569. StoragePath string
  2570. Basename string
  2571. VirtualDirname string
  2572. StorageDirname string
  2573. Ext string
  2574. MimeType string
  2575. Filesize int64
  2576. Permission string
  2577. LastModTime string
  2578. LastModUnix int64
  2579. CreationTime string
  2580. CreationUnix int64
  2581. IsDirectory bool
  2582. Owner string
  2583. }
  2584. result := fileProperties{}
  2585. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2586. if err != nil {
  2587. utils.SendErrorResponse(w, err.Error())
  2588. return
  2589. }
  2590. vpath, err := utils.PostPara(r, "path")
  2591. if err != nil {
  2592. utils.SendErrorResponse(w, "path not defined")
  2593. return
  2594. }
  2595. vrootID, subpath, _ := filesystem.GetIDFromVirtualPath(vpath)
  2596. fsh, err := GetFsHandlerByUUID(vrootID)
  2597. if err != nil {
  2598. utils.SendErrorResponse(w, err.Error())
  2599. return
  2600. }
  2601. fshAbs := fsh.FileSystemAbstraction
  2602. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2603. if err != nil {
  2604. utils.SendErrorResponse(w, err.Error())
  2605. return
  2606. }
  2607. fileStat, err := fshAbs.Stat(rpath)
  2608. if err != nil {
  2609. utils.SendErrorResponse(w, err.Error())
  2610. return
  2611. }
  2612. fileMime := "text/directory"
  2613. if !fileStat.IsDir() {
  2614. m, _, err := filesystem.GetMime(rpath)
  2615. if err != nil {
  2616. fileMime = mime.TypeByExtension(filepath.Ext(rpath))
  2617. } else {
  2618. fileMime = m
  2619. }
  2620. }
  2621. filesize := fileStat.Size()
  2622. //Get file overall size if this is folder
  2623. if fileStat.IsDir() {
  2624. if fsh.IsNetworkDrive() {
  2625. filesize = -1
  2626. } else {
  2627. //Check if du exists
  2628. usefallback := true //Use fallback
  2629. if fsh.IsLocalDrive() {
  2630. //Try using native syscall to grab directory size
  2631. nativeSize, err := filesystem.GetDirectorySizeNative(rpath)
  2632. if err == nil {
  2633. usefallback = false
  2634. filesize = nativeSize
  2635. }
  2636. }
  2637. if usefallback {
  2638. // invalid platform. walk the whole file system
  2639. var size int64 = 0
  2640. fshAbs.Walk(rpath, func(_ string, info os.FileInfo, err error) error {
  2641. if err != nil {
  2642. return err
  2643. }
  2644. if !info.IsDir() {
  2645. size += info.Size()
  2646. }
  2647. return err
  2648. })
  2649. filesize = size
  2650. }
  2651. }
  2652. }
  2653. //Get the file creation time. Not every platform / file system records one,
  2654. //in which case it is left empty for the client to grey out
  2655. creationTimeString := ""
  2656. var creationUnix int64 = 0
  2657. if creationTime, ok := filesystem.GetFileCreationTime(fileStat); ok {
  2658. creationTimeString = time.Unix(creationTime, 0).Format("2006-01-02 15:04:05")
  2659. creationUnix = creationTime
  2660. }
  2661. //Get file owner
  2662. owner := userinfo.GetFileOwner(fsh, vpath)
  2663. if owner == "" {
  2664. //Handle special virtual roots
  2665. owner = "Unknown"
  2666. }
  2667. result = fileProperties{
  2668. VirtualPath: vpath,
  2669. StoragePath: filepath.ToSlash(filepath.Clean(rpath)),
  2670. Basename: filepath.Base(rpath),
  2671. VirtualDirname: virtualDirname(vpath),
  2672. StorageDirname: filepath.ToSlash(filepath.Dir(rpath)),
  2673. Ext: filepath.Ext(rpath),
  2674. MimeType: fileMime,
  2675. Filesize: filesize,
  2676. Permission: fileStat.Mode().Perm().String(),
  2677. LastModTime: fileStat.ModTime().Format("2006-01-02 15:04:05"),
  2678. LastModUnix: fileStat.ModTime().Unix(),
  2679. CreationTime: creationTimeString,
  2680. CreationUnix: creationUnix,
  2681. IsDirectory: fileStat.IsDir(),
  2682. Owner: owner,
  2683. }
  2684. jsonString, _ := json.Marshal(result)
  2685. utils.SendJSONResponse(w, string(jsonString))
  2686. }
  2687. /*
  2688. Storage info
  2689. Where the file actually is. Most drives have nothing to add beyond the
  2690. storage path already in the properties dialog, but a drive that spreads
  2691. its files over several hosts does: cluster:/ answers with the nodes and
  2692. volumes holding a copy of the file. The abstraction resolves it
  2693. (arozfs.StorageInfoProvider), so this handler stays the same whatever
  2694. kind of drive the file is on.
  2695. */
  2696. func system_fs_getStorageInfo(w http.ResponseWriter, r *http.Request) {
  2697. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2698. if err != nil {
  2699. utils.SendErrorResponse(w, "User not logged in")
  2700. return
  2701. }
  2702. vpath, err := utils.PostPara(r, "path")
  2703. if err != nil {
  2704. utils.SendErrorResponse(w, "path not defined")
  2705. return
  2706. }
  2707. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  2708. if err != nil {
  2709. utils.SendErrorResponse(w, err.Error())
  2710. return
  2711. }
  2712. //Reading a file the user has no access to must not leak where it is kept
  2713. if !userinfo.CanRead(vpath) {
  2714. utils.SendErrorResponse(w, "Permission denied")
  2715. return
  2716. }
  2717. provider, ok := fsh.FileSystemAbstraction.(arozfs.StorageInfoProvider)
  2718. if !ok {
  2719. //Nothing to add for this kind of drive
  2720. utils.SendErrorResponse(w, "Storage info not supported by this drive")
  2721. return
  2722. }
  2723. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  2724. if err != nil {
  2725. utils.SendErrorResponse(w, err.Error())
  2726. return
  2727. }
  2728. info, err := provider.StorageInfo(rpath)
  2729. if err != nil {
  2730. utils.SendErrorResponse(w, err.Error())
  2731. return
  2732. }
  2733. js, _ := json.Marshal(info)
  2734. utils.SendJSONResponse(w, string(js))
  2735. }
  2736. /*
  2737. List directory in the given path
  2738. Usage: Pass in dir like the following examples:
  2739. AOR:/Desktop <= Open /user/{username}/Desktop
  2740. S1:/ <= Open {uuid=S1}/
  2741. */
  2742. func system_fs_handleList(w http.ResponseWriter, r *http.Request) {
  2743. currentDir, err := utils.PostPara(r, "dir")
  2744. if err != nil {
  2745. utils.SendErrorResponse(w, err.Error())
  2746. return
  2747. }
  2748. sortMode, _ := utils.PostPara(r, "sort")
  2749. showHidden, _ := utils.PostPara(r, "showHidden")
  2750. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2751. if err != nil {
  2752. //user not logged in. Redirect to login page.
  2753. utils.SendErrorResponse(w, "User not logged in")
  2754. return
  2755. }
  2756. if currentDir == "" {
  2757. utils.SendErrorResponse(w, "Invalid dir given.")
  2758. return
  2759. }
  2760. // Pad a slash at the end of currentDir if not exists
  2761. if !strings.HasSuffix(currentDir, "/") {
  2762. currentDir = currentDir + "/"
  2763. }
  2764. fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
  2765. if err != nil {
  2766. utils.SendErrorResponse(w, err.Error())
  2767. return
  2768. }
  2769. fshAbs := fsh.FileSystemAbstraction
  2770. realpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2771. if err != nil {
  2772. utils.SendErrorResponse(w, err.Error())
  2773. return
  2774. }
  2775. if !fshAbs.FileExists(realpath) {
  2776. //Path not exists
  2777. userRoot, _ := fshAbs.VirtualPathToRealPath("/", userinfo.Username)
  2778. if filepath.Clean(realpath) == filepath.Clean(userRoot) || realpath == "" {
  2779. //Initiate user folder (Initiaed in user object)
  2780. err = fshAbs.MkdirAll(userRoot, 0775)
  2781. if err != nil {
  2782. systemWideLogger.PrintAndLog("File System", "Unable to create user root on "+fsh.UUID+": "+err.Error(), nil)
  2783. utils.SendErrorResponse(w, "Unable to create user root folder due to file system error")
  2784. return
  2785. }
  2786. } else {
  2787. //Folder not exists
  2788. systemWideLogger.PrintAndLog("File System", "Requested path: "+realpath+" does not exists", nil)
  2789. utils.SendErrorResponse(w, "Folder not exists")
  2790. return
  2791. }
  2792. }
  2793. if sortMode == "" {
  2794. sortMode = "default"
  2795. }
  2796. files, err := fshAbs.ReadDir(realpath)
  2797. if err != nil {
  2798. utils.SendErrorResponse(w, "Readdir Failed: "+strings.ReplaceAll(err.Error(), "\\", "/"))
  2799. systemWideLogger.PrintAndLog("File System", "Unable to read dir: "+err.Error(), err)
  2800. return
  2801. }
  2802. //Remapping use parsed list
  2803. parsedFilelist := map[string]filesystem.FileData{}
  2804. //Sorting use list
  2805. realpathList := []string{}
  2806. fileInfoList := []fs.FileInfo{}
  2807. for _, f := range files {
  2808. //Check if it is hidden file
  2809. isHidden, _ := hidden.IsHidden(f.Name(), false)
  2810. if showHidden != "true" && isHidden {
  2811. //Skipping hidden files
  2812. continue
  2813. }
  2814. //Check if this file contains invalid characters
  2815. if !utils.FilenameIsWebSafe(f.Name()) {
  2816. continue
  2817. }
  2818. //Check if this is an aodb file
  2819. if f.Name() == "aofs.db" || f.Name() == "aofs.db.lock" {
  2820. //Database file (reserved)
  2821. continue
  2822. }
  2823. //Check if it is shortcut file. If yes, render a shortcut data struct
  2824. var shortCutInfo *arozfs.ShortcutData = nil
  2825. if filepath.Ext(f.Name()) == ".shortcut" {
  2826. //This is a shortcut file
  2827. fcontent, err := fshAbs.ReadFile(arozfs.ToSlash(filepath.Join(realpath, f.Name())))
  2828. if err != nil {
  2829. shortCutInfo = nil
  2830. } else {
  2831. shorcutData, err := shortcut.ReadShortcut(fcontent)
  2832. if err != nil {
  2833. shortCutInfo = nil
  2834. } else {
  2835. shortCutInfo = shorcutData
  2836. }
  2837. }
  2838. }
  2839. statInfo, err := f.Info()
  2840. if err != nil {
  2841. continue
  2842. }
  2843. thisvPath, _ := fshAbs.RealPathToVirtualPath(filepath.Join(realpath, f.Name()), userinfo.Username)
  2844. thisFile := filesystem.FileData{
  2845. Filename: f.Name(),
  2846. Filepath: currentDir + f.Name(),
  2847. Realpath: filepath.ToSlash(filepath.Join(realpath, f.Name())),
  2848. IsDir: f.IsDir(),
  2849. Filesize: statInfo.Size(),
  2850. Displaysize: filesystem.GetFileDisplaySize(statInfo.Size(), 2),
  2851. ModTime: statInfo.ModTime().Unix(),
  2852. IsShared: shareManager.FileIsShared(userinfo, thisvPath),
  2853. Shortcut: shortCutInfo,
  2854. }
  2855. parsedFilelist[currentDir+f.Name()] = thisFile
  2856. realpathList = append(realpathList, currentDir+f.Name())
  2857. fileInfoList = append(fileInfoList, statInfo)
  2858. }
  2859. //Sort the filelist
  2860. sortedRealpathList := fssort.SortFileList(realpathList, fileInfoList, sortMode)
  2861. results := []filesystem.FileData{}
  2862. for _, thisRpath := range sortedRealpathList {
  2863. val, ok := parsedFilelist[thisRpath]
  2864. if ok {
  2865. results = append(results, val)
  2866. }
  2867. }
  2868. jsonString, _ := json.Marshal(results)
  2869. utils.SendJSONResponse(w, string(jsonString))
  2870. }
  2871. // Handle getting a hash from a given contents in the given path
  2872. func system_fs_handleDirHash(w http.ResponseWriter, r *http.Request) {
  2873. currentDir, err := utils.GetPara(r, "dir")
  2874. if err != nil {
  2875. utils.SendErrorResponse(w, "Invalid dir given")
  2876. return
  2877. }
  2878. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2879. if err != nil {
  2880. utils.SendErrorResponse(w, "User not logged in")
  2881. return
  2882. }
  2883. fsh, subpath, err := GetFSHandlerSubpathFromVpath(currentDir)
  2884. if err != nil {
  2885. utils.SendErrorResponse(w, "Unable to resolve target directory")
  2886. return
  2887. }
  2888. fshAbs := fsh.FileSystemAbstraction
  2889. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2890. if err != nil {
  2891. utils.SendErrorResponse(w, "Invalid dir given")
  2892. return
  2893. }
  2894. //The folder might have been removed by another window or client
  2895. if !fshAbs.FileExists(rpath) || !fshAbs.IsDir(rpath) {
  2896. utils.SendErrorResponse(w, "Directory not found")
  2897. return
  2898. }
  2899. //Get a list of files in this directory
  2900. /*
  2901. currentDir = filepath.ToSlash(filepath.Clean(rpath)) + "/"
  2902. filesInDir, err := fshAbs.Glob(currentDir + "*")
  2903. if err != nil {
  2904. utils.SendErrorResponse(w, err.Error())
  2905. return
  2906. }
  2907. filenames := []string{}
  2908. for _, file := range filesInDir {
  2909. if len(filepath.Base(file)) > 0 && string([]rune(filepath.Base(file))[0]) != "." {
  2910. //Ignore hidden files
  2911. filenames = append(filenames, filepath.Base(file))
  2912. }
  2913. }
  2914. */
  2915. finfos, err := fshAbs.ReadDir(rpath)
  2916. if err != nil {
  2917. utils.SendErrorResponse(w, err.Error())
  2918. return
  2919. }
  2920. filenames := []string{}
  2921. for _, fi := range finfos {
  2922. isHiddenFile, _ := hidden.IsHidden(fi.Name(), false)
  2923. if len(fi.Name()) > 0 && !isHiddenFile {
  2924. //Ignore hidden files
  2925. filenames = append(filenames, fi.Name())
  2926. }
  2927. }
  2928. sort.Strings(filenames)
  2929. //Build a hash base on the filelist
  2930. h := sha256.New()
  2931. h.Write([]byte(strings.Join(filenames, ",")))
  2932. utils.SendTextResponse(w, hex.EncodeToString((h.Sum(nil))))
  2933. }
  2934. /*
  2935. File zipping and unzipping functions
  2936. */
  2937. // Handle all zip related API
  2938. func system_fs_zipHandler(w http.ResponseWriter, r *http.Request) {
  2939. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  2940. if err != nil {
  2941. utils.SendErrorResponse(w, err.Error())
  2942. return
  2943. }
  2944. opr, err := utils.PostPara(r, "opr")
  2945. if err != nil {
  2946. utils.SendErrorResponse(w, "Invalid opr or opr not defined")
  2947. return
  2948. }
  2949. vsrc, _ := utils.PostPara(r, "src")
  2950. if vsrc == "" {
  2951. utils.SendErrorResponse(w, "Invalid src paramter")
  2952. return
  2953. }
  2954. vdest, _ := utils.PostPara(r, "dest")
  2955. rdest := ""
  2956. //Convert source path from JSON string to object
  2957. virtualSourcePaths := []string{}
  2958. err = json.Unmarshal([]byte(vsrc), &virtualSourcePaths)
  2959. if err != nil {
  2960. utils.SendErrorResponse(w, err.Error())
  2961. return
  2962. }
  2963. //Check each of the path
  2964. realSourcePaths := []string{}
  2965. sourceFshs := []*filesystem.FileSystemHandler{}
  2966. for _, vpath := range virtualSourcePaths {
  2967. thisSrcFsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  2968. if err != nil {
  2969. utils.SendErrorResponse(w, "Unable to resolve file: "+vpath)
  2970. return
  2971. }
  2972. thisSrcFshAbs := thisSrcFsh.FileSystemAbstraction
  2973. thisrpath, err := thisSrcFshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  2974. if err != nil || !thisSrcFshAbs.FileExists(thisrpath) {
  2975. utils.SendErrorResponse(w, "File not exists: "+vpath)
  2976. return
  2977. }
  2978. realSourcePaths = append(realSourcePaths, thisrpath)
  2979. sourceFshs = append(sourceFshs, thisSrcFsh)
  2980. }
  2981. ///Convert dest to real if given
  2982. var destFsh *filesystem.FileSystemHandler = nil
  2983. var subpath string = ""
  2984. var filename string = ""
  2985. if vdest != "" {
  2986. //Given target virtual dest
  2987. destFsh, subpath, err = GetFSHandlerSubpathFromVpath(rdest)
  2988. if err != nil {
  2989. utils.SendErrorResponse(w, err.Error())
  2990. return
  2991. }
  2992. } else {
  2993. //Given no virtual dest. Zip to tmp:/
  2994. filename = utils.Int64ToString(time.Now().Unix()) + ".zip"
  2995. destFsh, subpath, err = GetFSHandlerSubpathFromVpath(filepath.Join("tmp:/", filename))
  2996. if err != nil {
  2997. utils.SendErrorResponse(w, err.Error())
  2998. return
  2999. }
  3000. }
  3001. rdest, _ = destFsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3002. destFshAbs := destFsh.FileSystemAbstraction
  3003. zipOutput := rdest
  3004. zipDestFsh := destFsh
  3005. if destFsh.RequireBuffer {
  3006. zipOutput = getFsBufferFilepath(rdest, false)
  3007. zipDestFsh = nil
  3008. }
  3009. if opr == "tmpzipAsync" {
  3010. /*
  3011. Same as tmpzip, but the zipping runs in the background and reports
  3012. its progress into the file operation task list, so the caller can
  3013. follow it on /system/file_system/ongoing instead of staring at a
  3014. request that returns nothing until the archive is finished.
  3015. */
  3016. if filename == "" {
  3017. utils.SendErrorResponse(w, "tmpzipAsync does not support a custom destination")
  3018. return
  3019. }
  3020. vzipPath := "tmp:/" + filename
  3021. task := NewOngoingFileOperation(userinfo, "zip", virtualSourcePaths, vzipPath)
  3022. go runTmpZipTask(task.ID, sourceFshs, realSourcePaths, zipDestFsh, zipOutput, destFsh, rdest)
  3023. js, _ := json.Marshal(map[string]string{
  3024. "oprid": task.ID,
  3025. "dest": vzipPath,
  3026. })
  3027. utils.SendJSONResponse(w, string(js))
  3028. return
  3029. }
  3030. if opr == "zip" {
  3031. //Check if destination location exists
  3032. if rdest == "" || !destFshAbs.FileExists(filepath.Dir(zipOutput)) {
  3033. utils.SendErrorResponse(w, "Invalid dest location")
  3034. return
  3035. }
  3036. //OK. Create the zip at the desired location
  3037. err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
  3038. if err != nil {
  3039. utils.SendErrorResponse(w, err.Error())
  3040. return
  3041. }
  3042. utils.SendOK(w)
  3043. } else if opr == "tmpzip" {
  3044. //Zip to tmp folder
  3045. err := filesystem.ArozZipFile(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false)
  3046. if err != nil {
  3047. utils.SendErrorResponse(w, err.Error())
  3048. return
  3049. }
  3050. //Send the tmp filename to the user
  3051. utils.SendTextResponse(w, "tmp:/"+filename)
  3052. }
  3053. if destFsh.RequireBuffer {
  3054. //Write the buffer zip file to destination
  3055. f, _ := os.Open(zipOutput)
  3056. destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
  3057. f.Close()
  3058. os.Remove(zipOutput)
  3059. }
  3060. cleanFsBufferFileFromList(realSourcePaths)
  3061. }
  3062. /*
  3063. runTmpZipTask builds a temporary zip in the background, keeping the file
  3064. operation record of oprId up to date so the front end can poll its progress.
  3065. The progress handler returns the task's control signal, which is what lets a
  3066. user cancel the archive from the file operation list: ArozZipFileWithProgress
  3067. stops and reports the cancellation as an error, and finishTmpZipTask turns that
  3068. back into a cancelled record.
  3069. */
  3070. func runTmpZipTask(oprId string, sourceFshs []*filesystem.FileSystemHandler, realSourcePaths []string, zipDestFsh *filesystem.FileSystemHandler, zipOutput string, destFsh *filesystem.FileSystemHandler, rdest string) {
  3071. err := filesystem.ArozZipFileWithProgress(sourceFshs, realSourcePaths, zipDestFsh, zipOutput, false, func(currentFilename string, _ int, _ int, progress float64) int {
  3072. sig, _ := UpdateOngoingFileOperation(oprId, currentFilename, math.Ceil(progress))
  3073. return sig
  3074. })
  3075. if err != nil {
  3076. systemWideLogger.PrintAndLog("File System", "Zipping request failed: "+err.Error(), err)
  3077. finishTmpZipTask(oprId, err)
  3078. return
  3079. }
  3080. if destFsh.RequireBuffer {
  3081. //Write the buffer zip file to destination
  3082. f, _ := os.Open(zipOutput)
  3083. err = destFsh.FileSystemAbstraction.WriteStream(rdest, f, 0775)
  3084. f.Close()
  3085. os.Remove(zipOutput)
  3086. if err != nil {
  3087. systemWideLogger.PrintAndLog("File System", "Zip write to remote file system with driver "+destFsh.Filesystem+" failed", err)
  3088. finishTmpZipTask(oprId, err)
  3089. return
  3090. }
  3091. }
  3092. cleanFsBufferFileFromList(realSourcePaths)
  3093. finishTmpZipTask(oprId, nil)
  3094. }
  3095. // finishTmpZipTask closes off the task record of a background tmp zip.
  3096. // Pass in the error the zipping ended with, or nil when it completed.
  3097. func finishTmpZipTask(oprId string, err error) {
  3098. if err != nil {
  3099. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Error, err.Error())
  3100. return
  3101. }
  3102. SetFileOperationTaskEnded(oprId, filesystem.FsOpr_Continue, "")
  3103. }
  3104. // Manage file version history
  3105. func system_fs_FileVersionHistory(w http.ResponseWriter, r *http.Request) {
  3106. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3107. if err != nil {
  3108. utils.SendErrorResponse(w, err.Error())
  3109. return
  3110. }
  3111. path, err := utils.PostPara(r, "path")
  3112. if err != nil {
  3113. utils.SendErrorResponse(w, "Invalid path given")
  3114. return
  3115. }
  3116. fsh, subpath, err := GetFSHandlerSubpathFromVpath(path)
  3117. if err != nil {
  3118. if err != nil {
  3119. utils.SendErrorResponse(w, "Invalid path given")
  3120. return
  3121. }
  3122. }
  3123. fshAbs := fsh.FileSystemAbstraction
  3124. opr, _ := utils.PostPara(r, "opr")
  3125. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3126. if err != nil {
  3127. utils.SendErrorResponse(w, "Unable to translate virtual path")
  3128. return
  3129. }
  3130. if opr == "" {
  3131. //List file history
  3132. fileVersionData, err := localversion.GetFileVersionData(fsh, rpath)
  3133. if err != nil {
  3134. utils.SendErrorResponse(w, "Unable to load version information: "+err.Error())
  3135. return
  3136. }
  3137. js, _ := json.Marshal(fileVersionData)
  3138. utils.SendJSONResponse(w, string(js))
  3139. } else if opr == "delete" {
  3140. //Delete file history of given history ID
  3141. historyID, err := utils.PostPara(r, "histid")
  3142. if err != nil {
  3143. utils.SendErrorResponse(w, "Invalid history id given")
  3144. return
  3145. }
  3146. err = localversion.RemoveFileHistory(fsh, rpath, historyID)
  3147. if err != nil {
  3148. utils.SendErrorResponse(w, err.Error())
  3149. return
  3150. }
  3151. utils.SendOK(w)
  3152. } else if opr == "deleteAll" {
  3153. //Delete all file history of given vpath
  3154. err = localversion.RemoveAllRelatedFileHistory(fsh, rpath)
  3155. if err != nil {
  3156. utils.SendErrorResponse(w, err.Error())
  3157. return
  3158. }
  3159. utils.SendOK(w)
  3160. } else if opr == "restore" {
  3161. //Restore file history of given history ID
  3162. historyID, err := utils.PostPara(r, "histid")
  3163. if err != nil {
  3164. utils.SendErrorResponse(w, "Invalid history id given")
  3165. return
  3166. }
  3167. err = localversion.RestoreFileHistory(fsh, rpath, historyID)
  3168. if err != nil {
  3169. utils.SendErrorResponse(w, err.Error())
  3170. return
  3171. }
  3172. utils.SendOK(w)
  3173. } else if opr == "new" {
  3174. //Create a new snapshot of this file
  3175. err = localversion.CreateFileSnapshot(fsh, rpath)
  3176. if err != nil {
  3177. utils.SendErrorResponse(w, err.Error())
  3178. return
  3179. }
  3180. utils.SendOK(w)
  3181. } else {
  3182. utils.SendErrorResponse(w, "Unknown opr")
  3183. }
  3184. }
  3185. func system_fs_clearVersionHistories() {
  3186. allFsh := GetAllLoadedFsh()
  3187. for _, fsh := range allFsh {
  3188. if fsh.ReadOnly {
  3189. continue
  3190. }
  3191. if !nightlyShouldMaintainFsh(fsh) {
  3192. //A drive shared by the cluster is cleaned by the master node
  3193. continue
  3194. }
  3195. localversion.CleanExpiredVersionBackups(fsh, fsh.Path, 30*86400)
  3196. }
  3197. }
  3198. // Handle cache rendering with websocket pipeline
  3199. func system_fs_handleCacheRender(w http.ResponseWriter, r *http.Request) {
  3200. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3201. vpath, err := utils.GetPara(r, "folder")
  3202. if err != nil {
  3203. utils.SendErrorResponse(w, "Invalid folder paramter")
  3204. return
  3205. }
  3206. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  3207. if err != nil {
  3208. utils.SendErrorResponse(w, "Unable to resolve target directory")
  3209. return
  3210. }
  3211. rpath, _ := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3212. //Get folder sort mode
  3213. sortMode := "default"
  3214. folder := arozfs.Clean(vpath)
  3215. if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
  3216. sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
  3217. }
  3218. //Perform cache rendering
  3219. thumbRenderHandler.HandleLoadCache(w, r, fsh, rpath, sortMode)
  3220. }
  3221. // Handle loading of one thumbnail
  3222. func system_fs_handleThumbnailLoad(w http.ResponseWriter, r *http.Request) {
  3223. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3224. vpath, err := utils.GetPara(r, "vpath")
  3225. if err != nil {
  3226. utils.SendErrorResponse(w, "vpath not defined")
  3227. return
  3228. }
  3229. byteMode, _ := utils.GetPara(r, "bytes")
  3230. isByteMode := byteMode == "true"
  3231. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vpath)
  3232. if err != nil {
  3233. if isByteMode {
  3234. http.NotFound(w, r)
  3235. return
  3236. }
  3237. utils.SendErrorResponse(w, "Unable to resolve target directory")
  3238. return
  3239. }
  3240. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3241. if err != nil {
  3242. if isByteMode {
  3243. http.NotFound(w, r)
  3244. return
  3245. }
  3246. utils.SendErrorResponse(w, err.Error())
  3247. return
  3248. }
  3249. if isByteMode {
  3250. thumbnailBytes, err := thumbRenderHandler.LoadCacheAsBytes(fsh, vpath, userinfo.Username, false)
  3251. if err != nil {
  3252. http.NotFound(w, r)
  3253. return
  3254. }
  3255. filetype := http.DetectContentType(thumbnailBytes)
  3256. w.Header().Add("Content-Type", filetype)
  3257. w.Write(thumbnailBytes)
  3258. } else {
  3259. thumbnailPath, err := thumbRenderHandler.LoadCache(fsh, rpath, false)
  3260. if err != nil {
  3261. utils.SendErrorResponse(w, err.Error())
  3262. return
  3263. }
  3264. js, _ := json.Marshal(thumbnailPath)
  3265. utils.SendJSONResponse(w, string(js))
  3266. }
  3267. }
  3268. // Handle file thumbnail caching
  3269. func system_fs_handleFolderCache(w http.ResponseWriter, r *http.Request) {
  3270. userinfo, _ := userHandler.GetUserInfoFromRequest(w, r)
  3271. vfolderpath, err := utils.GetPara(r, "folder")
  3272. if err != nil {
  3273. utils.SendErrorResponse(w, "folder not defined")
  3274. return
  3275. }
  3276. fsh, _, err := GetFSHandlerSubpathFromVpath(vfolderpath)
  3277. if err != nil {
  3278. utils.SendErrorResponse(w, "unable to resolve path")
  3279. return
  3280. }
  3281. thumbRenderHandler.BuildCacheForFolder(fsh, vfolderpath, userinfo.Username)
  3282. utils.SendOK(w)
  3283. }
  3284. // Handle the get and set of sort mode of a particular folder
  3285. func system_fs_handleFolderSortModePreference(w http.ResponseWriter, r *http.Request) {
  3286. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3287. if err != nil {
  3288. utils.SendErrorResponse(w, "User not logged in")
  3289. return
  3290. }
  3291. folder, err := utils.PostPara(r, "folder")
  3292. if err != nil {
  3293. utils.SendErrorResponse(w, "Invalid folder given")
  3294. return
  3295. }
  3296. opr, _ := utils.PostPara(r, "opr")
  3297. folder = arozfs.Clean(folder)
  3298. if opr == "" || opr == "get" {
  3299. sortMode := "default"
  3300. if sysdb.KeyExists("fs-sortpref", userinfo.Username+"/"+folder) {
  3301. sysdb.Read("fs-sortpref", userinfo.Username+"/"+folder, &sortMode)
  3302. }
  3303. js, err := json.Marshal(sortMode)
  3304. if err != nil {
  3305. utils.SendErrorResponse(w, err.Error())
  3306. return
  3307. }
  3308. utils.SendJSONResponse(w, string(js))
  3309. } else if opr == "set" {
  3310. sortMode, err := utils.PostPara(r, "mode")
  3311. if err != nil {
  3312. utils.SendErrorResponse(w, "Invalid sort mode given")
  3313. return
  3314. }
  3315. if !utils.StringInArray(fssort.ValidSortModes, sortMode) {
  3316. utils.SendErrorResponse(w, "Not supported sort mode: "+sortMode)
  3317. return
  3318. }
  3319. sysdb.Write("fs-sortpref", userinfo.Username+"/"+folder, sortMode)
  3320. utils.SendOK(w)
  3321. } else {
  3322. utils.SendErrorResponse(w, "Invalid opr mode")
  3323. return
  3324. }
  3325. }
  3326. // Handle setting and loading of file permission on Linux
  3327. func system_fs_handleFilePermission(w http.ResponseWriter, r *http.Request) {
  3328. file, err := utils.PostPara(r, "file")
  3329. if err != nil {
  3330. utils.SendErrorResponse(w, "Invalid file")
  3331. return
  3332. }
  3333. //Translate the file to real path
  3334. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3335. if err != nil {
  3336. utils.SendErrorResponse(w, "User not logged in")
  3337. return
  3338. }
  3339. fsh, subpath, err := GetFSHandlerSubpathFromVpath(file)
  3340. if err != nil {
  3341. utils.SendErrorResponse(w, err.Error())
  3342. return
  3343. }
  3344. fshAbs := fsh.FileSystemAbstraction
  3345. rpath, err := fshAbs.VirtualPathToRealPath(subpath, userinfo.Username)
  3346. if err != nil {
  3347. utils.SendErrorResponse(w, err.Error())
  3348. return
  3349. }
  3350. newMode, _ := utils.PostPara(r, "mode")
  3351. if newMode == "" {
  3352. //Read the file mode
  3353. //Check if the file exists
  3354. if !fshAbs.FileExists(rpath) {
  3355. utils.SendErrorResponse(w, "File not exists!")
  3356. return
  3357. }
  3358. //Read the file permission
  3359. filePermission, err := fsp.GetFilePermissions(fsh, rpath)
  3360. if err != nil {
  3361. utils.SendErrorResponse(w, err.Error())
  3362. return
  3363. }
  3364. //Send the file permission to client
  3365. js, _ := json.Marshal(filePermission)
  3366. utils.SendJSONResponse(w, string(js))
  3367. } else {
  3368. //Set the file mode
  3369. //Check if the file exists
  3370. if !filesystem.FileExists(rpath) {
  3371. utils.SendErrorResponse(w, "File not exists!")
  3372. return
  3373. }
  3374. //Check if windows. If yes, ignore this request
  3375. if runtime.GOOS == "windows" {
  3376. utils.SendErrorResponse(w, "Windows host not supported")
  3377. return
  3378. }
  3379. //Check if this user has permission to change the file permission
  3380. //Aka user must be 1. This is his own folder or 2. Admin
  3381. fsh, _ := userinfo.GetFileSystemHandlerFromVirtualPath(file)
  3382. if fsh.Hierarchy == "user" {
  3383. //Always ok as this is owned by the user
  3384. } else if fsh.Hierarchy == "public" {
  3385. //Require admin
  3386. if !userinfo.IsAdmin() {
  3387. utils.SendErrorResponse(w, "Permission Denied")
  3388. return
  3389. }
  3390. } else {
  3391. //Not implemeneted. Require admin
  3392. if !userinfo.IsAdmin() {
  3393. utils.SendErrorResponse(w, "Permission Denied")
  3394. return
  3395. }
  3396. }
  3397. //Be noted that if the system is not running in sudo mode,
  3398. //File permission change might not works.
  3399. err := fsp.SetFilePermisson(fsh, rpath, newMode)
  3400. if err != nil {
  3401. utils.SendErrorResponse(w, err.Error())
  3402. return
  3403. } else {
  3404. utils.SendOK(w)
  3405. }
  3406. }
  3407. }
  3408. // Clear the old files inside the tmp file
  3409. func system_fs_clearOldTmpFiles() {
  3410. filesToBeDelete := []string{}
  3411. tmpAbs, _ := filepath.Abs(*tmp_directory)
  3412. filepath.Walk(*tmp_directory, func(path string, info os.FileInfo, err error) error {
  3413. if filepath.Base(path) != "aofs.db" && filepath.Base(path) != "aofs.db.lock" {
  3414. //Check if root folders. Do not delete root folders
  3415. parentAbs, _ := filepath.Abs(filepath.Dir(path))
  3416. if tmpAbs == parentAbs {
  3417. //Root folder. Do not remove
  3418. return nil
  3419. }
  3420. //Get its modification time
  3421. modTime, err := filesystem.GetModTime(path)
  3422. if err != nil {
  3423. return nil
  3424. }
  3425. //Check if mod time is more than 24 hours ago
  3426. if time.Now().Unix()-modTime > int64(*maxTempFileKeepTime) {
  3427. //Delete OK
  3428. filesToBeDelete = append(filesToBeDelete, path)
  3429. }
  3430. }
  3431. return nil
  3432. })
  3433. //Remove all files from the delete list
  3434. for _, fileToBeDelete := range filesToBeDelete {
  3435. os.RemoveAll(fileToBeDelete)
  3436. }
  3437. }
  3438. /*
  3439. File System Utilities for Buffered type FS
  3440. These functions help create a local representation of file
  3441. buffer from remote file systems like webdav or SMB
  3442. **REMEMBER TO CLEAR THE BUFFER FILES YOURSELF**
  3443. Example Usage
  3444. //Replace a destination path (for file create) with local buffer filepath
  3445. if destFsh.RequireBuffer {
  3446. dest = getFsBufferFilepath(outputFilename)
  3447. }
  3448. //Buffer a remote file to local first before doing any advance file operations
  3449. if thisSrcFsh.RequireBuffer {
  3450. localBufferFilepath, err := bufferRemoteFileToLocal(fsh, remoteRealSrc)
  3451. if err != nil{
  3452. //Handle Error
  3453. }
  3454. }
  3455. //Clean a list of source files that contains local buffer files
  3456. clearnFsBufferFileFromList(realSourceFiles)
  3457. */
  3458. // Generate a random buffer filepath. Remember to delete file after usage
  3459. func getFsBufferFilepath(originalFilename string, keepOriginalName bool) string {
  3460. thisBuffFilename := uuid.NewV4().String()
  3461. tmpDir := filepath.Join(*tmp_directory, "fsBuff")
  3462. targetFile := filepath.Join(tmpDir, thisBuffFilename+filepath.Ext(originalFilename))
  3463. if keepOriginalName {
  3464. targetFile = filepath.Join(tmpDir, thisBuffFilename, filepath.Base(originalFilename))
  3465. }
  3466. os.MkdirAll(filepath.Dir(targetFile), 0775)
  3467. return filepath.ToSlash(targetFile)
  3468. }
  3469. // Generate a buffer filepath and buffer the remote file to local. Remember to remove file after done.
  3470. func bufferRemoteFileToLocal(targetFsh *filesystem.FileSystemHandler, rpath string, keepOriginalName bool) (string, error) {
  3471. newBufferFilename := getFsBufferFilepath(rpath, keepOriginalName)
  3472. src, err := targetFsh.FileSystemAbstraction.ReadStream(rpath)
  3473. if err != nil {
  3474. systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
  3475. return "", err
  3476. }
  3477. defer src.Close()
  3478. dest, err := os.OpenFile(newBufferFilename, os.O_CREATE|os.O_WRONLY, 0775)
  3479. if err != nil {
  3480. systemWideLogger.PrintAndLog("File System", "Buffer from remote to local failed: "+err.Error(), err)
  3481. return "", err
  3482. }
  3483. io.Copy(dest, src)
  3484. dest.Close()
  3485. return newBufferFilename, nil
  3486. }
  3487. // Check if a file is buffer filepath
  3488. func isFsBufferFilepath(filename string) bool {
  3489. tmpDir := filepath.Join(*tmp_directory, "fsBuff")
  3490. filenameAbs, _ := filepath.Abs(filename)
  3491. filenameAbs = filepath.ToSlash(filenameAbs)
  3492. tmpDirAbs, _ := filepath.Abs(tmpDir)
  3493. tmpDirAbs = filepath.ToSlash(tmpDirAbs)
  3494. return strings.HasPrefix(filenameAbs, tmpDirAbs)
  3495. }
  3496. func cleanFsBufferFileFromList(filelist []string) {
  3497. for _, thisFilepath := range filelist {
  3498. if isFsBufferFilepath(thisFilepath) {
  3499. os.RemoveAll(thisFilepath)
  3500. folderContent, _ := os.ReadDir(filepath.Dir(thisFilepath))
  3501. if len(folderContent) == 0 {
  3502. //Nothing in this folder. Remove it
  3503. os.Remove(filepath.Dir(thisFilepath))
  3504. }
  3505. }
  3506. }
  3507. }
  3508. /*
  3509. File operation task book keeping
  3510. All file operations of all users are tracked in wsConnectionStore, keyed by
  3511. the operation id. Finished records are kept around for a while so the file
  3512. operation dialog can still show the completed entries to the user before
  3513. they are cleared (either by the user or by the janitor below).
  3514. */
  3515. // Handle all the on going task requests.
  3516. // Accept parameter: flag={continue / pause / cancel / remove / clear}
  3517. // When no flag is given, the ongoing task list of this user is returned.
  3518. // Pass in all=true to include the recently finished tasks in the listing.
  3519. func system_fs_HandleOnGoingTasks(w http.ResponseWriter, r *http.Request) {
  3520. //Get the user information
  3521. userinfo, err := userHandler.GetUserInfoFromRequest(w, r)
  3522. if err != nil {
  3523. utils.SendErrorResponse(w, "User not logged in")
  3524. return
  3525. }
  3526. statusFlag, _ := utils.PostPara(r, "flag")
  3527. oprid, _ := utils.PostPara(r, "oprid")
  3528. if statusFlag == "" {
  3529. //Also accept the flag as a GET parameter for easier polling
  3530. statusFlag, _ = utils.GetPara(r, "flag")
  3531. oprid, _ = utils.GetPara(r, "oprid")
  3532. }
  3533. if statusFlag == "" {
  3534. //No flag defined. Print all operations of this user
  3535. includeFinished, _ := utils.GetPara(r, "all")
  3536. js, _ := MarshalFileOperationForUser(userinfo.Username, includeFinished == "true")
  3537. utils.SendJSONResponse(w, string(js))
  3538. return
  3539. }
  3540. if statusFlag == "clear" {
  3541. //Clear all the finished records of this user
  3542. ClearFinishedFileOperationForUser(userinfo.Username)
  3543. utils.SendOK(w)
  3544. return
  3545. }
  3546. if oprid == "" {
  3547. utils.SendErrorResponse(w, "oprid is empty or not set")
  3548. return
  3549. }
  3550. err = ApplyFileOperationControl(userinfo.Username, statusFlag, oprid)
  3551. if err != nil {
  3552. utils.SendErrorResponse(w, err.Error())
  3553. return
  3554. }
  3555. utils.SendOK(w)
  3556. }
  3557. // ApplyFileOperationControl applies a control signal to one of the file operations
  3558. // owned by the given user. Supported commands are continue, pause, cancel and remove.
  3559. func ApplyFileOperationControl(username string, command string, oprid string) error {
  3560. if command == "clear" {
  3561. ClearFinishedFileOperationForUser(username)
  3562. return nil
  3563. }
  3564. //Get the operation record
  3565. oprRecord, err := GetOngoingFileOperationByOprID(oprid)
  3566. if err != nil {
  3567. return err
  3568. }
  3569. //Only the owner of this operation can control it
  3570. if oprRecord.Owner != username {
  3571. return errors.New("permission denied")
  3572. }
  3573. fileOprTaskLock.Lock()
  3574. defer fileOprTaskLock.Unlock()
  3575. switch command {
  3576. case "continue":
  3577. //Continue the file operation
  3578. oprRecord.FileOperationSignal = filesystem.FsOpr_Continue
  3579. case "pause":
  3580. //Pause the file operation until the flag is set to other status
  3581. oprRecord.FileOperationSignal = filesystem.FsOpr_Pause
  3582. case "cancel":
  3583. //Cancel and stop the operation
  3584. oprRecord.FileOperationSignal = filesystem.FsOpr_Cancel
  3585. if taskIsFinished(oprRecord) {
  3586. //Already finished. Nothing left to cancel, just drop the record
  3587. wsConnectionStore.Delete(oprid)
  3588. }
  3589. case "remove":
  3590. //Remove a finished record from the listing
  3591. if !taskIsFinished(oprRecord) {
  3592. return errors.New("task is still running")
  3593. }
  3594. wsConnectionStore.Delete(oprid)
  3595. default:
  3596. return errors.New("unsupported operation")
  3597. }
  3598. return nil
  3599. }
  3600. // NewOngoingFileOperation creates and registers a new file operation task record.
  3601. // The size of each source file is resolved here so the front end can show the
  3602. // transferred / total size of every file in the operation.
  3603. func NewOngoingFileOperation(userinfo *user.User, operation string, sourceFiles []string, vdestFile string) *fileOperationTask {
  3604. /*
  3605. Sizing a source folder means walking it. For a move that the storage can
  3606. satisfy with a rename that walk costs far more than the move itself, and
  3607. it would only feed a bar that jumps straight to done, so it is skipped.
  3608. Should the rename fail after all and the files end up being streamed, the
  3609. first progress report fills the real sizes in, see
  3610. UpdateOngoingFileOperationSubtask.
  3611. */
  3612. resolveSizes := !fileOperationCanBeRenamed(operation, sourceFiles, vdestFile)
  3613. subtasks := []*fileOperationSubtask{}
  3614. totalSize := int64(0)
  3615. for _, vsrc := range sourceFiles {
  3616. thisSize, thisIsDir := resolveFileOperationSourceInfo(userinfo, vsrc, resolveSizes)
  3617. totalSize += thisSize
  3618. subtasks = append(subtasks, &fileOperationSubtask{
  3619. Filename: filepath.Base(strings.TrimSuffix(vsrc, "/")),
  3620. Src: arozfs.ToSlash(vsrc),
  3621. IsDir: thisIsDir,
  3622. Size: thisSize,
  3623. Done: 0,
  3624. Progress: 0,
  3625. Status: FsTask_Pending,
  3626. })
  3627. }
  3628. thisTask := fileOperationTask{
  3629. ID: strconv.Itoa(int(time.Now().Unix())) + "_" + uuid.NewV4().String(),
  3630. Owner: userinfo.Username,
  3631. Operation: operation,
  3632. Src: arozfs.ToSlash(filepath.Dir(sourceFiles[0])),
  3633. Dest: arozfs.ToSlash(vdestFile),
  3634. Progress: 0.0,
  3635. LatestFile: arozfs.ToSlash(filepath.Base(sourceFiles[0])),
  3636. FileOperationSignal: filesystem.FsOpr_Continue,
  3637. Files: subtasks,
  3638. TotalSize: totalSize,
  3639. DoneSize: 0,
  3640. StartTime: time.Now().Unix(),
  3641. Status: FsTask_Ongoing,
  3642. }
  3643. wsConnectionStore.Store(thisTask.ID, &thisTask)
  3644. startFinishedFileOperationJanitor()
  3645. return &thisTask
  3646. }
  3647. // resolveFileOperationSourceInfo returns the size of a source file in bytes and
  3648. // whether it is a folder. Set resolveSize to false to skip the folder walk when
  3649. // the size is not worth what it costs to work out.
  3650. // The size is 0 when it cannot be determined.
  3651. func resolveFileOperationSourceInfo(userinfo *user.User, vsrc string, resolveSize bool) (int64, bool) {
  3652. fsh, subpath, err := GetFSHandlerSubpathFromVpath(vsrc)
  3653. if err != nil {
  3654. return 0, false
  3655. }
  3656. rpath, err := fsh.FileSystemAbstraction.VirtualPathToRealPath(subpath, userinfo.Username)
  3657. if err != nil {
  3658. return 0, false
  3659. }
  3660. isDir := fsh.FileSystemAbstraction.IsDir(rpath)
  3661. if !resolveSize {
  3662. return 0, isDir
  3663. }
  3664. if isDir {
  3665. size, _ := fsh.GetDirctorySizeFromRealPath(rpath, false)
  3666. return size, true
  3667. }
  3668. return fsh.FileSystemAbstraction.GetFileSize(rpath), false
  3669. }
  3670. // fileOperationCanBeRenamed reports whether an operation is a move that the
  3671. // storage holding both of its ends can carry out on its own, in which case no
  3672. // bytes travel through ArozOS and the operation finishes right away.
  3673. func fileOperationCanBeRenamed(operation string, sourceFiles []string, vdestFile string) bool {
  3674. if operation != "move" {
  3675. return false
  3676. }
  3677. destFsh, _, err := GetFSHandlerSubpathFromVpath(vdestFile)
  3678. if err != nil || destFsh.ReadOnly {
  3679. return false
  3680. }
  3681. for _, vsrc := range sourceFiles {
  3682. srcFsh, _, err := GetFSHandlerSubpathFromVpath(vsrc)
  3683. if err != nil || srcFsh.ReadOnly || !filesystem.SameFileSystem(srcFsh, destFsh) {
  3684. return false
  3685. }
  3686. }
  3687. return true
  3688. }
  3689. // GetAllFileOperationForUser returns all the file operation records of the given user.
  3690. // Set includeFinished to true to also return the recently finished records.
  3691. // The records are live objects that the running operations keep updating, so
  3692. // read them through MarshalFileOperationForUser unless you hold fileOprTaskLock.
  3693. func GetAllFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
  3694. fileOprTaskLock.RLock()
  3695. defer fileOprTaskLock.RUnlock()
  3696. return collectFileOperationForUser(username, includeFinished)
  3697. }
  3698. // MarshalFileOperationForUser serializes the file operation records of a user
  3699. // while holding the task lock, so the running operations cannot update a record
  3700. // half way through the serialization.
  3701. func MarshalFileOperationForUser(username string, includeFinished bool) ([]byte, error) {
  3702. fileOprTaskLock.RLock()
  3703. defer fileOprTaskLock.RUnlock()
  3704. return json.Marshal(collectFileOperationForUser(username, includeFinished))
  3705. }
  3706. // collectFileOperationForUser gathers the task records of a user in operation id
  3707. // order. The caller is expected to be holding fileOprTaskLock.
  3708. func collectFileOperationForUser(username string, includeFinished bool) []*fileOperationTask {
  3709. results := []*fileOperationTask{}
  3710. wsConnectionStore.Range(func(key, value interface{}) bool {
  3711. taskInfo := value.(*fileOperationTask)
  3712. if taskInfo.Owner == username && (includeFinished || !taskIsFinished(taskInfo)) {
  3713. results = append(results, taskInfo)
  3714. }
  3715. return true
  3716. })
  3717. sort.Slice(results, func(i, j int) bool {
  3718. return results[i].ID < results[j].ID
  3719. })
  3720. return results
  3721. }
  3722. // ClearFinishedFileOperationForUser removes all the finished records of a user
  3723. func ClearFinishedFileOperationForUser(username string) {
  3724. fileOprTaskLock.Lock()
  3725. defer fileOprTaskLock.Unlock()
  3726. wsConnectionStore.Range(func(key, value interface{}) bool {
  3727. taskInfo := value.(*fileOperationTask)
  3728. if taskInfo.Owner == username && taskIsFinished(taskInfo) {
  3729. wsConnectionStore.Delete(key)
  3730. }
  3731. return true
  3732. })
  3733. }
  3734. // Get an ongoing task record
  3735. func GetOngoingFileOperationByOprID(oprid string) (*fileOperationTask, error) {
  3736. object, ok := wsConnectionStore.Load(oprid)
  3737. if !ok {
  3738. return nil, errors.New("task not exists")
  3739. }
  3740. return object.(*fileOperationTask), nil
  3741. }
  3742. // Set or update an ongoing task record
  3743. func SetOngoingFileOperation(opr *fileOperationTask) {
  3744. wsConnectionStore.Store(opr.ID, opr)
  3745. }
  3746. // Update the status of an ongoing task record, return latest status code and error if any
  3747. func UpdateOngoingFileOperation(oprid string, currentFile string, progress float64) (int, error) {
  3748. t, err := GetOngoingFileOperationByOprID(oprid)
  3749. if err != nil {
  3750. return 0, err
  3751. }
  3752. fileOprTaskLock.Lock()
  3753. t.LatestFile = currentFile
  3754. t.Progress = progress
  3755. t.DoneSize = int64(float64(t.TotalSize) * progress / 100)
  3756. //Zip and unzip report one progress value for the whole operation. Spread it
  3757. //over the source files in order so the dialog can still show them one by one.
  3758. remaining := t.DoneSize
  3759. for _, thisSubtask := range t.Files {
  3760. if remaining >= thisSubtask.Size {
  3761. thisSubtask.Done = thisSubtask.Size
  3762. thisSubtask.Progress = 100
  3763. thisSubtask.Status = FsTask_Completed
  3764. remaining -= thisSubtask.Size
  3765. } else {
  3766. thisSubtask.Done = remaining
  3767. if thisSubtask.Size > 0 {
  3768. thisSubtask.Progress = float64(remaining) / float64(thisSubtask.Size) * 100
  3769. } else {
  3770. thisSubtask.Progress = progress
  3771. }
  3772. thisSubtask.Status = FsTask_Ongoing
  3773. remaining = 0
  3774. }
  3775. }
  3776. sig := t.FileOperationSignal
  3777. fileOprTaskLock.Unlock()
  3778. return sig, nil
  3779. }
  3780. /*
  3781. UpdateOngoingFileOperationSubtask records how many bytes of one source file are
  3782. already processed and recalculates the progress of the whole operation from it.
  3783. It returns the current control signal and the overall progress of the operation.
  3784. bytesTotal is the size the file operation itself reports for this source, which
  3785. wins over the size resolved when the task was created: a folder can grow or
  3786. shrink between the two, and the transfer knows better than the earlier walk.
  3787. */
  3788. func UpdateOngoingFileOperationSubtask(oprid string, subtaskIndex int, currentFile string, bytesDone int64, bytesTotal int64) (int, float64, error) {
  3789. t, err := GetOngoingFileOperationByOprID(oprid)
  3790. if err != nil {
  3791. return 0, 0, err
  3792. }
  3793. fileOprTaskLock.Lock()
  3794. if currentFile != "" {
  3795. t.LatestFile = currentFile
  3796. }
  3797. if subtaskIndex >= 0 && subtaskIndex < len(t.Files) {
  3798. thisSubtask := t.Files[subtaskIndex]
  3799. thisSubtask.Status = FsTask_Ongoing
  3800. if bytesTotal > 0 && bytesTotal != thisSubtask.Size {
  3801. //Keep the operation total in step with the corrected file size
  3802. t.TotalSize += bytesTotal - thisSubtask.Size
  3803. thisSubtask.Size = bytesTotal
  3804. }
  3805. if bytesDone > thisSubtask.Size {
  3806. bytesDone = thisSubtask.Size
  3807. }
  3808. thisSubtask.Done = bytesDone
  3809. if thisSubtask.Size > 0 {
  3810. thisSubtask.Progress = float64(bytesDone) / float64(thisSubtask.Size) * 100
  3811. }
  3812. }
  3813. t.DoneSize = sumFileOperationDoneSize(t)
  3814. recalculateFileOperationProgress(t)
  3815. sig := t.FileOperationSignal
  3816. progress := t.Progress
  3817. fileOprTaskLock.Unlock()
  3818. return sig, progress, nil
  3819. }
  3820. /*
  3821. recalculateFileOperationProgress works out the overall progress of a task from
  3822. the bytes of the files inside it, so copying a 1 KB text file next to a 4 GB
  3823. image no longer moves the bar half way when the text file lands.
  3824. Operations whose sizes could not be resolved fall back to weighting every source
  3825. file equally. The caller is expected to be holding fileOprTaskLock.
  3826. */
  3827. func recalculateFileOperationProgress(t *fileOperationTask) {
  3828. if t.TotalSize > 0 {
  3829. progress := float64(t.DoneSize) / float64(t.TotalSize) * 100
  3830. if progress > 100 {
  3831. progress = 100
  3832. }
  3833. t.Progress = progress
  3834. return
  3835. }
  3836. if len(t.Files) == 0 {
  3837. return
  3838. }
  3839. totalProgress := float64(0)
  3840. for _, thisSubtask := range t.Files {
  3841. totalProgress += thisSubtask.Progress
  3842. }
  3843. t.Progress = totalProgress / float64(len(t.Files))
  3844. }
  3845. // MarkFileOperationSubtaskEnded marks one source file inside a task as finished.
  3846. // Pass in an empty errmsg for a successful completion.
  3847. func MarkFileOperationSubtaskEnded(oprid string, subtaskIndex int, errmsg string) {
  3848. t, err := GetOngoingFileOperationByOprID(oprid)
  3849. if err != nil {
  3850. return
  3851. }
  3852. fileOprTaskLock.Lock()
  3853. defer fileOprTaskLock.Unlock()
  3854. if subtaskIndex < 0 || subtaskIndex >= len(t.Files) {
  3855. return
  3856. }
  3857. thisSubtask := t.Files[subtaskIndex]
  3858. if errmsg != "" {
  3859. thisSubtask.Status = FsTask_Error
  3860. thisSubtask.Error = errmsg
  3861. } else {
  3862. thisSubtask.Status = FsTask_Completed
  3863. thisSubtask.Progress = 100
  3864. thisSubtask.Done = thisSubtask.Size
  3865. }
  3866. t.DoneSize = sumFileOperationDoneSize(t)
  3867. recalculateFileOperationProgress(t)
  3868. }
  3869. // SetFileOperationTaskEnded closes off a task record with the given ending signal.
  3870. // The record is kept in the store so the front end can still render the result.
  3871. func SetFileOperationTaskEnded(oprid string, endingSignal int, errmsg string) {
  3872. t, err := GetOngoingFileOperationByOprID(oprid)
  3873. if err != nil {
  3874. return
  3875. }
  3876. fileOprTaskLock.Lock()
  3877. defer fileOprTaskLock.Unlock()
  3878. t.EndTime = time.Now().Unix()
  3879. t.Error = errmsg
  3880. switch {
  3881. case endingSignal == filesystem.FsOpr_Cancel || t.FileOperationSignal == filesystem.FsOpr_Cancel:
  3882. //A file operation stopped by the user reports the abort as an error on
  3883. //its way out. That is not a failure, so record it as a cancellation.
  3884. t.Status = FsTask_Cancelled
  3885. t.Error = ""
  3886. for _, thisSubtask := range t.Files {
  3887. if thisSubtask.Status == FsTask_Error {
  3888. thisSubtask.Status = FsTask_Cancelled
  3889. thisSubtask.Error = ""
  3890. }
  3891. }
  3892. case errmsg != "" || endingSignal == filesystem.FsOpr_Error:
  3893. t.Status = FsTask_Error
  3894. default:
  3895. t.Status = FsTask_Completed
  3896. t.Progress = 100
  3897. t.DoneSize = t.TotalSize
  3898. for _, thisSubtask := range t.Files {
  3899. if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
  3900. thisSubtask.Status = FsTask_Completed
  3901. thisSubtask.Progress = 100
  3902. thisSubtask.Done = thisSubtask.Size
  3903. }
  3904. }
  3905. }
  3906. //Any file still marked as running at this point will never move again
  3907. for _, thisSubtask := range t.Files {
  3908. if thisSubtask.Status == FsTask_Pending || thisSubtask.Status == FsTask_Ongoing {
  3909. thisSubtask.Status = t.Status
  3910. }
  3911. }
  3912. }
  3913. // taskIsFinished checks if a task record has stopped running.
  3914. // The caller is expected to be holding fileOprTaskLock when required.
  3915. func taskIsFinished(t *fileOperationTask) bool {
  3916. return t.Status != FsTask_Ongoing && t.Status != FsTask_Pending
  3917. }
  3918. // sumFileOperationDoneSize sums up the transferred bytes of all the files in a task.
  3919. // The caller is expected to be holding fileOprTaskLock.
  3920. func sumFileOperationDoneSize(t *fileOperationTask) int64 {
  3921. total := int64(0)
  3922. for _, thisSubtask := range t.Files {
  3923. total += thisSubtask.Done
  3924. }
  3925. return total
  3926. }
  3927. // startFinishedFileOperationJanitor starts the background cleaner that tidies up
  3928. // the finished task records, see clearExpiredFileOperationRecords.
  3929. func startFinishedFileOperationJanitor() {
  3930. if !fileOprJanitorStarted.CompareAndSwap(false, true) {
  3931. return
  3932. }
  3933. go func() {
  3934. for {
  3935. time.Sleep(fileOprJanitorInterval)
  3936. clearExpiredFileOperationRecords()
  3937. }
  3938. }()
  3939. }
  3940. /*
  3941. clearExpiredFileOperationRecords drops the finished task records that are no
  3942. longer worth keeping.
  3943. An operation that finished without an error is cleared on its own shortly
  3944. after it ended, whether or not anyone was watching it: closing the browser
  3945. tab half way through a copy must not leave the record behind forever.
  3946. A failed operation is kept instead, so the user can still find out what went
  3947. wrong after reopening the desktop from this or from another machine. Only the
  3948. newest fileOprErrorRecordLimit failures of each user are kept.
  3949. */
  3950. func clearExpiredFileOperationRecords() {
  3951. expireTime := time.Now().Unix() - fileOprFinishedRecordTTL
  3952. errorRecords := map[string][]*fileOperationTask{}
  3953. fileOprTaskLock.Lock()
  3954. defer fileOprTaskLock.Unlock()
  3955. wsConnectionStore.Range(func(key, value interface{}) bool {
  3956. taskInfo := value.(*fileOperationTask)
  3957. if !taskIsFinished(taskInfo) || taskInfo.EndTime == 0 {
  3958. //Still running, leave it alone
  3959. return true
  3960. }
  3961. if taskInfo.Status == FsTask_Error {
  3962. errorRecords[taskInfo.Owner] = append(errorRecords[taskInfo.Owner], taskInfo)
  3963. return true
  3964. }
  3965. if taskInfo.EndTime < expireTime {
  3966. wsConnectionStore.Delete(key)
  3967. }
  3968. return true
  3969. })
  3970. //Trim the failure backlog of each user down to the newest few
  3971. for _, records := range errorRecords {
  3972. if len(records) <= fileOprErrorRecordLimit {
  3973. continue
  3974. }
  3975. sort.Slice(records, func(i, j int) bool {
  3976. return records[i].EndTime > records[j].EndTime
  3977. })
  3978. for _, expiredRecord := range records[fileOprErrorRecordLimit:] {
  3979. wsConnectionStore.Delete(expiredRecord.ID)
  3980. }
  3981. }
  3982. }