agi.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582
  1. package agi
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "fmt"
  6. "io"
  7. "net/http"
  8. "os"
  9. "path/filepath"
  10. "strings"
  11. "sync"
  12. "time"
  13. "github.com/robertkrimen/otto"
  14. uuid "github.com/satori/go.uuid"
  15. "imuslab.com/arozos/mod/agi/static"
  16. apt "imuslab.com/arozos/mod/apt"
  17. "imuslab.com/arozos/mod/email"
  18. "imuslab.com/arozos/mod/filesystem"
  19. "imuslab.com/arozos/mod/filesystem/arozfs"
  20. metadata "imuslab.com/arozos/mod/filesystem/metadata"
  21. "imuslab.com/arozos/mod/git"
  22. "imuslab.com/arozos/mod/info/logger"
  23. "imuslab.com/arozos/mod/meetroom"
  24. notification "imuslab.com/arozos/mod/notification"
  25. "imuslab.com/arozos/mod/share"
  26. "imuslab.com/arozos/mod/sharedspace"
  27. "imuslab.com/arozos/mod/time/nightly"
  28. user "imuslab.com/arozos/mod/user"
  29. "imuslab.com/arozos/mod/utils"
  30. )
  31. /*
  32. ArOZ Online Javascript Gateway Interface (AGI)
  33. author: tobychui
  34. This script load plugins written in Javascript and run them in VM inside golang
  35. DO NOT CONFUSE PLUGIN WITH SUBSERVICE :))
  36. */
  37. var (
  38. AgiVersion string = "3.8" //Defination of the agi runtime version. Update this when new function is added
  39. //AGI Internal Error Standard
  40. errExitcall = errors.New("errExit")
  41. errTimeout = errors.New("errTimeout")
  42. // agiLogger is a stdout-only fallback used when no system-wide logger is
  43. // available. Scripts should use g.Option.Logger when present.
  44. agiLogger, _ = logger.NewTmpLogger()
  45. )
  46. type AgiPackage struct {
  47. InitRoot string //The initialization of the root for the module that request this package
  48. }
  49. type AgiSysInfo struct {
  50. //System information
  51. BuildVersion string
  52. InternalVersion string
  53. LoadedModule []string
  54. //System Handlers
  55. Logger *logger.Logger
  56. UserHandler *user.UserHandler
  57. ReservedTables []string
  58. PackageManager *apt.AptPackageManager
  59. ModuleRegisterParser func(sourceFolder string, jsonModuleConfig string) error //sourceFolder is the web app folder the init.agi lives in, e.g. "web/Office"
  60. ModuleFolderResolver func(folder string) []string //Returns the names of the modules registered from a source folder
  61. ModuleListProvider func(username string) string //Returns JSON of accessible modules for a user
  62. ExtIconRegisterParser func(ext, iconPath string) //Called when registerExtensionIcon() fires in an init.agi
  63. FileSystemRender *metadata.RenderHandler
  64. ShareManager *share.Manager
  65. NightlyManager *nightly.TaskManager
  66. MeetRoomManager *meetroom.Manager //MeetRoom rooms for the meetroom lib (nil disables the lib)
  67. SharedSpaceManager *sharedspace.Manager //Shared collaboration spaces for the sharedspace lib (nil disables the lib)
  68. GitManager *git.Manager //Version control backend for the git lib (nil disables the lib)
  69. EmailManager *email.Manager //IMAP / SMTP mail client backend for the email lib (nil disables the lib)
  70. ClusterProvider ClusterProvider //Cluster namespace / events for the cluster lib (nil disables the lib)
  71. //NotificationSender routes a notification raised by an AGI script into the
  72. //ArozOS core notification system (nil disables the notification lib).
  73. NotificationSender func(*notification.NotificationPayload) error
  74. //Scanning Roots
  75. StartupRoot string
  76. ActivateScope []string
  77. TempFolderPath string
  78. }
  79. type Gateway struct {
  80. ReservedTables []string
  81. NightlyScripts []string
  82. //AllowAccessPkgs map[string][]AgiPackage
  83. LoadedAGILibrary map[string]AgiLibInjectionIntergface
  84. Option *AgiSysInfo
  85. endpointStats map[string]*EndpointStats // per-UUID execution statistics (in-memory)
  86. statsMux sync.RWMutex // guards endpointStats
  87. vmReg *vmRegistry // live VM lifecycle registry
  88. }
  89. func NewGateway(option AgiSysInfo) (*Gateway, error) {
  90. //Handle startup registration of ajgi modules
  91. gatewayObject := Gateway{
  92. ReservedTables: option.ReservedTables,
  93. NightlyScripts: []string{},
  94. LoadedAGILibrary: map[string]AgiLibInjectionIntergface{},
  95. Option: &option,
  96. endpointStats: make(map[string]*EndpointStats),
  97. vmReg: newVMRegistry(),
  98. }
  99. //Start all WebApps Registration
  100. gatewayObject.InitiateAllWebAppModules()
  101. gatewayObject.RegisterNightlyOperations()
  102. //Load all the other libs entry points into the memoary
  103. gatewayObject.LoadAllFunctionalModules()
  104. return &gatewayObject, nil
  105. }
  106. func (g *Gateway) RegisterNightlyOperations() {
  107. g.Option.NightlyManager.RegisterNightlyTask(func() {
  108. //This function will execute nightly
  109. for _, scriptFile := range g.NightlyScripts {
  110. if static.IsValidAGIScript(scriptFile) {
  111. //Valid script file. Execute it with system
  112. for _, username := range g.Option.UserHandler.GetAuthAgent().ListUsers() {
  113. userinfo, err := g.Option.UserHandler.GetUserInfoFromUsername(username)
  114. if err != nil {
  115. continue
  116. }
  117. if static.CheckUserAccessToScript(userinfo, filepath.Join("./web", scriptFile), "./web", g.Option.ModuleFolderResolver) {
  118. //This user can access the module that provide this script.
  119. //Execute this script on his account.
  120. logger.PrintAndLog("Agi", "[AGI_Nightly] WIP ("+scriptFile+")", nil)
  121. }
  122. }
  123. } else {
  124. //Invalid script. Skipping
  125. logger.PrintAndLog("Agi", "[AGI_Nightly] Invalid script file: "+scriptFile, nil)
  126. }
  127. }
  128. })
  129. }
  130. func (g *Gateway) InitiateAllWebAppModules() {
  131. startupScripts, _ := filepath.Glob(filepath.ToSlash(filepath.Clean(g.Option.StartupRoot)) + "/*/init.agi")
  132. for _, script := range startupScripts {
  133. err := g.RunWebAppInitScript(script)
  134. if err != nil {
  135. logger.PrintAndLog("Agi", "[AGI] Load Failed: "+script+". Skipping.", nil)
  136. logger.PrintAndLog("Agi", fmt.Sprint(err), nil)
  137. continue
  138. }
  139. }
  140. }
  141. // Run a web app's init.agi (e.g. "web/Office/init.agi") with system scope only.
  142. // The script path is what ties the modules it registers to their source folder.
  143. func (g *Gateway) RunWebAppInitScript(script string) error {
  144. scriptContentByte, err := os.ReadFile(script)
  145. if err != nil {
  146. return err
  147. }
  148. scriptContent := string(scriptContentByte)
  149. logger.PrintAndLog("Agi", "[AGI] Gateway script loaded ("+script+")", nil)
  150. //Create a new vm for this request
  151. vm := otto.New()
  152. //Only allow non user based operations
  153. g.injectStandardLibs(vm, script, "./web/")
  154. g.injectAppdataLibFunctions(&static.AgiLibInjectionPayload{
  155. VM: vm,
  156. })
  157. _, err = vm.Run(scriptContent)
  158. return err
  159. }
  160. func (g *Gateway) RaiseError(err error) {
  161. if err == nil {
  162. return
  163. }
  164. logger.PrintAndLog("Agi", "[AGI] Runtime Error "+err.Error(), nil)
  165. //To be implemented
  166. }
  167. // Check if this table is restricted table. Return true if the access is valid
  168. func (g *Gateway) filterDBTable(tablename string, existsCheck bool) bool {
  169. //Check if table is restricted
  170. if utils.StringInArray(g.ReservedTables, tablename) {
  171. return false
  172. }
  173. //Check if table exists
  174. if existsCheck {
  175. if !g.Option.UserHandler.GetDatabase().TableExists(tablename) {
  176. return false
  177. }
  178. }
  179. return true
  180. }
  181. // Handle request from RESTFUL API
  182. func (g *Gateway) APIHandler(w http.ResponseWriter, r *http.Request, thisuser *user.User) {
  183. scriptContent, err := utils.PostPara(r, "script")
  184. if err != nil {
  185. w.WriteHeader(http.StatusBadRequest)
  186. w.Write([]byte("400 - Bad Request (Missing script content)"))
  187. return
  188. }
  189. g.ExecuteAGIScript(scriptContent, nil, "", "", w, r, thisuser)
  190. }
  191. // Handle user requests
  192. func (g *Gateway) InterfaceHandler(w http.ResponseWriter, r *http.Request, thisuser *user.User) {
  193. //Get user object from the request
  194. //startupRoot := g.Option.StartupRoot
  195. //startupRoot = filepath.ToSlash(filepath.Clean(startupRoot))
  196. //Get the script files for the plugin
  197. scriptFile, err := utils.GetPara(r, "script")
  198. if err != nil {
  199. w.WriteHeader(http.StatusInternalServerError)
  200. w.Write([]byte("500 - Internal Server Error: Invalid script path"))
  201. return
  202. }
  203. scriptFile = static.SpecialURIDecode(scriptFile)
  204. //Check if the script path exists
  205. scriptExists := false
  206. scriptScope := "./web/"
  207. for _, thisScope := range g.Option.ActivateScope {
  208. thisScope = arozfs.ToSlash(filepath.Clean(thisScope))
  209. if utils.FileExists(arozfs.ToSlash(filepath.Join(thisScope, scriptFile))) {
  210. scriptExists = true
  211. scriptFile = arozfs.ToSlash(filepath.Join(thisScope, scriptFile))
  212. scriptScope = thisScope
  213. break
  214. }
  215. }
  216. if !scriptExists {
  217. w.WriteHeader(http.StatusInternalServerError)
  218. w.Write([]byte("500 - Internal Server Error: Script not exists"))
  219. return
  220. }
  221. //Check for user permission on this module
  222. if !static.CheckUserAccessToScript(thisuser, scriptFile, scriptScope, g.Option.ModuleFolderResolver) {
  223. moduleName := static.GetScriptRoot(scriptFile, scriptScope)
  224. w.WriteHeader(http.StatusForbidden)
  225. if g.Option.BuildVersion == "development" {
  226. w.Write([]byte("403 Forbidden: User do not have permission to access " + moduleName))
  227. } else {
  228. w.Write([]byte("403 Forbidden"))
  229. }
  230. return
  231. }
  232. //Check the given file is actually agi script
  233. if !(filepath.Ext(scriptFile) == ".agi" || filepath.Ext(scriptFile) == ".js") {
  234. w.WriteHeader(http.StatusForbidden)
  235. if g.Option.BuildVersion == "development" {
  236. w.Write([]byte("AGI script must have file extension of .agi or .js"))
  237. } else {
  238. w.Write([]byte("403 Forbidden"))
  239. }
  240. return
  241. }
  242. //Get the content of the script
  243. scriptContentByte, err := os.ReadFile(scriptFile)
  244. if err != nil {
  245. w.WriteHeader(http.StatusInternalServerError)
  246. w.Write([]byte("500 - Internal Server Error: Script load error =>" + err.Error()))
  247. return
  248. }
  249. scriptContent := string(scriptContentByte)
  250. g.ExecuteAGIScript(scriptContent, nil, scriptFile, scriptScope, w, r, thisuser)
  251. }
  252. /*
  253. Executing the given AGI Script contents. Requires:
  254. scriptContent: The AGI command sequence
  255. scriptFile: The filepath of the script file
  256. scriptScope: The scope of the script file, aka the module base path
  257. w / r : Web request and response writer
  258. thisuser: userObject
  259. */
  260. func (g *Gateway) ExecuteAGIScript(scriptContent string, fsh *filesystem.FileSystemHandler, scriptFile string, scriptScope string, w http.ResponseWriter, r *http.Request, thisuser *user.User) {
  261. // Check if developer debug mode is requested via URL query param (set AGI_DEV=true in ao_module)
  262. devMode := r.URL.Query().Get("agi_devmode") == "true"
  263. //Create a new vm for this request
  264. vm := otto.New()
  265. vm.Interrupt = make(chan func(), 1) // required for force-stop support
  266. //Inject standard libs into the vm; capture execID for registry correlation
  267. execID := g.injectStandardLibs(vm, scriptFile, scriptScope)
  268. releaseLibResources := g.injectUserFunctions(vm, fsh, scriptFile, scriptScope, thisuser, w, r)
  269. username := ""
  270. if thisuser != nil {
  271. username = thisuser.Username
  272. }
  273. // Register in the VM lifecycle registry so it can be listed and force-stopped
  274. g.vmReg.register(&VMRecord{
  275. ExecID: execID,
  276. ScriptFile: scriptFile,
  277. Username: username,
  278. StartTime: time.Now(),
  279. interruptCh: vm.Interrupt,
  280. })
  281. defer func() {
  282. g.vmReg.unregister(execID)
  283. //Release library resources (open SQLite handles, etc.) on every exit path
  284. releaseLibResources()
  285. if caught := recover(); caught != nil {
  286. switch caught {
  287. case errForceStop:
  288. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM %s force-stopped (script: %s, user: %s)", execID, scriptFile, username), nil)
  289. w.WriteHeader(http.StatusServiceUnavailable)
  290. w.Write([]byte("503 - Script execution was force-terminated"))
  291. case errExitcall:
  292. // exit() in AGI script — clean early termination, not an error.
  293. // check anything else in the buffered response and send it before returning, if needed.
  294. value, err := vm.Get("HTTP_RESP")
  295. if err == nil {
  296. valueString, err := value.ToString()
  297. if err == nil && valueString != "" {
  298. w.Write([]byte(valueString))
  299. }
  300. }
  301. default:
  302. panic(caught) // re-panic anything we don't own
  303. }
  304. }
  305. }()
  306. //Detect cotent type
  307. contentType := r.Header.Get("Content-type")
  308. if strings.Contains(contentType, "application/json") {
  309. //For people who use Angular
  310. body, _ := io.ReadAll(r.Body)
  311. fields := map[string]interface{}{}
  312. json.Unmarshal(body, &fields)
  313. for k, v := range fields {
  314. vm.Set(k, v)
  315. }
  316. vm.Set("POST_data", string(body))
  317. } else {
  318. r.ParseForm()
  319. //Insert all paramters into the vm
  320. for k, v := range r.PostForm {
  321. if len(v) == 1 {
  322. vm.Set(k, v[0])
  323. } else {
  324. vm.Set(k, v)
  325. }
  326. }
  327. }
  328. _, err := vm.Run(scriptContent)
  329. if err != nil {
  330. username := ""
  331. if thisuser != nil {
  332. username = thisuser.Username
  333. }
  334. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI][%s] Script error in %s (user: %s): %s", execID, scriptFile, username, err.Error()), nil)
  335. if devMode {
  336. // Return a detailed JSON error payload for developer inspection
  337. errMsg := err.Error()
  338. stackTrace := errMsg
  339. if ottoErr, ok := err.(*otto.Error); ok {
  340. stackTrace = ottoErr.String()
  341. }
  342. errPayload, _ := json.Marshal(map[string]interface{}{
  343. "error": true,
  344. "message": errMsg,
  345. "stacktrace": stackTrace,
  346. "script": scriptFile,
  347. "user": username,
  348. })
  349. w.Header().Set("Content-Type", "application/json")
  350. w.WriteHeader(http.StatusInternalServerError)
  351. w.Write(errPayload)
  352. } else {
  353. scriptpath, _ := filepath.Abs(scriptFile)
  354. g.RenderErrorTemplate(w, err.Error(), scriptpath)
  355. }
  356. return
  357. }
  358. //Get the return valu from the script
  359. value, err := vm.Get("HTTP_RESP")
  360. if err != nil {
  361. utils.SendTextResponse(w, "")
  362. return
  363. }
  364. valueString, err := value.ToString()
  365. //Get respond header type from the vm
  366. header, _ := vm.Get("HTTP_HEADER")
  367. headerString, _ := header.ToString()
  368. if headerString != "" {
  369. w.Header().Set("Content-Type", headerString)
  370. }
  371. w.Write([]byte(valueString))
  372. }
  373. /*
  374. Execute AGI script with given user information
  375. scriptFile must be realpath resolved by fsa VirtualPathToRealPath function
  376. Pass in http.Request pointer to enable serverless GET / POST request
  377. */
  378. // ExecuteAGIScriptAsUser runs an AGI script on behalf of targetUser.
  379. // Returns (execID, output, error) where execID matches the EXECUTION_ID
  380. // constant injected into the script's VM environment.
  381. func (g *Gateway) ExecuteAGIScriptAsUser(fsh *filesystem.FileSystemHandler, scriptFile string, targetUser *user.User, w http.ResponseWriter, r *http.Request) (string, string, error) {
  382. //Create a new vm for this request
  383. vm := otto.New()
  384. //Inject standard libs into the vm; capture the execution ID for log correlation.
  385. execID := g.injectStandardLibs(vm, scriptFile, "")
  386. releaseLibResources := g.injectUserFunctions(vm, fsh, scriptFile, "", targetUser, w, r)
  387. if r != nil {
  388. //Inject serverless script to enable access to GET / POST paramters
  389. g.injectServerlessFunctions(vm, scriptFile, "", targetUser, r)
  390. }
  391. //Inject interrupt Channel
  392. vm.Interrupt = make(chan func(), 1)
  393. // Register in the VM lifecycle registry
  394. g.vmReg.register(&VMRecord{
  395. ExecID: execID,
  396. ScriptFile: scriptFile,
  397. Username: targetUser.Username,
  398. StartTime: time.Now(),
  399. interruptCh: vm.Interrupt,
  400. })
  401. //Create a panic recovery logic
  402. defer func() {
  403. g.vmReg.unregister(execID)
  404. //Release library resources (open SQLite handles, etc.) on every exit path
  405. releaseLibResources()
  406. if caught := recover(); caught != nil {
  407. if caught == errTimeout {
  408. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] Execution timeout: %s (user: %s)", scriptFile, targetUser.Username), nil)
  409. return
  410. } else if caught == errExitcall {
  411. //Exit gracefully
  412. return
  413. } else if caught == errForceStop {
  414. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM %s force-stopped (script: %s, user: %s)", execID, scriptFile, targetUser.Username), nil)
  415. if w != nil {
  416. w.WriteHeader(http.StatusServiceUnavailable)
  417. w.Write([]byte("503 - Script execution was force-terminated"))
  418. }
  419. } else {
  420. //Something screwed. Return Internal Server Error
  421. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI] VM crash in %s (user: %s): %v", scriptFile, targetUser.Username, caught), nil)
  422. if w != nil {
  423. devMode := r != nil && r.URL.Query().Get("agi_devmode") == "true"
  424. if devMode {
  425. errPayload, _ := json.Marshal(map[string]interface{}{
  426. "error": true,
  427. "message": fmt.Sprintf("VM crash: %v", caught),
  428. "stacktrace": fmt.Sprintf("VM crash: %v", caught),
  429. "script": scriptFile,
  430. "user": targetUser.Username,
  431. })
  432. w.Header().Set("Content-Type", "application/json")
  433. w.WriteHeader(http.StatusInternalServerError)
  434. w.Write(errPayload)
  435. } else {
  436. w.WriteHeader(http.StatusInternalServerError)
  437. w.Write([]byte("500 - ECMA VM crashed due to unknown reason"))
  438. }
  439. }
  440. }
  441. }
  442. }()
  443. //Create a max runtime of 5 minutes
  444. go func() {
  445. time.Sleep(300 * time.Second) // Stop after 300 seconds
  446. vm.Interrupt <- func() {
  447. panic(errTimeout)
  448. }
  449. }()
  450. //Try to read the script content.
  451. // When fsh is nil (e.g. app-root scripts), fall back to reading from the OS filesystem.
  452. var scriptContent []byte
  453. var err error
  454. if fsh != nil {
  455. scriptContent, err = fsh.FileSystemAbstraction.ReadFile(scriptFile)
  456. } else {
  457. scriptContent, err = os.ReadFile(scriptFile)
  458. }
  459. if err != nil {
  460. return execID, "", err
  461. }
  462. _, err = vm.Run(scriptContent)
  463. if err != nil {
  464. logger.PrintAndLog("Agi", fmt.Sprintf("[AGI][%s] Script error in %s (user: %s): %s", execID, scriptFile, targetUser.Username, err.Error()), nil)
  465. return execID, "", err
  466. }
  467. //Get the return value from the script
  468. value, err := vm.Get("HTTP_RESP")
  469. if err != nil {
  470. return execID, "", err
  471. }
  472. if w != nil {
  473. //Serverless: Get respond header type from the vm
  474. header, _ := vm.Get("HTTP_HEADER")
  475. headerString, _ := header.ToString()
  476. if headerString != "" {
  477. w.Header().Set("Content-Type", headerString)
  478. }
  479. }
  480. valueString, err := value.ToString()
  481. if err != nil {
  482. return execID, "", err
  483. }
  484. return execID, valueString, nil
  485. }
  486. /*
  487. Get user specific tmp filepath for buffering remote file. Return filepath and closer
  488. tempFilepath, closerFunction := g.getUserSpecificTempFilePath(u, "myfile.txt")
  489. //Do something with it, after done
  490. closerFunction();
  491. */
  492. func (g *Gateway) getUserSpecificTempFilePath(u *user.User, filename string) (string, func()) {
  493. uuid := uuid.NewV4().String()
  494. tmpFileLocation := filepath.Join(g.Option.TempFolderPath, "agiBuff", u.Username, uuid, filepath.Base(filename))
  495. os.MkdirAll(filepath.Dir(tmpFileLocation), 0775)
  496. return tmpFileLocation, func() {
  497. os.RemoveAll(filepath.Dir(tmpFileLocation))
  498. }
  499. }
  500. /*
  501. Buffer remote reosurces to local by fsh and rpath. Return buffer filepath on local device and its closer function
  502. */
  503. func (g *Gateway) bufferRemoteResourcesToLocal(fsh *filesystem.FileSystemHandler, u *user.User, rpath string) (string, func(), error) {
  504. buffFile, closerFunc := g.getUserSpecificTempFilePath(u, rpath)
  505. f, err := fsh.FileSystemAbstraction.ReadStream(rpath)
  506. if err != nil {
  507. return "", nil, err
  508. }
  509. defer f.Close()
  510. dest, err := os.OpenFile(buffFile, os.O_CREATE|os.O_RDWR, 0775)
  511. if err != nil {
  512. return "", nil, err
  513. }
  514. io.Copy(dest, f)
  515. dest.Close()
  516. return buffFile, func() {
  517. closerFunc()
  518. }, nil
  519. }