subservice.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617
  1. package subservice
  2. import (
  3. "encoding/json"
  4. "errors"
  5. "fmt"
  6. "net/http"
  7. "net/url"
  8. "os"
  9. "os/exec"
  10. "path/filepath"
  11. "runtime"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "time"
  16. "imuslab.com/arozos/mod/info/logger"
  17. modules "imuslab.com/arozos/mod/modules"
  18. "imuslab.com/arozos/mod/network/reverseproxy"
  19. "imuslab.com/arozos/mod/network/websocketproxy"
  20. user "imuslab.com/arozos/mod/user"
  21. )
  22. /*
  23. ArOZ Online System - Dynamic Subsystem loading services
  24. author: tobychui
  25. This module load in ArOZ Online Subservice using authorized reverse proxy channel.
  26. Please see the demo subservice module for more information on implementing a subservice module.
  27. */
  28. type SubService struct {
  29. Port int //Port that this subservice use
  30. ServiceDir string //The directory where the service is located
  31. Path string //Path that this subservice is located
  32. RpEndpoint string //Reverse Proxy Endpoint
  33. ProxyHandler *reverseproxy.ReverseProxy //Reverse Proxy Object
  34. Info modules.ModuleInfo //Module information for this subservice
  35. Process *exec.Cmd //The CMD runtime object of the process
  36. }
  37. type SubServiceRouter struct {
  38. ReservePaths []string
  39. RunningSubService []SubService
  40. BasePort int
  41. listenPort int
  42. userHandler *user.UserHandler
  43. moduleHandler *modules.ModuleHandler
  44. logger *logger.Logger
  45. }
  46. func NewSubServiceRouter(ReservePaths []string, basePort int, userHandler *user.UserHandler, moduleHandler *modules.ModuleHandler, parentPort int) *SubServiceRouter {
  47. //Create a service logger
  48. thisLogger, _ := logger.NewLogger("subserv", "system/logs/subservice", true)
  49. return &SubServiceRouter{
  50. ReservePaths: ReservePaths,
  51. RunningSubService: []SubService{},
  52. BasePort: basePort,
  53. listenPort: parentPort,
  54. userHandler: userHandler,
  55. moduleHandler: moduleHandler,
  56. logger: thisLogger,
  57. }
  58. }
  59. // Load and start all the subservices inside this rootpath
  60. func (sr *SubServiceRouter) LoadSubservicesFromRootPath(rootpath string) {
  61. scanningPath := filepath.ToSlash(filepath.Clean(rootpath)) + "/*"
  62. subservices, _ := filepath.Glob(scanningPath)
  63. for _, servicePath := range subservices {
  64. if !fileExists(servicePath + "/.disabled") {
  65. //Only enable module with no suspended config file
  66. err := sr.Launch(servicePath, true)
  67. if err != nil {
  68. sr.logger.PrintAndLog("Subservice", "Failed to start subservice: "+filepath.Base(servicePath)+" "+err.Error(), err)
  69. //log.Println(err)
  70. }
  71. }
  72. }
  73. }
  74. func (sr *SubServiceRouter) Launch(servicePath string, startupMode bool) error {
  75. //Get the executable name from its path
  76. servicePath = filepath.ToSlash(servicePath)
  77. binaryname := filepath.Base(servicePath)
  78. serviceRoot := filepath.Base(servicePath)
  79. binaryExecPath := filepath.ToSlash(binaryname)
  80. if runtime.GOOS == "windows" {
  81. binaryExecPath = binaryExecPath + ".exe"
  82. } else {
  83. binaryExecPath = binaryExecPath + "_" + runtime.GOOS + "_" + runtime.GOARCH
  84. }
  85. //Check if startscript exists. If no, try to launch the binaries
  86. if fileExists(servicePath + "/.startscript") {
  87. //Launch from start.bat or start.sh
  88. if !(fileExists(servicePath+"/start.sh") || fileExists(servicePath+"/start.bat")) {
  89. //log.Println("Failed to load subservice: " + serviceRoot + ", .startscript flag is TRUE but no start script found")
  90. return errors.New(".startscript flag is TRUE but no start script found")
  91. }
  92. startScriptName := "start.sh"
  93. if runtime.GOOS == "windows" {
  94. startScriptName = "start.bat"
  95. }
  96. binaryExecPath = startScriptName
  97. } else {
  98. //No startscript defined. Start from binary files if exists
  99. if runtime.GOOS == "windows" && !fileExists(servicePath+"/"+binaryExecPath) {
  100. if startupMode {
  101. //log.Println("Failed to load subservice: " + serviceRoot)
  102. return errors.New("Subservice executable not exists " + servicePath + "/" + binaryExecPath + ". Skipping this service")
  103. } else {
  104. return errors.New("Subservice executable " + servicePath + "/" + binaryExecPath + ". Skipping this service")
  105. }
  106. } else if runtime.GOOS == "linux" {
  107. //Check if service installed using which
  108. cmd := exec.Command("which", serviceRoot)
  109. searchResults, _ := cmd.CombinedOutput()
  110. if len(strings.TrimSpace(string(searchResults))) == 0 {
  111. //This is not installed. Check if it exists as a binary (aka ./myservice)
  112. if !fileExists(servicePath + "/" + binaryExecPath) {
  113. if startupMode {
  114. //log.Println("Package not installed. " + serviceRoot)
  115. return errors.New("Package not installed")
  116. } else {
  117. return errors.New("Package not installed")
  118. }
  119. }
  120. }
  121. } else if runtime.GOOS == "darwin" {
  122. //Skip the whereis approach that linux use
  123. if !fileExists(servicePath + "/" + binaryExecPath) {
  124. return errors.New("Subservice executable not exists " + servicePath + "/" + binaryExecPath + ". Skipping this service")
  125. }
  126. }
  127. }
  128. //Check if the suspend file exists. If yes, clear it
  129. if fileExists(servicePath + "/.disabled") {
  130. os.Remove(servicePath + "/.disabled")
  131. }
  132. //Check if there are config files that replace the -info tag. If yes, use it instead.
  133. out := []byte{}
  134. if fileExists(servicePath + "/moduleInfo.json") {
  135. launchConfig, err := os.ReadFile(servicePath + "/moduleInfo.json")
  136. if err != nil {
  137. if startupMode {
  138. logger.PrintAndLog("Subservice", fmt.Sprint("Failed to read moduleInfo.json: "+binaryname, err), nil)
  139. os.Exit(1)
  140. } else {
  141. return errors.New("Failed to read moduleInfo.json: " + binaryname)
  142. }
  143. }
  144. out = launchConfig
  145. } else {
  146. infocmd := exec.Command(servicePath+"/"+binaryExecPath, "-info")
  147. launchConfig, err := infocmd.CombinedOutput()
  148. if err != nil {
  149. sr.logger.PrintAndLog("Subservice", "Missing module startup info for "+servicePath, errors.New("Startup flag -info return no JSON string and moduleInfo.json does not exists for "+servicePath))
  150. if startupMode {
  151. logger.PrintAndLog("Subservice", fmt.Sprint("Unable to start service: "+binaryname, err), nil)
  152. os.Exit(1)
  153. } else {
  154. return errors.New("Unable to start service: " + binaryname)
  155. }
  156. }
  157. out = launchConfig
  158. }
  159. //Clean the module info and append it into the module list
  160. serviceLaunchInfo := strings.TrimSpace(string(out))
  161. thisModuleInfo := modules.ModuleInfo{}
  162. err := json.Unmarshal([]byte(serviceLaunchInfo), &thisModuleInfo)
  163. if err != nil {
  164. if startupMode {
  165. logger.PrintAndLog("Subservice", fmt.Sprint("Failed to load subservice: "+serviceRoot+"\n", err.Error()), nil)
  166. os.Exit(1)
  167. } else {
  168. return errors.New("Failed to load subservice: " + serviceRoot)
  169. }
  170. }
  171. var thisSubService SubService
  172. if fileExists(servicePath + "/.noproxy") {
  173. //Adaptive mode. This is designed for modules that do not designed with ArOZ Online in mind.
  174. //Ignore proxy setup and startup the application
  175. absolutePath, _ := filepath.Abs(servicePath + "/" + binaryExecPath)
  176. if fileExists(servicePath + "/.startscript") {
  177. initPath := servicePath + "/start.sh"
  178. if runtime.GOOS == "windows" {
  179. initPath = servicePath + "/start.bat"
  180. }
  181. if !fileExists(initPath) {
  182. if startupMode {
  183. logger.PrintAndLog("Subservice", "start.sh not found. Unable to startup service "+serviceRoot, nil)
  184. os.Exit(1)
  185. } else {
  186. return errors.New("start.sh not found. Unable to startup service " + serviceRoot)
  187. }
  188. }
  189. absolutePath, _ = filepath.Abs(initPath)
  190. }
  191. cmd := exec.Command(absolutePath)
  192. cmd.Stdout = os.Stdout
  193. cmd.Stderr = os.Stderr
  194. cmd.Dir = filepath.ToSlash(servicePath + "/")
  195. //Spawn a new go routine to run this subservice
  196. go func(cmdObject *exec.Cmd) {
  197. if err := cmd.Start(); err != nil {
  198. panic(err)
  199. }
  200. }(cmd)
  201. //Create the servie object
  202. thisSubService = SubService{
  203. Path: binaryExecPath,
  204. Info: thisModuleInfo,
  205. ServiceDir: serviceRoot,
  206. Process: cmd,
  207. }
  208. sr.logger.PrintAndLog("Subservice", "Starting service "+serviceRoot+" in compatibility mode", nil)
  209. } else {
  210. //Create a proxy for this service
  211. //Get proxy endpoint from startDir dir
  212. rProxyEndpoint := filepath.Dir(thisModuleInfo.StartDir)
  213. //Check if this path is reversed
  214. if stringInSlice(rProxyEndpoint, sr.ReservePaths) || rProxyEndpoint == "" {
  215. if startupMode {
  216. logger.PrintAndLog("Subservice", serviceRoot+" service try to request system reserve path as Reverse Proxy endpoint.", nil)
  217. os.Exit(1)
  218. } else {
  219. return errors.New(serviceRoot + " service try to request system reserve path as Reverse Proxy endpoint.")
  220. }
  221. }
  222. //Assign a port for this subservice
  223. thisServicePort := sr.GetNextUsablePort()
  224. //Run the subservice with the given port
  225. absolutePath, _ := filepath.Abs(servicePath + "/" + binaryExecPath)
  226. if fileExists(servicePath + "/.startscript") {
  227. initPath := servicePath + "/start.sh"
  228. if runtime.GOOS == "windows" {
  229. initPath = servicePath + "/start.bat"
  230. }
  231. if !fileExists(initPath) {
  232. if startupMode {
  233. logger.PrintAndLog("Subservice", "start.sh not found. Unable to startup service "+serviceRoot, nil)
  234. os.Exit(1)
  235. } else {
  236. return errors.New(serviceRoot + "start.sh not found. Unable to startup service " + serviceRoot)
  237. }
  238. }
  239. absolutePath, _ = filepath.Abs(initPath)
  240. }
  241. servicePort := ":" + intToString(thisServicePort)
  242. if fileExists(filepath.Join(servicePath, "/.intport")) {
  243. servicePort = intToString(thisServicePort)
  244. }
  245. cmd := exec.Command(absolutePath, "-port", servicePort, "-rpt", "http://localhost:"+intToString(sr.listenPort)+"/api/ajgi/interface")
  246. cmd.Stdout = os.Stdout
  247. cmd.Stderr = os.Stderr
  248. cmd.Dir = filepath.ToSlash(servicePath + "/")
  249. //log.Println(cmd.Dir,binaryExecPath)
  250. //Spawn a new go routine to run this subservice
  251. go func(cmd *exec.Cmd) {
  252. if err := cmd.Start(); err != nil {
  253. panic(err)
  254. }
  255. }(cmd)
  256. //Create a subservice object for this subservice
  257. thisSubService = SubService{
  258. Port: thisServicePort,
  259. Path: binaryExecPath,
  260. ServiceDir: serviceRoot,
  261. RpEndpoint: rProxyEndpoint,
  262. Info: thisModuleInfo,
  263. Process: cmd,
  264. }
  265. sr.logger.PrintAndLog("Subservice", "Subservice Registered: "+thisModuleInfo.Name, nil)
  266. //Create a new proxy object
  267. path, _ := url.Parse("http://localhost:" + intToString(thisServicePort))
  268. proxy := reverseproxy.NewReverseProxy(path)
  269. thisSubService.ProxyHandler = proxy
  270. }
  271. //Append this subservice into the list
  272. sr.RunningSubService = append(sr.RunningSubService, thisSubService)
  273. //Append this module into the loaded module list
  274. thisModuleInfo.SourceFolder = filepath.ToSlash(filepath.Clean(servicePath))
  275. sr.moduleHandler.LoadedModule = append(sr.moduleHandler.LoadedModule, &thisModuleInfo)
  276. return nil
  277. }
  278. func (sr *SubServiceRouter) HandleListing(w http.ResponseWriter, r *http.Request) {
  279. //List all subservice running in the background
  280. type visableInfo struct {
  281. Port int
  282. ServiceDir string
  283. Path string
  284. RpEndpoint string
  285. ProcessID int
  286. Info modules.ModuleInfo
  287. }
  288. type disabledServiceInfo struct {
  289. ServiceDir string
  290. Path string
  291. }
  292. enabled := []visableInfo{}
  293. disabled := []disabledServiceInfo{}
  294. for _, thisSubservice := range sr.RunningSubService {
  295. enabled = append(enabled, visableInfo{
  296. Port: thisSubservice.Port,
  297. Path: thisSubservice.Path,
  298. ServiceDir: thisSubservice.ServiceDir,
  299. RpEndpoint: thisSubservice.RpEndpoint,
  300. ProcessID: thisSubservice.Process.Process.Pid,
  301. Info: thisSubservice.Info,
  302. })
  303. }
  304. disabledModules, _ := filepath.Glob("subservice/*/.disabled")
  305. for _, modFile := range disabledModules {
  306. thisdsi := new(disabledServiceInfo)
  307. thisdsi.ServiceDir = filepath.Base(filepath.Dir(modFile))
  308. thisdsi.Path = filepath.Base(filepath.Dir(modFile))
  309. if runtime.GOOS == "windows" {
  310. thisdsi.Path = thisdsi.Path + ".exe"
  311. }
  312. disabled = append(disabled, *thisdsi)
  313. }
  314. jsonString, err := json.Marshal(struct {
  315. Enabled []visableInfo
  316. Disabled []disabledServiceInfo
  317. }{
  318. Enabled: enabled,
  319. Disabled: disabled,
  320. })
  321. if err != nil {
  322. sr.logger.PrintAndLog("Subservice", "Unable to list subservice folder", err)
  323. }
  324. sendJSONResponse(w, string(jsonString))
  325. }
  326. // Kill the subservice that is currently running
  327. func (sr *SubServiceRouter) HandleKillSubService(w http.ResponseWriter, r *http.Request) {
  328. userinfo, _ := sr.userHandler.GetUserInfoFromRequest(w, r)
  329. //Require admin permission
  330. if !userinfo.IsAdmin() {
  331. sendErrorResponse(w, "Permission denied")
  332. return
  333. }
  334. //OK. Get paramters
  335. serviceDir, _ := mv(r, "serviceDir", true)
  336. //moduleName, _ := mv(r, "moduleName", true)
  337. err := sr.KillSubService(serviceDir)
  338. if err != nil {
  339. sendErrorResponse(w, err.Error())
  340. } else {
  341. sendOK(w)
  342. }
  343. }
  344. func (sr *SubServiceRouter) HandleStartSubService(w http.ResponseWriter, r *http.Request) {
  345. userinfo, _ := sr.userHandler.GetUserInfoFromRequest(w, r)
  346. //Require admin permission
  347. if !userinfo.IsAdmin() {
  348. sendErrorResponse(w, "Permission denied")
  349. return
  350. }
  351. //OK. Get which dir to start
  352. serviceDir, _ := mv(r, "serviceDir", true)
  353. err := sr.StartSubService(serviceDir)
  354. if err != nil {
  355. sendErrorResponse(w, err.Error())
  356. } else {
  357. sendOK(w)
  358. }
  359. }
  360. // Check if the user has permission to access such proxy module
  361. func (sr *SubServiceRouter) CheckUserPermissionOnSubservice(ss *SubService, u *user.User) bool {
  362. moduleName := ss.Info.Name
  363. return u.GetModuleAccessPermission(moduleName)
  364. }
  365. // Check if the target is reverse proxy. If yes, return the proxy handler and the rewritten url in string
  366. func (sr *SubServiceRouter) CheckIfReverseProxyPath(r *http.Request) (bool, *reverseproxy.ReverseProxy, string, *SubService) {
  367. requestURL := r.URL.Path
  368. for _, subservice := range sr.RunningSubService {
  369. thisServiceProxyEP := subservice.RpEndpoint
  370. if thisServiceProxyEP != "" {
  371. if len(requestURL) > len(thisServiceProxyEP)+1 && requestURL[1:len(thisServiceProxyEP)+1] == thisServiceProxyEP {
  372. //This is a proxy path. Generate the rewrite URL
  373. //Get all GET paramters from URL
  374. values := r.URL.Query()
  375. counter := 0
  376. parsedGetTail := ""
  377. for k, v := range values {
  378. if counter == 0 {
  379. parsedGetTail = "?" + k + "=" + url.QueryEscape(v[0])
  380. } else {
  381. parsedGetTail = parsedGetTail + "&" + k + "=" + url.QueryEscape(v[0])
  382. }
  383. counter++
  384. }
  385. return true, subservice.ProxyHandler, requestURL[len(thisServiceProxyEP)+1:] + parsedGetTail, &subservice
  386. }
  387. }
  388. }
  389. return false, nil, "", &SubService{}
  390. }
  391. func (sr *SubServiceRouter) Close() {
  392. //Handle shutdown of subprocesses. Kill all of them
  393. for _, subservice := range sr.RunningSubService {
  394. cmd := subservice.Process
  395. if cmd != nil {
  396. if runtime.GOOS == "windows" {
  397. //Force kill with the power of CMD
  398. kill := exec.Command("TASKKILL", "/T", "/F", "/PID", strconv.Itoa(cmd.Process.Pid))
  399. //kill.Stderr = os.Stderr
  400. //kill.Stdout = os.Stdout
  401. kill.Run()
  402. } else {
  403. //Send sigkill to process
  404. cmd.Process.Kill()
  405. }
  406. }
  407. }
  408. }
  409. func (sr *SubServiceRouter) KillSubService(serviceDir string) error {
  410. //Remove them from the system
  411. ssi := -1
  412. moduleName := ""
  413. for i, ss := range sr.RunningSubService {
  414. if ss.ServiceDir == serviceDir {
  415. ssi = i
  416. moduleName = ss.Info.Name
  417. //Kill the module cmd
  418. cmd := ss.Process
  419. if cmd != nil {
  420. if runtime.GOOS == "windows" {
  421. //Force kill with the power of CMD
  422. kill := exec.Command("TASKKILL", "/T", "/F", "/PID", strconv.Itoa(cmd.Process.Pid))
  423. kill.Run()
  424. } else {
  425. err := cmd.Process.Kill()
  426. if err != nil {
  427. return err
  428. }
  429. }
  430. }
  431. //Write a suspended file into the module
  432. os.WriteFile("subservice/"+ss.ServiceDir+"/.disabled", []byte(""), 0755)
  433. }
  434. }
  435. //Pop this service from running Subservice
  436. if ssi != -1 {
  437. i := ssi
  438. copy(sr.RunningSubService[i:], sr.RunningSubService[i+1:])
  439. sr.RunningSubService = sr.RunningSubService[:len(sr.RunningSubService)-1]
  440. }
  441. //Pop the related module from the loadedModule list
  442. mi := -1
  443. for i, m := range sr.moduleHandler.LoadedModule {
  444. if m.Name == moduleName {
  445. mi = i
  446. }
  447. }
  448. if mi != -1 {
  449. i := mi
  450. copy(sr.moduleHandler.LoadedModule[i:], sr.moduleHandler.LoadedModule[i+1:])
  451. sr.moduleHandler.LoadedModule = sr.moduleHandler.LoadedModule[:len(sr.moduleHandler.LoadedModule)-1]
  452. }
  453. return nil
  454. }
  455. func (sr *SubServiceRouter) StartSubService(serviceDir string) error {
  456. if fileExists("subservice/" + serviceDir) {
  457. err := sr.Launch("subservice/"+serviceDir, false)
  458. if err != nil {
  459. return err
  460. }
  461. } else {
  462. return errors.New("Subservice directory not exists.")
  463. }
  464. //Sort the list
  465. sort.Slice(sr.moduleHandler.LoadedModule, func(i, j int) bool {
  466. return sr.moduleHandler.LoadedModule[i].Name < sr.moduleHandler.LoadedModule[j].Name
  467. })
  468. sort.Slice(sr.RunningSubService, func(i, j int) bool {
  469. return sr.RunningSubService[i].Info.Name < sr.RunningSubService[j].Info.Name
  470. })
  471. return nil
  472. }
  473. // Get a list of subservice roots in realpath
  474. func (sr *SubServiceRouter) GetSubserviceRoot() []string {
  475. subserviceRoots := []string{}
  476. for _, subService := range sr.RunningSubService {
  477. subserviceRoots = append(subserviceRoots, subService.Path)
  478. }
  479. return subserviceRoots
  480. }
  481. // Scan and get the next avaible port for subservice from its basePort
  482. func (sr *SubServiceRouter) GetNextUsablePort() int {
  483. basePort := sr.BasePort
  484. for sr.CheckIfPortInUse(basePort) {
  485. basePort++
  486. }
  487. return basePort
  488. }
  489. func (sr *SubServiceRouter) CheckIfPortInUse(port int) bool {
  490. for _, service := range sr.RunningSubService {
  491. if service.Port == port {
  492. return true
  493. }
  494. }
  495. return false
  496. }
  497. func (sr *SubServiceRouter) HandleRoutingRequest(w http.ResponseWriter, r *http.Request, proxy *reverseproxy.ReverseProxy, subserviceObject *SubService, rewriteURL string) {
  498. u, _ := sr.userHandler.GetUserInfoFromRequest(w, r)
  499. if !sr.CheckUserPermissionOnSubservice(subserviceObject, u) {
  500. //Permission denied
  501. http.NotFound(w, r)
  502. return
  503. }
  504. //Perform reverse proxy serving
  505. r.URL, _ = url.Parse(rewriteURL)
  506. token, _ := sr.userHandler.GetAuthAgent().NewTokenFromRequest(w, r)
  507. r.Header.Set("aouser", u.Username)
  508. r.Header.Set("aotoken", token)
  509. r.Header.Set("X-Forwarded-Host", r.Host)
  510. if r.Header["Upgrade"] != nil && r.Header["Upgrade"][0] == "websocket" {
  511. //Handle WebSocket request. Forward the custom Upgrade header and rewrite origin
  512. r.Header.Set("A-Upgrade", "websocket")
  513. u, _ := url.Parse("ws://localhost:" + strconv.Itoa(subserviceObject.Port) + r.URL.String())
  514. wspHandler := websocketproxy.NewProxy(u)
  515. wspHandler.ServeHTTP(w, r)
  516. return
  517. }
  518. r.Host = r.URL.Host
  519. err := proxy.ServeHTTP(w, r)
  520. if err != nil {
  521. //Check if it is cancelling events.
  522. if !strings.Contains(err.Error(), "cancel") {
  523. sr.logger.PrintAndLog("Subservice", subserviceObject.Info.Name+" IS NOT RESPONDING!", err)
  524. sr.RestartSubService(subserviceObject)
  525. }
  526. }
  527. }
  528. // Handle fail start over when the remote target is not responding
  529. func (sr *SubServiceRouter) RestartSubService(ss *SubService) {
  530. go func(ss *SubService) {
  531. //Kill the original subservice
  532. sr.KillSubService(ss.ServiceDir)
  533. sr.logger.PrintAndLog("Subservice", "RESTARTING SUBSERVICE "+ss.Info.Name+" IN 10 SECOUNDS", nil)
  534. time.Sleep(10000 * time.Millisecond)
  535. sr.StartSubService(ss.ServiceDir)
  536. sr.logger.PrintAndLog("Subservice", "SUBSERVICE "+ss.Info.Name+" RESTARTED", nil)
  537. }(ss)
  538. }