fix: unpack executable assets inside SASJS_ROOT, and remove them

.github/scripts/smoke-executable.mjs

@@ -53,6 +53,14 @@ const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'sasjs-smoke-'))
 const port = await freePort()
 const baseUrl = `http://127.0.0.1:${port}`
 
+// pkg's default cache, in the home directory. Captured before the executable
+// starts so a folder left by an earlier run on the runner is not read as this
+// one's (sasjs/server#523).
+const homeCacheDir = path.join(os.homedir(), '.cache', 'pkg')
+const homeCacheBefore = new Set(
+  fs.existsSync(homeCacheDir) ? fs.readdirSync(homeCacheDir) : []
+)
+
 const env = {
   ...process.env,
   MODE: 'desktop',
@@ -185,8 +193,60 @@ const checkServedViews = async () => {
   console.log('the executable served /ddc-host and /ddc-host.js')
 }
 
-const deadline = Date.now() + READY_TIMEOUT_MS
+/**
+ * Where pkg unpacks the assets it serves (sasjs/server#523).
+ *
+ * pkg extracts a snapshot file the first time the executable asks for it, into
+ * whatever `PKG_NATIVE_CACHE_PATH` names AT THAT MOMENT - so this is the check
+ * that the redirect from `setProcessVariables` happens before anything reads a
+ * snapshot file. A source checkout and the npm package run no pkg prelude at
+ * all, which is why no unit test can see this.
+ */
+const checkPkgCache = () => {
+  const tmpFolder = path.join(env.SASJS_ROOT, 'tmp')
+  const folders = fs.existsSync(tmpFolder)
+    ? fs.readdirSync(tmpFolder).filter((entry) => entry.startsWith('pkg-cache-'))
+    : []
+
+  if (folders.length !== 1)
+    fail(
+      `expected one pkg cache folder under ${tmpFolder}, found ${folders.length}`
+    )
+
+  const countFiles = (folder) => {
+    let total = 0
 
+    for (const entry of fs.readdirSync(folder, { withFileTypes: true })) {
+      if (entry.isDirectory()) total += countFiles(path.join(folder, entry.name))
+      else if (entry.isFile()) total += 1
+    }
+
+    return total
+  }
+
+  const files = countFiles(path.join(tmpFolder, folders[0]))
+
+  // An empty folder means the assets came from somewhere else - the redirect
+  // happened but nothing was unpacked into it.
+  if (files === 0)
+    fail(`the pkg cache folder ${folders[0]} is empty`)
+
+  const homeCacheNow = new Set(
+    fs.existsSync(homeCacheDir) ? fs.readdirSync(homeCacheDir) : []
+  )
+  const leaked = [...homeCacheNow].filter((entry) => !homeCacheBefore.has(entry))
+
+  if (leaked.length > 0)
+    fail(
+      `the executable unpacked into the home cache instead: ${leaked.join(', ')}`
+    )
+
+  console.log(
+    `the executable unpacked ${files} file(s) into ${path.join(tmpFolder, folders[0])}, not ${homeCacheDir}`
+  )
+}
+
+const deadline = Date.now() + READY_TIMEOUT_MS
 while (Date.now() < deadline) {
   if (exited) fail(`the executable exited before it was ready (${JSON.stringify(exited)})`)
 
@@ -208,6 +268,7 @@ while (Date.now() < deadline) {
         )
 
       await checkServedViews()
+      checkPkgCache()
       finish(output.slice(-TAIL_LINES))
       break
     }

README.md

@@ -15,8 +15,7 @@ SASjs Server is available in two modes - Desktop (without authentication) and Se
 
 ## Installation
 
-Two channels - the npm package and the standalone executable. Both read the same
-environment variables and serve the same API.
+Two channels - the npm package and the standalone executable. Both read the same environment variables and serve the same API.
 
 ### npm
 

api/src/types/system/process.d.ts

@@ -8,6 +8,8 @@ declare namespace NodeJS {
     sasjsRoot: string
     logsLoc: string
     logsUUID: string
+    /** Present only in an executable built by pkg. */
+    pkg?: unknown
     sessionController?: import('../../controllers/internal').SessionController
     sasSessionController?: import('../../controllers/internal').SASSessionController
     appStreamConfig: import('../').AppStreamConfig

api/src/utils/index.ts

@@ -32,6 +32,7 @@ export * from './oidcClient'
 export * from './oidcGroupSync'
 export * from './oidcUser'
 export * from './parseLogToArray'
+export * from './pkgCache'
 export * from './provisionUserHome'
 export * from './removeTokensInDB'
 export * from './routeAccess'

api/src/utils/pkgCache.ts

@@ -0,0 +1,159 @@
+import { randomUUID } from 'crypto'
+import fs from 'fs'
+import path from 'path'
+
+/**
+ * The prefix pkg's extracted-asset cache takes inside `<SASJS_ROOT>/tmp`.
+ *
+ * `pkg` appends its own `pkg/tmp-XXXXXX` below the directory it is given, so the
+ * instance's folder is removed whole rather than file by file.
+ */
+const cacheFolderPrefix = 'pkg-cache-'
+
+/**
+ * How often a running instance refreshes its folder's timestamp.
+ *
+ * The sweep below removes folders by AGE, because no pid check can tell a live
+ * instance from a dead one across process namespaces - under Docker this server
+ * is pid 1 in every container. A heartbeat is what a live instance leaves
+ * instead: as long as it is running, its folder is never old.
+ */
+const heartbeatMs = 60 * 60 * 1000
+
+/** How old a folder no instance is refreshing has to be before a start removes it. */
+export const staleAfterMs = 24 * 60 * 60 * 1000
+
+const isPackaged = () => Boolean(process.pkg)
+
+export interface SweepPkgCachesOptions {
+  tmpFolder: string
+  now?: number
+  staleAfter?: number
+}
+
+/**
+ * Removes cache folders that no running instance is refreshing.
+ *
+ * A process that dies without running its `exit` handler - SIGKILL from a
+ * supervisor, an OOM kill, a crash - leaves its folder behind, and this is what
+ * keeps that from accumulating. Only folders this module named are considered,
+ * and a folder younger than `staleAfter` is left alone whatever it is.
+ */
+export const sweepStalePkgCaches = ({
+  tmpFolder,
+  now = Date.now(),
+  staleAfter = staleAfterMs
+}: SweepPkgCachesOptions) => {
+  let entries: string[]
+
+  try {
+    entries = fs.readdirSync(tmpFolder)
+  } catch {
+    return // the folder is created lazily by configurePkgCache, so a missing one is fine
+  }
+
+  entries.forEach((entry) => {
+    if (!entry.startsWith(cacheFolderPrefix)) return
+
+    const folder = path.join(tmpFolder, entry)
+
+    try {
+      const { mtimeMs } = fs.statSync(folder)
+
+      if (now - mtimeMs < staleAfter) return
+
+      fs.rmSync(folder, { recursive: true, force: true })
+    } catch {
+      // a folder that cannot be inspected or removed costs disk, not correctness
+    }
+  })
+}
+
+export interface PkgCacheOptions {
+  packaged?: boolean
+  root?: string
+  now?: number
+}
+
+/**
+ * Directs pkg's extracted-asset cache into this instance's own folder inside
+ * SASJS_ROOT, and removes it when the process exits.
+ *
+ * A packaged executable reads its bundled files from a snapshot inside the
+ * binary, and pkg externalises each one it is asked for into `pkgCacheDir()` - `~/.cache/pkg`
+ * unless `PKG_NATIVE_CACHE_PATH` says otherwise - under a random `tmp-XXXXXX`
+ * name. It registers that folder's removal on `beforeExit`, which is not emitted
+ * for `process.exit()` (yao-pkg/pkg#52), and this server's shutdown path ends
+ * with exactly that call. Every start that refreshed the drive therefore left a
+ * folder holding the 192 macro files it copied in the service account's home,
+ * where nothing looks and nothing cleans - sasjs/server#523.
+ *
+ * Three decisions make the leak go away without depending on upstream:
+ *
+ * - the cache lives in the data root the operator already watches, not in a
+ *   home directory nobody does;
+ * - the folder name carries a random id as well as the pid, so two instances -
+ *   including two containers that are both pid 1 - never share one, and an exit
+ *   can remove exactly its own;
+ * - liveness is a timestamp the instance refreshes hourly, not a pid probe, so a
+ *   sweep can run without a namespace-aware view of the process table.
+ *
+ * A `PKG_NATIVE_CACHE_PATH` the operator has set is left alone: on a host whose
+ * SASJS_ROOT is small or read-only, that is their call to make.
+ *
+ * Note the scope: pkg resolves the directory it extracts NATIVE ADDONS into when
+ * its prelude loads, before any application code runs, so `~/.cache/pkg` can
+ * still appear for a native addon. This server ships none - the assets here are
+ * the ones the application reads.
+ *
+ * @returns the folder pkg is directed to, or undefined when this process is not
+ * a pkg executable or the operator has set the variable themselves.
+ */
+export const configurePkgCache = ({
+  packaged = isPackaged(),
+  root = process.sasjsRoot,
+  now = Date.now()
+}: PkgCacheOptions = {}): string | undefined => {
+  if (!packaged) return
+  if (process.env.PKG_NATIVE_CACHE_PATH) return
+
+  const tmpFolder = path.join(root, 'tmp')
+  const cacheFolder = path.join(
+    tmpFolder,
+    `${cacheFolderPrefix}${process.pid}-${randomUUID()}`
+  )
+
+  sweepStalePkgCaches({ tmpFolder, now })
+
+  fs.mkdirSync(cacheFolder, { recursive: true })
+
+  process.env.PKG_NATIVE_CACHE_PATH = cacheFolder
+
+  // A directory's mtime moves when its own entries change, and pkg writes into
+  // the folder BELOW this one - so the heartbeat has to set the time explicitly.
+  const heartbeat = setInterval(() => {
+    try {
+      const time = new Date()
+
+      fs.utimesSync(cacheFolder, time, time)
+    } catch {
+      // the folder has gone; the exit handler below has nothing left to do
+    }
+  }, heartbeatMs)
+
+  heartbeat.unref()
+
+  // `exit` runs on `process.exit()` as well as on a drained event loop, and the
+  // removal is synchronous, so it completes before the process goes.
+  process.once('exit', () => {
+    clearInterval(heartbeat)
+
+    try {
+      fs.rmSync(cacheFolder, { recursive: true, force: true })
+    } catch {
+      // there is nothing useful to do about it on the way out
+    }
+  })
+
+  return cacheFolder
+}

api/src/utils/setProcessVariables.ts

@@ -6,7 +6,14 @@ import {
   fileExists
 } from '@sasjs/utils'
 import dotenv from 'dotenv'
-import { connectDB, getDesktopFields, ModeType, RunTimeType, SECRETS } from '.'
+import {
+  connectDB,
+  configurePkgCache,
+  getDesktopFields,
+  ModeType,
+  RunTimeType,
+  SECRETS
+} from '.'
 
 export const setProcessVariables = async () => {
   const { execPath } = process
@@ -69,6 +76,11 @@ export const setProcessVariables = async () => {
 
   process.sasjsRoot = getRealPath(absPath)
 
+  // pkg extracts the assets it serves from its snapshot into a cache directory
+  // of its own choosing, and leaves it behind when this server exits - so point
+  // it at a folder this instance owns inside the data root (sasjs/server#523).
+  const pkgCacheFolder = configurePkgCache()
+
   const { DRIVE_LOCATION } = process.env
   const absDrivePath = getAbsolutePath(
     DRIVE_LOCATION ?? path.join(process.sasjsRoot, 'drive'),
@@ -94,4 +106,6 @@ export const setProcessVariables = async () => {
   process.logger.info('sasDrive: ', process.driveLoc)
   process.logger.info('sasLogs: ', process.logsLoc)
   process.logger.info('runTimes: ', process.runTimes)
+
+  if (pkgCacheFolder) process.logger.info('pkg cache: ', pkgCacheFolder)
 }