feat: support auto-delete runs on completion

api/config/development.mjs

@@ -7,7 +7,7 @@ export default {
   },
   port: 8082,
   privateDirectoryUrl: 'http://localhost:8080',
-  privateEventsUrl: 'http://localhost:8084',
+  privateEventsUrl: 'http://localhost:8083',
   secretKeys: {
     identities: 'secret-identities',
     events: 'secret-events'

api/config/test.mjs

@@ -7,7 +7,7 @@ export default {
   },
   port: 8082,
   privateDirectoryUrl: 'http://localhost:8080',
-  privateEventsUrl: 'http://localhost:8084',
+  privateEventsUrl: 'http://localhost:8083',
   secretKeys: {
     identities: 'secret-identities',
     events: 'secret-events'

api/package.json

@@ -18,7 +18,6 @@
     "@types/memoizee": "^0.4.11",
     "@types/multer": "^1.4.13",
     "@types/resolve-path": "^1.4.3",
-    "@types/semver": "^7.5.8",
     "@types/ws": "^8.5.12"
   },
   "dependencies": {
@@ -38,7 +37,7 @@
     "multer": "^2.0.1",
     "nanoid": "^5.0.8",
     "resolve-path": "^1.4.0",
-    "semver": "^7.6.3",
+    "semver": "^7.7.4",
     "tmp-promise": "^3.0.3",
     "ws": "^8.18.0"
   }

api/src/admin.ts → api/src/admin/router.ts

@@ -2,7 +2,8 @@ import { resolve } from 'node:path'
 import { readFile } from 'node:fs/promises'
 import { Router } from 'express'
 import { reqOrigin, session } from '@data-fair/lib-express'
-import getApiDoc from './utils/api-docs.ts'
+import { getStatus } from './status.ts'
+import getApiDoc from '../misc/utils/api-docs.ts'
 
 const router = Router()
 export default router
@@ -15,12 +16,17 @@ router.use(async (req, res, next) => {
 
 let info = { version: process.env.NODE_ENV }
 if (process.env.NODE_ENV === 'production') {
-  info = JSON.parse(await readFile(resolve(import.meta.dirname, '../../BUILD.json'), 'utf8'))
+  info = JSON.parse(await readFile(resolve(import.meta.dirname, '../../../BUILD.json'), 'utf8'))
 }
 router.get('/info', (req, res) => {
   res.send(info)
 })
 
+router.get('/status', async (req, res) => {
+  const status = await getStatus(req)
+  res.send(status)
+})
+
 // Get the full API documentation of the service
 router.get('/api-docs.json', async (req, res) => {
   res.json(getApiDoc(reqOrigin(req)))

api/src/admin/status.ts

@@ -0,0 +1,55 @@
+import type { Request } from 'express'
+import mongo from '#mongo'
+import config from '#config'
+import fs from 'node:fs/promises'
+
+const mongoStatus = async () => { await mongo.db.command({ ping: 1 }) }
+const volumeStatus = async () => {
+  await fs.writeFile(`${config.dataDir}/check-access.txt`, 'ok')
+}
+
+export const getStatus = async (req: Request) =>
+  runHealthChecks(req, [
+    { fn: mongoStatus, name: 'mongodb' },
+    { fn: volumeStatus, name: 'data volume' }
+  ])
+
+// Helper functions
+const getSingleStatus = async (req: Request, fn: (req: Request) => Promise<void>, name: string) => {
+  const start = performance.now()
+  try {
+    await fn(req)
+    return {
+      status: 'ok',
+      name,
+      timeInMs: Math.round(performance.now() - start)
+    }
+  } catch (err) {
+    return {
+      status: 'error',
+      message: err instanceof Error ? err.message : String(err),
+      name,
+      timeInMs: Math.round(performance.now() - start)
+    }
+  }
+}
+const runHealthChecks = async (
+  req: Request,
+  checks: Array<{ fn: (req: Request) => Promise<void>; name: string }>
+) => {
+  let results
+  try {
+    results = await Promise.all(checks.map(({ fn, name }) => getSingleStatus(req, fn, name)))
+  } catch (err: any) {
+    return {
+      status: 'error',
+      details: err.toString()
+    }
+  }
+  const errors = results.filter(r => r.status === 'error')
+  return {
+    status: errors.length ? 'error' : 'ok',
+    message: errors.length ? ('Problem with : ' + errors.map(s => s.name).join(', ')) : 'Service is ok',
+    details: results
+  }
+}

api/src/app.ts

@@ -1,13 +1,13 @@
 import { resolve } from 'node:path'
 import express from 'express'
 import { session, errorHandler, createSiteMiddleware, createSpaMiddleware, defaultNonceCSPDirectives } from '@data-fair/lib-express/index.js'
-import identitiesRouter from './routers/identities.ts'
-import limitsRouter from './routers/limits.ts'
-import pluginsRegistryRouter from './routers/plugins-registry.ts'
-import pluginsRouter from './routers/plugins.ts'
-import processingsRouter from './routers/processings.ts'
-import runsRouter from './routers/runs.ts'
-import adminRouter from './admin.ts'
+import identitiesRouter from './misc/routers/identities.ts'
+import limitsRouter from './limits/router.ts'
+import pluginsRegistryRouter from './plugins-registry/router.ts'
+import pluginsRouter from './plugins/router.ts'
+import processingsRouter from './processings/router.ts'
+import runsRouter from './runs/router.ts'
+import adminRouter from './admin/router.ts'
 import config, { uiConfig } from '#config'
 
 export const app = express()

api/src/utils/api-docs.ts → api/src/misc/utils/api-docs.ts

@@ -4,7 +4,7 @@ import { type Plugin, resolvedSchema as PluginSchema } from '#types/plugin/index
 import { readFileSync } from 'node:fs'
 import path from 'path'
 
-const packageJson = JSON.parse(readFileSync(path.resolve(import.meta.dirname, '../../package.json'), 'utf-8'))
+const packageJson = JSON.parse(readFileSync(path.resolve(import.meta.dirname, '../../../package.json'), 'utf-8'))
 
 // CTRL + K CTRL + 4 to fold operations levels
 

api/src/routers/plugins.ts → api/src/plugins/router.ts

@@ -15,7 +15,7 @@ import multer from 'multer'
 import { session, httpError } from '@data-fair/lib-express'
 import mongo from '#mongo'
 import config from '#config'
-import permissions from '../utils/permissions.ts'
+import permissions from '../misc/utils/permissions.ts'
 
 // @ts-ignore
 const ajv = ajvFormats(new Ajv({ strict: false }))
@@ -264,6 +264,7 @@ router.delete('/:id', permissions.isSuperAdmin, async (req, res) => {
   await fs.remove(path.join(pluginsDir, req.params.id))
   await fs.remove(path.join(pluginsDir, req.params.id + '-config.json'))
   await fs.remove(path.join(pluginsDir, req.params.id + '-access.json'))
+  await fs.remove(path.join(pluginsDir, req.params.id + '-metadata.json'))
   res.status(204).send()
 })
 

api/src/routers/processings.ts → api/src/processings/router.ts

@@ -15,15 +15,15 @@ import eventsQueue from '@data-fair/lib-node/events-queue.js'
 import { reqOrigin, session } from '@data-fair/lib-express/index.js'
 import { httpError } from '@data-fair/lib-utils/http-errors.js'
 import { createNext } from '@data-fair/processings-shared/runs.ts'
-import { applyProcessing, deleteProcessing } from '../utils/runs.ts'
+import { applyProcessing, deleteProcessing } from '../runs/service.ts'
 import { cipher, decipher } from '@data-fair/processings-shared/cipher.ts'
 import mongo from '#mongo'
 import config from '#config'
 import locks from '#locks'
 import { resolvedSchema as processingSchema } from '#types/processing/index.ts'
-import getApiDoc from '../utils/api-docs.ts'
-import findUtils from '../utils/find.ts'
-import permissions from '../utils/permissions.ts'
+import getApiDoc from '../misc/utils/api-docs.ts'
+import findUtils from '../misc/utils/find.ts'
+import permissions from '../misc/utils/permissions.ts'
 
 const router = Router()
 export default router

api/src/routers/runs.ts → api/src/runs/router.ts

@@ -5,8 +5,8 @@ import * as wsEmitter from '@data-fair/lib-node/ws-emitter.js'
 import { session } from '@data-fair/lib-express/index.js'
 import { Router } from 'express'
 import mongo from '#mongo'
-import findUtils from '../utils/find.ts'
-import permissions from '../utils/permissions.ts'
+import findUtils from '../misc/utils/find.ts'
+import permissions from '../misc/utils/permissions.ts'
 
 const router = Router()
 export default router

api/src/server.ts

@@ -13,7 +13,7 @@ import config from '#config'
 import mongo from '#mongo'
 import locks from '#locks'
 import http from 'http'
-import permissions from './utils/permissions.ts'
+import permissions from './misc/utils/permissions.ts'
 
 const exec = promisify(execCallback)
 const server = http.createServer(app)

api/types/run/schema.js

@@ -100,6 +100,10 @@ export default {
       items: {
         $ref: 'https://github.com/data-fair/processings/permission'
       }
+    },
+    deleteOnComplete: {
+      type: 'boolean',
+      description: 'If true, the run will be automatically deleted after completion'
     }
   }
 }

dev/resources/nginx.conf

@@ -98,11 +98,11 @@ http {
         }
 
         location /openapi-viewer {
-            proxy_pass http://localhost:8083;
+            proxy_pass http://localhost:8084;
         }
 
         location /events/ {
-            proxy_pass http://localhost:8084;
+            proxy_pass http://localhost:8083;
         }
     }
 }

docker-compose.yml

@@ -17,7 +17,7 @@ services:
     profiles:
       - dev
       - test
-    image: ghcr.io/data-fair/simple-directory:master
+    image: ghcr.io/data-fair/simple-directory:8
     network_mode: host
     depends_on:
       - mongo
@@ -52,41 +52,47 @@ services:
         condition: service_healthy
     environment:
       - PORT=8081
-      - DEBUG=files
       - MONGO_URL=mongodb://localhost:27017/data-fair
-      - OPENAPI_VIEWER_URL=http://localhost:5600/openapi-viewer/
       - DIRECTORY_URL=http://localhost:5600/simple-directory
-      - NOTIFY_URL=http://localhost:5600/events
+      - PRIVATE_EVENTS_URL=http://localhost:8083
+      - PRIVATE_DIRECTORY_URL=http://localhost:8080
       - PUBLIC_URL=http://localhost:5600/data-fair
       - WS_PUBLIC_URL=ws://localhost:5600/data-fair
-      - EXTRA_NAV_ITEMS=[{"id":"events","can":"admin","iframe":"http://localhost:5600/events/embed/events","basePath":"/events","icon":"mdi-clipboard-text-clock","title":"Traçabilité"}, {"id":"processings","can":"contrib","iframe":"http://localhost:5600/processings/processings","basePath":"/processings","icon":"mdi-cog-transfer-outline","title":"Traitements périodiques"}]
-      - EXTRA_ADMIN_NAV_ITEMS=[{"id":"processings","iframe":"http://localhost:5600/processings/admin","basePath":"/processings","icon":"mdi-cog-transfer-outline","title":"Plugins - Traitements"},{"id":"processingsAdminDoc","href":"http://localhost:5600/openapi-viewer?urlType=processingsAdmin","icon":"mdi-cog-transfer-outline","title":"API Traitements périodiques"}]
       - OBSERVER_ACTIVE=false
+
+      - PRIVATE_PROCESSINGS_URL=http://localhost:8082
+      - SECRET_EVENTS=secret-events
+      - EXTRA_ADMIN_NAV_ITEMS=[{"id":"processingsAdminDoc","href":"http://localhost:5600/openapi-viewer?urlType=processingsAdmin","icon":"mdi-cog-transfer-outline","title":"API Traitements périodiques"}]
+      - OPENAPI_VIEWER_URL=http://localhost:5600/openapi-viewer/
       - OPENAPI_VIEWER_V2=true
 
-  openapi-viewer:
-    profiles:
-      - dev
-    image: ghcr.io/data-fair/openapi-viewer:master
-    ports:
-      - 8083:8080
-    environment:
-      - USE_SIMPLE_DIRECTORY=true
-      - ALLOWED_URLS={"processings":"http://localhost:5600/processings/api/v1/api-docs.json","processingsId":"http://localhost:5600/processings/api/v1/processings/{id}/api-docs.json","processingsAdmin":"http://localhost:5600/processings/api/v1/admin/api-docs.json"}
+  #####
+  # optional services
+  #####
 
   events:
     profiles:
       - dev
     image: ghcr.io/data-fair/events:main
     network_mode: host
     environment:
-      - PORT=8084
+      - PORT=8083
+      - MONGO_URL=mongodb://localhost:27017/data-fair-events
       - PRIVATE_DIRECTORY_URL=http://localhost:5600/simple-directory
       - SECRET_IDENTITIES=secret-identities
       - SECRET_EVENTS=secret-events
       - SECRET_SENDMAILS=secret-sendmails
       - OBSERVER_ACTIVE=false
 
+  openapi-viewer:
+    profiles:
+      - dev
+    image: ghcr.io/data-fair/openapi-viewer:master
+    environment:
+      - PORT=8084
+      - USE_SIMPLE_DIRECTORY=true
+      - ALLOWED_URLS={"processings":"http://localhost:5600/processings/api/v1/api-docs.json","processingsId":"http://localhost:5600/processings/api/v1/processings/{id}/api-docs.json","processingsAdmin":"http://localhost:5600/processings/api/v1/admin/api-docs.json"}
+
   #####
   # db and search engine
   #####