diff --git a/packages/deploy/package.json b/packages/deploy/package.json index 6df709b8d..66b82f276 100644 --- a/packages/deploy/package.json +++ b/packages/deploy/package.json @@ -26,6 +26,7 @@ "license": "ISC", "devDependencies": { "@inquirer/testing": "^2.1.53", + "@openfn/lexicon": "workspace:^", "@types/json-diff": "^1.0.3", "@types/jsonpath": "^0.2.4", "@types/mock-fs": "^4.13.4", diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 34640c486..d4b5b80fc 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -1,11 +1,6 @@ -export type StateJob = { - id: string; - name: string; - adaptor: string; - body: string; - project_credential_id: string | null; - delete?: boolean; -}; +import { Provisioner } from '@openfn/lexicon/lightning'; + +export type StateJob = Provisioner.Job; export type SpecJobBody = | string @@ -14,138 +9,127 @@ export type SpecJobBody = content: string; }; -export type SpecJob = { +export type SpecJob = Omit< + Provisioner.Job, + 'id' | 'body' | 'project_credential_id' | 'keychain_credential_id' | 'delete' +> & { id?: string; - name: string; - adaptor: string; body: SpecJobBody; credential: string | null; }; -export type StateKafkaHost = [string, string]; - -export type StateKafkaConfiguration = { - hosts: StateKafkaHost[]; - topics: string[]; - initial_offset_reset_policy: string; - connect_timeout: number; -}; - -export type SpecKafkaConfiguration = { - hosts: string[]; - topics: string[]; - initial_offset_reset_policy: string; - connect_timeout: number; -}; +export type StateTrigger = Provisioner.Trigger; -export type WebhookResponse = { - error_code: number | null; - success_code: number | null; -}; +export type SpecTrigger = Omit; -export type WebhookReply = 'before_start' | 'after_completion'; +export type StateKafkaHost = [string, string]; -export type SpecTrigger = { - type: string; - cron_expression?: string; - cron_cursor_job?: string; - webhook_reply?: WebhookReply; - webhook_response?: WebhookResponse | null; - enabled?: boolean; - kafka_configuration?: SpecKafkaConfiguration; -}; +export type StateKafkaConfiguration = Provisioner.KafkaConfiguration; -export type StateTrigger = { - id: string; - type: string; - cron_expression?: string; - cron_cursor_job_id?: string | null; - webhook_reply?: WebhookReply; - webhook_response?: WebhookResponse | null; - delete?: boolean; - enabled?: boolean; - kafka_configuration?: StateKafkaConfiguration; -}; - -export type StateEdge = { - id: string; - condition_type: string; - condition_expression: string | null; - condition_label: string | null; - source_job_id: string | null; - source_trigger_id: string | null; - target_job_id: string; - enabled?: boolean; +export type SpecKafkaConfiguration = Omit< + Provisioner.KafkaConfiguration, + 'hosts' +> & { + hosts: string[]; }; -export type SpecEdge = { - condition_type: string; +export type StateEdge = Provisioner.Edge; + +export type SpecEdge = Omit< + Provisioner.Edge, + | 'id' + | 'condition_expression' + | 'condition_label' + | 'source_job_id' + | 'source_trigger_id' + | 'target_job_id' +> & { condition_expression: string | null; condition_label: string | null; source_job?: string | null; source_trigger?: string | null; target_job: string | null; - enabled?: boolean; -}; - -export type WorkflowSpec = { - id?: string; - name: string; - jobs?: Record; - triggers?: Record; - edges?: Record; -}; - -export type CredentialSpec = { - name: string; - owner: string; }; -export type CredentialState = { - id: string; - name: string; - owner: string; -}; +export type CredentialState = Provisioner.Credential; -export type CollectionSpec = { - name: string; -}; +export type CredentialSpec = Omit; -export type CollectionState = { - id: string; - name: string; - delete?: boolean; -}; +export type CollectionState = Provisioner.Collection; -export interface ProjectSpec { - name: string; - description: string; - workflows: Record; - credentials: Record; - collections: Record; -} +export type CollectionSpec = Omit; -export interface WorkflowState { - id: string; - name: string; +export interface WorkflowState + extends Omit< + Provisioner.Workflow, + 'jobs' | 'triggers' | 'edges' | 'deleted_at' + > { jobs: Record>; triggers: Record>; edges: Record>; - delete?: boolean; - project_id?: string; - - inserted_at?: string; - updated_at?: string; - deleted_at?: string; + deleted_at?: string | null; } -export interface ProjectState { - id: string; - name: string; +export type WorkflowSpec = Omit< + Provisioner.Workflow, + | 'id' + | 'jobs' + | 'triggers' + | 'edges' + | 'delete' + | 'project_id' + | 'version_history' + | 'concurrency' + | 'lock_version' + | 'inserted_at' + | 'updated_at' + | 'deleted_at' +> & { + id?: string; + jobs?: Record; + triggers?: Record; + edges?: Record; +}; + +export interface ProjectState + extends Omit< + Provisioner.Project, + | 'description' + | 'workflows' + | 'project_credentials' + | 'collections' + | 'scheduled_deletion' + | 'history_retention_period' + | 'dataclip_retention_period' + > { description: string; workflows: Record; project_credentials: Record; collections: Record; + scheduled_deletion?: string | null; + history_retention_period?: string | null; + dataclip_retention_period?: string | null; +} + +export interface ProjectSpec + extends Omit< + Provisioner.Project, + | 'id' + | 'description' + | 'workflows' + | 'project_credentials' + | 'collections' + | 'inserted_at' + | 'updated_at' + | 'scheduled_deletion' + | 'history_retention_period' + | 'dataclip_retention_period' + | 'parent_id' + > { + description: string; + workflows: Record; + credentials: Record; + collections: Record; } export interface ProjectPayload { diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts index bab1bdc9a..b856cc2d1 100644 --- a/packages/lexicon/lightning.d.ts +++ b/packages/lexicon/lightning.d.ts @@ -309,10 +309,20 @@ export namespace Provisioner { delete?: boolean; }; + export type WebhookResponse = { + error_code: number | null; + success_code: number | null; + }; + + export type WebhookReply = 'before_start' | 'after_completion'; + export type Trigger = { id: string; type: string; cron_expression?: string; + cron_cursor_job?: string; + webhook_reply?: WebhookReply; + webhook_response?: WebhookResponse | null; delete?: boolean; enabled?: boolean; kafka_configuration?: KafkaConfiguration; diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts index 6193ec2df..8ba1ba222 100644 --- a/packages/lexicon/portability.d.ts +++ b/packages/lexicon/portability.d.ts @@ -78,7 +78,11 @@ export interface Trigger extends Step { enabled?: boolean; - webhook_reply?: string; + webhook_reply?: 'before_start' | 'after_completion'; + webhook_response?: { + error_code?: number; + success_code?: number; + }; cron_cursor_job_id?: string; /** Allow arbitrary properties on trigger nodes (as configuration options) */ diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index b6ed9324c..4649675ca 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -120,7 +120,8 @@ export const mapWorkflow = ( // TODO what do we do if the condition is disabled? // I don't think that's the same as edge condition false? Object.values(workflow.triggers).forEach((trigger: Provisioner.Trigger) => { - const { type, enabled, ...otherProps } = trigger; + const { type, enabled, webhook_reply, webhook_response, ...otherProps } = + trigger; if (!mapped.start) { mapped.start = type; } @@ -132,6 +133,7 @@ export const mapWorkflow = ( id: type, type, enabled, + webhook_reply, openfn: renameKeys(otherProps, { id: 'uuid' }), next: connectedEdges.reduce((obj: any, edge) => { const target = Object.values(jobs).find( diff --git a/packages/project/src/util/version.ts b/packages/project/src/util/version.ts index df2d70913..767469abd 100644 --- a/packages/project/src/util/version.ts +++ b/packages/project/src/util/version.ts @@ -59,6 +59,7 @@ export const generateHash = ( 'cron_expression', 'enabled', 'webhook_reply', + 'webhook_response', 'cron_cursor_job_id', ].sort(); diff --git a/packages/project/test/canonical.test.ts b/packages/project/test/canonical.test.ts new file mode 100644 index 000000000..ec8e8bc90 --- /dev/null +++ b/packages/project/test/canonical.test.ts @@ -0,0 +1,212 @@ +import test from 'ava'; +import { ProjectSpec } from '@openfn/lexicon'; +import { Project } from '../src/Project'; + +/** + * This file tests a kitchen sink, canonical v2 project spec file + * + * It should build it without type errors, then serialize to json and yaml formats + */ +const project: ProjectSpec = { + id: 'kitchen-sink', + name: 'Kitchen Sink Test', + description: 'Everything including the kitchen sink', + schema_version: '4.0', + credentials: [{ owner: 'admin@openfn.org', name: 'secret-squirrel' }], + collections: ['nut-stash'], + workflows: [ + { + id: 'wf-webhook', + name: 'Webhook Workflow', + start: 'webhook', + options: { timeout: 60_000, run_memory_limit_mb: 512 }, + steps: [ + { + id: 'webhook', + name: 'Webhook Trigger', + type: 'webhook', + enabled: true, + webhook_reply: 'before_start', + webhook_response: { + success_code: 202, + error_code: 500, + }, + next: 'fetch', + }, + { + id: 'fetch', + name: 'Fetch Data', + adaptor: '@openfn/language-http@latest', + expression: 'get("/data");', + next: { + transform: true, + log: 'state.data.length > 0', + archive: { + condition: '!state.errors', + label: 'No errors', + disabled: false, + }, + }, + }, + { + id: 'transform', + name: 'Transform', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => state);', + }, + { + id: 'log', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => { console.log(state); return state; });', + }, + { + id: 'archive', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => state);', + }, + ], + }, + { + id: 'wf-cron', + name: 'Cron Workflow', + schema_version: '1.0', + start: 'cron', + steps: [ + { + id: 'cron', + name: 'Cron Trigger', + type: 'cron', + enabled: false, + cron_expression: '0 0 * * *', + cron_cursor_job_id: 'cron-job', + webhook_reply: 'after_completion', + next: { 'cron-job': true }, + }, + { + id: 'cron-job', + name: 'Daily Sync', + adaptor: '@openfn/language-dhis2@latest', + expression: 'create("trackedEntityInstances", state.data);', + }, + ], + }, + ], +}; + +test('create a canonical project', (t) => { + const p = new Project(project); + + t.is(p.id, 'kitchen-sink'); + t.is(p.name, 'Kitchen Sink Test'); + t.is(p.description, 'Everything including the kitchen sink'); + + t.is(p.workflows.length, 2); + t.deepEqual(p.credentials, [ + { owner: 'admin@openfn.org', name: 'secret-squirrel' }, + ]); + t.deepEqual(p.collections, ['nut-stash']); + + const [webhook, cron] = p.workflows; + + t.is(webhook.id, 'wf-webhook'); + t.is(webhook.name, 'Webhook Workflow'); + t.is(webhook.start, 'webhook'); + t.is(webhook.steps.length, 5); + t.is(webhook.steps[0].id, 'webhook'); + t.is(webhook.steps[0].type, 'webhook'); + + t.is(cron.id, 'wf-cron'); + t.is(cron.steps.length, 2); + t.is(cron.steps[0].type, 'cron'); + t.is(cron.steps[0].cron_expression, '0 0 * * *'); +}); + +test('convert to v2 yaml', (t) => { + const p = new Project(project); + const yaml = p.serialize('project') as string; + + const expected = `id: kitchen-sink +name: Kitchen Sink Test +schema_version: '4.0' +description: Everything including the kitchen sink +collections: + - nut-stash +credentials: + - owner: admin@openfn.org + name: secret-squirrel +openfn: {} +options: {} +workflows: + - id: wf-webhook + name: Webhook Workflow + start: webhook + options: + timeout: 60000 + run_memory_limit_mb: 512 + steps: + - id: archive + adaptor: '@openfn/language-common@latest' + expression: fn(state => state); + - id: fetch + name: Fetch Data + adaptor: '@openfn/language-http@latest' + expression: get("/data"); + next: + transform: true + log: state.data.length > 0 + archive: + condition: '!state.errors' + label: No errors + disabled: false + - id: log + adaptor: '@openfn/language-common@latest' + expression: fn(state => { console.log(state); return state; }); + - id: transform + name: Transform + adaptor: '@openfn/language-common@latest' + expression: fn(state => state); + - id: webhook + name: Webhook Trigger + type: webhook + enabled: true + webhook_reply: before_start + webhook_response: + success_code: 202 + error_code: 500 + next: fetch + history: [] + - id: wf-cron + name: Cron Workflow + schema_version: '1.0' + start: cron + steps: + - id: cron + name: Cron Trigger + type: cron + enabled: false + cron_expression: 0 0 * * * + cron_cursor_job_id: cron-job + webhook_reply: after_completion + next: + cron-job: true + - id: cron-job + name: Daily Sync + adaptor: '@openfn/language-dhis2@latest' + expression: create("trackedEntityInstances", state.data); + history: []`.trim(); + + t.is(yaml.trim(), expected); +}); + +// skipped because serialized step order is different +test.skip('convert to v2 json', (t) => { + const p = new Project(project); + const json = p.serialize('project', { format: 'json' }) as string; + + t.deepEqual(json, project); +}); + +// I'd like to load the canonical json then convert it into json format +// but dropping all state keys +// that means supporting Project.serialize('project', { state: false }) +test.todo('roundtrip'); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fa9301662..6ecfa4b03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -332,6 +332,9 @@ importers: '@inquirer/testing': specifier: ^2.1.53 version: 2.1.53(@types/node@18.19.130) + '@openfn/lexicon': + specifier: workspace:^ + version: link:../lexicon '@types/json-diff': specifier: ^1.0.3 version: 1.0.3