diff --git a/.changeset/dull-meals-stare.md b/.changeset/dull-meals-stare.md new file mode 100644 index 000000000..3e036c38a --- /dev/null +++ b/.changeset/dull-meals-stare.md @@ -0,0 +1,8 @@ +--- +'@openfn/lexicon': minor +'@openfn/project': minor +'@openfn/deploy': minor +'@openfn/cli': minor +--- + +Add support for channels diff --git a/.changeset/frank-canyons-report.md b/.changeset/frank-canyons-report.md new file mode 100644 index 000000000..bd6da2ea7 --- /dev/null +++ b/.changeset/frank-canyons-report.md @@ -0,0 +1,5 @@ +--- +'@openfn/deploy': minor +--- + +Add support for webhook_response_config diff --git a/.changeset/salty-areas-juggle.md b/.changeset/salty-areas-juggle.md new file mode 100644 index 000000000..b954927d9 --- /dev/null +++ b/.changeset/salty-areas-juggle.md @@ -0,0 +1,5 @@ +--- +'@openfn/lexicon': minor +--- + +Support `cron_cursor_job_id`, `webhook_reply` and `webhook_response_config` in Provisioner types diff --git a/.changeset/shy-needles-attack.md b/.changeset/shy-needles-attack.md new file mode 100644 index 000000000..703e0b7f3 --- /dev/null +++ b/.changeset/shy-needles-attack.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': minor +--- + +Support webhook responses in sync & deploy diff --git a/.changeset/wide-seals-tease.md b/.changeset/wide-seals-tease.md new file mode 100644 index 000000000..3d7dc5b75 --- /dev/null +++ b/.changeset/wide-seals-tease.md @@ -0,0 +1,5 @@ +--- +'@openfn/project': patch +--- + +Support more trigger keys diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index 474b8a148..69e28e20f 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -164,6 +164,13 @@ function mergeTriggers( if (specTrigger.type === 'webhook' && specTrigger.webhook_reply) { trigger.webhook_reply = specTrigger.webhook_reply; } + if ( + specTrigger.type === 'webhook' && + specTrigger.webhook_response_config + ) { + trigger.webhook_response_config = + specTrigger.webhook_response_config; + } if (specTrigger.type === 'cron') { trigger.cron_expression = specTrigger.cron_expression; @@ -202,6 +209,13 @@ function mergeTriggers( if (specTrigger!.type === 'webhook' && specTrigger!.webhook_reply) { trigger.webhook_reply = specTrigger!.webhook_reply; } + if ( + specTrigger!.type === 'webhook' && + specTrigger!.webhook_response_config + ) { + trigger.webhook_response_config = + specTrigger!.webhook_response_config; + } if (specTrigger!.type === 'cron') { trigger.cron_expression = specTrigger!.cron_expression; @@ -360,6 +374,58 @@ export function mergeSpecIntoState( } ) ); + const nextChannels = Object.fromEntries( + splitZip(oldState.channels || {}, spec.channels || {}).map( + ([channelKey, stateChannel, specChannel]) => { + if (specChannel && !stateChannel) { + return [ + channelKey, + { + id: crypto.randomUUID(), + name: specChannel.name, + destination_url: specChannel.destination_url, + enabled: specChannel.enabled, + destination_credential_id: + specChannel.destination_credential && + getStateJobCredential( + specChannel.destination_credential, + nextCredentials + ), + }, + ]; + } + + if (specChannel && stateChannel) { + return [ + channelKey, + { + id: stateChannel.id, + name: specChannel.name, + destination_url: specChannel.destination_url, + enabled: specChannel.enabled, + destination_credential_id: + specChannel.destination_credential && + getStateJobCredential( + specChannel.destination_credential, + nextCredentials + ), + }, + ]; + } + + if (!specChannel && stateChannel) { + return [channelKey, { id: stateChannel.id, delete: true }]; + } + + throw new DeployError( + `Invalid channel spec or corrupted state for channel: ${ + stateChannel?.name || specChannel?.name + }`, + 'VALIDATION_ERROR' + ); + } + ) + ); const nextWorkflows = Object.fromEntries( splitZip(oldState.workflows, spec.workflows).map( @@ -428,6 +494,7 @@ export function mergeSpecIntoState( workflows: nextWorkflows, project_credentials: nextCredentials, collections: nextCollections, + channels: nextChannels, }; if (spec.description) projectState.description = spec.description; @@ -476,9 +543,12 @@ export function getStateFromProjectPayload( const collections = reduceByKey('name', project.collections || []); + const channels = reduceByKey('name', project.channels || []); + return { ...project, collections, + channels, project_credentials, workflows, }; @@ -547,9 +617,18 @@ export function mergeProjectPayloadIntoState( ) ); + const nextChannels = Object.fromEntries( + idKeyPairs(project.channels || [], state.channels || {}).map( + ([key, nextChannel, _state]) => { + return [key, nextChannel]; + } + ) + ); + return { ...project, collections: nextCollections, + channels: nextChannels, project_credentials: nextCredentials, workflows: nextWorkflows, }; @@ -595,12 +674,17 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { state.collections ); - const { collections: _, ...stateWithoutCollections } = state; + const channels: ProjectPayload['channels'] = Object.values( + state.channels || {} + ); + + const { collections: _, channels: __, ...stateWithoutOptionals } = state; return { - ...stateWithoutCollections, + ...stateWithoutOptionals, project_credentials, workflows, ...(collections.length > 0 && { collections }), + ...(channels.length > 0 && { channels }), }; } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 2cba41176..d8dbf4444 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -38,6 +38,11 @@ export type SpecKafkaConfiguration = { connect_timeout: number; }; +export type WebhookResponseConfig = { + error_code: number | null; + success_code: number | null; +}; + export type WebhookReply = 'before_start' | 'after_completion'; export type SpecTrigger = { @@ -45,6 +50,7 @@ export type SpecTrigger = { cron_expression?: string; cron_cursor_job?: string; webhook_reply?: WebhookReply; + webhook_response_config?: WebhookResponseConfig | null; enabled?: boolean; kafka_configuration?: SpecKafkaConfiguration; }; @@ -55,6 +61,7 @@ export type StateTrigger = { cron_expression?: string; cron_cursor_job_id?: string | null; webhook_reply?: WebhookReply; + webhook_response_config?: WebhookResponseConfig | null; delete?: boolean; enabled?: boolean; kafka_configuration?: StateKafkaConfiguration; @@ -110,12 +117,29 @@ export type CollectionState = { delete?: boolean; }; +export type ChannelSpec = { + name: string; + destination_url: string; + enabled: boolean; + destination_credential: string | null; +}; + +export type ChannelState = { + id: string; + name: string; + destination_url: string; + enabled: boolean; + destination_credential_id: string | null; + delete?: boolean; +}; + export interface ProjectSpec { name: string; description: string; workflows: Record; credentials: Record; collections: Record; + channels: Record; } export interface WorkflowState { @@ -139,6 +163,7 @@ export interface ProjectState { workflows: Record; project_credentials: Record; collections: Record; + channels: Record; } export interface ProjectPayload { @@ -146,6 +171,7 @@ export interface ProjectPayload { name: string; description: string; collections?: Concrete[]; + channels?: Concrete[]; project_credentials: Concrete[]; workflows: { id: string; diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index 130f61963..6c79750f5 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -129,6 +129,12 @@ export async function parseAndValidate( } } + if (pair.key && pair.key.value === 'channels') { + if (pair.value.value === null) { + return doc.createPair('channels', {}); + } + } + if (pair.key && pair.key.value === 'jobs') { if (pair.value.value === null) { errors.push({ diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index 42ccdf4f3..982c2b70e 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -5,6 +5,7 @@ export function fullExampleSpec() { name: 'my project', description: 'some helpful description', collections: {}, + channels: {}, credentials: {}, workflows: { 'workflow-one': { @@ -58,6 +59,7 @@ export function fullExampleState() { name: 'my project', description: 'some helpful description', collections: {}, + channels: {}, project_credentials: {}, workflows: { 'workflow-one': { @@ -254,6 +256,7 @@ export const lightningProjectState = { name: 'collection-one', }, }, + channels: {}, project_credentials: { 'email@test.com-Basic-Auth': { id: '25f48989-d349-4eb8-99c3-923ebba5b116', diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index 4d6b91e4e..8b55dbcb9 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -92,6 +92,7 @@ test('toNextState adding a job', (t) => { description: 'my test project', project_credentials: {}, collections: {}, + channels: {}, }); }); @@ -131,6 +132,7 @@ test('toNextState deleting a credential', (t) => { delete: true, }, }, + channels: {}, }); }); @@ -159,6 +161,7 @@ test('toNextState with empty state', (t) => { description: 'some helpful description', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: jp.query(result, '$..workflows["workflow-one"].id')[0], @@ -219,6 +222,7 @@ test('toNextState with no changes', (t) => { description: 'for the humans', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -335,6 +339,7 @@ test('toNextState with a new job', (t) => { description: 'some other description', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -639,6 +644,125 @@ test('toNextState omits webhook_reply on existing trigger when not specified', ( t.false('webhook_reply' in result.workflows.w.triggers.t); }); +test('toNextState sets webhook_response_config when specified', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: { + w: { + name: 'workflow', + jobs: {}, + triggers: { + t: { + type: 'webhook', + webhook_response_config: { success_code: 200, error_code: 400 }, + }, + }, + edges: {}, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + t.deepEqual(result.workflows.w.triggers.t.webhook_response_config, { + success_code: 200, + error_code: 400, + }); +}); + +test('toNextState omits webhook_response_config when not specified', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: { + w: { + name: 'workflow', + jobs: {}, + triggers: { + t: { type: 'webhook' }, + }, + edges: {}, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + t.false('webhook_response_config' in result.workflows.w.triggers.t); +}); + +test('toNextState sets webhook_response_config on existing trigger', (t) => { + const triggerId = 'aaa-bbb-ccc'; + const state = { + workflows: { + w: { + id: 'wf-1', + name: 'workflow', + jobs: {}, + triggers: { + t: { id: triggerId, type: 'webhook', enabled: true }, + }, + edges: {}, + }, + }, + }; + const spec = { + name: 'my project', + workflows: { + w: { + name: 'workflow', + jobs: {}, + triggers: { + t: { + type: 'webhook', + webhook_response_config: { success_code: 201, error_code: 500 }, + }, + }, + edges: {}, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + t.is(result.workflows.w.triggers.t.id, triggerId); + t.deepEqual(result.workflows.w.triggers.t.webhook_response_config, { + success_code: 201, + error_code: 500, + }); +}); + +test('toNextState omits webhook_response_config on existing trigger when not specified', (t) => { + const triggerId = 'aaa-bbb-ccc'; + const state = { + workflows: { + w: { + id: 'wf-1', + name: 'workflow', + jobs: {}, + triggers: { + t: { id: triggerId, type: 'webhook', enabled: true }, + }, + edges: {}, + }, + }, + }; + const spec = { + name: 'my project', + workflows: { + w: { + name: 'workflow', + jobs: {}, + triggers: { + t: { type: 'webhook' }, + }, + edges: {}, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + t.false('webhook_response_config' in result.workflows.w.triggers.t); +}); + test('toNextState sets cron_cursor_job_id on existing trigger', (t) => { const triggerId = 'aaa-bbb-ccc'; const jobId = 'job-uuid-111'; @@ -663,10 +787,18 @@ test('toNextState sets cron_cursor_job_id on existing trigger', (t) => { w: { name: 'workflow', jobs: { - 'job-a': { name: 'job a', adaptor: '@openfn/language-http', body: 'fn()' }, + 'job-a': { + name: 'job a', + adaptor: '@openfn/language-http', + body: 'fn()', + }, }, triggers: { - t: { type: 'cron', cron_expression: '0 * * * *', cron_cursor_job: 'job-a' }, + t: { + type: 'cron', + cron_expression: '0 * * * *', + cron_cursor_job: 'job-a', + }, }, edges: {}, }, @@ -702,7 +834,11 @@ test('toNextState omits cron_cursor_job_id on existing trigger when not specifie w: { name: 'workflow', jobs: { - 'job-a': { name: 'job a', adaptor: '@openfn/language-http', body: 'fn()' }, + 'job-a': { + name: 'job a', + adaptor: '@openfn/language-http', + body: 'fn()', + }, }, triggers: { t: { type: 'cron', cron_expression: '0 * * * *' }, @@ -824,6 +960,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { name: 'project', project_credentials: {}, collections: {}, + channels: {}, workflows: { a: { id: 'wf-a', @@ -876,3 +1013,264 @@ test('toProjectPayload drops empty collections key', (t) => { t.deepEqual(payload, expectedPayload); }); + +test('toNextState adding a channel', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: null, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + const channel = result.channels['webhook-out']; + t.truthy(channel.id); + t.is(channel.name, 'webhook-out'); + t.is(channel.destination_url, 'https://example.com/hook'); + t.is(channel.enabled, true); + t.is(channel.destination_credential_id, null); +}); + +test('toNextState updating a channel preserves id', (t) => { + const channelId = 'aaa-bbb-ccc'; + const state = { + workflows: {}, + channels: { + 'webhook-out': { + id: channelId, + name: 'webhook-out', + destination_url: 'https://old.example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://new.example.com/hook', + enabled: false, + destination_credential: null, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + t.is(result.channels['webhook-out'].id, channelId); + t.is( + result.channels['webhook-out'].destination_url, + 'https://new.example.com/hook' + ); + t.is(result.channels['webhook-out'].enabled, false); +}); + +test('toNextState deleting a channel', (t) => { + const channelId = 'aaa-bbb-ccc'; + const state = { + workflows: {}, + channels: { + 'webhook-out': { + id: channelId, + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + channels: {}, + }; + + const result = mergeSpecIntoState(state, spec); + + t.deepEqual(result.channels['webhook-out'], { + id: channelId, + delete: true, + }); +}); + +test('toNextState resolves channel destination_credential to id', (t) => { + const state = { + workflows: {}, + project_credentials: { + 'me-auth': { + id: 'cred-id-123', + name: 'auth', + owner: 'me', + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + credentials: { + 'me-auth': { name: 'auth', owner: 'me' }, + }, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'me-auth', + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + t.is( + result.channels['webhook-out'].destination_credential_id, + 'cred-id-123' + ); +}); + +test('toNextState throws when channel references unknown credential', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'missing', + }, + }, + }; + + t.throws(() => mergeSpecIntoState(state, spec), { + message: 'Could not find a credential with name: missing', + }); +}); + +test('getStateFromProjectPayload reads channels', (t) => { + const project = { + id: 'xyz', + name: 'project', + workflows: [], + project_credentials: [], + channels: [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ], + }; + + const state = getStateFromProjectPayload(project); + + t.deepEqual(state.channels, { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }); +}); + +test('toProjectPayload drops empty channels key', (t) => { + const projectState = { + ...lightningProjectState, + channels: {}, + }; + + const payload = toProjectPayload(projectState); + + t.false('channels' in payload); +}); + +test('toProjectPayload includes channels when non-empty', (t) => { + const projectState = { + ...lightningProjectState, + channels: { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + + const payload = toProjectPayload(projectState); + + t.deepEqual(payload.channels, [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]); +}); + +test('mergeProjectIntoState preserves channels from payload', (t) => { + const state = { + id: 'p-1', + name: 'p', + description: '', + workflows: {}, + project_credentials: {}, + collections: {}, + channels: { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const payload: ProjectPayload = { + id: 'p-1', + name: 'p', + description: '', + workflows: [], + project_credentials: [], + channels: [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://updated.example.com/hook', + enabled: false, + destination_credential_id: null, + }, + ], + }; + + const result = mergeProjectPayloadIntoState(state, payload); + + t.deepEqual(result.channels, { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://updated.example.com/hook', + enabled: false, + destination_credential_id: null, + }, + }); +}); diff --git a/packages/deploy/test/validator.test.ts b/packages/deploy/test/validator.test.ts index 79e88706e..d963db1b0 100644 --- a/packages/deploy/test/validator.test.ts +++ b/packages/deploy/test/validator.test.ts @@ -130,6 +130,46 @@ test('allow empty workflows', async (t) => { }); }); +test('allows null channels by converting to empty map', async (t) => { + const doc = ` + name: project-name + channels: + workflows: + workflow-one: + name: workflow one + `; + + const result = await parseAndValidate(doc, 'spec.yaml'); + + t.is(result.errors.length, 0); + t.deepEqual(result.doc.channels, {}); +}); + +test('parses channels with destination_credential', async (t) => { + const doc = ` + name: project-name + channels: + webhook-out: + name: webhook-out + destination_url: https://example.com/hook + enabled: true + destination_credential: my-cred + workflows: + workflow-one: + name: workflow one + `; + + const result = await parseAndValidate(doc, 'spec.yaml'); + + t.is(result.errors.length, 0); + t.deepEqual(result.doc.channels!['webhook-out'], { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'my-cred', + }); +}); + test('adds the file content into the job body from the specified path', async (t) => { // Step 1: Create a temporary file that the YAML will reference const fileContent = 'fn(state => state.data);'; diff --git a/packages/lexicon/index.d.ts b/packages/lexicon/index.d.ts index 5ee3e64b9..4b29c3821 100644 --- a/packages/lexicon/index.d.ts +++ b/packages/lexicon/index.d.ts @@ -1,2 +1,3 @@ export * from './core'; +export * from './portability'; export * as lightning from './lighting'; diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts index bab1bdc9a..aa93a7304 100644 --- a/packages/lexicon/lightning.d.ts +++ b/packages/lexicon/lightning.d.ts @@ -1,5 +1,6 @@ import type { LogLevel, SanitizePolicies } from '@openfn/logger'; import { LegacyJob, State } from './core'; +import { Channel } from './portability'; export const API_VERSION: number; @@ -252,6 +253,8 @@ export namespace Provisioner { // should be an array of something? collections: any[]; + channels?: Channel[]; + // serverside metadata inserted_at?: string; updated_at?: string; @@ -285,6 +288,8 @@ export namespace Provisioner { inserted_at?: string; updated_at?: string; deleted_at: string | null; + + positions?: Record; } export type Collection = { @@ -309,10 +314,18 @@ export namespace Provisioner { delete?: boolean; }; + export type WebhookResponseConfig = { + error_code: number | null; + success_code: number | null; + }; + export type Trigger = { id: string; type: string; cron_expression?: string; + cron_cursor_job_id?: string | null; + webhook_reply?: 'before_start' | 'after_completion'; + webhook_response_config?: WebhookResponseConfig | null; delete?: boolean; enabled?: boolean; kafka_configuration?: KafkaConfiguration; diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts index 6193ec2df..1bd985cd5 100644 --- a/packages/lexicon/portability.d.ts +++ b/packages/lexicon/portability.d.ts @@ -24,6 +24,8 @@ export interface ProjectSpec { credentials?: Credential[]; collections?: string[]; + + channels?: Channel[]; } export interface WorkflowSpec { @@ -78,11 +80,12 @@ export interface Trigger extends Step { enabled?: boolean; - webhook_reply?: string; + webhook_reply?: 'before_start' | 'after_completion'; + webhook_response_config?: { + error_code?: number; + success_code?: number; + }; cron_cursor_job_id?: string; - - /** Allow arbitrary properties on trigger nodes (as configuration options) */ - [option: string]: any; } // TODO credential should just be an id string in the near future @@ -96,3 +99,12 @@ export interface Job extends Step { expression?: string; configuration?: object | string; } + +export type Channel = { + id: string; + name: string; + destination_url: string; + enabled: boolean; + destination_credential_id?: string; + delete?: boolean; +}; diff --git a/packages/project/CHANGELOG.md b/packages/project/CHANGELOG.md index 8aed302d3..0929ce629 100644 --- a/packages/project/CHANGELOG.md +++ b/packages/project/CHANGELOG.md @@ -1,5 +1,18 @@ # @openfn/project +## 0.16.0 + +### Minor Changes + +- aade583: Add support for channels + +### Patch Changes + +- d43bab9: Support more trigger keys +- Updated dependencies [aade583] +- Updated dependencies [d43bab9] + - @openfn/lexicon@2.2.0 + ## 0.15.2 ### Patch Changes diff --git a/packages/project/package.json b/packages/project/package.json index 4ce92a645..27e68bc3f 100644 --- a/packages/project/package.json +++ b/packages/project/package.json @@ -1,6 +1,6 @@ { "name": "@openfn/project", - "version": "0.15.2", + "version": "0.16.0", "description": "Read, serialize, replicate and sync OpenFn projects", "scripts": { "test": "pnpm ava", diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 3691bacde..749600b26 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -76,6 +76,8 @@ export class Project { collections: any; + channels?: l.Channel[]; + credentials: Credential[]; sandbox?: SandboxMeta; @@ -161,6 +163,7 @@ export class Project { this.options = data.options; this.workflows = data.workflows?.map(maybeCreateWorkflow) ?? []; this.collections = data.collections; + this.channels = data.channels; this.credentials = data.credentials ?? []; this.sandbox = data.sandbox; } diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 47e0d62d0..b689bb57f 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -169,11 +169,12 @@ export function merge( target.credentials ), collections: source.collections ?? target.collections, + channels: source.channels ?? target.channels, }; // with project level props merging, target goes into source because we want to preserve the target props. return new Project( - baseMerge(target, source, ['collections'], assigns as any) + baseMerge(target, source, ['collections', 'channels'], assigns as any) ); } diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index b6ed9324c..16c98ab21 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -6,6 +6,7 @@ import { Provisioner } from '@openfn/lexicon/lightning'; import { Project } from '../Project'; import renameKeys from '../util/rename-keys'; import slugify from '../util/slugify'; +import omitNil from '../util/omit-nil'; import ensureJson from '../util/ensure-json'; import getCredentialName from '../util/get-credential-name'; @@ -29,6 +30,7 @@ export default ( workflows, project_credentials = [], collections, + channels, inserted_at, updated_at, parent_id, @@ -46,6 +48,7 @@ export default ( name, description: description ?? undefined, collections, + channels, credentials, options, config: config as l.WorkspaceConfig, @@ -120,7 +123,15 @@ export const mapWorkflow = ( // TODO what do we do if the condition is disabled? // I don't think that's the same as edge condition false? Object.values(workflow.triggers).forEach((trigger: Provisioner.Trigger) => { - const { type, enabled, ...otherProps } = trigger; + const { + type, + enabled, + cron_expression, + cron_cursor_job_id, + webhook_reply, + webhook_response_config, + ...otherProps + } = trigger; if (!mapped.start) { mapped.start = type; } @@ -128,23 +139,29 @@ export const mapWorkflow = ( const connectedEdges = Object.values(edges).filter( (e) => e.source_trigger_id === trigger.id ); - mapped.steps.push({ - id: type, - type, - enabled, - openfn: renameKeys(otherProps, { id: 'uuid' }), - next: connectedEdges.reduce((obj: any, edge) => { - const target = Object.values(jobs).find( - (j) => j.id === edge.target_job_id - ); - if (!target) { - throw new Error(`Failed to find ${edge.target_job_id}`); - } - // we use the name, not the id, to reference - obj[slugify(target.name)] = mapEdge(edge); - return obj; - }, {}), - } as l.Trigger); + mapped.steps.push( + omitNil({ + id: type, + type, + enabled, + cron_expression, + cron_cursor_job_id, + webhook_reply, + webhook_response_config, + openfn: renameKeys(otherProps, { id: 'uuid' }), + next: connectedEdges.reduce((obj: any, edge) => { + const target = Object.values(jobs).find( + (j) => j.id === edge.target_job_id + ); + if (!target) { + throw new Error(`Failed to find ${edge.target_job_id}`); + } + // we use the name, not the id, to reference + obj[slugify(target.name)] = mapEdge(edge); + return obj; + }, {}), + }) as l.Trigger + ); }); Object.values(workflow.jobs).forEach((step: Provisioner.Job) => { diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index 644147bb2..7805910a1 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -34,7 +34,7 @@ export default function ( } = project.openfn ?? {}; const state = omitBy( - pick(project, ['name', 'description', 'collections']), + pick(project, ['name', 'description', 'collections', 'channels']), isNil ) as Provisioner.Project; @@ -117,10 +117,12 @@ export const mapWorkflow = ( if (s.type) { isTrigger = true; + + const { type, id, next, openfn, ...rest } = s; node = { + ...rest, type: s.type ?? 'webhook', // this is mostly for tests - enabled: s.enabled, - ...renameKeys(s.openfn, { uuid: 'id' }), + ...renameKeys(openfn, { uuid: 'id' }), } as Provisioner.Trigger; wfState.triggers[node.type] = node; } else { diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts index 8192041f8..d970aedbe 100644 --- a/packages/project/src/serialize/to-project.ts +++ b/packages/project/src/serialize/to-project.ts @@ -33,6 +33,7 @@ export default (project: Project, options: ToProjectOptions = {}) => { description: project.description, collections: project.collections, + channels: project.channels, credentials: project.credentials, openfn: omitBy(project.openfn, isNil), diff --git a/packages/project/src/util/version.ts b/packages/project/src/util/version.ts index df2d70913..c9e8f2f27 100644 --- a/packages/project/src/util/version.ts +++ b/packages/project/src/util/version.ts @@ -41,7 +41,6 @@ export const generateHash = ( // this means we can match keys with lightning // and everything gets cleaner const wfState = mapWorkflow(workflow); - // These are the keys we hash against const wfKeys = ['name', 'positions'].sort(); @@ -56,9 +55,10 @@ export const generateHash = ( const triggerKeys = [ 'type', - 'cron_expression', 'enabled', + 'cron_expression', 'webhook_reply', + 'webhook_response_config', 'cron_cursor_job_id', ].sort(); diff --git a/packages/project/test/canonical.test.ts b/packages/project/test/canonical.test.ts new file mode 100644 index 000000000..0bdda3be6 --- /dev/null +++ b/packages/project/test/canonical.test.ts @@ -0,0 +1,229 @@ +import test from 'ava'; +import { ProjectSpec } from '@openfn/lexicon'; +import { Project } from '../src/Project'; + +/** + * This file tests a kitchen sink, canonical v2 project spec file + * + * It should build it without type errors, then serialize to json and yaml formats + */ +const project: ProjectSpec = { + id: 'kitchen-sink', + name: 'Kitchen Sink Test', + description: 'Everything including the kitchen sink', + schema_version: '4.0', + credentials: [{ owner: 'admin@openfn.org', name: 'secret-squirrel' }], + collections: ['nut-stash'], + channels: [ + { + id: 'proxy', + name: 'My Proxy', + destination_url: 'https://proxy.openfn.org', + enabled: true, + delete: false, + destination_credential_id: 'secret', + }, + ], + workflows: [ + { + id: 'wf-webhook', + name: 'Webhook Workflow', + start: 'webhook', + options: { timeout: 60_000, run_memory_limit_mb: 512 }, + steps: [ + { + id: 'webhook', + name: 'Webhook Trigger', + type: 'webhook', + enabled: true, + webhook_reply: 'before_start', + webhook_response_config: { + success_code: 202, + error_code: 500, + }, + next: 'fetch', + }, + { + id: 'fetch', + name: 'Fetch Data', + adaptor: '@openfn/language-http@latest', + expression: 'get("/data");', + next: { + transform: true, + log: 'state.data.length > 0', + archive: { + condition: '!state.errors', + label: 'No errors', + disabled: false, + }, + }, + }, + { + id: 'transform', + name: 'Transform', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => state);', + }, + { + id: 'log', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => { console.log(state); return state; });', + }, + { + id: 'archive', + adaptor: '@openfn/language-common@latest', + expression: 'fn(state => state);', + }, + ], + }, + { + id: 'wf-cron', + name: 'Cron Workflow', + schema_version: '1.0', + start: 'cron', + steps: [ + { + id: 'cron', + name: 'Cron Trigger', + type: 'cron', + enabled: false, + cron_expression: '0 0 * * *', + cron_cursor_job_id: 'cron-job', + webhook_reply: 'after_completion', + next: { 'cron-job': true }, + }, + { + id: 'cron-job', + name: 'Daily Sync', + adaptor: '@openfn/language-dhis2@latest', + expression: 'create("trackedEntityInstances", state.data);', + }, + ], + }, + ], +}; + +test('create a canonical project', (t) => { + const p = new Project(project); + + t.is(p.id, 'kitchen-sink'); + t.is(p.name, 'Kitchen Sink Test'); + t.is(p.description, 'Everything including the kitchen sink'); + + t.is(p.workflows.length, 2); + t.deepEqual(p.credentials, [ + { owner: 'admin@openfn.org', name: 'secret-squirrel' }, + ]); + t.deepEqual(p.collections, ['nut-stash']); + + const [webhook, cron] = p.workflows; + + t.is(webhook.id, 'wf-webhook'); + t.is(webhook.name, 'Webhook Workflow'); + t.is(webhook.start, 'webhook'); + t.is(webhook.steps.length, 5); + t.is(webhook.steps[0].id, 'webhook'); + t.is(webhook.steps[0].type, 'webhook'); + + t.is(cron.id, 'wf-cron'); + t.is(cron.steps.length, 2); + t.is(cron.steps[0].type, 'cron'); + t.is(cron.steps[0].cron_expression, '0 0 * * *'); +}); + +test('convert to v2 yaml', (t) => { + const p = new Project(project); + const yaml = p.serialize('project') as string; + + const expected = `id: kitchen-sink +name: Kitchen Sink Test +schema_version: '4.0' +description: Everything including the kitchen sink +collections: + - nut-stash +channels: + - id: proxy + name: My Proxy + destination_url: https://proxy.openfn.org + enabled: true + delete: false + destination_credential_id: secret +credentials: + - owner: admin@openfn.org + name: secret-squirrel +openfn: {} +options: {} +workflows: + - id: wf-webhook + name: Webhook Workflow + start: webhook + options: + timeout: 60000 + run_memory_limit_mb: 512 + steps: + - id: archive + adaptor: '@openfn/language-common@latest' + expression: fn(state => state); + - id: fetch + name: Fetch Data + adaptor: '@openfn/language-http@latest' + expression: get("/data"); + next: + transform: true + log: state.data.length > 0 + archive: + condition: '!state.errors' + label: No errors + disabled: false + - id: log + adaptor: '@openfn/language-common@latest' + expression: fn(state => { console.log(state); return state; }); + - id: transform + name: Transform + adaptor: '@openfn/language-common@latest' + expression: fn(state => state); + - id: webhook + name: Webhook Trigger + type: webhook + enabled: true + webhook_reply: before_start + webhook_response_config: + success_code: 202 + error_code: 500 + next: fetch + history: [] + - id: wf-cron + name: Cron Workflow + schema_version: '1.0' + start: cron + steps: + - id: cron + name: Cron Trigger + type: cron + enabled: false + cron_expression: 0 0 * * * + cron_cursor_job_id: cron-job + webhook_reply: after_completion + next: + cron-job: true + - id: cron-job + name: Daily Sync + adaptor: '@openfn/language-dhis2@latest' + expression: create("trackedEntityInstances", state.data); + history: []`.trim(); + + t.is(yaml.trim(), expected); +}); + +// skipped because serialized step order is different +test.skip('convert to v2 json', (t) => { + const p = new Project(project); + const json = p.serialize('project', { format: 'json' }) as string; + + t.deepEqual(json, project); +}); + +// I'd like to load the canonical json then convert it into json format +// but dropping all state keys +// that means supporting Project.serialize('project', { state: false }) +test.todo('roundtrip'); diff --git a/packages/project/test/fixtures/sample-v2-project.ts b/packages/project/test/fixtures/sample-v2-project.ts index 8c11c46f6..92dfd7323 100644 --- a/packages/project/test/fixtures/sample-v2-project.ts +++ b/packages/project/test/fixtures/sample-v2-project.ts @@ -27,6 +27,8 @@ export const json: SerializedProject = { name: 'b', id: 'b', configuration: 'admin@openfn.org|My Credential', + // TODO it's unclear why we have a type error here?? + // @ts-ignore openfn: { uuid: 3 }, expression: 'fn()', adaptor: 'common', @@ -35,6 +37,8 @@ export const json: SerializedProject = { id: 'trigger', openfn: { uuid: 2 }, type: 'webhook', + // TODO it's unclear why we have a type error here?? + // @ts-ignore next: { b: { openfn: { uuid: 4 } } }, }, ], diff --git a/packages/project/test/gen/generator.test.ts b/packages/project/test/gen/generator.test.ts index 94e522b96..4d2a2b44c 100644 --- a/packages/project/test/gen/generator.test.ts +++ b/packages/project/test/gen/generator.test.ts @@ -8,7 +8,7 @@ const LOG_OUTPUTS = false; // Generate a workflow with a fixed UUID seed // Pass test context to log the result -const gen = (src: string, t?: ExecutionContext, options = {}) => { +const gen = (src: string, t?: ExecutionContext, options = {}): any => { const result = generateWorkflow(src, { uuidSeed: 1, printErrors: false, @@ -176,7 +176,6 @@ test('it should generate a simple workflow with trailing space', (t) => { }); test("it should fail if there's a space on an edge", (t) => { - const result = gen('a -b'); t.throws(() => gen('a-'), { message: /parsing failed!/i, }); @@ -444,7 +443,7 @@ test('it should generate a webhook trigger', (t) => { test('it should generate a node with a prop', (t) => { const result = gen('a(expression=y)-b', t); - const expected = _.cloneDeep(fixtures.ab); + const expected: any = _.cloneDeep(fixtures.ab); expected.steps[0].expression = 'y'; t.deepEqual(result, expected); @@ -452,7 +451,7 @@ test('it should generate a node with a prop', (t) => { test('it should generate a node with a prop with an underscore', (t) => { const result = gen('a(project_credential_id=y)-b', t); - const expected = _.cloneDeep(fixtures.ab); + const expected: any = _.cloneDeep(fixtures.ab); expected.steps[0].openfn = { uuid: 1, project_credential_id: 'y', @@ -486,7 +485,7 @@ test('it should save unexpected props to .openfn', (t) => { test('it should generate a node with two props', (t) => { const result = gen('a(adaptor=j,expression=k)-b', t); - const expected = _.cloneDeep(fixtures.ab); + const expected: any = _.cloneDeep(fixtures.ab); expected.steps[0].adaptor = 'j'; expected.steps[0].expression = 'k'; @@ -495,7 +494,7 @@ test('it should generate a node with two props', (t) => { test('it should treat quotes specially', (t) => { const result = gen('a(expression="fn()")-b', t); - const expected = _.cloneDeep(fixtures.ab); + const expected: any = _.cloneDeep(fixtures.ab); expected.steps[0].expression = 'fn()'; t.deepEqual(result, expected); @@ -623,40 +622,12 @@ a-b #zz }); test('it should generate a project with seeded uuids', (t) => { - const result = generateProject('x', ['a-b'], { + const result: any = generateProject('x', ['a-b'], { openfnUuid: true, uuidSeed: 1000, uuid: 123, }); - const expected = { - id: 'workflow', - name: 'Workflow', - history: [], - steps: [ - { - id: 'a', - name: 'a', - openfn: { - uuid: 'A', - }, - next: { - b: { - openfn: { - uuid: 'AB', - }, - }, - }, - }, - { - id: 'b', - name: 'b', - openfn: { - uuid: 'B', - }, - }, - ], - }; t.deepEqual(result.openfn, { uuid: 123, }); @@ -710,6 +681,7 @@ test('it should generate a simple workflow with mapped uuids', (t) => { test('it should generate a project with mapped uuids', (t) => { const result = generateProject('x', ['a-b'], { openfnUuid: true, + // @ts-ignore uuidMap: [ { a: 'A', diff --git a/packages/project/test/merge/map-uuids.test.ts b/packages/project/test/merge/map-uuids.test.ts index f0b7b0d07..b559faf57 100644 --- a/packages/project/test/merge/map-uuids.test.ts +++ b/packages/project/test/merge/map-uuids.test.ts @@ -1,21 +1,20 @@ -import * as l from '@openfn/lexicon'; - import test from 'ava'; import mapUUIDs from '../../src/merge/map-uuids'; import generateWorkflow from '../../src/gen/generator'; import Workflow from '../../src/Workflow'; -const gen = (src) => generateWorkflow(src, { uuidSeed: 1 }); +const gen = (src: string) => generateWorkflow(src, { uuidSeed: 1 }); let idgen = 0; -const createSingleNode = (name, uuid) => +const createSingleNode = (name: string, uuid?: string | number) => new Workflow({ id: `wf-${++idgen}}`, steps: [ { id: name, name, + // @ts-ignore openfn: { uuid: uuid ?? crypto.randomUUID() }, }, ], diff --git a/packages/project/test/merge/merge-project.test.ts b/packages/project/test/merge/merge-project.test.ts index f8932490f..2ec99089d 100644 --- a/packages/project/test/merge/merge-project.test.ts +++ b/packages/project/test/merge/merge-project.test.ts @@ -1,5 +1,7 @@ import test from 'ava'; import { randomUUID } from 'node:crypto'; +import type { CredentialState } from '@openfn/lexicon'; + import Project from '../../src'; import { merge, @@ -7,16 +9,15 @@ import { replaceCredentials, } from '../../src/merge/merge-project'; import { generateWorkflow } from '../../src/gen/generator'; -import { Credential } from '../../src/Project'; let idgen = 0; // go over each node in a workflow and add a new uuid // does not mutate -const assignUUIDs = (workflow, generator = randomUUID) => ({ +const assignUUIDs = (workflow: any, generator: any = randomUUID) => ({ id: 'wf', ...workflow, - steps: workflow.steps.map((s) => { + steps: workflow.steps.map((s: any) => { const step = { ...s, openfn: { @@ -25,7 +26,7 @@ const assignUUIDs = (workflow, generator = randomUUID) => ({ }; if (s.next) { // TODO this reduction isn't quite right - step.next = Object.keys(s.next).reduce((obj, key) => { + step.next = Object.keys(s.next).reduce((obj: any, key) => { obj[key] = { condition: true, openfn: { @@ -39,7 +40,7 @@ const assignUUIDs = (workflow, generator = randomUUID) => ({ }), }); -const createProject = (workflow, id = 'a', extra = {}) => +const createProject = (workflow: any, id = 'a', extra = {}) => new Project({ id, name: id, @@ -50,7 +51,7 @@ const createProject = (workflow, id = 'a', extra = {}) => ...extra, }); -const createStep = (id, props) => ({ +const createStep = (id: string) => ({ id, adaptor: 'common', expression: 'fn(s => s)', @@ -80,12 +81,11 @@ test('Preserve the name and UUID of the target project', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); - const step = result.workflows[0].steps[0]; + const result: any = merge(staging, main); // Ensure that the result has the name and UUID of main t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); }); // In this test,two projects with the same credential "id" @@ -134,7 +134,7 @@ test('Merge new credentials into the target', (t) => { }); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); t.deepEqual(result.credentials, [ // Keep the original credential @@ -151,6 +151,69 @@ test('Merge new credentials into the target', (t) => { ]); }); +test('replace mode: source channels override target channels', (t) => { + const wf = { + steps: [ + { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, + ], + }; + const wf_a = assignUUIDs(wf); + const wf_b = assignUUIDs(wf); + + const targetChannels = [ + { + id: 'chan-target', + name: 'target-channel', + destination_url: 'https://target.example.com', + enabled: true, + destination_credential_id: null, + }, + ]; + const sourceChannels = [ + { + id: 'chan-source', + name: 'source-channel', + destination_url: 'https://source.example.com', + enabled: false, + destination_credential_id: null, + }, + ]; + + const target = createProject(wf_a, 'a', { channels: targetChannels }); + const source = createProject(wf_b, 'b', { channels: sourceChannels }); + + const result = merge(source, target, { mode: REPLACE_MERGE }); + + t.deepEqual(result.channels, sourceChannels); +}); + +test('replace mode: target channels preserved when source has none', (t) => { + const wf = { + steps: [ + { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, + ], + }; + const wf_a = assignUUIDs(wf); + const wf_b = assignUUIDs(wf); + + const targetChannels = [ + { + id: 'chan-target', + name: 'target-channel', + destination_url: 'https://target.example.com', + enabled: true, + destination_credential_id: null, + }, + ]; + + const target = createProject(wf_a, 'a', { channels: targetChannels }); + const source = createProject(wf_b, 'b'); + + const result = merge(source, target, { mode: REPLACE_MERGE }); + + t.deepEqual(result.channels, targetChannels); +}); + test('replace mode: replace the name and UUID of the target project', (t) => { const wf = { steps: [ @@ -171,12 +234,11 @@ test('replace mode: replace the name and UUID of the target project', (t) => { const local = createProject(wf_b, 'b'); // merge staging into main - const result = merge(local, remote, { mode: REPLACE_MERGE }); - const step = result.workflows[0].steps[0]; + const result: any = merge(local, remote, { mode: REPLACE_MERGE }); // Ensure that the result has the name and UUID of local t.is(result.name, 'b'); - t.is(result.openfn.uuid, local.openfn.uuid); + t.is(result.openfn.uuid, local.openfn!.uuid); }); test('merge a simple change between single-step workflows with preserved uuids', (t) => { @@ -204,13 +266,13 @@ test('merge a simple change between single-step workflows with preserved uuids', const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); const step = result.workflows[0].steps[0]; // The resulting project should basically be main but with a different adaptor t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); t.is(step.adaptor, wf_b.steps[0].adaptor); t.is(step.openfn.uuid, wf_a.steps[0].openfn.uuid); @@ -243,7 +305,7 @@ test('merge with history (prefers source history)', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); t.deepEqual(result.workflows[0].history, ['a', 'b']); }); @@ -273,13 +335,13 @@ test('merge a simple change between single-step workflows with preserved numeric const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); const step = result.workflows[0].steps[0]; // The resulting project should basically be main but with a different adaptor t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); t.is(step.adaptor, wf_b.steps[0].adaptor); t.is(step.openfn.uuid, wf_a.steps[0].openfn.uuid); @@ -310,12 +372,12 @@ test('merge a new step into an existing workflow', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); // The resulting project should have: const [x, y] = result.workflows[0].steps; t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); t.deepEqual(x, wf_a.steps[0]); t.deepEqual(y, wf_b.steps[1]); @@ -353,11 +415,11 @@ test('merge with an edge and no changes', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); // console.log(JSON.stringify(result.workflow, null, 2)); t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); const step = result.workflows[0].steps[0]; // The step and edge should be totally unchanged @@ -398,11 +460,11 @@ test('merge with a change to an edge condition', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); // console.log(JSON.stringify(result.workflow, null, 2)); t.is(result.name, 'a'); - t.is(result.openfn.uuid, main.openfn.uuid); + t.is(result.openfn.uuid, main.openfn!.uuid); const step = result.workflows[0].steps[0]; t.deepEqual(step.next.y.openfn, wf_a.steps[0].next.y.openfn); @@ -434,7 +496,7 @@ test('remove a step from an existing workflow', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); // The resulting project should have no steps t.is(result.workflows[0].steps.length, 0); @@ -465,7 +527,7 @@ test('merge an id change in a single step with preserved uuids', (t) => { const staging = createProject(wf_b, 'b'); // merge staging into main - const result = merge(staging, main); + const result: any = merge(staging, main); // The resulting project should have: const step = result.workflows[0].steps[0]; @@ -480,7 +542,7 @@ test('merge an id change in a single step with preserved uuids', (t) => { test('should merge two projects and preserve edge id', (t) => { const source = generateWorkflow(`a-b`).set('a-b', { condition: true, - }); + } as any); const target = generateWorkflow(`a-b`); t.not(source.getUUID('a-b'), target.getUUID('a-b')); @@ -489,7 +551,9 @@ test('should merge two projects and preserve edge id', (t) => { createProject(target.workflow) ); + // @ts-ignore const resultEdge = result.workflows[0].steps[0].next['b']; + // preserve edge condition from source t.is(resultEdge.condition, true); // preserve edge id from target @@ -512,7 +576,7 @@ test('merge a new workflow', (t) => { t.is(main.workflows.length, 1); t.is(staging.workflows.length, 2); - const result = merge(staging, main); + const result: any = merge(staging, main); t.is(result.workflows.length, 2); }); @@ -532,7 +596,7 @@ test('merge a new workflow with onlyUpdated: true', (t) => { t.is(main.workflows.length, 1); t.is(staging.workflows.length, 2); - const result = merge(staging, main, { onlyUpdated: true }); + const result: any = merge(staging, main, { onlyUpdated: true }); t.is(result.workflows.length, 2); }); @@ -552,7 +616,7 @@ test('remove a workflow', (t) => { t.is(main.workflows.length, 2); t.is(staging.workflows.length, 1); - const result = merge(staging, main); + const result: any = merge(staging, main); t.is(result.workflows.length, 1); }); @@ -572,7 +636,7 @@ test('remove a workflow with onlyUpdated: true', (t) => { t.is(main.workflows.length, 2); t.is(staging.workflows.length, 1); - const result = merge(staging, main, { onlyUpdated: true }); + const result: any = merge(staging, main, { onlyUpdated: true }); t.is(result.workflows.length, 1); }); @@ -656,13 +720,9 @@ test('id match: preserve target uuid', (t) => { t.is(result.workflows.length, 1); // we expect every thing in target to be overridden except the uuid + // @ts-ignore t.is(result.workflows[0].openfn.uuid, target.openfn.uuid); }); -// should preserve UUID if id changes - -// should generate a UUID if name, adaptor and expression fail - -// should generate a UUID if change is ambiguous test('options: no mappings & removeUnmapped=false', (t) => { const source_project = createProject([ @@ -825,12 +885,12 @@ test('options: onlyUpdated with no changed workflows', (t) => { generateWorkflow('@id a a-b', { history: true }), generateWorkflow('@id b x-y', { history: true }), ]); - const target = createProject([ + const target: any = createProject([ generateWorkflow('@id a a-b', { history: true }), generateWorkflow('@id b x-y', { history: true }), ]); - const result = merge(source, target, { + const result: any = merge(source, target, { onlyUpdated: true, mode: 'replace', }); @@ -858,13 +918,15 @@ test('options: onlyUpdated with 1 changed, 1 unchanged workflow', (t) => { ]); // Scribble on both workflows + // @ts-ignore target.workflows[0].jam = 'jar'; + // @ts-ignore target.workflows[1].jam = 'jar'; // change the source source.workflows[0].steps[0].expression = 'fn()'; - const result = merge(source, target, { + const result: any = merge(source, target, { onlyUpdated: true, // Set this to mode replace and use UUIDs as a proxy for @@ -887,7 +949,7 @@ test.todo('options: only changed and 1 workflow'); test.todo('options: only changed, and 1 changed, 1 unchanged workflow'); test('replaceCredentials: preserves target credentials with their UUIDs', (t) => { - const targetCreds: Credential[] = [ + const targetCreds: CredentialState[] = [ { uuid: 'target-uuid-1', name: 'cred1', owner: 'user1' }, { uuid: 'target-uuid-2', name: 'cred2', owner: 'user1' }, ]; @@ -900,10 +962,10 @@ test('replaceCredentials: preserves target credentials with their UUIDs', (t) => }); test('replaceCredentials: adds new credentials from source without their UUIDs', (t) => { - const sourceCreds: Credential[] = [ + const sourceCreds: CredentialState[] = [ { uuid: 'source-uuid-1', name: 'newcred', owner: 'user1' }, ]; - const targetCreds: Credential[] = [ + const targetCreds: CredentialState[] = [ { uuid: 'target-uuid-1', name: 'existingcred', owner: 'user1' }, ]; @@ -921,10 +983,10 @@ test('replaceCredentials: adds new credentials from source without their UUIDs', }); test('replaceCredentials: does not duplicate credentials with same name/owner', (t) => { - const sourceCreds: Credential[] = [ + const sourceCreds: CredentialState[] = [ { uuid: 'source-uuid-1', name: 'samecred', owner: 'user1' }, ]; - const targetCreds: Credential[] = [ + const targetCreds: CredentialState[] = [ { uuid: 'target-uuid-1', name: 'samecred', owner: 'user1' }, ]; @@ -938,10 +1000,10 @@ test('replaceCredentials: does not duplicate credentials with same name/owner', }); test('replaceCredentials: treats credentials with different owners as different', (t) => { - const sourceCreds: Credential[] = [ + const sourceCreds: CredentialState[] = [ { uuid: 'source-uuid-1', name: 'cred1', owner: 'user2' }, ]; - const targetCreds: Credential[] = [ + const targetCreds: CredentialState[] = [ { uuid: 'target-uuid-1', name: 'cred1', owner: 'user1' }, ]; @@ -955,12 +1017,12 @@ test('replaceCredentials: treats credentials with different owners as different' }); test('replaceCredentials: handles multiple new and existing credentials', (t) => { - const sourceCreds: Credential[] = [ + const sourceCreds: CredentialState[] = [ { uuid: 'source-uuid-1', name: 'existing', owner: 'user1' }, { uuid: 'source-uuid-2', name: 'new1', owner: 'user1' }, { uuid: 'source-uuid-3', name: 'new2', owner: 'user2' }, ]; - const targetCreds: Credential[] = [ + const targetCreds: CredentialState[] = [ { uuid: 'target-uuid-1', name: 'existing', owner: 'user1' }, { uuid: 'target-uuid-2', name: 'old', owner: 'user1' }, ]; diff --git a/packages/project/test/parse/from-app-state.test.ts b/packages/project/test/parse/from-app-state.test.ts index e90421796..f1db229bb 100644 --- a/packages/project/test/parse/from-app-state.test.ts +++ b/packages/project/test/parse/from-app-state.test.ts @@ -54,6 +54,29 @@ test('should create a Project from prov state with collections', (t) => { t.deepEqual(project.collections, []); }); +test('should create a Project from prov state with channels', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + const stateWithChannels: any = { ...state, channels }; + + const project = fromAppState(stateWithChannels, meta); + + t.deepEqual(project.channels, channels); +}); + +test('project channels is undefined when missing from state', (t) => { + const project = fromAppState(state, meta); + + t.is(project.channels, undefined); +}); + test('should create a Project from prov state with sandbox stuff', (t) => { const stateWithSandbox = { ...state, @@ -63,7 +86,7 @@ test('should create a Project from prov state with sandbox stuff', (t) => { }; const project = fromAppState(stateWithSandbox, meta, { format: 'json' }); - t.is(project.sandbox.parentId, 'abc'); + t.is(project.sandbox!.parentId, 'abc'); t.is(project.options.env, 'dev'); t.is(project.options.color, 'red'); }); @@ -81,14 +104,18 @@ test('should create a Project from prov state with positions', (t) => { // the provisioner right now doesn't include positions // - but one day it will, and Project needs to be able to sync it newState.workflows['my-workflow'].positions = { - x: 1, - y: 1, + step1: { + x: 1, + y: 1, + }, }; const project = fromAppState(newState, meta); - t.deepEqual(project.workflows[0].openfn.positions, { - x: 1, - y: 1, + t.deepEqual(project.workflows[0].openfn!.positions, { + step1: { + x: 1, + y: 1, + }, }); }); @@ -161,6 +188,7 @@ test('mapWorkflow: map a cron trigger', (t) => { id: '1234', type: 'cron', cron_expression: '0 1 0 0', + cron_cursor_job_id: 'x', enabled: true, }, }, @@ -174,15 +202,30 @@ test('mapWorkflow: map a cron trigger', (t) => { type: 'cron', next: {}, enabled: true, + cron_expression: '0 1 0 0', + cron_cursor_job_id: 'x', openfn: { uuid: '1234', - cron_expression: '0 1 0 0', }, }); }); test('mapWorkflow: map a webhook trigger', (t) => { - const mapped = mapWorkflow(state.workflows['my-workflow']); + const mapped = mapWorkflow({ + ...state.workflows['my-workflow'], + triggers: { + webhook: { + id: '4a06289c-15aa-4662-8dc6-f0aaacd8a058', + type: 'webhook', + enabled: true, + webhook_reply: 'before_start', + webhook_response_config: { + success_code: 202, + error_code: 500, + }, + }, + }, + }); const [trigger] = mapped.steps; @@ -190,6 +233,11 @@ test('mapWorkflow: map a webhook trigger', (t) => { id: 'webhook', type: 'webhook', enabled: true, + webhook_reply: 'before_start', + webhook_response_config: { + success_code: 202, + error_code: 500, + }, next: { 'transform-data': { condition: 'always', @@ -387,7 +435,7 @@ test('mapEdge: map conditions', (t) => { // TODO the workflow yaml is not a project yaml // so this test doesn't work // I'll need to pull the project yaml, with uuids, to get this to work -test.skip('mapWorkflow: map edge conditions', (t) => { +test.skip('mapWorkflow: map edge conditions', () => { // TODO for yaml like this: const yaml = ` workflows: @@ -439,12 +487,10 @@ workflows: condition_expression: state.ok == 22 `; - const project = fromAppState(yaml, meta, { + fromAppState(yaml, meta, { format: 'yaml', }); - console.log(project.workflows['my-workflow'].steps); - const { next } = project.workflows['my-workflow'].steps[1]; - console.log({ next }); + // const { next } = project.workflows['my-workflow'].steps[1]; // make sure that the condition_types get mapped to condition // also make sure that custom conditions work (both ways) }); diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index d554f7eff..134ca901b 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -18,6 +18,7 @@ function mockFile(path: string, content: string | object) { content = JSON.stringify(content); } + // @ts-ignore files[path] = content; mock(files); } @@ -55,12 +56,12 @@ test.serial('should include multiple workflows (legacy format)', async (t) => { t.is(project.workflows.length, 2); - const wf1 = project.getWorkflow('workflow-1'); + const wf1 = project.getWorkflow('workflow-1')!; t.truthy(wf1); t.is(wf1.id, 'workflow-1'); t.is(wf1.name, 'Workflow 1'); - const wf2 = project.getWorkflow('workflow-2'); + const wf2 = project.getWorkflow('workflow-2')!; t.truthy(wf2); t.is(wf2.id, 'workflow-2'); t.is(wf2.name, 'Workflow 2'); @@ -85,7 +86,7 @@ test.serial('should load a workflow expression (legacy format)', async (t) => { const project = await parseProject({ root: '/ws' }); t.is(project.workflows.length, 1); - const wf = project.getWorkflow('my-workflow'); + const wf = project.getWorkflow('my-workflow')!; t.truthy(wf); @@ -136,6 +137,7 @@ test.serial( t.is(project.workflows.length, 1); const [wf] = project.workflows; + // @ts-ignore t.is(typeof wf.steps[1].next.c, 'object'); } ); diff --git a/packages/project/test/parse/from-path.test.ts b/packages/project/test/parse/from-path.test.ts index dd47a5e56..5fdf99772 100644 --- a/packages/project/test/parse/from-path.test.ts +++ b/packages/project/test/parse/from-path.test.ts @@ -18,7 +18,7 @@ test.serial('should load a v1 state json', async (t) => { const project = await fromPath('/p1/main@openfn.org.json'); t.is(project.name, proj.name); - t.deepEqual(project.openfn.uuid, proj.openfn.uuid); + t.deepEqual(project.openfn!.uuid, proj.openfn!.uuid); // TODO this isn't quite right for a few reasons // will investigate later @@ -27,12 +27,13 @@ test.serial('should load a v1 state json', async (t) => { test.serial('should load a v1 state yaml', async (t) => { mock({ + // @ts-ignore '/p1/main@openfn.org.yaml': proj.serialize('state', { format: 'yaml' }), }); const project = await fromPath('/p1/main@openfn.org.yaml'); t.is(project.name, proj.name); - t.deepEqual(project.openfn.uuid, proj.openfn.uuid); + t.deepEqual(project.openfn!.uuid, proj.openfn!.uuid); // TODO this isn't quite right for a few reasons // will investigate later @@ -46,7 +47,7 @@ test.serial('should load a v2 project yaml', async (t) => { const project = await fromPath('/p1/main@openfn.org.yaml'); t.is(project.id, proj.id); - t.deepEqual(project.openfn.uuid, '1234'); + t.deepEqual(project.openfn!.uuid, '1234'); t.is(project.workflows.length, 1); }); @@ -57,12 +58,13 @@ test.serial('should load a v2 project json', async (t) => { const project = await fromPath('/p1/main@openfn.org.json'); t.is(project.id, proj.id); - t.deepEqual(project.openfn.uuid, '1234'); + t.deepEqual(project.openfn!.uuid, '1234'); t.is(project.workflows.length, 1); }); test.serial('should use workspace config', async (t) => { mock({ + // @ts-ignore '/p1/main@openfn.org.yaml': proj.serialize('state', { format: 'yaml' }), }); const config = { @@ -89,7 +91,7 @@ test.serial('should use workspace config', async (t) => { x: 1234, }); - t.deepEqual(project.openfn.uuid, proj.openfn.uuid); + t.deepEqual(project.openfn!.uuid, proj.openfn!.uuid); }); test('extractAliasFromFilename: should extract alias from alias@domain.yaml format', (t) => { diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index e1ecd5bc5..aa0db4f96 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -61,7 +61,7 @@ test('import from a v1 state as JSON', async (t) => { // make a few basic assertions about the project t.is(proj.id, 'my-workflow'); t.is(proj.name, 'My Workflow'); - t.is(proj.openfn.uuid, 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'); + t.is(proj.openfn!.uuid, 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'); t.is(proj.options.retention_policy, 'retain_all'); t.is(proj.workflows.length, 1); @@ -73,7 +73,7 @@ test('import from a v1 state as YAML', async (t) => { // make a few basic assertions about the project t.is(proj.id, 'aaa'); t.is(proj.name, 'aaa'); - t.is(proj.openfn.uuid, '1234'); + t.is(proj.openfn!.uuid, '1234'); t.is(proj.options.retention_policy, 'retain_all'); t.is(proj.workflows.length, 1); @@ -94,12 +94,13 @@ test('import a legacy v2 project (cli.version === 2) as JSON', async (t) => { }); test('import from a v2 project as JSON', async (t) => { + // @ts-ignore const proj = await Project.from('project', v2.json, { alias: 'main' }); t.is(proj.id, 'my-project'); t.is(proj.name, 'My Project'); t.is(proj.cli.alias, 'main'); - t.is(proj.sandbox.parentId, 'abcd'); + t.is(proj.sandbox!.parentId, 'abcd'); t.is(proj.options.env, 'dev'); t.is(proj.options.color, 'red'); t.is(proj.openfn!.uuid, '1234'); @@ -153,6 +154,7 @@ test('import from a v2 project as JSON', async (t) => { }); test('import from a v2 project with alias', async (t) => { + // @ts-ignore const proj = await Project.from('project', v2.json, { alias: 'staging' }); t.is(proj.id, 'my-project'); @@ -167,7 +169,7 @@ test('import from a v2 project as YAML', async (t) => { t.is(proj.cli.alias, 'main'); t.is(proj.openfn!.uuid, '1234'); t.is(proj.openfn!.endpoint, 'https://app.openfn.org'); - t.is(proj.sandbox.parentId, 'abcd'); + t.is(proj.sandbox!.parentId, 'abcd'); t.is(proj.options.env, 'dev'); t.is(proj.options.color, 'red'); diff --git a/packages/project/test/project.test.ts b/packages/project/test/project.test.ts index 7322ac173..30379676e 100644 --- a/packages/project/test/project.test.ts +++ b/packages/project/test/project.test.ts @@ -193,10 +193,7 @@ test('should convert a state file to a project and back again', async (t) => { t.is(project.openfn?.uuid, state.id); t.is(project.name, state.name); - // TODO: this hack is needed right now to serialize the state as json - project.config.formats.project = 'json'; - - const newState = project.serialize('state'); + const newState = project.serialize('state', { format: 'json' }); t.deepEqual(newState, state); }); @@ -219,10 +216,10 @@ test('should merge two projects', (t) => { const result = Project.merge(staging, main); t.is(result.name, 'a'); - const mergedStep = result.workflows[0].get('a'); + const mergedStep: any = result.workflows[0].get('a'); t.is(mergedStep.expression, 'b()'); - t.is(mergedStep.openfn.uuid, wf_a.get('a').openfn.uuid); + t.is(mergedStep.openfn.uuid, wf_a.get('a').openfn!.uuid); }); test('should return UUIDs for everything', async (t) => { diff --git a/packages/project/test/serialize/to-app-state.test.ts b/packages/project/test/serialize/to-app-state.test.ts index f27adc615..c007f114c 100644 --- a/packages/project/test/serialize/to-app-state.test.ts +++ b/packages/project/test/serialize/to-app-state.test.ts @@ -180,7 +180,7 @@ test('should serialize workflow positions', (t) => { }, }); - const state = toAppState(project); + const state = toAppState(project) as Provisioner.Project_v1; t.deepEqual(state.workflows['wf'].positions, { step: { x: 1, @@ -193,7 +193,7 @@ test('should serialize workflow positions', (t) => { // gets written back to state test('should write openfn keys to objects', (t) => { const openfn = { x: 1 }; - const data = { + const data: any = { id: 'my-project', openfn, workflows: [ @@ -227,7 +227,8 @@ test('should write openfn keys to objects', (t) => { }, }); - const state = toAppState(project); + // Disable typing because we're doing wierd stuff + const state = toAppState(project) as any; t.is(state.x, 1); t.is(state.workflows['wf'].x, 1); t.is(state.workflows['wf'].jobs.step.x, 1); @@ -270,7 +271,9 @@ test('should handle credentials', (t) => { ], }; - const state = toAppState(new Project(data), { format: 'json' }); + const state = toAppState(new Project(data), { + format: 'json', + }) as Provisioner.Project_v1; const { step } = state.workflows['wf'].jobs; t.is(step.keychain_credential_id, 'k'); t.is(step.project_credential_id, '123'); @@ -380,7 +383,9 @@ test('should ignore workflow start keys', (t) => { ], }; - const state = toAppState(new Project(data), { format: 'json' }); + const state = toAppState(new Project(data), { + format: 'json', + }) as any; t.falsy(state.workflows['wf'].start); }); @@ -415,7 +420,9 @@ test('should handle edge labels', (t) => { ], }; - const state = toAppState(new Project(data), { format: 'json' }); + const state = toAppState(new Project(data), { + format: 'json', + }) as Provisioner.Project_v1; t.is(state.workflows.wf.edges['trigger->step'].condition_label, 'hello'); }); @@ -427,7 +434,9 @@ c-p `; const project = generateProject('proj', [wf], { uuidSeed: 1 }); - const state = toAppState(project, { format: 'json' }); + const state = toAppState(project, { + format: 'json', + }) as Provisioner.Project_v1; const jobs = Object.keys(state.workflows['wf'].jobs); // short be sorted by name @@ -450,7 +459,9 @@ a-(condition=x)-f uuidSeed: 1, // ensure predictable UUIDS }); - const state = toAppState(project, { format: 'json' }); + const state = toAppState(project, { + format: 'json', + }) as Provisioner.Project_v1; const { 'a->b': a_b, 'a->c': a_c, @@ -475,9 +486,45 @@ a-(condition=x)-f t.is(a_f.condition_expression, 'x'); }); +test('should serialize channels to app state', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + const project = new Project( + { + id: 'p', + // @ts-ignore - channels is opaque (any) on the Project type + channels, + workflows: [], + }, + { formats: { project: 'json' } } + ); + + const state = toAppState(project, { format: 'json' }) as Provisioner.Project; + + t.deepEqual(state.channels, channels); +}); + +test('should omit channels from app state when not set', (t) => { + const project = new Project( + { id: 'p', workflows: [] }, + { formats: { project: 'json' } } + ); + + const state = toAppState(project, { format: 'json' }) as Provisioner.Project; + + t.false('channels' in state); +}); + test('should convert a project back to app state in json', (t) => { // this is a serialized project file - const data = { + const data: any = { name: 'aaa', description: 'a project', credentials: [ @@ -567,7 +614,7 @@ test('should convert a project back to app state in json', (t) => { // We probably need to force alphabetical sorting on yaml keys test.skip('should convert a project back to app state in yaml', (t) => { // this is a serialized project file - const data = { + const data: any = { name: 'aaa', description: 'a project', credentials: [], diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index c24c1ad43..7ce5c01f9 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -58,6 +58,7 @@ test('extractWorkflow: single simple workflow with an edge', (t) => { id: 'step1', next: { step2: { + // @ts-ignore condition: true, openfn: { // should be excluded! @@ -121,6 +122,7 @@ test('extractWorkflow: single simple workflow with random edge property', (t) => steps: [ { ...step, + // @ts-ignore foo: 'bar', }, ], @@ -268,10 +270,11 @@ test('extractWorkflow: single simple workflow with custom root', (t) => { }, ], }, + // @ts-ignore config ); - const { path, content } = extractWorkflow(project, 'my-workflow'); + const { path } = extractWorkflow(project, 'my-workflow'); t.is(path, 'openfn/wfs/my-workflow/my-workflow.json'); }); diff --git a/packages/project/test/serialize/to-project.test.ts b/packages/project/test/serialize/to-project.test.ts index 53ebe769e..14aa18a89 100644 --- a/packages/project/test/serialize/to-project.test.ts +++ b/packages/project/test/serialize/to-project.test.ts @@ -1,11 +1,11 @@ import * as l from '@openfn/lexicon'; import test from 'ava'; import { Project } from '../../src/Project'; -import generateWorkflow, { generateProject } from '../../src/gen/generator'; +import generateWorkflow from '../../src/gen/generator'; import * as v2 from '../fixtures/sample-v2-project'; -const createProject = (props: Partial = {}) => { +const createProject = (props: Partial = {}) => { const proj = new Project({ id: 'my-project', name: 'My Project', @@ -86,7 +86,7 @@ test('should exclude null values in yaml', (t) => { }); // force some null values into the workflow structure - proj.workflows[0].steps[1].openfn.keychain_credential_id = null; + proj.workflows[0].steps[1].openfn!.keychain_credential_id = null; const yaml = proj.serialize('project', { format: 'yaml' }); t.deepEqual(yaml, v2.yaml); @@ -95,9 +95,35 @@ test('should exclude null values in yaml', (t) => { test('should include sandboxy metadata', (t) => { const proj = createProject({}); - const json = proj.serialize('project', { format: 'json' }); + const json: any = proj.serialize('project', { format: 'json' }); t.is(json.sandbox.parentId, 'abcd'); t.is(json.options.env, 'dev'); t.is(json.options.color, 'red'); }); + +test('should include channels in serialized project', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + // @ts-ignore - channels is opaque (any) on Project + const proj = createProject({ channels }); + + const json = proj.serialize('project', { format: 'json' }); + + t.deepEqual((json as any).channels, channels); +}); + +test('should omit channels from serialized project when unset', (t) => { + const proj = createProject(); + + const json = proj.serialize('project', { format: 'json' }); + + t.false('channels' in (json as any)); +}); diff --git a/packages/project/test/util/base-merge.test.ts b/packages/project/test/util/base-merge.test.ts index d06f371c1..de55ccb46 100644 --- a/packages/project/test/util/base-merge.test.ts +++ b/packages/project/test/util/base-merge.test.ts @@ -35,7 +35,7 @@ test('full merge: additional source keys', (t) => { test('full merge: existing target props & additional source keys', (t) => { const target = { key: 'one', title: 'target', openfn: { uuid: '' } }; - const source = { key: 'two', title: 'source', props: { isNew: true } }; + const source: any = { key: 'two', title: 'source', props: { isNew: true } }; const result = baseMerge(target, source); t.deepEqual(result, { @@ -72,7 +72,7 @@ test('partial merge: existing target props', (t) => { test('partial merge: additional source keys', (t) => { const target = { key: 'one', title: 'target' }; const source = { key: 'two', title: 'source', props: { isNew: true } }; - const result = baseMerge(target, source, ['key', 'props']); + const result = baseMerge(target, source, ['key', 'props'] as any); t.deepEqual(result, { key: 'two', @@ -84,7 +84,7 @@ test('partial merge: additional source keys', (t) => { test('partial merge: existing target props & additional source keys', (t) => { const target = { key: 'one', title: 'target', openfn: { uuid: '' } }; const source = { key: 'two', title: 'source', props: { isNew: true } }; - const result = baseMerge(target, source, ['key', 'props']); + const result = baseMerge(target, source as any, ['key', 'props']); t.deepEqual(result, { key: 'two', @@ -97,7 +97,7 @@ test('partial merge: existing target props & additional source keys', (t) => { test('assign', (t) => { const target = { key: 'one', title: 'target', openfn: { uuid: '' } }; const source = { key: 'two', title: 'source', props: { isNew: true } }; - const result = baseMerge(target, source, ['key', 'props'], { + const result = baseMerge(target, source as any, ['key', 'props'], { something: 'an assign prop', }); @@ -111,9 +111,13 @@ test('assign', (t) => { }); test('special: arrays are shallow merged', (t) => { - const target = { key: 'one', title: 'target', colours: ['red', 'blue', 'green'] }; + const target = { + key: 'one', + title: 'target', + colours: ['red', 'blue', 'green'], + }; const source = { key: 'two', title: 'source', colours: ['green', 'yellow'] }; - const result = baseMerge(target, source, ['key', 'props', 'colours']); + const result = baseMerge(target, source, ['key', 'props', 'colours'] as any); t.deepEqual(result, { key: 'two', @@ -134,7 +138,7 @@ test('special: objects are shallow merged', (t) => { title: 'source', opts: { isDeleted: true, time: 'yesterday' }, }; - const result = baseMerge(target, source, ['key', 'props', 'opts']); + const result = baseMerge(target, source as any, ['key', 'props', 'opts']); t.deepEqual(result, { key: 'two', diff --git a/packages/project/test/util/find-changed-workflows.test.ts b/packages/project/test/util/find-changed-workflows.test.ts index 2ccd969cc..ddca796d7 100644 --- a/packages/project/test/util/find-changed-workflows.test.ts +++ b/packages/project/test/util/find-changed-workflows.test.ts @@ -46,7 +46,7 @@ test('should return 1 removed workflow', (t) => { }; // remove workflow b - project.workflows.pop() + project.workflows.pop(); const changed = findChangedWorkflows(project); t.is(changed.length, 1); @@ -55,7 +55,7 @@ test('should return 1 removed workflow', (t) => { test('should return 1 added workflow', (t) => { const project = generateProject('proj', ['@id a a-b', '@id b x-y']); - const [a, b] = project.workflows; + const [a] = project.workflows; project.cli.forked_from = { [a.id]: generateHash(a), diff --git a/packages/project/test/util/get-credential-name.test.ts b/packages/project/test/util/get-credential-name.test.ts index e25374ddc..15ebee731 100644 --- a/packages/project/test/util/get-credential-name.test.ts +++ b/packages/project/test/util/get-credential-name.test.ts @@ -1,9 +1,9 @@ import test from 'ava'; import getCredentialName, { parse } from '../../src/util/get-credential-name'; -import { Credential } from '../../src/Project'; +import { CredentialState } from '@openfn/lexicon'; test('should generate a credential name', (t) => { - const cred: Credential = { + const cred: CredentialState = { uuid: '', owner: 'admin@openfn.org', name: 'my credential', diff --git a/packages/project/test/util/uuid.test.ts b/packages/project/test/util/uuid.test.ts index d356a76cf..ba0b73236 100644 --- a/packages/project/test/util/uuid.test.ts +++ b/packages/project/test/util/uuid.test.ts @@ -7,7 +7,7 @@ let idGen = 0; // builders const b = { - step: (id, props: Parital = {}) => ({ + step: (id: string, props: Partial = {}) => ({ id, openfn: { uuid: ++idGen }, ...props, @@ -37,6 +37,7 @@ test('getUuidForStep: should return null if there are no steps', (t) => { }); test('getUuidForStep: should return null if a matching step does not exist', (t) => { + // @ts-ignore const proj = b.project([b.step('a'), b.step('b')]); const result = getUuidForStep(proj, 'w', 'z'); @@ -45,6 +46,7 @@ test('getUuidForStep: should return null if a matching step does not exist', (t) test('getUuidForStep: should get a UUID for a step', (t) => { const target = b.step('b'); + // @ts-ignore const proj = b.project([b.step('a'), target, b.step('c')]); const result = getUuidForStep(proj, 'w', 'b'); @@ -54,7 +56,7 @@ test('getUuidForStep: should get a UUID for a step', (t) => { test("getUuidForEdge: throw if workflow doesn't exist", (t) => { const proj = b.project(); - t.throws(() => getUuidForStep(proj, 'xxx', 'a', 'b')); + t.throws(() => getUuidForStep(proj, 'xxx', 'a')); }); test('getUuidForEdge: should return null if there are no steps', (t) => { @@ -68,6 +70,7 @@ test('getUuidForEdge: should return null if no edge exists', (t) => { const x = b.step('x', { next: { y: { + // @ts-ignore condition: true, openfn: { uuid: 'x-y', @@ -76,6 +79,7 @@ test('getUuidForEdge: should return null if no edge exists', (t) => { }, }); const y = b.step('y'); + // @ts-ignore const proj = b.project([x, y]); const result = getUuidForEdge(proj, 'w', 'a', 'b'); @@ -86,6 +90,7 @@ test('getUuidForEdge: should get a UUID for an edge', (t) => { const x = b.step('x', { next: { y: { + // @ts-ignore condition: true, openfn: { uuid: 'x-y', @@ -94,6 +99,7 @@ test('getUuidForEdge: should get a UUID for an edge', (t) => { }, }); const y = b.step('y'); + // @ts-ignore const proj = b.project([x, y]); const result = getUuidForEdge(proj, 'w', 'x', 'y'); diff --git a/packages/project/test/util/version-workflow.test.ts b/packages/project/test/util/version-workflow.test.ts index 6b0dd0543..de724c969 100644 --- a/packages/project/test/util/version-workflow.test.ts +++ b/packages/project/test/util/version-workflow.test.ts @@ -1,7 +1,6 @@ import test from 'ava'; import { generateHash, parse } from '../../src/util/version'; import Project, { generateWorkflow } from '../../src'; -import Workflow from '../../src/Workflow'; // this is an actual lightning workflow state, copied verbatim // todo already out of data as the version will change soon @@ -62,6 +61,7 @@ test('match lightning version', async (t) => { const [expected] = example.version_history; // load the project from v1 state + // @ts-ignore const proj = await Project.from('state', { workflows: [example], }); diff --git a/packages/project/test/workflow.test.ts b/packages/project/test/workflow.test.ts index 008db9f94..5e2758241 100644 --- a/packages/project/test/workflow.test.ts +++ b/packages/project/test/workflow.test.ts @@ -3,7 +3,7 @@ import test from 'ava'; import Workflow from '../src/Workflow'; import { generateWorkflow } from '../src'; -const simpleWorkflow = { +const simpleWorkflow: any = { id: 'my-workflow', history: [], name: 'My Workflow', @@ -106,7 +106,7 @@ test('a Workflow class can serialize to JSON', (t) => { test('get - get a step', (t) => { const w = new Workflow(simpleWorkflow); - const step = w.get('a'); + const step: any = w.get('a'); t.deepEqual(step, simpleWorkflow.steps[0]); }); @@ -140,7 +140,7 @@ test('set properties on a step', (t) => { w.set('a', { adaptor: 'salesforce' }); - const step = w.get('a'); + const step: any = w.get('a'); t.is(step.adaptor, 'salesforce'); }); @@ -149,10 +149,10 @@ test('set properties on a step, with a chain', (t) => { w.set('a', { adaptor: 'salesforce' }).set('b', { adaptor: 'dhis2' }); - const a = w.get('a'); + const a: any = w.get('a'); t.is(a.adaptor, 'salesforce'); - const b = w.get('b'); + const b: any = w.get('b'); t.is(b.adaptor, 'dhis2'); }); @@ -161,7 +161,7 @@ test('set properties on an edge', (t) => { w.set('a-c', { condition: '!state.error' }); - const edge = w.get('a-c'); + const edge: any = w.get('a-c'); t.is(edge.condition, '!state.error'); }); diff --git a/packages/project/test/workspace.test.ts b/packages/project/test/workspace.test.ts index 7131fff15..ec8c6f973 100644 --- a/packages/project/test/workspace.test.ts +++ b/packages/project/test/workspace.test.ts @@ -2,7 +2,7 @@ import mock from 'mock-fs'; import { jsonToYaml, Workspace } from '../src'; import test from 'ava'; -const gen = (uuid: any, alias: string, id: string, domain: string) => +const gen = (uuid: any, _alias: string, id: string, _domain: string) => jsonToYaml({ id, name: id.toUpperCase(), @@ -287,6 +287,7 @@ test('load project meta', (t) => { test('load v2 projects with multiple matching ids', (t) => { const ws = new Workspace('/ws4'); + // @ts-ignore t.is(ws.projects.length, 3); }); diff --git a/packages/project/tsconfig.json b/packages/project/tsconfig.json index 9ad186134..77e4d2e8f 100644 --- a/packages/project/tsconfig.json +++ b/packages/project/tsconfig.json @@ -1,6 +1,6 @@ { "extends": "../../tsconfig.common", - "include": ["src/**/*.ts"], + "include": ["src/**/*.ts", "test/**/*test.ts"], "compilerOptions": { "module": "ESNext", "sourceMap": true