diff --git a/.changeset/dull-meals-stare.md b/.changeset/dull-meals-stare.md new file mode 100644 index 000000000..3e036c38a --- /dev/null +++ b/.changeset/dull-meals-stare.md @@ -0,0 +1,8 @@ +--- +'@openfn/lexicon': minor +'@openfn/project': minor +'@openfn/deploy': minor +'@openfn/cli': minor +--- + +Add support for channels diff --git a/packages/deploy/src/stateTransform.ts b/packages/deploy/src/stateTransform.ts index 0cf633080..69e28e20f 100644 --- a/packages/deploy/src/stateTransform.ts +++ b/packages/deploy/src/stateTransform.ts @@ -374,6 +374,58 @@ export function mergeSpecIntoState( } ) ); + const nextChannels = Object.fromEntries( + splitZip(oldState.channels || {}, spec.channels || {}).map( + ([channelKey, stateChannel, specChannel]) => { + if (specChannel && !stateChannel) { + return [ + channelKey, + { + id: crypto.randomUUID(), + name: specChannel.name, + destination_url: specChannel.destination_url, + enabled: specChannel.enabled, + destination_credential_id: + specChannel.destination_credential && + getStateJobCredential( + specChannel.destination_credential, + nextCredentials + ), + }, + ]; + } + + if (specChannel && stateChannel) { + return [ + channelKey, + { + id: stateChannel.id, + name: specChannel.name, + destination_url: specChannel.destination_url, + enabled: specChannel.enabled, + destination_credential_id: + specChannel.destination_credential && + getStateJobCredential( + specChannel.destination_credential, + nextCredentials + ), + }, + ]; + } + + if (!specChannel && stateChannel) { + return [channelKey, { id: stateChannel.id, delete: true }]; + } + + throw new DeployError( + `Invalid channel spec or corrupted state for channel: ${ + stateChannel?.name || specChannel?.name + }`, + 'VALIDATION_ERROR' + ); + } + ) + ); const nextWorkflows = Object.fromEntries( splitZip(oldState.workflows, spec.workflows).map( @@ -442,6 +494,7 @@ export function mergeSpecIntoState( workflows: nextWorkflows, project_credentials: nextCredentials, collections: nextCollections, + channels: nextChannels, }; if (spec.description) projectState.description = spec.description; @@ -490,9 +543,12 @@ export function getStateFromProjectPayload( const collections = reduceByKey('name', project.collections || []); + const channels = reduceByKey('name', project.channels || []); + return { ...project, collections, + channels, project_credentials, workflows, }; @@ -561,9 +617,18 @@ export function mergeProjectPayloadIntoState( ) ); + const nextChannels = Object.fromEntries( + idKeyPairs(project.channels || [], state.channels || {}).map( + ([key, nextChannel, _state]) => { + return [key, nextChannel]; + } + ) + ); + return { ...project, collections: nextCollections, + channels: nextChannels, project_credentials: nextCredentials, workflows: nextWorkflows, }; @@ -609,12 +674,17 @@ export function toProjectPayload(state: ProjectState): ProjectPayload { state.collections ); - const { collections: _, ...stateWithoutCollections } = state; + const channels: ProjectPayload['channels'] = Object.values( + state.channels || {} + ); + + const { collections: _, channels: __, ...stateWithoutOptionals } = state; return { - ...stateWithoutCollections, + ...stateWithoutOptionals, project_credentials, workflows, ...(collections.length > 0 && { collections }), + ...(channels.length > 0 && { channels }), }; } diff --git a/packages/deploy/src/types.ts b/packages/deploy/src/types.ts index 1857e6428..d8dbf4444 100644 --- a/packages/deploy/src/types.ts +++ b/packages/deploy/src/types.ts @@ -117,12 +117,29 @@ export type CollectionState = { delete?: boolean; }; +export type ChannelSpec = { + name: string; + destination_url: string; + enabled: boolean; + destination_credential: string | null; +}; + +export type ChannelState = { + id: string; + name: string; + destination_url: string; + enabled: boolean; + destination_credential_id: string | null; + delete?: boolean; +}; + export interface ProjectSpec { name: string; description: string; workflows: Record; credentials: Record; collections: Record; + channels: Record; } export interface WorkflowState { @@ -146,6 +163,7 @@ export interface ProjectState { workflows: Record; project_credentials: Record; collections: Record; + channels: Record; } export interface ProjectPayload { @@ -153,6 +171,7 @@ export interface ProjectPayload { name: string; description: string; collections?: Concrete[]; + channels?: Concrete[]; project_credentials: Concrete[]; workflows: { id: string; diff --git a/packages/deploy/src/validator.ts b/packages/deploy/src/validator.ts index 130f61963..6c79750f5 100644 --- a/packages/deploy/src/validator.ts +++ b/packages/deploy/src/validator.ts @@ -129,6 +129,12 @@ export async function parseAndValidate( } } + if (pair.key && pair.key.value === 'channels') { + if (pair.value.value === null) { + return doc.createPair('channels', {}); + } + } + if (pair.key && pair.key.value === 'jobs') { if (pair.value.value === null) { errors.push({ diff --git a/packages/deploy/test/fixtures.ts b/packages/deploy/test/fixtures.ts index 42ccdf4f3..982c2b70e 100644 --- a/packages/deploy/test/fixtures.ts +++ b/packages/deploy/test/fixtures.ts @@ -5,6 +5,7 @@ export function fullExampleSpec() { name: 'my project', description: 'some helpful description', collections: {}, + channels: {}, credentials: {}, workflows: { 'workflow-one': { @@ -58,6 +59,7 @@ export function fullExampleState() { name: 'my project', description: 'some helpful description', collections: {}, + channels: {}, project_credentials: {}, workflows: { 'workflow-one': { @@ -254,6 +256,7 @@ export const lightningProjectState = { name: 'collection-one', }, }, + channels: {}, project_credentials: { 'email@test.com-Basic-Auth': { id: '25f48989-d349-4eb8-99c3-923ebba5b116', diff --git a/packages/deploy/test/stateTransform.test.ts b/packages/deploy/test/stateTransform.test.ts index 997dc9c6e..8b55dbcb9 100644 --- a/packages/deploy/test/stateTransform.test.ts +++ b/packages/deploy/test/stateTransform.test.ts @@ -92,6 +92,7 @@ test('toNextState adding a job', (t) => { description: 'my test project', project_credentials: {}, collections: {}, + channels: {}, }); }); @@ -131,6 +132,7 @@ test('toNextState deleting a credential', (t) => { delete: true, }, }, + channels: {}, }); }); @@ -159,6 +161,7 @@ test('toNextState with empty state', (t) => { description: 'some helpful description', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: jp.query(result, '$..workflows["workflow-one"].id')[0], @@ -219,6 +222,7 @@ test('toNextState with no changes', (t) => { description: 'for the humans', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -335,6 +339,7 @@ test('toNextState with a new job', (t) => { description: 'some other description', project_credentials: {}, collections: {}, + channels: {}, workflows: { 'workflow-one': { id: '8124e88c-566f-472f-be38-363e588af55a', @@ -955,6 +960,7 @@ test('getStateFromProjectPayload with minimal project', (t) => { name: 'project', project_credentials: {}, collections: {}, + channels: {}, workflows: { a: { id: 'wf-a', @@ -1007,3 +1013,264 @@ test('toProjectPayload drops empty collections key', (t) => { t.deepEqual(payload, expectedPayload); }); + +test('toNextState adding a channel', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: null, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + const channel = result.channels['webhook-out']; + t.truthy(channel.id); + t.is(channel.name, 'webhook-out'); + t.is(channel.destination_url, 'https://example.com/hook'); + t.is(channel.enabled, true); + t.is(channel.destination_credential_id, null); +}); + +test('toNextState updating a channel preserves id', (t) => { + const channelId = 'aaa-bbb-ccc'; + const state = { + workflows: {}, + channels: { + 'webhook-out': { + id: channelId, + name: 'webhook-out', + destination_url: 'https://old.example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://new.example.com/hook', + enabled: false, + destination_credential: null, + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + t.is(result.channels['webhook-out'].id, channelId); + t.is( + result.channels['webhook-out'].destination_url, + 'https://new.example.com/hook' + ); + t.is(result.channels['webhook-out'].enabled, false); +}); + +test('toNextState deleting a channel', (t) => { + const channelId = 'aaa-bbb-ccc'; + const state = { + workflows: {}, + channels: { + 'webhook-out': { + id: channelId, + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + channels: {}, + }; + + const result = mergeSpecIntoState(state, spec); + + t.deepEqual(result.channels['webhook-out'], { + id: channelId, + delete: true, + }); +}); + +test('toNextState resolves channel destination_credential to id', (t) => { + const state = { + workflows: {}, + project_credentials: { + 'me-auth': { + id: 'cred-id-123', + name: 'auth', + owner: 'me', + }, + }, + }; + const spec = { + name: 'my project', + workflows: {}, + credentials: { + 'me-auth': { name: 'auth', owner: 'me' }, + }, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'me-auth', + }, + }, + }; + + const result = mergeSpecIntoState(state, spec); + + t.is( + result.channels['webhook-out'].destination_credential_id, + 'cred-id-123' + ); +}); + +test('toNextState throws when channel references unknown credential', (t) => { + const state = { workflows: {} }; + const spec = { + name: 'my project', + workflows: {}, + channels: { + 'webhook-out': { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'missing', + }, + }, + }; + + t.throws(() => mergeSpecIntoState(state, spec), { + message: 'Could not find a credential with name: missing', + }); +}); + +test('getStateFromProjectPayload reads channels', (t) => { + const project = { + id: 'xyz', + name: 'project', + workflows: [], + project_credentials: [], + channels: [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ], + }; + + const state = getStateFromProjectPayload(project); + + t.deepEqual(state.channels, { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }); +}); + +test('toProjectPayload drops empty channels key', (t) => { + const projectState = { + ...lightningProjectState, + channels: {}, + }; + + const payload = toProjectPayload(projectState); + + t.false('channels' in payload); +}); + +test('toProjectPayload includes channels when non-empty', (t) => { + const projectState = { + ...lightningProjectState, + channels: { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + + const payload = toProjectPayload(projectState); + + t.deepEqual(payload.channels, [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]); +}); + +test('mergeProjectIntoState preserves channels from payload', (t) => { + const state = { + id: 'p-1', + name: 'p', + description: '', + workflows: {}, + project_credentials: {}, + collections: {}, + channels: { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + }, + }; + const payload: ProjectPayload = { + id: 'p-1', + name: 'p', + description: '', + workflows: [], + project_credentials: [], + channels: [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://updated.example.com/hook', + enabled: false, + destination_credential_id: null, + }, + ], + }; + + const result = mergeProjectPayloadIntoState(state, payload); + + t.deepEqual(result.channels, { + 'webhook-out': { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://updated.example.com/hook', + enabled: false, + destination_credential_id: null, + }, + }); +}); diff --git a/packages/deploy/test/validator.test.ts b/packages/deploy/test/validator.test.ts index 79e88706e..d963db1b0 100644 --- a/packages/deploy/test/validator.test.ts +++ b/packages/deploy/test/validator.test.ts @@ -130,6 +130,46 @@ test('allow empty workflows', async (t) => { }); }); +test('allows null channels by converting to empty map', async (t) => { + const doc = ` + name: project-name + channels: + workflows: + workflow-one: + name: workflow one + `; + + const result = await parseAndValidate(doc, 'spec.yaml'); + + t.is(result.errors.length, 0); + t.deepEqual(result.doc.channels, {}); +}); + +test('parses channels with destination_credential', async (t) => { + const doc = ` + name: project-name + channels: + webhook-out: + name: webhook-out + destination_url: https://example.com/hook + enabled: true + destination_credential: my-cred + workflows: + workflow-one: + name: workflow one + `; + + const result = await parseAndValidate(doc, 'spec.yaml'); + + t.is(result.errors.length, 0); + t.deepEqual(result.doc.channels!['webhook-out'], { + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential: 'my-cred', + }); +}); + test('adds the file content into the job body from the specified path', async (t) => { // Step 1: Create a temporary file that the YAML will reference const fileContent = 'fn(state => state.data);'; diff --git a/packages/lexicon/lightning.d.ts b/packages/lexicon/lightning.d.ts index f95be1d88..aa93a7304 100644 --- a/packages/lexicon/lightning.d.ts +++ b/packages/lexicon/lightning.d.ts @@ -1,5 +1,6 @@ import type { LogLevel, SanitizePolicies } from '@openfn/logger'; import { LegacyJob, State } from './core'; +import { Channel } from './portability'; export const API_VERSION: number; @@ -252,6 +253,8 @@ export namespace Provisioner { // should be an array of something? collections: any[]; + channels?: Channel[]; + // serverside metadata inserted_at?: string; updated_at?: string; diff --git a/packages/lexicon/portability.d.ts b/packages/lexicon/portability.d.ts index b13534043..1bd985cd5 100644 --- a/packages/lexicon/portability.d.ts +++ b/packages/lexicon/portability.d.ts @@ -24,6 +24,8 @@ export interface ProjectSpec { credentials?: Credential[]; collections?: string[]; + + channels?: Channel[]; } export interface WorkflowSpec { @@ -97,3 +99,12 @@ export interface Job extends Step { expression?: string; configuration?: object | string; } + +export type Channel = { + id: string; + name: string; + destination_url: string; + enabled: boolean; + destination_credential_id?: string; + delete?: boolean; +}; diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index 3691bacde..749600b26 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -76,6 +76,8 @@ export class Project { collections: any; + channels?: l.Channel[]; + credentials: Credential[]; sandbox?: SandboxMeta; @@ -161,6 +163,7 @@ export class Project { this.options = data.options; this.workflows = data.workflows?.map(maybeCreateWorkflow) ?? []; this.collections = data.collections; + this.channels = data.channels; this.credentials = data.credentials ?? []; this.sandbox = data.sandbox; } diff --git a/packages/project/src/merge/merge-project.ts b/packages/project/src/merge/merge-project.ts index 47e0d62d0..b689bb57f 100644 --- a/packages/project/src/merge/merge-project.ts +++ b/packages/project/src/merge/merge-project.ts @@ -169,11 +169,12 @@ export function merge( target.credentials ), collections: source.collections ?? target.collections, + channels: source.channels ?? target.channels, }; // with project level props merging, target goes into source because we want to preserve the target props. return new Project( - baseMerge(target, source, ['collections'], assigns as any) + baseMerge(target, source, ['collections', 'channels'], assigns as any) ); } diff --git a/packages/project/src/parse/from-app-state.ts b/packages/project/src/parse/from-app-state.ts index f98c11969..16c98ab21 100644 --- a/packages/project/src/parse/from-app-state.ts +++ b/packages/project/src/parse/from-app-state.ts @@ -30,6 +30,7 @@ export default ( workflows, project_credentials = [], collections, + channels, inserted_at, updated_at, parent_id, @@ -47,6 +48,7 @@ export default ( name, description: description ?? undefined, collections, + channels, credentials, options, config: config as l.WorkspaceConfig, diff --git a/packages/project/src/serialize/to-app-state.ts b/packages/project/src/serialize/to-app-state.ts index bcde55620..7805910a1 100644 --- a/packages/project/src/serialize/to-app-state.ts +++ b/packages/project/src/serialize/to-app-state.ts @@ -34,7 +34,7 @@ export default function ( } = project.openfn ?? {}; const state = omitBy( - pick(project, ['name', 'description', 'collections']), + pick(project, ['name', 'description', 'collections', 'channels']), isNil ) as Provisioner.Project; diff --git a/packages/project/src/serialize/to-project.ts b/packages/project/src/serialize/to-project.ts index 8192041f8..d970aedbe 100644 --- a/packages/project/src/serialize/to-project.ts +++ b/packages/project/src/serialize/to-project.ts @@ -33,6 +33,7 @@ export default (project: Project, options: ToProjectOptions = {}) => { description: project.description, collections: project.collections, + channels: project.channels, credentials: project.credentials, openfn: omitBy(project.openfn, isNil), diff --git a/packages/project/test/canonical.test.ts b/packages/project/test/canonical.test.ts index 3bd25a514..0bdda3be6 100644 --- a/packages/project/test/canonical.test.ts +++ b/packages/project/test/canonical.test.ts @@ -14,6 +14,16 @@ const project: ProjectSpec = { schema_version: '4.0', credentials: [{ owner: 'admin@openfn.org', name: 'secret-squirrel' }], collections: ['nut-stash'], + channels: [ + { + id: 'proxy', + name: 'My Proxy', + destination_url: 'https://proxy.openfn.org', + enabled: true, + delete: false, + destination_credential_id: 'secret', + }, + ], workflows: [ { id: 'wf-webhook', @@ -131,6 +141,13 @@ schema_version: '4.0' description: Everything including the kitchen sink collections: - nut-stash +channels: + - id: proxy + name: My Proxy + destination_url: https://proxy.openfn.org + enabled: true + delete: false + destination_credential_id: secret credentials: - owner: admin@openfn.org name: secret-squirrel diff --git a/packages/project/test/merge/merge-project.test.ts b/packages/project/test/merge/merge-project.test.ts index c95c96bea..2ec99089d 100644 --- a/packages/project/test/merge/merge-project.test.ts +++ b/packages/project/test/merge/merge-project.test.ts @@ -151,6 +151,69 @@ test('Merge new credentials into the target', (t) => { ]); }); +test('replace mode: source channels override target channels', (t) => { + const wf = { + steps: [ + { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, + ], + }; + const wf_a = assignUUIDs(wf); + const wf_b = assignUUIDs(wf); + + const targetChannels = [ + { + id: 'chan-target', + name: 'target-channel', + destination_url: 'https://target.example.com', + enabled: true, + destination_credential_id: null, + }, + ]; + const sourceChannels = [ + { + id: 'chan-source', + name: 'source-channel', + destination_url: 'https://source.example.com', + enabled: false, + destination_credential_id: null, + }, + ]; + + const target = createProject(wf_a, 'a', { channels: targetChannels }); + const source = createProject(wf_b, 'b', { channels: sourceChannels }); + + const result = merge(source, target, { mode: REPLACE_MERGE }); + + t.deepEqual(result.channels, sourceChannels); +}); + +test('replace mode: target channels preserved when source has none', (t) => { + const wf = { + steps: [ + { id: 'x', name: 'X', adaptor: 'common', expression: 'fn(s => s)' }, + ], + }; + const wf_a = assignUUIDs(wf); + const wf_b = assignUUIDs(wf); + + const targetChannels = [ + { + id: 'chan-target', + name: 'target-channel', + destination_url: 'https://target.example.com', + enabled: true, + destination_credential_id: null, + }, + ]; + + const target = createProject(wf_a, 'a', { channels: targetChannels }); + const source = createProject(wf_b, 'b'); + + const result = merge(source, target, { mode: REPLACE_MERGE }); + + t.deepEqual(result.channels, targetChannels); +}); + test('replace mode: replace the name and UUID of the target project', (t) => { const wf = { steps: [ diff --git a/packages/project/test/parse/from-app-state.test.ts b/packages/project/test/parse/from-app-state.test.ts index bfa551b86..f1db229bb 100644 --- a/packages/project/test/parse/from-app-state.test.ts +++ b/packages/project/test/parse/from-app-state.test.ts @@ -54,6 +54,29 @@ test('should create a Project from prov state with collections', (t) => { t.deepEqual(project.collections, []); }); +test('should create a Project from prov state with channels', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + const stateWithChannels: any = { ...state, channels }; + + const project = fromAppState(stateWithChannels, meta); + + t.deepEqual(project.channels, channels); +}); + +test('project channels is undefined when missing from state', (t) => { + const project = fromAppState(state, meta); + + t.is(project.channels, undefined); +}); + test('should create a Project from prov state with sandbox stuff', (t) => { const stateWithSandbox = { ...state, diff --git a/packages/project/test/serialize/to-app-state.test.ts b/packages/project/test/serialize/to-app-state.test.ts index fe0729ed4..c007f114c 100644 --- a/packages/project/test/serialize/to-app-state.test.ts +++ b/packages/project/test/serialize/to-app-state.test.ts @@ -486,6 +486,42 @@ a-(condition=x)-f t.is(a_f.condition_expression, 'x'); }); +test('should serialize channels to app state', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + const project = new Project( + { + id: 'p', + // @ts-ignore - channels is opaque (any) on the Project type + channels, + workflows: [], + }, + { formats: { project: 'json' } } + ); + + const state = toAppState(project, { format: 'json' }) as Provisioner.Project; + + t.deepEqual(state.channels, channels); +}); + +test('should omit channels from app state when not set', (t) => { + const project = new Project( + { id: 'p', workflows: [] }, + { formats: { project: 'json' } } + ); + + const state = toAppState(project, { format: 'json' }) as Provisioner.Project; + + t.false('channels' in state); +}); + test('should convert a project back to app state in json', (t) => { // this is a serialized project file const data: any = { diff --git a/packages/project/test/serialize/to-project.test.ts b/packages/project/test/serialize/to-project.test.ts index 6beb8990c..14aa18a89 100644 --- a/packages/project/test/serialize/to-project.test.ts +++ b/packages/project/test/serialize/to-project.test.ts @@ -101,3 +101,29 @@ test('should include sandboxy metadata', (t) => { t.is(json.options.env, 'dev'); t.is(json.options.color, 'red'); }); + +test('should include channels in serialized project', (t) => { + const channels = [ + { + id: 'chan-1', + name: 'webhook-out', + destination_url: 'https://example.com/hook', + enabled: true, + destination_credential_id: null, + }, + ]; + // @ts-ignore - channels is opaque (any) on Project + const proj = createProject({ channels }); + + const json = proj.serialize('project', { format: 'json' }); + + t.deepEqual((json as any).channels, channels); +}); + +test('should omit channels from serialized project when unset', (t) => { + const proj = createProject(); + + const json = proj.serialize('project', { format: 'json' }); + + t.false('channels' in (json as any)); +});