Skip to content

Types rethink #1408

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
josephjclark wants to merge 5 commits into from
Closed

Types rethink #1408

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
91579cf
update portability spec
josephjclark May 14, 2026
54a0010
add canonical v2 test
josephjclark May 14, 2026
7e417c2
project: support extra state keys
josephjclark May 14, 2026
38424b2
add lexicon to deploy and start moving
josephjclark May 14, 2026
bad4288
update types
josephjclark May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions packages/deploy/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
"license": "ISC",
"devDependencies": {
"@inquirer/testing": "^2.1.53",
"@openfn/lexicon": "workspace:^",
"@types/json-diff": "^1.0.3",
"@types/jsonpath": "^0.2.4",
"@types/mock-fs": "^4.13.4",
Expand Down
202 changes: 93 additions & 109 deletions packages/deploy/src/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,6 @@
export type StateJob = {
id: string;
name: string;
adaptor: string;
body: string;
project_credential_id: string | null;
delete?: boolean;
};
import { Provisioner } from '@openfn/lexicon/lightning';

export type StateJob = Provisioner.Job;

export type SpecJobBody =
| string
Expand All @@ -14,138 +9,127 @@ export type SpecJobBody =
content: string;
};

export type SpecJob = {
export type SpecJob = Omit<
Provisioner.Job,
'id' | 'body' | 'project_credential_id' | 'keychain_credential_id' | 'delete'
> & {
id?: string;
name: string;
adaptor: string;
body: SpecJobBody;
credential: string | null;
};

export type StateKafkaHost = [string, string];

export type StateKafkaConfiguration = {
hosts: StateKafkaHost[];
topics: string[];
initial_offset_reset_policy: string;
connect_timeout: number;
};

export type SpecKafkaConfiguration = {
hosts: string[];
topics: string[];
initial_offset_reset_policy: string;
connect_timeout: number;
};
export type StateTrigger = Provisioner.Trigger;

export type WebhookResponse = {
error_code: number | null;
success_code: number | null;
};
export type SpecTrigger = Omit<Provisioner.Trigger, 'id'>;

export type WebhookReply = 'before_start' | 'after_completion';
export type StateKafkaHost = [string, string];

export type SpecTrigger = {
type: string;
cron_expression?: string;
cron_cursor_job?: string;
webhook_reply?: WebhookReply;
webhook_response?: WebhookResponse | null;
enabled?: boolean;
kafka_configuration?: SpecKafkaConfiguration;
};
export type StateKafkaConfiguration = Provisioner.KafkaConfiguration;

export type StateTrigger = {
id: string;
type: string;
cron_expression?: string;
cron_cursor_job_id?: string | null;
webhook_reply?: WebhookReply;
webhook_response?: WebhookResponse | null;
delete?: boolean;
enabled?: boolean;
kafka_configuration?: StateKafkaConfiguration;
};

export type StateEdge = {
id: string;
condition_type: string;
condition_expression: string | null;
condition_label: string | null;
source_job_id: string | null;
source_trigger_id: string | null;
target_job_id: string;
enabled?: boolean;
export type SpecKafkaConfiguration = Omit<
Provisioner.KafkaConfiguration,
'hosts'
> & {
hosts: string[];
};

export type SpecEdge = {
condition_type: string;
export type StateEdge = Provisioner.Edge;

export type SpecEdge = Omit<
Provisioner.Edge,
| 'id'
| 'condition_expression'
| 'condition_label'
| 'source_job_id'
| 'source_trigger_id'
| 'target_job_id'
> & {
condition_expression: string | null;
condition_label: string | null;
source_job?: string | null;
source_trigger?: string | null;
target_job: string | null;
enabled?: boolean;
};

export type WorkflowSpec = {
id?: string;
name: string;
jobs?: Record<string | symbol, SpecJob>;
triggers?: Record<string | symbol, SpecTrigger>;
edges?: Record<string | symbol, SpecEdge>;
};

export type CredentialSpec = {
name: string;
owner: string;
};

export type CredentialState = {
id: string;
name: string;
owner: string;
};
export type CredentialState = Provisioner.Credential;

export type CollectionSpec = {
name: string;
};
export type CredentialSpec = Omit<Provisioner.Credential, 'id'>;

export type CollectionState = {
id: string;
name: string;
delete?: boolean;
};
export type CollectionState = Provisioner.Collection;

export interface ProjectSpec {
name: string;
description: string;
workflows: Record<string | symbol, WorkflowSpec>;
credentials: Record<string | symbol, CredentialSpec>;
collections: Record<string | symbol, CollectionSpec>;
}
export type CollectionSpec = Omit<Provisioner.Collection, 'id' | 'delete'>;

export interface WorkflowState {
id: string;
name: string;
export interface WorkflowState
extends Omit<
Provisioner.Workflow,
'jobs' | 'triggers' | 'edges' | 'deleted_at'
> {
jobs: Record<string | symbol, Concrete<StateJob>>;
triggers: Record<string | symbol, Concrete<StateTrigger>>;
edges: Record<string | symbol, Concrete<StateEdge>>;
delete?: boolean;
project_id?: string;

inserted_at?: string;
updated_at?: string;
deleted_at?: string;
deleted_at?: string | null;
}

export interface ProjectState {
id: string;
name: string;
export type WorkflowSpec = Omit<
Provisioner.Workflow,
| 'id'
| 'jobs'
| 'triggers'
| 'edges'
| 'delete'
| 'project_id'
| 'version_history'
| 'concurrency'
| 'lock_version'
| 'inserted_at'
| 'updated_at'
| 'deleted_at'
> & {
id?: string;
jobs?: Record<string | symbol, SpecJob>;
triggers?: Record<string | symbol, SpecTrigger>;
edges?: Record<string | symbol, SpecEdge>;
};

export interface ProjectState
extends Omit<
Provisioner.Project,
| 'description'
| 'workflows'
| 'project_credentials'
| 'collections'
| 'scheduled_deletion'
| 'history_retention_period'
| 'dataclip_retention_period'
> {
description: string;
workflows: Record<string | symbol, WorkflowState>;
project_credentials: Record<string | symbol, CredentialState>;
collections: Record<string | symbol, CollectionState>;
scheduled_deletion?: string | null;
history_retention_period?: string | null;
dataclip_retention_period?: string | null;
}

export interface ProjectSpec
Copy link
Copy Markdown
Collaborator Author

@josephjclark josephjclark May 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this doens't work for me at all. It's too hard to reconcile these types right now, sadly

extends Omit<
Provisioner.Project,
| 'id'
| 'description'
| 'workflows'
| 'project_credentials'
| 'collections'
| 'inserted_at'
| 'updated_at'
| 'scheduled_deletion'
| 'history_retention_period'
| 'dataclip_retention_period'
| 'parent_id'
> {
description: string;
workflows: Record<string | symbol, WorkflowSpec>;
credentials: Record<string | symbol, CredentialSpec>;
collections: Record<string | symbol, CollectionSpec>;
}

export interface ProjectPayload {
Expand Down
10 changes: 10 additions & 0 deletions packages/lexicon/lightning.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,10 +309,20 @@ export namespace Provisioner {
delete?: boolean;
};

export type WebhookResponse = {
error_code: number | null;
success_code: number | null;
};

export type WebhookReply = 'before_start' | 'after_completion';

export type Trigger = {
id: string;
type: string;
cron_expression?: string;
cron_cursor_job?: string;
webhook_reply?: WebhookReply;
webhook_response?: WebhookResponse | null;
delete?: boolean;
enabled?: boolean;
kafka_configuration?: KafkaConfiguration;
Expand Down
6 changes: 5 additions & 1 deletion packages/lexicon/portability.d.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ export interface Trigger extends Step {

enabled?: boolean;

webhook_reply?: string;
webhook_reply?: 'before_start' | 'after_completion';
webhook_response?: {
error_code?: number;
success_code?: number;
};
cron_cursor_job_id?: string;

/** Allow arbitrary properties on trigger nodes (as configuration options) */
Expand Down
4 changes: 3 additions & 1 deletion packages/project/src/parse/from-app-state.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,8 @@ export const mapWorkflow = (
// TODO what do we do if the condition is disabled?
// I don't think that's the same as edge condition false?
Object.values(workflow.triggers).forEach((trigger: Provisioner.Trigger) => {
const { type, enabled, ...otherProps } = trigger;
const { type, enabled, webhook_reply, webhook_response, ...otherProps } =
trigger;
if (!mapped.start) {
mapped.start = type;
}
Expand All @@ -132,6 +133,7 @@ export const mapWorkflow = (
id: type,
type,
enabled,
webhook_reply,
openfn: renameKeys(otherProps, { id: 'uuid' }),
next: connectedEdges.reduce((obj: any, edge) => {
const target = Object.values(jobs).find(
Expand Down
1 change: 1 addition & 0 deletions packages/project/src/util/version.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export const generateHash = (
'cron_expression',
'enabled',
'webhook_reply',
'webhook_response',
'cron_cursor_job_id',
].sort();

Expand Down
Loading
Loading