diff --git a/.changeset/moody-ducks-warn.md b/.changeset/moody-ducks-warn.md new file mode 100644 index 000000000..7e1a6a94f --- /dev/null +++ b/.changeset/moody-ducks-warn.md @@ -0,0 +1,5 @@ +--- +'@openfn/project': patch +--- + +Support forked_from metadata key in openfn.yaml diff --git a/packages/cli/src/projects/checkout.ts b/packages/cli/src/projects/checkout.ts index 33369e34c..1bb2886db 100644 --- a/packages/cli/src/projects/checkout.ts +++ b/packages/cli/src/projects/checkout.ts @@ -10,7 +10,7 @@ import * as o from '../options'; import * as po from './options'; import type { Opts } from './options'; -import { tidyWorkflowDir } from './util'; +import { tidyWorkflowDir, updateForkedFrom } from './util'; export type CheckoutOptions = Pick< Opts, @@ -69,7 +69,11 @@ export const handler = async (options: CheckoutOptions, logger: Logger) => { await tidyWorkflowDir(currentProject!, switchProject); } + // write the forked from map + updateForkedFrom(switchProject); + // expand project into directory + // TODO: only write files with a diff const files: any = switchProject.serialize('fs'); for (const f in files) { if (files[f]) { diff --git a/packages/cli/src/projects/deploy.ts b/packages/cli/src/projects/deploy.ts index 25fa87a93..42bc34e8f 100644 --- a/packages/cli/src/projects/deploy.ts +++ b/packages/cli/src/projects/deploy.ts @@ -1,6 +1,8 @@ import yargs from 'yargs'; import Project from '@openfn/project'; import c from 'chalk'; +import { writeFile } from 'node:fs/promises'; +import path from 'node:path'; import * as o from '../options'; import * as o2 from './options'; @@ -10,6 +12,7 @@ import { fetchProject, serialize, getSerializePath, + updateForkedFrom, } from './util'; import { build, ensure } from '../util/command-builders'; @@ -64,6 +67,34 @@ export const command: yargs.CommandModule = { handler: ensure('project-deploy', options), }; +export const hasRemoteDiverged = ( + local: Project, + remote: Project +): string[] | null => { + let diverged: string[] | null = null; + + const refs = local.cli.forked_from ?? {}; + + // for each workflow, check that the local fetched_from is the head of the remote history + for (const wf of local.workflows) { + if (wf.id in refs) { + const forkedVersion = refs[wf.id]; + const remoteVersion = remote.getWorkflow(wf.id)?.history.at(-1); + if (forkedVersion !== remoteVersion) { + diverged ??= []; + diverged.push(wf.id); + } + } else { + // TODO what if there's no forked from for this workflow? + // Do we assume divergence because we don't know? Do we warn? + } + } + + // TODO what if a workflow is removed locally? + + return diverged; +}; + export async function handler(options: DeployOptions, logger: Logger) { logger.warn( 'WARNING: the project deploy command is in BETA and may not be stable. Use cautiously on production projects.' @@ -132,32 +163,41 @@ Pass --force to override this error and deploy anyway.`); // Skip divergence testing if the remote has no history in its workflows // (this will only happen on older versions of lightning) + // TODO now maybe skip if there's no forked_from const skipVersionTest = - localProject.workflows.find((wf) => wf.history.length === 0) || + // localProject.workflows.find((wf) => wf.history.length === 0) || remoteProject.workflows.find((wf) => wf.history.length === 0); + // localProject.workflows.forEach((w) => console.log(w.history)); + if (skipVersionTest) { logger.warn( 'Skipping compatibility check as no local version history detected' ); logger.warn('Pushing these changes may overrite changes made to the app'); - } else if (!localProject.canMergeInto(remoteProject!)) { - if (!options.force) { - logger.error(`Error: Projects have diverged! + } else { + const divergentWorkflows = hasRemoteDiverged(localProject, remoteProject!); + if (divergentWorkflows) { + logger.warn( + `The following workflows have diverged: ${divergentWorkflows}` + ); + if (!options.force) { + logger.error(`Error: Projects have diverged! -The remote project has been edited since the local project was branched. Changes may be lost. + The remote project has been edited since the local project was branched. Changes may be lost. -Pass --force to override this error and deploy anyway.`); - return; + Pass --force to override this error and deploy anyway.`); + return; + } else { + logger.warn( + 'Remote project has not diverged from local project! Pushing anyway as -f passed' + ); + } } else { - logger.warn( - 'Remote project has not diverged from local project! Pushing anyway as -f passed' + logger.info( + 'Remote project has not diverged from local project - it is safe to deploy 🎉' ); } - } else { - logger.info( - 'Remote project has not diverged from local project - it is safe to deploy 🎉' - ); } logger.info('Merging changes into remote project'); @@ -180,6 +220,8 @@ Pass --force to override this error and deploy anyway.`); // TODO not totally sold on endpoint handling right now config.endpoint ??= localProject.openfn?.endpoint!; + // TODO: I want to report diff HERE, after the merged state and stuff has been built + if (options.dryRun) { logger.always('dryRun option set: skipping upload step'); } else { @@ -218,6 +260,14 @@ Pass --force to override this error and deploy anyway.`); merged.config ); + // TODO why isn't this right? oh, because the outpu path isn't quite right + updateForkedFrom(finalProject); + const configData = finalProject.generateConfig(); + await writeFile( + path.resolve(options.workspace, configData.path), + configData.content + ); + const finalOutputPath = getSerializePath(localProject, options.workspace!); logger.debug('Updating local project at ', finalOutputPath); await serialize(finalProject, finalOutputPath); @@ -267,3 +317,4 @@ export const reportDiff = (local: Project, remote: Project, logger: Logger) => { return diffs; }; +``; diff --git a/packages/cli/src/projects/fetch.ts b/packages/cli/src/projects/fetch.ts index 37cf67373..d1d618c79 100644 --- a/packages/cli/src/projects/fetch.ts +++ b/packages/cli/src/projects/fetch.ts @@ -331,6 +331,8 @@ To ignore this error and override the local file, pass --force (-f) options.force || // The user forced the checkout !hasAnyHistory; // the remote project has no history (can happen in old apps) + // TODO temporarily force skip + // TODO canMergeInto needs to return a reason if (!skipVersionCheck && !remoteProject.canMergeInto(localProject!)) { // TODO allow rename throw new Error('Error! An incompatible project exists at this location'); diff --git a/packages/cli/src/projects/util.ts b/packages/cli/src/projects/util.ts index be6f8b1a0..d367be7a2 100644 --- a/packages/cli/src/projects/util.ts +++ b/packages/cli/src/projects/util.ts @@ -216,3 +216,14 @@ export async function tidyWorkflowDir( // Return and sort for testing return toRemove.sort(); } + +export const updateForkedFrom = (proj: Project) => { + proj.cli.forked_from = proj.workflows.reduce((obj: any, wf) => { + if (wf.history.length) { + obj[wf.id] = wf.history.at(-1); + } + return obj; + }, {}); + + return proj; +}; diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index 44fcb0fa3..fac0c1e29 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -3,7 +3,7 @@ import { createMockLogger } from '@openfn/logger'; import { handler as checkoutHandler } from '../../src/projects/checkout'; import mock from 'mock-fs'; import fs from 'fs'; -import { jsonToYaml, Workspace } from '@openfn/project'; +import { jsonToYaml, Workspace, yamlToJson } from '@openfn/project'; test.beforeEach(() => { mock({ @@ -28,6 +28,7 @@ test.beforeEach(() => { { name: 'simple-workflow', id: 'wf-id', + history: ['a'], jobs: [ { name: 'Transform data to FHIR standard', @@ -56,6 +57,7 @@ test.beforeEach(() => { { name: 'another-workflow', id: 'another-id', + history: ['b'], jobs: [ { name: 'Transform data to FHIR standard', @@ -83,6 +85,7 @@ test.beforeEach(() => { }, ], }), + // TODO this is actually a v1 state file for some reason, which is wierd '/ws/.projects/project@app.openfn.org.yaml': jsonToYaml({ id: '', name: 'My Project', @@ -90,6 +93,7 @@ test.beforeEach(() => { { name: 'simple-workflow-main', id: 'wf-id-main', + version_history: ['a'], jobs: [ { name: 'Transform data to FHIR standard', @@ -118,6 +122,7 @@ test.beforeEach(() => { { name: 'another-workflow-main', id: 'another-id', + version_history: ['b'], jobs: [ { name: 'Transform data to FHIR standard', @@ -217,6 +222,25 @@ test.serial('checkout: same id as active', async (t) => { ); }); +test.serial( + 'checkout: writes forked_from based on version history', + async (t) => { + const bcheckout = new Workspace('/ws'); + t.is(bcheckout.activeProject!.id, 'my-project'); + + await checkoutHandler( + { command: 'project-checkout', project: 'my-project', workspace: '/ws' }, + logger + ); + + const openfn = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8')); + t.deepEqual(openfn.project.forked_from, { + 'simple-workflow-main': 'a', + 'another-workflow-main': 'b', + }); + } +); + test.serial('checkout: switching to and back between projects', async (t) => { // before checkout. my-project is active and expanded const bcheckout = new Workspace('/ws'); diff --git a/packages/cli/test/projects/deploy.test.ts b/packages/cli/test/projects/deploy.test.ts index 903dabf52..47ddf0d45 100644 --- a/packages/cli/test/projects/deploy.test.ts +++ b/packages/cli/test/projects/deploy.test.ts @@ -10,6 +10,7 @@ import createLightningServer, { import { handler as deployHandler, + hasRemoteDiverged, reportDiff, } from '../../src/projects/deploy'; import { myProject_yaml, myProject_v1 } from './fixtures'; @@ -278,3 +279,53 @@ test.serial.skip( t.truthy(expectedLog); } ); + +test('hasRemoteDiverged: 1 workflow, no diverged', (t) => { + const local = { + workflows: [ + { + id: 'w', + }, + ], + cli: { + forked_from: { + w: 'a', + }, + }, + } as unknown as Project; + + const remote = { + getWorkflow: () => ({ + id: 'w', + history: ['a'], + }), + } as unknown as Project; + + const diverged = hasRemoteDiverged(local, remote); + t.falsy(diverged); +}); + +test('hasRemoteDiverged: 1 workflow, 1 diverged', (t) => { + const local = { + workflows: [ + { + id: 'w', + }, + ], + cli: { + forked_from: { + w: 'w', + }, + }, + } as unknown as Project; + + const remote = { + getWorkflow: () => ({ + id: 'w', + history: ['a', 'b'], + }), + } as unknown as Project; + + const diverged = hasRemoteDiverged(local, remote); + t.deepEqual(diverged, ['w']); +}); diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index 187da091f..07e9b6490 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -118,6 +118,7 @@ export interface ProjectMeta { env?: string; inserted_at?: string; updated_at?: string; + forked_from?: Record; [key: string]: unknown; } diff --git a/packages/project/src/Project.ts b/packages/project/src/Project.ts index e1c425ed8..e4800fd0b 100644 --- a/packages/project/src/Project.ts +++ b/packages/project/src/Project.ts @@ -12,7 +12,7 @@ import { getUuidForEdge, getUuidForStep } from './util/uuid'; import { merge, MergeProjectOptions } from './merge/merge-project'; import { diff as projectDiff } from './util/project-diff'; import { Workspace } from './Workspace'; -import { buildConfig } from './util/config'; +import { buildConfig, extractConfig } from './util/config'; import { Provisioner } from '@openfn/lexicon/lightning'; import { SandboxMeta, UUID, WorkspaceConfig } from '@openfn/lexicon'; @@ -31,6 +31,7 @@ type UUIDMap = { type CLIMeta = { version?: number; alias?: string; + forked_from?: Record; }; export class Project { @@ -255,6 +256,14 @@ export class Project { } return true; } + + /** + * Generates the contents of the openfn.yaml file, + * plus its file path + */ + generateConfig() { + return extractConfig(this); + } } export default Project; diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index e1066f981..07b976a0e 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -27,8 +27,8 @@ class Workflow { this.workflow = clone(workflow); - // history needs to be on workflow object. - this.workflow.history = workflow.history?.length ? workflow.history : []; + // history needs to be on workflow object + this.workflow.history = workflow.history ?? []; const { id, @@ -71,6 +71,10 @@ class Workflow { this.workflow.start = s; } + get history() { + return this.workflow.history ?? []; + } + _buildIndex() { for (const step of this.workflow.steps) { const s = step as any; @@ -191,10 +195,6 @@ class Workflow { this.workflow.history?.push(versionHash); } - get history() { - return this.workflow.history ?? []; - } - // return true if the current workflow can be merged into the target workflow without losing any changes canMergeInto(target: Workflow) { const thisHistory = diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index d1ff194d2..1ced41426 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -12,6 +12,7 @@ import { } from '../util/config'; import { omit } from 'lodash-es'; import { Logger } from '@openfn/logger'; +import omitNil from '../util/omit-nil'; export type FromFsConfig = { root: string; @@ -33,9 +34,12 @@ export const parseProject = async (options: FromFsConfig) => { const proj: any = { id: context.project?.id, name: context.project?.name, - openfn: omit(context.project, ['id']), + openfn: omit(context.project, ['id', 'forked_from']), config: config, workflows: [], + cli: omitNil({ + forked_from: context.project.forked_from, + }), }; // now find all the workflows diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index c1d80c433..17bd59ecc 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -30,6 +30,11 @@ export const extractConfig = (source: Project, format?: 'yaml' | 'json') => { if (source.name) { project.name = source.name; } + + if (source.cli.forked_from && Object.keys(source.cli.forked_from).length) { + project.forked_from = source.cli.forked_from; + } + const workspace = { ...source.config, }; diff --git a/packages/project/src/util/omit-nil.ts b/packages/project/src/util/omit-nil.ts index 40a8974e8..1e05f887f 100644 --- a/packages/project/src/util/omit-nil.ts +++ b/packages/project/src/util/omit-nil.ts @@ -1,8 +1,10 @@ import { omitBy, isNil } from 'lodash-es'; -export const omitNil = (obj: any, key: string) => { - if (obj[key]) { +export const omitNil = (obj: any, key?: string) => { + if (key && obj[key]) { obj[key] = omitBy(obj[key], isNil); + } else { + return omitBy(obj, isNil); } }; export default omitNil; diff --git a/packages/project/test/fixtures/sample-v2-project.ts b/packages/project/test/fixtures/sample-v2-project.ts index b8202ecf9..4029eb90f 100644 --- a/packages/project/test/fixtures/sample-v2-project.ts +++ b/packages/project/test/fixtures/sample-v2-project.ts @@ -33,7 +33,7 @@ export const json: SerializedProject = { name: 'Workflow', id: 'workflow', openfn: { uuid: 1 }, - history: [], + history: ['a', 'b'], start: 'trigger', }, ], @@ -72,7 +72,9 @@ workflows: id: workflow openfn: uuid: 1 - history: [] + history: + - a + - b start: trigger sandbox: parentId: abcd diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index 6a0785f52..a3f8f7639 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -22,7 +22,7 @@ function mockFile(path: string, content: string | object) { mock(files); } -test.serial('should include multiple workflows', async (t) => { +test.serial('should include multiple workflows (legacy format)', async (t) => { mockFile('/ws/openfn.yaml', buildConfig()); mockFile('/ws/workflows/workflow-1/workflow-1.yaml', { @@ -66,7 +66,7 @@ test.serial('should include multiple workflows', async (t) => { t.is(wf2.name, 'Workflow 2'); }); -test.serial('should load a workflow expression', async (t) => { +test.serial('should load a workflow expression (legacy format)', async (t) => { mockFile('/ws/openfn.yaml', buildConfig()); mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { @@ -104,7 +104,7 @@ test.serial( ); test.serial( - 'should load a workflow from the file system and expand shorthand links', + 'should load a workflow from the file system and expand shorthand links (legacy format)', async (t) => { mockFile('/ws/openfn.yaml', buildConfig()); @@ -139,3 +139,33 @@ test.serial( t.is(typeof wf.steps[1].next.c, 'object'); } ); + +test.serial('should track forked_from', async (t) => { + mockFile('/ws/openfn.yaml', { + workspace: buildConfig(), + project: { + uuid: '', + forked_from: { + w1: 'abcd', + }, + }, + }); + + mockFile('/ws/workflows/workflow-1/workflow-1.yaml', { + id: 'workflow-1', + name: 'Workflow 1', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }); + + mockFile('/ws/workflows/workflow-1/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); + + t.deepEqual(project.cli.forked_from, { w1: 'abcd' }); + t.falsy(project.openfn!.forked_from); +}); diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index 6a328b1bb..c41b4dca6 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -26,6 +26,9 @@ workflows: lock_version: 1 deleted_at: null concurrency: null + version_history: + - a + - b jobs: transform-data: name: Transform data @@ -93,7 +96,7 @@ test('import from a v2 project as JSON', async (t) => { openfn: { uuid: 1, }, - history: [], + history: ['a', 'b'], start: 'trigger', steps: [ { @@ -152,7 +155,7 @@ test('import from a v2 project as YAML', async (t) => { uuid: 1, }, start: 'trigger', - history: [], + history: ['a', 'b'], steps: [ { name: 'b', diff --git a/packages/project/test/serialize/to-app-state.test.ts b/packages/project/test/serialize/to-app-state.test.ts index de6e4117e..b6a7fa570 100644 --- a/packages/project/test/serialize/to-app-state.test.ts +++ b/packages/project/test/serialize/to-app-state.test.ts @@ -262,6 +262,43 @@ test('should handle credentials', (t) => { t.is(step.project_credential_id, 'p'); }); +test.only('should ignore forked_from', (t) => { + const data = { + id: 'my-project', + workflows: [ + { + id: 'wf', + name: 'wf', + steps: [ + { + id: 'trigger', + type: 'webhook', + next: { + step: {}, + }, + }, + { + id: 'step', + expression: '.', + configuration: 'p', + openfn: { + keychain_credential_id: 'k', + }, + }, + ], + }, + ], + cli: { + forked_form: { wf: 'a' }, + }, + }; + const proj = new Project(data); + console.log(proj); + const state = toAppState(proj, { format: 'json' }); + console.log(state); + t.falsy((state as any).forked_form); +}); + test('should ignore workflow start keys', (t) => { const data = { id: 'my-project', diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index 63c29c4d2..484a0e244 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -233,4 +233,58 @@ test('toFs: extract a project with 1 workflow and 1 step', (t) => { t.is(files['workflows/my-workflow/step.js'], 'fn(s => s)'); }); +test('toFs: extract a project with forked_from meta', (t) => { + const project = new Project( + { + name: 'My Project', + workflows: [ + { + id: 'my-workflow', + steps: [step], + }, + ], + cli: { + forked_from: 'abcd', + }, + }, + { + formats: { + openfn: 'json', // for easier testing + workflow: 'json', + }, + } + ); + + const files = toFs(project); + + // Ensure that all the right files have been created + t.deepEqual(Object.keys(files), [ + 'openfn.json', + 'workflows/my-workflow/my-workflow.json', + 'workflows/my-workflow/step.js', + ]); + + // rough test on the file contents + // (this should be validated in more detail by each step) + const config = JSON.parse(files['openfn.json']); + t.deepEqual(config, { + workspace: { + credentials: 'credentials.yaml', + formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, + dirs: { projects: '.projects', workflows: 'workflows' }, + }, + project: { + id: 'my-project', + name: 'My Project', + forked_from: 'abcd', + }, + }); + + const workflow = JSON.parse(files['workflows/my-workflow/my-workflow.json']); + t.is(workflow.id, 'my-workflow'); + t.is(workflow.steps.length, 1); + + t.is(files['workflows/my-workflow/step.js'], 'fn(s => s)'); +}); + // TODO we need many more tests on this, with options diff --git a/packages/project/test/serialize/to-project.test.ts b/packages/project/test/serialize/to-project.test.ts index 76eeced78..ea4f1cba5 100644 --- a/packages/project/test/serialize/to-project.test.ts +++ b/packages/project/test/serialize/to-project.test.ts @@ -37,6 +37,9 @@ const createProject = (props: Partial = {}) => { // hack delete proj.workflows[0].steps[0].name; proj.workflows[0].start = 'trigger'; + + // add some history + proj.workflows[0].workflow.history = ['a', 'b']; return proj; }; diff --git a/packages/project/test/util/config.test.ts b/packages/project/test/util/config.test.ts index db6837c09..02cdb5ffd 100644 --- a/packages/project/test/util/config.test.ts +++ b/packages/project/test/util/config.test.ts @@ -29,6 +29,8 @@ project: env: dev inserted_at: 2025-10-21T17:10:57Z updated_at: 2025-10-21T17:10:57Z + forked_from: + w1: abcd `; const result = loadWorkspaceFile(yaml); @@ -51,6 +53,9 @@ project: env: 'dev', inserted_at: '2025-10-21T17:10:57Z', updated_at: '2025-10-21T17:10:57Z', + forked_from: { + w1: 'abcd', + }, }); }); @@ -161,6 +166,47 @@ test('generate openfn.yaml', (t) => { openfn: { uuid: 1234, }, + cli: { + forked_from: 'abcd', + }, + }, + { + formats: { + openfn: 'yaml', + }, + } + ); + const result = extractConfig(proj); + t.is(result.path, 'openfn.yaml'), + t.deepEqual( + result.content, + `project: + uuid: 1234 + id: my-project + name: My Project + forked_from: abcd +workspace: + credentials: credentials.yaml + formats: + openfn: yaml + project: yaml + workflow: yaml + dirs: + projects: .projects + workflows: workflows +` + ); +}); + +test("exclude forked_from if it's not set", (t) => { + const proj = new Project( + { + id: 'my-project', + name: 'My Project', + openfn: { + uuid: 1234, + }, + cli: {}, }, { formats: {