Fix version history

.changeset/moody-ducks-warn.md

@@ -0,0 +1,5 @@
+---
+'@openfn/project': patch
+---
+
+Support forked_from metadata key in openfn.yaml

packages/cli/src/projects/checkout.ts

@@ -10,7 +10,7 @@ import * as o from '../options';
 import * as po from './options';
 
 import type { Opts } from './options';
-import { tidyWorkflowDir } from './util';
+import { tidyWorkflowDir, updateForkedFrom } from './util';
 
 export type CheckoutOptions = Pick<
   Opts,
@@ -69,7 +69,11 @@ export const handler = async (options: CheckoutOptions, logger: Logger) => {
     await tidyWorkflowDir(currentProject!, switchProject);
   }
 
+  // write the forked from map
+  updateForkedFrom(switchProject);
+
   // expand project into directory
+  // TODO: only write files with a diff
   const files: any = switchProject.serialize('fs');
   for (const f in files) {
     if (files[f]) {

packages/cli/src/projects/deploy.ts

@@ -1,6 +1,8 @@
 import yargs from 'yargs';
 import Project from '@openfn/project';
 import c from 'chalk';
+import { writeFile } from 'node:fs/promises';
+import path from 'node:path';
 
 import * as o from '../options';
 import * as o2 from './options';
@@ -10,6 +12,7 @@ import {
   fetchProject,
   serialize,
   getSerializePath,
+  updateForkedFrom,
 } from './util';
 import { build, ensure } from '../util/command-builders';
 
@@ -64,6 +67,34 @@ export const command: yargs.CommandModule<DeployOptions> = {
   handler: ensure('project-deploy', options),
 };
 
+export const hasRemoteDiverged = (
+  local: Project,
+  remote: Project
+): string[] | null => {
+  let diverged: string[] | null = null;
+
+  const refs = local.cli.forked_from ?? {};
+
+  // for each workflow, check that the local fetched_from is the head of the remote history
+  for (const wf of local.workflows) {
+    if (wf.id in refs) {
+      const forkedVersion = refs[wf.id];
+      const remoteVersion = remote.getWorkflow(wf.id)?.history.at(-1);
+      if (forkedVersion !== remoteVersion) {
+        diverged ??= [];
+        diverged.push(wf.id);
+      }
+    } else {
+      // TODO what if there's no forked from for this workflow?
+      // Do we assume divergence because we don't know? Do we warn?
+    }
+  }
+
+  // TODO what if a workflow is removed locally?
+
+  return diverged;
+};
+
 export async function handler(options: DeployOptions, logger: Logger) {
   logger.warn(
     'WARNING: the project deploy command is in BETA and may not be stable. Use cautiously on production projects.'
@@ -132,32 +163,41 @@ Pass --force to override this error and deploy anyway.`);
 
   // Skip divergence testing if the remote has no history in its workflows
   // (this will only happen on older versions of lightning)
+  // TODO now maybe skip if there's no forked_from
   const skipVersionTest =
-    localProject.workflows.find((wf) => wf.history.length === 0) ||
+    // localProject.workflows.find((wf) => wf.history.length === 0) ||
     remoteProject.workflows.find((wf) => wf.history.length === 0);
 
+  // localProject.workflows.forEach((w) => console.log(w.history));
+
   if (skipVersionTest) {
     logger.warn(
       'Skipping compatibility check as no local version history detected'
     );
     logger.warn('Pushing these changes may overrite changes made to the app');
-  } else if (!localProject.canMergeInto(remoteProject!)) {
-    if (!options.force) {
-      logger.error(`Error: Projects have diverged!
+  } else {
+    const divergentWorkflows = hasRemoteDiverged(localProject, remoteProject!);
+    if (divergentWorkflows) {
+      logger.warn(
+        `The following workflows have diverged: ${divergentWorkflows}`
+      );
+      if (!options.force) {
+        logger.error(`Error: Projects have diverged!
 
-The remote project has been edited since the local project was branched. Changes may be lost.
+  The remote project has been edited since the local project was branched. Changes may be lost.
 
-Pass --force to override this error and deploy anyway.`);
-      return;
+  Pass --force to override this error and deploy anyway.`);
+        return;
+      } else {
+        logger.warn(
+          'Remote project has not diverged from local project! Pushing anyway as -f passed'
+        );
+      }
     } else {
-      logger.warn(
-        'Remote project has not diverged from local project! Pushing anyway as -f passed'
+      logger.info(
+        'Remote project has not diverged from local project - it is safe to deploy 🎉'
       );
     }
-  } else {
-    logger.info(
-      'Remote project has not diverged from local project - it is safe to deploy 🎉'
-    );
   }
 
   logger.info('Merging changes into remote project');
@@ -180,6 +220,8 @@ Pass --force to override this error and deploy anyway.`);
   // TODO not totally sold on endpoint handling right now
   config.endpoint ??= localProject.openfn?.endpoint!;
 
+  // TODO: I want to report diff HERE, after the merged state and stuff has been built
+
   if (options.dryRun) {
     logger.always('dryRun option set: skipping upload step');
   } else {
@@ -218,6 +260,14 @@ Pass --force to override this error and deploy anyway.`);
       merged.config
     );
 
+    // TODO why isn't this right? oh, because the outpu path isn't quite right
+    updateForkedFrom(finalProject);
+    const configData = finalProject.generateConfig();
+    await writeFile(
+      path.resolve(options.workspace, configData.path),
+      configData.content
+    );
+
     const finalOutputPath = getSerializePath(localProject, options.workspace!);
     logger.debug('Updating local project at ', finalOutputPath);
     await serialize(finalProject, finalOutputPath);
@@ -267,3 +317,4 @@ export const reportDiff = (local: Project, remote: Project, logger: Logger) => {
 
   return diffs;
 };
+``;

packages/cli/src/projects/fetch.ts

@@ -331,6 +331,8 @@ To ignore this error and override the local file, pass --force (-f)
       options.force || // The user forced the checkout
       !hasAnyHistory; // the remote project has no history (can happen in old apps)
 
+    // TODO temporarily force skip
+    // TODO canMergeInto needs to return a reason
     if (!skipVersionCheck && !remoteProject.canMergeInto(localProject!)) {
       // TODO allow rename
       throw new Error('Error! An incompatible project exists at this location');

packages/cli/src/projects/util.ts

@@ -216,3 +216,14 @@ export async function tidyWorkflowDir(
   // Return and sort for testing
   return toRemove.sort();
 }
+
+export const updateForkedFrom = (proj: Project) => {
+  proj.cli.forked_from = proj.workflows.reduce((obj: any, wf) => {
+    if (wf.history.length) {
+      obj[wf.id] = wf.history.at(-1);
+    }
+    return obj;
+  }, {});
+
+  return proj;
+};

packages/cli/test/projects/checkout.test.ts

@@ -3,7 +3,7 @@ import { createMockLogger } from '@openfn/logger';
 import { handler as checkoutHandler } from '../../src/projects/checkout';
 import mock from 'mock-fs';
 import fs from 'fs';
-import { jsonToYaml, Workspace } from '@openfn/project';
+import { jsonToYaml, Workspace, yamlToJson } from '@openfn/project';
 
 test.beforeEach(() => {
   mock({
@@ -28,6 +28,7 @@ test.beforeEach(() => {
         {
           name: 'simple-workflow',
           id: 'wf-id',
+          history: ['a'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -56,6 +57,7 @@ test.beforeEach(() => {
         {
           name: 'another-workflow',
           id: 'another-id',
+          history: ['b'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -83,13 +85,15 @@ test.beforeEach(() => {
         },
       ],
     }),
+    // TODO this is actually a v1 state file for some reason, which is wierd
     '/ws/.projects/project@app.openfn.org.yaml': jsonToYaml({
       id: '<uuid:main>',
       name: 'My Project',
       workflows: [
         {
           name: 'simple-workflow-main',
           id: 'wf-id-main',
+          version_history: ['a'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -118,6 +122,7 @@ test.beforeEach(() => {
         {
           name: 'another-workflow-main',
           id: 'another-id',
+          version_history: ['b'],
           jobs: [
             {
               name: 'Transform data to FHIR standard',
@@ -217,6 +222,25 @@ test.serial('checkout: same id as active', async (t) => {
   );
 });
 
+test.serial(
+  'checkout: writes forked_from based on version history',
+  async (t) => {
+    const bcheckout = new Workspace('/ws');
+    t.is(bcheckout.activeProject!.id, 'my-project');
+
+    await checkoutHandler(
+      { command: 'project-checkout', project: 'my-project', workspace: '/ws' },
+      logger
+    );
+
+    const openfn = yamlToJson(fs.readFileSync('/ws/openfn.yaml', 'utf8'));
+    t.deepEqual(openfn.project.forked_from, {
+      'simple-workflow-main': 'a',
+      'another-workflow-main': 'b',
+    });
+  }
+);
+
 test.serial('checkout: switching to and back between projects', async (t) => {
   // before checkout. my-project is active and expanded
   const bcheckout = new Workspace('/ws');

packages/cli/test/projects/deploy.test.ts

@@ -10,6 +10,7 @@ import createLightningServer, {
 
 import {
   handler as deployHandler,
+  hasRemoteDiverged,
   reportDiff,
 } from '../../src/projects/deploy';
 import { myProject_yaml, myProject_v1 } from './fixtures';
@@ -278,3 +279,53 @@ test.serial.skip(
     t.truthy(expectedLog);
   }
 );
+
+test('hasRemoteDiverged: 1 workflow, no diverged', (t) => {
+  const local = {
+    workflows: [
+      {
+        id: 'w',
+      },
+    ],
+    cli: {
+      forked_from: {
+        w: 'a',
+      },
+    },
+  } as unknown as Project;
+
+  const remote = {
+    getWorkflow: () => ({
+      id: 'w',
+      history: ['a'],
+    }),
+  } as unknown as Project;
+
+  const diverged = hasRemoteDiverged(local, remote);
+  t.falsy(diverged);
+});
+
+test('hasRemoteDiverged: 1 workflow, 1 diverged', (t) => {
+  const local = {
+    workflows: [
+      {
+        id: 'w',
+      },
+    ],
+    cli: {
+      forked_from: {
+        w: 'w',
+      },
+    },
+  } as unknown as Project;
+
+  const remote = {
+    getWorkflow: () => ({
+      id: 'w',
+      history: ['a', 'b'],
+    }),
+  } as unknown as Project;
+
+  const diverged = hasRemoteDiverged(local, remote);
+  t.deepEqual(diverged, ['w']);
+});

packages/lexicon/core.d.ts

@@ -118,6 +118,7 @@ export interface ProjectMeta {
   env?: string;
   inserted_at?: string;
   updated_at?: string;
+  forked_from?: Record<string, string>;
 
   [key: string]: unknown;
 }

packages/project/src/Project.ts

@@ -12,7 +12,7 @@ import { getUuidForEdge, getUuidForStep } from './util/uuid';
 import { merge, MergeProjectOptions } from './merge/merge-project';
 import { diff as projectDiff } from './util/project-diff';
 import { Workspace } from './Workspace';
-import { buildConfig } from './util/config';
+import { buildConfig, extractConfig } from './util/config';
 import { Provisioner } from '@openfn/lexicon/lightning';
 import { SandboxMeta, UUID, WorkspaceConfig } from '@openfn/lexicon';
 
@@ -31,6 +31,7 @@ type UUIDMap = {
 type CLIMeta = {
   version?: number;
   alias?: string;
+  forked_from?: Record<string, string>;
 };
 
 export class Project {
@@ -255,6 +256,14 @@ export class Project {
     }
     return true;
   }
+
+  /**
+   * Generates the contents of the openfn.yaml file,
+   * plus its file path
+   */
+  generateConfig() {
+    return extractConfig(this);
+  }
 }
 
 export default Project;

packages/project/src/Workflow.ts

@@ -27,8 +27,8 @@ class Workflow {
 
     this.workflow = clone(workflow);
 
-    // history needs to be on workflow object.
-    this.workflow.history = workflow.history?.length ? workflow.history : [];
+    // history needs to be on workflow object
+    this.workflow.history = workflow.history ?? [];
 
     const {
       id,
@@ -71,6 +71,10 @@ class Workflow {
     this.workflow.start = s;
   }
 
+  get history() {
+    return this.workflow.history ?? [];
+  }
+
   _buildIndex() {
     for (const step of this.workflow.steps) {
       const s = step as any;
@@ -191,10 +195,6 @@ class Workflow {
     this.workflow.history?.push(versionHash);
   }
 
-  get history() {
-    return this.workflow.history ?? [];
-  }
-
   // return true if the current workflow can be merged into the target workflow without losing any changes
   canMergeInto(target: Workflow) {
     const thisHistory =