CLI: native collections support

.changeset/brave-islands-search.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Support aliases on all project subcommands

.changeset/tiny-forks-brake.md

@@ -0,0 +1,5 @@
+---
+'@openfn/cli': minor
+---
+
+Auto-load collections adptor when using collections

.changeset/wicked-plants-push.md

@@ -0,0 +1,5 @@
+---
+'@openfn/lightning-mock': minor
+---
+
+Add basic collections support (GET only)

integration-tests/cli/test/execute-workflow.test.ts

@@ -1,9 +1,24 @@
 import test from 'ava';
 import { rm, mkdir } from 'node:fs/promises';
 import path from 'node:path';
+
+import createLightningServer from '@openfn/lightning-mock';
+
 import run from '../src/run';
 import { getJSON } from '../src/util';
 
+// set up a lightning mock
+let server: any;
+
+const port = 8968;
+
+test.before(async () => {
+  server = await createLightningServer({ port });
+  server.collections.createCollection('stuff');
+  // Important: the collection value MUST be as string
+  server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' }));
+});
+
 const jobsPath = path.resolve('test/fixtures');
 
 // Note that these tests are STATEFUL
@@ -151,7 +166,6 @@ test.serial(
   `openfn ${jobsPath}/wf-creds.json --credentials ${jobsPath}/creds.json`,
   async (t) => {
     const { err, stdout, stderr } = await run(t.title);
-    console.log({ stdout, stderr });
     t.falsy(err);
 
     const out = getJSON();
@@ -285,3 +299,16 @@ test.serial(
     });
   }
 );
+
+// collections basic test
+test.serial(
+  `openfn ${jobsPath}/collections.json --endpoint http://localhost:${port} --api-key xyz`,
+  async (t) => {
+    const { err } = await run(t.title);
+    t.falsy(err);
+
+    const out = getJSON();
+
+    t.deepEqual(out.data, { id: 'x' });
+  }
+);

integration-tests/cli/test/fixtures/collections.json

@@ -0,0 +1,10 @@
+{
+  "workflow": {
+    "steps": [
+      {
+        "adaptor": "common",
+        "expression": "collections.get('stuff', 'x')"
+      }
+    ]
+  }
+}

packages/cli/src/collections/command.ts

@@ -88,14 +88,6 @@ const key = {
   },
 };
 
-const token = {
-  name: 'pat',
-  yargs: {
-    alias: ['token'],
-    description: 'Lightning Personal Access Token (PAT)',
-  },
-};
-
 const endpoint = {
   name: 'endpoint',
   yargs: {
@@ -160,7 +152,7 @@ const updatedAfter = {
 const getOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   pageSize,
   limit,
@@ -201,7 +193,7 @@ const dryRun = {
 const removeOptions = [
   collectionName,
   key,
-  token,
+  o.apiKey,
   endpoint,
   dryRun,
 
@@ -243,7 +235,7 @@ const setOptions = [
   override(key as any, {
     demand: false,
   }),
-  token,
+  o.apiKey,
   endpoint,
   value,
   items,

packages/cli/src/execute/command.ts

@@ -1,19 +1,22 @@
 import yargs from 'yargs';
-import { build, ensure } from '../util/command-builders';
+import { build, ensure, override } from '../util/command-builders';
 import * as o from '../options';
 
 import type { Opts } from '../options';
 
 export type ExecuteOptions = Required<
   Pick<
     Opts,
+    | 'apiKey'
     | 'adaptors'
     | 'autoinstall'
     | 'baseDir'
     | 'cacheSteps'
     | 'command'
     | 'compile'
     | 'credentials'
+    | 'collectionsEndpoint'
+    | 'collectionsVersion'
     | 'expandAdaptors'
     | 'end'
     | 'immutable'
@@ -44,10 +47,16 @@ const options = [
   o.expandAdaptors, // order is important
 
   o.adaptors,
+  override(o.apiKey, {
+    description: 'API token for collections',
+    alias: ['collections-api-key', 'collections-token', 'pat'],
+  }),
   o.autoinstall,
   o.cacheSteps,
   o.compile,
   o.credentials,
+  o.collectionsEndpoint,
+  o.collectionsVersion,
   o.end,
   o.ignoreImports,
   o.immutable,

packages/cli/src/options.ts

@@ -31,6 +31,8 @@ export type Opts = {
   configPath?: string;
   confirm?: boolean;
   credentials?: string;
+  collectionsEndpoint?: string;
+  collectionsVersion?: string;
   describe?: string;
   end?: string; // workflow end node
   expandAdaptors?: boolean; // for unit tests really
@@ -136,12 +138,17 @@ export const autoinstall: CLIOption = {
   },
 };
 
-export const apikey: CLIOption = {
+export const apiKey: CLIOption = {
   name: 'apikey',
   yargs: {
-    alias: ['key', 'pat', 'token'],
+    alias: ['pat', 'token', 'api-key'],
     description:
-      '[beta only] API Key, Personal Access Token (Pat), or other access token',
+      'API Key, Personal Access Token (PAT), or other access token from Lightning',
+  },
+  ensure: (opts: any) => {
+    if (!opts.apikey) {
+      opts.apiKey = process.env.OPENFN_API_KEY;
+    }
   },
 };
 
@@ -240,6 +247,23 @@ export const configPath: CLIOption = {
   },
 };
 
+export const collectionsVersion: CLIOption = {
+  name: 'collections-version',
+  yargs: {
+    description:
+      'The version of the collections adaptor to use. Defaults to latest. Use OPENFN_COLLECTIONS_VERSION env.',
+  },
+};
+
+export const collectionsEndpoint: CLIOption = {
+  name: 'collections-endpoint',
+  yargs: {
+    alias: ['endpoint'],
+    description:
+      'The Lightning server to use for collections. Will use the project endpoint if available. Use OPENFN_COLLECTIONS_ENDPOINT env.',
+  },
+};
+
 export const credentials: CLIOption = {
   name: 'credentials',
   yargs: {

packages/cli/src/projects/fetch.ts

@@ -35,7 +35,7 @@ export type FetchOptions = Pick<
 
 const options = [
   po.alias,
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   o.logJson,

packages/cli/src/projects/pull.ts

@@ -30,7 +30,7 @@ const options = [
   o2.workspace,
 
   // general options
-  o.apikey,
+  o.apiKey,
   o.endpoint,
   o.log,
   override(o.path, {

packages/cli/src/pull/command.ts

@@ -21,7 +21,7 @@ export type PullOptions = Required<
 >;
 
 const options = [
-  o.apikey,
+  o.apiKey,
   o.beta,
   o.configPath,
   o.endpoint,

packages/cli/src/util/load-plan.ts

@@ -331,11 +331,70 @@ const ensureAdaptors = (plan: CLIExecutionPlan) => {
   });
 };
 
+type ensureCollectionsOptions = {
+  endpoint?: string;
+  version?: string;
+  apiKey?: string;
+};
+
+const ensureCollections = (
+  plan: CLIExecutionPlan,
+  {
+    endpoint = 'https://app.openfn.org',
+    version = 'latest',
+    apiKey = 'null',
+  }: ensureCollectionsOptions = {},
+  logger?: Logger
+) => {
+  let collectionsFound = false;
+
+  Object.values(plan.workflow.steps)
+    .filter((step) => (step as any).expression?.match(/(collections\.)/))
+    .forEach((step) => {
+      const job = step as CLIJobNode;
+      if (
+        !job.adaptors?.find((v: string) =>
+          v.startsWith('@openfn/language-collections')
+        )
+      ) {
+        collectionsFound = true;
+        job.adaptors ??= [];
+        job.adaptors.push(
+          `@openfn/language-collections@${version || 'latest'}`
+        );
+
+        job.configuration = Object.assign({}, job.configuration, {
+          collections_endpoint: `${endpoint}/collections`,
+          collections_token: apiKey,
+        });
+      }
+    });
+
+  if (collectionsFound) {
+    if (!apiKey || apiKey === 'null') {
+      logger?.warn(
+        'WARNING: collections API was not set. Pass --api-key or OPENFN_API_KEY'
+      );
+    }
+    logger?.info(
+      `Configured collections to use endpoint ${endpoint} and API Key ending with ${apiKey?.substring(
+        apiKey.length - 10
+      )}`
+    );
+  }
+};
+
 const loadXPlan = async (
   plan: CLIExecutionPlan,
   options: Pick<
     Opts,
-    'monorepoPath' | 'baseDir' | 'expandAdaptors' | 'globals'
+    | 'monorepoPath'
+    | 'baseDir'
+    | 'expandAdaptors'
+    | 'globals'
+    | 'collectionsVersion'
+    | 'collectionsEndpoint'
+    | 'apiKey'
   >,
   logger: Logger,
   defaultName: string = ''
@@ -348,6 +407,15 @@ const loadXPlan = async (
     plan.workflow.name = defaultName;
   }
   ensureAdaptors(plan);
+  ensureCollections(
+    plan,
+    {
+      version: options.collectionsVersion,
+      apiKey: options.apiKey,
+      endpoint: options.collectionsEndpoint,
+    },
+    logger
+  );
 
   // import global functions
   // if globals is provided via cli argument. it takes precedence