Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ _mytests
.idea
.zed
docs/changelog.md
.generated

# Yarn files
.yarn/install-state.gz
Expand Down
33 changes: 29 additions & 4 deletions docs/reference.md
Original file line number Diff line number Diff line change
Expand Up @@ -249,16 +249,19 @@ DESCRIPTION
WARNING: Overwrites existing 'storage' directory.

USAGE
$ apify init [actorName] [-y]
$ apify init [actorName] [--dockerfile <value>] [-y]

ARGUMENTS
actorName Name of the Actor. If not provided, you will be prompted
for it.

FLAGS
-y, --yes Automatic yes to prompts; assume "yes" as answer to all
prompts. Note that in some cases, the command may still ask for
confirmation.
--dockerfile=<value> Path to a Dockerfile to use for
the Actor (e.g., "./Dockerfile" or
"./docker/Dockerfile").
-y, --yes Automatic yes to prompts;
assume "yes" as answer to all prompts. Note that in some
cases, the command may still ask for confirmation.
```

##### `apify run`
Expand Down Expand Up @@ -408,6 +411,8 @@ SUBCOMMANDS
actor calculate-memory Calculates the Actor’s dynamic
memory usage based on a memory expression from
actor.json, input data, and run options.
actor generate-types Generate TypeScript types from a
JSON schema file.
```

##### `apify actor calculate-memory`
Expand Down Expand Up @@ -464,6 +469,26 @@ FLAGS
charging without actually charging
```

##### `apify actor generate-types`

```sh
DESCRIPTION
Generate TypeScript types from a JSON schema file.

USAGE
$ apify actor generate-types <path> [-o <value>]
[-s]

ARGUMENTS
path Path to the JSON schema file.

FLAGS
-o, --output=<value> Directory where the generated files
should be outputted.
-s, --strict Whether generated interfaces should be
strict (no index signature [key: string]: unknown).
```

##### `apify actor get-input`

```sh
Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@
"istextorbinary": "~9.5.0",
"jju": "~1.4.0",
"js-levenshtein": "^1.1.6",
"json-schema-to-typescript": "^15.0.4",
"lodash.clonedeep": "^4.5.0",
"mime": "~4.1.0",
"open": "~11.0.0",
Expand Down
1 change: 1 addition & 0 deletions scripts/generate-cli-docs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ const categories: Record<string, CommandsInCategory[]> = {
{ command: Commands.actor },
{ command: Commands.actorCalculateMemory },
{ command: Commands.actorCharge },
{ command: Commands.actorGenerateTypes },
{ command: Commands.actorGetInput },
{ command: Commands.actorGetPublicUrl },
{ command: Commands.actorGetValue },
Expand Down
2 changes: 2 additions & 0 deletions src/commands/_register.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import type { BuiltApifyCommand } from '../lib/command-framework/apify-command.j
import { ActorIndexCommand } from './actor/_index.js';
import { ActorCalculateMemoryCommand } from './actor/calculate-memory.js';
import { ActorChargeCommand } from './actor/charge.js';
import { ActorGenerateTypesCommand } from './actor/generate-types.js';
import { ActorGetInputCommand } from './actor/get-input.js';
import { ActorGetPublicUrlCommand } from './actor/get-public-url.js';
import { ActorGetValueCommand } from './actor/get-value.js';
Expand Down Expand Up @@ -75,6 +76,7 @@ export const actorCommands = [
ActorGetInputCommand,
ActorChargeCommand,
ActorCalculateMemoryCommand,
ActorGenerateTypesCommand,

// top-level
HelpCommand,
Expand Down
2 changes: 2 additions & 0 deletions src/commands/actor/_index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
import { ActorCalculateMemoryCommand } from './calculate-memory.js';
import { ActorChargeCommand } from './charge.js';
import { ActorGenerateTypesCommand } from './generate-types.js';
import { ActorGetInputCommand } from './get-input.js';
import { ActorGetPublicUrlCommand } from './get-public-url.js';
import { ActorGetValueCommand } from './get-value.js';
Expand All @@ -21,6 +22,7 @@ export class ActorIndexCommand extends ApifyCommand<typeof ActorIndexCommand> {
ActorGetInputCommand,
ActorChargeCommand,
ActorCalculateMemoryCommand,
ActorGenerateTypesCommand,
];

async run() {
Expand Down
88 changes: 88 additions & 0 deletions src/commands/actor/generate-types.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
import process from 'node:process';

import type { JSONSchema4 } from 'json-schema';
import { compile } from 'json-schema-to-typescript';

import { ApifyCommand } from '../../lib/command-framework/apify-command.js';
import { Args } from '../../lib/command-framework/args.js';
import { Flags } from '../../lib/command-framework/flags.js';
import { LOCAL_CONFIG_PATH } from '../../lib/consts.js';
import { readAndValidateInputSchema } from '../../lib/input_schema.js';
import { success } from '../../lib/outputs.js';

export const BANNER_COMMENT = `
/* eslint-disable */
/* biome-ignore-all lint */
/* biome-ignore-all format */
/* prettier-ignore-start */
/**
* This file was automatically generated by json-schema-to-typescript.
* DO NOT MODIFY IT BY HAND. Instead, modify the source JSONSchema file,
* and run apify actor generate-types to regenerate this file.
*/
`;

export class ActorGenerateTypesCommand extends ApifyCommand<typeof ActorGenerateTypesCommand> {
static override name = 'generate-types' as const;

static override description = `Generate TypeScript types from an Actor input schema.

Reads the input schema from one of these locations (in priority order):
1. Object in '${LOCAL_CONFIG_PATH}' under "input" key
2. JSON file path in '${LOCAL_CONFIG_PATH}' "input" key
3. .actor/INPUT_SCHEMA.json
4. INPUT_SCHEMA.json

Optionally specify custom schema path to use.`;

static override flags = {
output: Flags.string({
char: 'o',
description: 'Directory where the generated files should be outputted.',
required: false,
default: './.generated/actor/',
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actor templates have typescript setup with only src as include, so you cannot import it right away and you have to adjust it. Maybe better idea is to generate it directly to src/.generated/actor/ ? cc @vladfrangu

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this might be best to discuss with @B4nan and @patrikbraborec too. Or maybe we wanna make it like how prisma handles it? so you'd have @apify/actor in node-modules which gets filled out by the cli?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so you'd have @apify/actor in node-modules which gets filled out by the cli?

I think changed this in v7, because it produced a lot of side effects, e.g. with caching. But we could explore this as well, maybe for our use case it would be fine.

If we won't go that way, I am slightly in favor of putting this into the src folder, the alternative would be dynamically appending the .generated folder to includes in tsconfig. But that would mean you end up with dist/src and dist/.generated if I'm not mistaken, which is a bit ugly (but also rather irrelevant).

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fwiw modifying tsconfig can be risky, people may have god knows what setups in their tsconfig files and we shouldn't randomly append our folder.

I think changed this in v7, because it produced a lot of side effects, e.g. with caching. But we could explore this as well, maybe for our use case it would be fine.

Can you help me understand what you mean by this? Did prisma change this or what are you referring to with v7?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just came back cuz I read that, holy heck I did not know.... But they also use the top level generated dir by def, interesting

}),
strict: Flags.boolean({
char: 's',
description: 'Whether generated interfaces should be strict (no index signature [key: string]: unknown).',
required: false,
default: true,
}),
};

static override args = {
path: Args.string({
required: false,
description: 'Optional path to the input schema file. If not provided, searches default locations.',
}),
};

async run() {
const { inputSchema, inputSchemaPath } = await readAndValidateInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
action: 'Generating types from',
});
Comment on lines +63 to +67
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

generate-type share a validation logic with validate-schema now


const name = inputSchemaPath ? path.basename(inputSchemaPath, path.extname(inputSchemaPath)) : 'input';

const result = await compile(inputSchema as JSONSchema4, name, {
bannerComment: BANNER_COMMENT,
maxItems: -1,
unknownAny: true,
format: true,
additionalProperties: !this.flags.strict,
$refOptions: { resolve: { external: false, file: false, http: false } },
});

const outputDir = path.resolve(process.cwd(), this.flags.output);
await mkdir(outputDir, { recursive: true });

const outputFile = path.join(outputDir, `${name}.ts`);
await writeFile(outputFile, result, 'utf-8');

success({ message: `Generated types written to ${outputFile}` });
}
}
22 changes: 4 additions & 18 deletions src/commands/validate-schema.ts
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
import process from 'node:process';

import { validateInputSchema } from '@apify/input_schema';

import { ApifyCommand } from '../lib/command-framework/apify-command.js';
import { Args } from '../lib/command-framework/args.js';
import { LOCAL_CONFIG_PATH } from '../lib/consts.js';
import { readInputSchema } from '../lib/input_schema.js';
import { info, success } from '../lib/outputs.js';
import { Ajv2019 } from '../lib/utils.js';
import { readAndValidateInputSchema } from '../lib/input_schema.js';
import { success } from '../lib/outputs.js';

export class ValidateInputSchemaCommand extends ApifyCommand<typeof ValidateInputSchemaCommand> {
static override name = 'validate-schema' as const;
Expand All @@ -30,23 +27,12 @@ Optionally specify custom schema path to validate.`;
static override hiddenAliases = ['vis'];

async run() {
const { inputSchema, inputSchemaPath } = await readInputSchema({
await readAndValidateInputSchema({
forcePath: this.args.path,
cwd: process.cwd(),
action: 'Validating',
});

if (!inputSchema) {
throw new Error(`Input schema has not been found at ${inputSchemaPath}.`);
}

if (inputSchemaPath) {
info({ message: `Validating input schema stored at ${inputSchemaPath}` });
} else {
info({ message: `Validating input schema embedded in '${LOCAL_CONFIG_PATH}'` });
}

const validator = new Ajv2019({ strict: false });
validateInputSchema(validator, inputSchema); // This one throws an error in a case of invalid schema.
success({ message: 'Input schema is valid.' });
}
}
43 changes: 41 additions & 2 deletions src/lib/input_schema.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@ import deepClone from 'lodash.clonedeep';
import { KEY_VALUE_STORE_KEYS } from '@apify/consts';
import { validateInputSchema } from '@apify/input_schema';

import { ACTOR_SPECIFICATION_FOLDER } from './consts.js';
import { warning } from './outputs.js';
import { ACTOR_SPECIFICATION_FOLDER, LOCAL_CONFIG_PATH } from './consts.js';
import { info, warning } from './outputs.js';
import { Ajv2019, getJsonFileContent, getLocalConfig, getLocalKeyValueStorePath } from './utils.js';

const DEFAULT_INPUT_SCHEMA_PATHS = [
Expand Down Expand Up @@ -70,6 +70,45 @@ export const readInputSchema = async (
};
};

/**
* Reads and validates input schema, logging appropriate info messages.
* Throws an error if the schema is not found or invalid.
*
* @param options.forcePath - Optional path to force reading from
* @param options.cwd - Current working directory
* @param options.action - Action description for the info message (e.g., "Validating", "Generating types from")
* @returns The validated input schema and its path
*/
export const readAndValidateInputSchema = async ({
forcePath,
cwd,
action,
}: {
forcePath?: string;
cwd: string;
action: string;
}): Promise<{ inputSchema: Record<string, unknown>; inputSchemaPath: string | null }> => {
const { inputSchema, inputSchemaPath } = await readInputSchema({
forcePath,
cwd,
});

if (!inputSchema) {
throw new Error(`Input schema has not been found at ${inputSchemaPath}.`);
}

if (inputSchemaPath) {
info({ message: `${action} input schema at ${inputSchemaPath}` });
} else {
info({ message: `${action} input schema embedded in '${LOCAL_CONFIG_PATH}'` });
}

const validator = new Ajv2019({ strict: false });
validateInputSchema(validator, inputSchema);

return { inputSchema, inputSchemaPath };
};

/**
* Goes to the Actor directory and creates INPUT.json file from the input schema prefills.

Expand Down
80 changes: 80 additions & 0 deletions test/__setup__/input-schemas/complex.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
{
"$schema": "https://apify-projects.github.io/actor-json-schemas/input.ide.json?v=0.6",
"title": "Example Actor Input Schema",
"description": "A sample input schema demonstrating different input field types.",
"type": "object",
"schemaVersion": 1,
"properties": {
"startUrls": {
"title": "Start URLs",
"description": "List of URLs to start crawling",
"type": "array",
"editor": "requestListSources",
"prefill": [
{ "url": "https://example.com" },
{ "url": "https://example.org" }
]
},
"searchQuery": {
"title": "Search query",
"description": "The keyword or phrase to search for",
"type": "string",
"editor": "textfield",
"minLength": 3,
"default": "apify"
},
"maxItems": {
"title": "Maximum items to fetch",
"description": "Limit the number of items the Actor will process",
"type": "integer",
"editor": "number",
"minimum": 1,
"default": 100
},
"includeImages": {
"title": "Include images",
"description": "Whether to include image data in the results",
"type": "boolean",
"editor": "checkbox",
"default": false
},
"crawlerType": {
"title": "Crawler type",
"description": "Select the crawling engine to use",
"type": "string",
"editor": "select",
"enum": ["cheerio", "puppeteer", "playwright"],
"enumTitles": [
"Cheerio crawler",
"Puppeteer browser",
"Playwright browser"
],
"default": "cheerio"
},
"proxyConfig": {
"title": "Proxy configuration",
"description": "Optional proxy settings to use while crawling",
"type": "object",
"editor": "json",
"properties": {
"useApifyProxy": {
"title": "Use Apify Proxy",
"description": "Enable Apify Proxy",
"type": "boolean"
},
"customProxyUrls": {
"title": "Custom proxy URLs",
"description": "List of custom proxy URLs",
"type": "array",
"editor": "json",
"items": {
"type": "string"
}
}
},
"required": ["useApifyProxy"],
"additionalProperties": false
}
},
"required": ["startUrls", "searchQuery"]
}
2 changes: 2 additions & 0 deletions test/__setup__/input-schemas/paths.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@ export const prefillsInputSchemaPath = fileURLToPath(new URL('./prefills.json',
export const unparsableInputSchemaPath = fileURLToPath(new URL('./unparsable.json', import.meta.url));

export const validInputSchemaPath = fileURLToPath(new URL('./valid.json', import.meta.url));

export const complexInputSchemaPath = fileURLToPath(new URL('./complex.json', import.meta.url));
Loading
Loading