Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 49 additions & 4 deletions packages/cli-tools/src/commands/deploy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,16 +80,61 @@ export const deploy = async ({
}: {
params: DeployParams;
upload: UploadIndividually | UploadWithBatch;
}): Promise<DeployResult> =>
await deployToCollection({
params,
upload,
collection: COLLECTION_DAPP
});

/**
* Prepares and uploads assets to a specified storage collection.
*
* This is a generalized version of {@link deploy} that accepts a `collection`
* parameter instead of targeting the hardcoded `#dapp` collection. It enables
* deploying files to arbitrary storage collections (e.g., `audio`, `images`).
*
* Steps:
* 1) Resolve source files to upload.
* 2) Ensure enough memory is available (via internal checks).
* 3) Upload files using the provided upload function.
*
* @param {Object} options
* @param {DeployParams} options.params - Deployment parameters (paths, config, etc.).
* @param {UploadIndividually | UploadWithBatch} options.upload - Upload strategy function.
* @param {string} options.collection - The target storage collection name.
* @returns {Promise<DeployResult>}
* - `{ result: 'skipped' }` when there are no files to upload.
* - `{ result: 'deployed', files }` when the upload completes.
*/
export const deployToCollection = async ({
params,
upload,
collection
}: {
params: DeployParams;
upload: UploadIndividually | UploadWithBatch;
collection: string;
}): Promise<DeployResult> => {
const prepareResult = await prepareDeploy(params);
const prepareResult = await prepareDeploy({...params, collection});

if (prepareResult.result === 'skipped') {
return {result: 'skipped'};
}

const {files, sourceAbsolutePath} = prepareResult;

const sourceFiles = prepareSourceFiles({files, sourceAbsolutePath});
let sourceFiles = prepareSourceFiles({files, sourceAbsolutePath});

if (collection !== COLLECTION_DAPP) {
sourceFiles = sourceFiles.map(({file, paths}) => ({
file,
paths: {
...paths,
fullPath: `/${collection}${paths.fullPath.startsWith('/') ? '' : '/'}${paths.fullPath}`
}
}));
}

const source: Omit<UploadFilesParams, 'collection'> = {
files: sourceFiles,
Expand All @@ -99,7 +144,7 @@ export const deploy = async ({

await uploadFiles({
...source,
collection: COLLECTION_DAPP,
collection,
upload
});

Expand Down Expand Up @@ -184,7 +229,7 @@ export const deployWithProposal = async ({
const prepareDeploy = async ({
assertMemory,
...rest
}: Omit<DeployParams, 'uploadFn'>): Promise<
}: Omit<DeployParams, 'uploadFn'> & {collection?: string}): Promise<
{result: 'skipped'} | {result: 'to-deploy'; files: FileDetails[]; sourceAbsolutePath: string}
> => {
const spinner = ora('Preparing deploy...').start();
Expand Down
39 changes: 29 additions & 10 deletions packages/cli-tools/src/services/deploy.prepare.services.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import {isNullish, nonNullish} from '@dfinity/utils';
import type {CliConfig, EncodingType} from '@junobuild/config';
import {COLLECTION_DAPP} from '../constants/storage.constants';
import type {Asset} from '@junobuild/storage';
import crypto from 'crypto';
import {fileTypeFromFile} from 'file-type';
Expand All @@ -23,10 +24,12 @@ export const prepareDeploy = async ({
config,
listAssets,
assertSourceDirExists,
includeAllFiles
includeAllFiles,
collection
}: {
config: CliConfig;
listAssets: ListAssets;
collection?: string;
} & PrepareDeployOptions): Promise<{
files: FileDetails[];
sourceAbsolutePath: string;
Expand All @@ -48,7 +51,8 @@ export const prepareDeploy = async ({
encoding,
precompress,
listAssets,
includeAllFiles
includeAllFiles,
collection
});

return {
Expand All @@ -60,16 +64,19 @@ export const prepareDeploy = async ({
const filterFilesToUpload = async ({
files,
sourceAbsolutePath,
listAssets
listAssets,
collection
}: {
files: FileDetails[];
sourceAbsolutePath: string;
listAssets: ListAssets;
collection?: string;
}): Promise<FileDetails[]> => {
const existingAssets = await listAssets({});

const promises = files.map(
async (file: FileDetails) => await fileNeedUpload({file, sourceAbsolutePath, existingAssets})
async (file: FileDetails) =>
await fileNeedUpload({file, sourceAbsolutePath, existingAssets, collection})
);
const results: Array<{file: FileDetails; upload: boolean}> = await Promise.all(promises);

Expand All @@ -84,21 +91,30 @@ const computeSha256 = async (file: string): Promise<string> => {
const fileNeedUpload = async ({
file,
existingAssets,
sourceAbsolutePath
sourceAbsolutePath,
collection
}: {
file: FileDetails;
existingAssets: Asset[];
sourceAbsolutePath: string;
collection?: string;
}): Promise<{
file: FileDetails;
upload: boolean;
}> => {
const effectiveFilePath = file.alternateFile ?? file.file;

let computedFullPath = fullPath({file: effectiveFilePath, sourceAbsolutePath});

// For non-#dapp collections (e.g. storage deploy), remote assets have a
// /{collection}/... prefix on their fullPath. We must apply the same prefix
// to the locally-computed path so the comparison matches.
if (nonNullish(collection) && collection !== COLLECTION_DAPP) {
computedFullPath = `/${collection}${computedFullPath.startsWith('/') ? '' : '/'}${computedFullPath}`;
}

// Is it a new file?
const asset = existingAssets.find(
({fullPath: f}) => f === fullPath({file: effectiveFilePath, sourceAbsolutePath})
);
const asset = existingAssets.find(({fullPath: f}) => f === computedFullPath);

if (isNullish(asset)) {
return {file, upload: true};
Expand Down Expand Up @@ -133,9 +149,11 @@ const prepareFiles = async ({
encoding,
precompress,
listAssets,
includeAllFiles
includeAllFiles,
collection
}: {
sourceAbsolutePath: string;
collection?: string;
} & {listAssets: ListAssets} & Pick<PrepareDeployOptions, 'includeAllFiles'> &
Required<Pick<CliConfig, 'ignore' | 'encoding' | 'precompress'>>): Promise<FileDetails[]> => {
const sourceFiles = listSourceFilesForDeploy({sourceAbsolutePath, ignore});
Expand Down Expand Up @@ -264,6 +282,7 @@ const prepareFiles = async ({
return await filterFilesToUpload({
files,
sourceAbsolutePath,
listAssets
listAssets,
collection
});
};
2 changes: 1 addition & 1 deletion packages/cli-tools/src/types/upload.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import type {FileAndPaths} from './deploy';
export interface UploadFilesParams {
files: FileAndPaths[];
sourceAbsolutePath: string;
collection: typeof COLLECTION_DAPP | typeof COLLECTION_CDN_RELEASES;
collection: typeof COLLECTION_DAPP | typeof COLLECTION_CDN_RELEASES | (string & {});
batchSize: number;
}

Expand Down
43 changes: 42 additions & 1 deletion packages/config/src/satellite/configs/satellite.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,38 @@ import {type Collections, CollectionsSchema} from './collections';
import {type DatastoreConfig, DatastoreConfigSchema} from './datastore.config';
import {type ModuleSettings, ModuleSettingsSchema} from './module.settings';

/**
* @see StorageDeployMapping
*/
export const StorageDeployMappingSchema = z.strictObject({
source: z.string(),
collection: z.string()
});

/**
* Maps a local source directory to a Juno Storage collection for deployment.
*
* Used by `juno storage deploy` to upload files from a local directory
* to a specific storage collection.
*
* @interface StorageDeployMapping
* @property {string} source - Local directory path relative to the project root.
* @property {string} collection - Target storage collection name.
*/
export interface StorageDeployMapping {
/**
* Local directory path relative to the project root.
* @type {string}
*/
source: string;

/**
* Target storage collection name.
* @type {string}
*/
collection: string;
}

/**
* @see SatelliteId
*/
Expand Down Expand Up @@ -71,7 +103,8 @@ const SatelliteConfigOptionsBaseSchema = z.object({
automation: AutomationConfigSchema.optional(),
assertions: SatelliteAssertionsSchema.optional(),
settings: ModuleSettingsSchema.optional(),
collections: CollectionsSchema.optional()
collections: CollectionsSchema.optional(),
deploy: z.array(StorageDeployMappingSchema).optional()
});

/**
Expand Down Expand Up @@ -163,6 +196,14 @@ export interface SatelliteConfigOptions {
* @optional
*/
collections?: Collections;

/**
* Optional configuration that maps local directories to storage collections for deployment.
* Used by `juno storage deploy` to upload files from local directories to storage collections.
* @type {StorageDeployMapping[]}
* @optional
*/
deploy?: StorageDeployMapping[];
}

/**
Expand Down
Loading