Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions supplement-readiness-assistant/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# Supplement Readiness Assistant

This module is a focused AI-Powered Research Assistant Suite slice for issue #16. It validates whether a manuscript supplement package is ready for reviewer and reproducibility use before submission.

The assistant checks:

- manuscript claims linked to the expected supplement artifacts
- dataset, code, notebook, and protocol manifest coverage
- checksum freshness and mismatch risk
- license and access metadata
- figure/table appendix support
- reproducibility rerun prerequisites
- privacy and restricted-data release blockers
- orphaned supplement files that reviewers cannot trace back to claims

It uses only Node.js built-ins and synthetic data.

## Run

```bash
node supplement-readiness-assistant/test.js
node supplement-readiness-assistant/demo.js
```

The demo writes reviewer artifacts to `supplement-readiness-assistant/reports/`.
104 changes: 104 additions & 0 deletions supplement-readiness-assistant/demo.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
const fs = require('fs');
const path = require('path');
const { evaluateSupplementReadiness } = require('./index');
const { blockedSupplementPackage } = require('./sample-data');

const reportDir = path.join(__dirname, 'reports');
fs.mkdirSync(reportDir, { recursive: true });

const result = evaluateSupplementReadiness(blockedSupplementPackage);
const packetPath = path.join(reportDir, 'supplement-readiness-packet.json');
const reportPath = path.join(reportDir, 'supplement-readiness-report.md');
const svgPath = path.join(reportDir, 'summary.svg');

fs.writeFileSync(packetPath, `${JSON.stringify(result, null, 2)}\n`);

const claimRows = result.claimResults
.map((claim) => `| ${claim.id} | ${claim.status} | ${claim.linkedArtifacts.join(', ') || 'none'} | ${claim.blockers.join(', ') || '-'} |`)
.join('\n');
const artifactRows = result.artifactResults
.map((artifact) => `| ${artifact.id} | ${artifact.kind} | ${artifact.status} | ${artifact.blockers.join(', ') || '-'} | ${artifact.warnings.join(', ') || '-'} |`)
.join('\n');
const actionRows = result.reviewerActions.map((action) => `- ${action}`).join('\n');

fs.writeFileSync(
reportPath,
`# Supplement Readiness Report

Status: ${result.status}
Readiness score: ${result.readinessScore}
Audit digest: ${result.auditDigest}

## Manuscript

Title: ${result.manuscript.title}
Domain: ${result.manuscript.domain}
Package: ${result.supplementPackage.id} ${result.supplementPackage.version}

## Coverage

- Claims ready: ${result.coverage.claimsReady}/${result.coverage.claimsTotal}
- Artifacts passing: ${result.coverage.artifactsPassing}/${result.coverage.artifactsTotal}
- Missing required artifact kinds: ${result.coverage.missingRequiredKinds.join(', ') || 'none'}

## Claim Review

| Claim | Status | Linked artifacts | Blockers |
| --- | --- | --- | --- |
${claimRows}

## Artifact Review

| Artifact | Kind | Status | Blockers | Warnings |
| --- | --- | --- | --- | --- |
${artifactRows}

## Blockers

${result.blockers.map((blocker) => `- ${blocker}`).join('\n')}

## Reviewer Actions

${actionRows}
`,
);

const blockerCount = result.blockers.length;
const warningCount = result.warnings.length;
const readyClaims = result.coverage.claimsReady;
const totalClaims = result.coverage.claimsTotal;
fs.writeFileSync(
svgPath,
`<svg xmlns="http://www.w3.org/2000/svg" width="960" height="540" viewBox="0 0 960 540">
<rect width="960" height="540" fill="#f8fafc"/>
<rect x="60" y="56" width="840" height="428" rx="18" fill="#ffffff" stroke="#cbd5e1"/>
<text x="92" y="112" font-family="Arial, sans-serif" font-size="32" font-weight="700" fill="#0f172a">Supplement Readiness Assistant</text>
<text x="92" y="154" font-family="Arial, sans-serif" font-size="18" fill="#475569">Status: ${result.status} | Score: ${result.readinessScore} | Claims: ${readyClaims}/${totalClaims}</text>
<g transform="translate(92 205)">
<rect width="230" height="132" rx="12" fill="#fee2e2" stroke="#ef4444"/>
<text x="24" y="54" font-family="Arial, sans-serif" font-size="42" font-weight="700" fill="#991b1b">${blockerCount}</text>
<text x="24" y="92" font-family="Arial, sans-serif" font-size="18" fill="#991b1b">publication blockers</text>
</g>
<g transform="translate(365 205)">
<rect width="230" height="132" rx="12" fill="#fef3c7" stroke="#f59e0b"/>
<text x="24" y="54" font-family="Arial, sans-serif" font-size="42" font-weight="700" fill="#92400e">${warningCount}</text>
<text x="24" y="92" font-family="Arial, sans-serif" font-size="18" fill="#92400e">review warnings</text>
</g>
<g transform="translate(638 205)">
<rect width="230" height="132" rx="12" fill="#dcfce7" stroke="#22c55e"/>
<text x="24" y="54" font-family="Arial, sans-serif" font-size="42" font-weight="700" fill="#166534">${readyClaims}</text>
<text x="24" y="92" font-family="Arial, sans-serif" font-size="18" fill="#166534">claims ready</text>
</g>
<text x="92" y="402" font-family="Arial, sans-serif" font-size="17" fill="#334155">Digest: ${result.auditDigest.slice(0, 32)}...</text>
<text x="92" y="434" font-family="Arial, sans-serif" font-size="17" fill="#334155">Reviewer packet: JSON + Markdown + SVG + MP4 demo</text>
</svg>
`,
);

console.log(`status=${result.status}`);
console.log(`readinessScore=${result.readinessScore}`);
console.log(`claimsReady=${result.coverage.claimsReady}/${result.coverage.claimsTotal}`);
console.log(`blockers=${result.blockers.length}`);
console.log(`warnings=${result.warnings.length}`);
console.log(`auditDigest=${result.auditDigest}`);
console.log(`reports=${reportDir}`);
265 changes: 265 additions & 0 deletions supplement-readiness-assistant/index.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,265 @@
const crypto = require('crypto');

const DEFAULT_POLICY = {
maxChecksumAgeDays: 21,
requiredArtifactKinds: ['dataset', 'code', 'notebook', 'protocol'],
licenseRequiredKinds: ['dataset', 'code', 'notebook'],
rerunRequiredClaimTypes: ['statistical', 'computational', 'figure'],
publicAccessModes: ['public', 'open'],
};

function asArray(value) {
return Array.isArray(value) ? value : [];
}

function stableJson(value) {
if (Array.isArray(value)) {
return `[${value.map(stableJson).join(',')}]`;
}
if (value && typeof value === 'object') {
return `{${Object.keys(value)
.sort()
.map((key) => `${JSON.stringify(key)}:${stableJson(value[key])}`)
.join(',')}}`;
}
return JSON.stringify(value);
}

function digest(value) {
return crypto.createHash('sha256').update(stableJson(value)).digest('hex');
}

function unique(values) {
return [...new Set(values.filter(Boolean))];
}

function indexById(items) {
const out = new Map();
for (const item of asArray(items)) {
if (item && item.id) {
out.set(item.id, item);
}
}
return out;
}

function hasPublicAccess(artifact, policy) {
return policy.publicAccessModes.includes(String(artifact.access || '').toLowerCase());
}

function evaluateArtifact(artifact, claimsById, policy) {
const blockers = [];
const warnings = [];
const actions = [];
const linkedClaims = asArray(artifact.linkedClaims);
const knownLinkedClaims = linkedClaims.filter((claimId) => claimsById.has(claimId));

if (!artifact.path) {
blockers.push('missing_path');
actions.push('add_artifact_path');
}

if (!artifact.checksum) {
blockers.push('missing_checksum');
actions.push('add_checksum');
} else if (artifact.expectedChecksum && artifact.checksum !== artifact.expectedChecksum) {
blockers.push('checksum_mismatch');
actions.push('regenerate_or_correct_checksum');
}

if (Number.isFinite(artifact.checksumAgeDays) && artifact.checksumAgeDays > policy.maxChecksumAgeDays) {
warnings.push('stale_checksum');
actions.push('refresh_checksum');
}

if (policy.licenseRequiredKinds.includes(artifact.kind) && !artifact.license) {
blockers.push('missing_license');
actions.push('add_license_metadata');
}

if (artifact.containsHumanData && hasPublicAccess(artifact, policy) && !artifact.deidentified) {
blockers.push('human_data_not_deidentified');
actions.push('redact_or_restrict_human_data');
}

if (artifact.embargoed && hasPublicAccess(artifact, policy)) {
blockers.push('embargoed_artifact_public');
actions.push('move_embargoed_artifact_to_restricted_access');
}

if (artifact.kind === 'notebook' && artifact.rerunnable === false) {
blockers.push('notebook_not_rerunnable');
actions.push('provide_rerun_environment');
}

if (artifact.kind === 'protocol' && !artifact.version) {
warnings.push('protocol_version_missing');
actions.push('add_protocol_version');
}

if (linkedClaims.length === 0) {
warnings.push('orphan_artifact');
actions.push('link_artifact_to_claim');
} else if (knownLinkedClaims.length !== linkedClaims.length) {
warnings.push('unknown_claim_link');
actions.push('repair_claim_links');
}

return {
id: artifact.id,
label: artifact.label,
kind: artifact.kind,
status: blockers.length ? 'block' : warnings.length ? 'warn' : 'pass',
linkedClaims: knownLinkedClaims,
blockers,
warnings,
actions: unique(actions),
};
}

function evaluateClaim(claim, artifactsById, policy) {
const blockers = [];
const warnings = [];
const actions = [];
const requiredArtifacts = asArray(claim.requiredArtifacts);
const linkedArtifacts = requiredArtifacts
.map((artifactId) => artifactsById.get(artifactId))
.filter(Boolean);
const missingArtifacts = requiredArtifacts.filter((artifactId) => !artifactsById.has(artifactId));

if (requiredArtifacts.length === 0) {
warnings.push('claim_has_no_required_supplements');
actions.push('declare_required_supplements_or_mark_textual_claim');
}

if (missingArtifacts.length > 0) {
blockers.push('missing_required_artifact');
actions.push('attach_missing_required_artifact');
}

if (asArray(claim.supplementLinks).length === 0) {
blockers.push('missing_supplement_link');
actions.push('add_claim_to_supplement_link');
}

const needsRerun = policy.rerunRequiredClaimTypes.includes(claim.type);
if (needsRerun) {
const hasRerunnableNotebook = linkedArtifacts.some(
(artifact) => artifact.kind === 'notebook' && artifact.rerunnable === true,
);
const hasCode = linkedArtifacts.some((artifact) => artifact.kind === 'code');
const hasDataset = linkedArtifacts.some((artifact) => artifact.kind === 'dataset');
if (!hasRerunnableNotebook || !hasCode || !hasDataset) {
blockers.push('rerun_prerequisites_incomplete');
actions.push('connect_dataset_code_and_rerunnable_notebook');
}
}

if (asArray(claim.figureRefs).length > 0) {
const hasAppendixSupport = linkedArtifacts.some((artifact) =>
asArray(artifact.appendixRefs).some((ref) => asArray(claim.figureRefs).includes(ref)),
);
if (!hasAppendixSupport) {
warnings.push('figure_appendix_support_not_linked');
actions.push('link_figure_or_table_appendix_support');
}
}

return {
id: claim.id,
text: claim.text,
type: claim.type,
status: blockers.length ? 'block' : warnings.length ? 'warn' : 'pass',
requiredArtifacts,
missingArtifacts,
linkedArtifacts: linkedArtifacts.map((artifact) => artifact.id),
blockers,
warnings,
actions: unique(actions),
};
}

function summarizeCoverage(claimResults, artifactResults, policy) {
const coveredClaims = claimResults.filter((claim) => claim.status !== 'block').length;
const presentKinds = new Set(artifactResults.map((artifact) => artifact.kind));
const missingKinds = policy.requiredArtifactKinds.filter((kind) => !presentKinds.has(kind));

return {
claimsTotal: claimResults.length,
claimsReady: coveredClaims,
claimsBlocked: claimResults.filter((claim) => claim.status === 'block').length,
artifactsTotal: artifactResults.length,
artifactsPassing: artifactResults.filter((artifact) => artifact.status === 'pass').length,
missingRequiredKinds: missingKinds,
};
}

function evaluateSupplementReadiness(input) {
const policy = { ...DEFAULT_POLICY, ...(input.policy || {}) };
const manuscript = input.manuscript || {};
const supplementPackage = input.supplementPackage || {};
const claims = asArray(manuscript.claims);
const artifacts = asArray(supplementPackage.artifacts);
const claimsById = indexById(claims);
const artifactsById = indexById(artifacts);

const claimResults = claims.map((claim) => evaluateClaim(claim, artifactsById, policy));
const artifactResults = artifacts.map((artifact) => evaluateArtifact(artifact, claimsById, policy));
const coverage = summarizeCoverage(claimResults, artifactResults, policy);
const globalBlockers = [];
const globalWarnings = [];
const reviewerActions = [];

for (const missingKind of coverage.missingRequiredKinds) {
globalBlockers.push(`missing_${missingKind}_artifact_kind`);
reviewerActions.push(`add_${missingKind}_artifact`);
}

for (const result of [...claimResults, ...artifactResults]) {
for (const blocker of result.blockers) {
globalBlockers.push(`${result.id}_${blocker}`);
}
for (const warning of result.warnings) {
globalWarnings.push(`${result.id}_${warning}`);
}
for (const action of result.actions) {
reviewerActions.push(`${result.id}_${action}`);
}
}

const possiblePenaltyUnits = Math.max(claimResults.length + artifactResults.length + coverage.missingRequiredKinds.length, 1);
const penalty = globalBlockers.length * 12 + globalWarnings.length * 4;
const readinessScore = Math.max(0, Math.round(100 - penalty / possiblePenaltyUnits));
const status = globalBlockers.length ? 'hold' : globalWarnings.length ? 'review_with_warnings' : 'ready';
const packet = {
manuscript: {
id: manuscript.id,
title: manuscript.title,
domain: manuscript.domain,
},
supplementPackage: {
id: supplementPackage.id,
version: supplementPackage.version,
accessMode: supplementPackage.accessMode,
},
status,
readinessScore,
coverage,
blockers: unique(globalBlockers),
warnings: unique(globalWarnings),
reviewerActions: unique(reviewerActions),
claimResults,
artifactResults,
};

return {
...packet,
auditDigest: digest(packet),
};
}

module.exports = {
DEFAULT_POLICY,
digest,
evaluateSupplementReadiness,
};
Binary file added supplement-readiness-assistant/reports/demo.mp4
Binary file not shown.
Loading