Skip to content
Merged

Sync #566

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
60 changes: 50 additions & 10 deletions apps/backend/routes/public/scanWebhook.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,34 @@ import { db, event, hashStringToUuid, normalizeHtmlWithVdom, generateShortId } f
export const scanWebhook = async () => {
console.log(JSON.stringify(event));
const { auditId, scanId, urlId, url, blockers, status, error } = event.body;

// Validate required fields
if (!auditId || !urlId) {
console.error("Missing required fields: auditId or urlId", { auditId, urlId });
return { success: false, message: 'Missing required fields: auditId and urlId are required' };
}

if (!scanId) {
console.error("Missing scanId in webhook payload - looking up from audit", { auditId, urlId });
}

await db.connect();

// If scanId is missing, look it up from the most recent scan for this audit
let effectiveScanId = scanId;
if (!effectiveScanId) {
const scanResult = await db.query({
text: `SELECT id FROM scans WHERE audit_id = $1 ORDER BY created_at DESC LIMIT 1`,
values: [auditId],
});
if (scanResult?.rows?.[0]?.id) {
effectiveScanId = scanResult.rows[0].id;
console.log(`Resolved scanId from audit: ${effectiveScanId}`);
} else {
console.error("Could not resolve scanId for audit", { auditId });
// Continue without scanId - we can still save blockers
}
}

const ignoredBlockerHashes = (await db.query({
text: `SELECT b.content_hash_id FROM ignored_blockers as ib LEFT OUTER JOIN blockers as b ON ib.blocker_id = b.id WHERE ib.audit_id=$1`,
Expand All @@ -22,16 +49,24 @@ export const scanWebhook = async () => {
};

// Atomic append using PostgreSQL's jsonb_concat (||) operator
await db.query({
text: `UPDATE "scans" SET "errors" = COALESCE("errors", '[]'::jsonb) || $1::jsonb WHERE "id"=$2`,
values: [JSON.stringify([errorEntry]), scanId],
});
if (effectiveScanId) {
await db.query({
text: `UPDATE "scans" SET "errors" = COALESCE("errors", '[]'::jsonb) || $1::jsonb WHERE "id"=$2`,
values: [JSON.stringify([errorEntry]), effectiveScanId],
});
}

console.log(`Scan error logged: ${errorType} - ${errorMessage}`);
};

// Helper function to update scan progress and status (atomic operation)
const updateScanProgress = async () => {
// Skip if no scanId available
if (!effectiveScanId) {
console.warn("Cannot update scan progress: no scanId available");
return { percentage: 0, isComplete: false, scannedCount: 0, totalPages: 0 };
}

// Use a single atomic UPDATE with RETURNING to avoid race conditions
// This atomically adds urlId to processed_pages if not present, then calculates progress
const result = (await db.query({
Expand All @@ -49,9 +84,14 @@ export const scanWebhook = async () => {
"processed_pages",
jsonb_array_length(COALESCE("processed_pages", '[]'::jsonb)) as scanned_count
`,
values: [JSON.stringify([urlId]), scanId],
values: [JSON.stringify([urlId]), effectiveScanId],
})).rows[0];

if (!result) {
console.error("Scan not found for id:", effectiveScanId);
return { percentage: 0, isComplete: false, scannedCount: 0, totalPages: 0 };
}

const totalPages = result.pages?.length || 0;
const scannedCount = result.scanned_count || 0;
const percentage = totalPages > 0 ? Math.min(Math.round((scannedCount / totalPages) * 100), 100) : 0;
Expand All @@ -60,7 +100,7 @@ export const scanWebhook = async () => {
// Second atomic update for percentage and status
await db.query({
text: `UPDATE "scans" SET "percentage"=$1, "status"=$2 WHERE "id"=$3`,
values: [percentage, isComplete ? 'complete' : 'processing', scanId],
values: [percentage, isComplete ? 'complete' : 'processing', effectiveScanId],
});

return { percentage, isComplete, scannedCount, totalPages };
Expand All @@ -86,10 +126,10 @@ export const scanWebhook = async () => {
// Only mark audit as failed if this is the only/last page, otherwise let other pages continue
if (isComplete) {
// Check if there were any successful pages by looking at blockers
const hasSuccessfulPages = (await db.query({
const hasSuccessfulPages = effectiveScanId ? (await db.query({
text: `SELECT COUNT(*) FROM "blockers" WHERE "scan_id"=$1`,
values: [scanId],
})).rows[0].count > 0;
values: [effectiveScanId],
})).rows[0].count > 0 : false;

await db.query({
text: `UPDATE "audits" SET "status"=$1, "response"=$2 WHERE "id"=$3`,
Expand Down Expand Up @@ -132,7 +172,7 @@ export const scanWebhook = async () => {
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
RETURNING "id"
`,
values: [auditId, JSON.stringify([]), blocker.node, contentNormalized, contentHashId, shortId, urlId, scanId],
values: [auditId, JSON.stringify([]), blocker.node, contentNormalized, contentHashId, shortId, urlId, effectiveScanId],
})).rows[0].id;

if (ignoredBlockerHashes.includes(contentHashId.replaceAll('-', ''))) {
Expand Down
2 changes: 1 addition & 1 deletion apps/frontend/src/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ export const App = () => {
apiKey={import.meta.env.VITE_POSTHOG_KEY}
options={{ api_host: "https://us.posthog.com" }}
>
<div aria-live="assertive" role="status" className="sr-only">
<div aria-live="assertive" aria-atomic className="sr-only">
{announceMessage}
</div>
<QueryClientProvider client={queryClient}>
Expand Down
3 changes: 2 additions & 1 deletion apps/frontend/src/components/AuditEmailSubscriptionInput.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ export const AuditEmailSubscriptionInput: React.FC<ChildProps> = ({
[]
);

const handleAddEmail = () => {
const handleAddEmail = (e:any) => {
e.preventDefault();
setEmails((prevEmails) => [
...prevEmails,
{
Expand Down
102 changes: 66 additions & 36 deletions apps/frontend/src/components/AuditHeader.tsx
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { ReactNode } from "react";
import { ReactNode, useState } from "react";
import styles from "./AuditHeader.module.scss";
import { Link, useNavigate } from "react-router-dom";
import { StyledButton } from "./StyledButton";
Expand All @@ -8,6 +8,7 @@ import { createLog } from "#src/utils/createLog.ts";
import { useGlobalStore } from "../utils";
import * as API from "aws-amplify/api";
import { QueryClient, useQueryClient } from "@tanstack/react-query";
import { SkeletonAuditHeader } from "./Skeleton";

interface AuditHeaderProps extends React.PropsWithChildren {
isShared: boolean;
Expand All @@ -24,6 +25,9 @@ export const AuditHeader = ({
}: AuditHeaderProps) => {
const navigate = useNavigate();
const { setAnnounceMessage } = useGlobalStore();
const [isScanning, setIsScanning] = useState(false);
const [isDeleting, setIsDeleting] = useState(false);
const [isRenaming, setIsRenaming] = useState(false);

const copyCurrentLocationToClipboard = async () => {
try {
Expand All @@ -43,36 +47,46 @@ export const AuditHeader = ({
};
const deleteAudit = async () => {
if (confirm(`Are you sure you want to delete this audit?`)) {
const response = await (
await API.post({
apiName: "auth",
path: "/deleteAudit",
options: { body: { id: auditId! } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audits"] });
// aria & logging
setAnnounceMessage(`Deleted audit ${audit.name}.`, "success");
await createLog(`Deleted audit ${audit.name}.`, auditId);
setIsDeleting(true);
try {
const response = await (
await API.post({
apiName: "auth",
path: "/deleteAudit",
options: { body: { id: auditId! } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audits"] });
// aria & logging
setAnnounceMessage(`Deleted audit ${audit.name}.`, "success");
await createLog(`Deleted audit ${audit.name}.`, auditId);

navigate("/audits");
navigate("/audits");
} finally {
setIsDeleting(false);
}
return;
}
};
const rescanAudit = async () => {
if (confirm(`Are you sure you want to re-scan this audit?`)) {
const response = await (
await API.post({
apiName: "auth",
path: "/rescanAudit",
options: { body: { id: auditId! } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audits"] });
// aria & logging
setAnnounceMessage(`Scanning audit ${audit.name}...`);
setIsScanning(true);
try {
const response = await (
await API.post({
apiName: "auth",
path: "/rescanAudit",
options: { body: { id: auditId! } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audits"] });
// aria & logging
setAnnounceMessage(`Scanning audit ${audit.name}...`);
} finally {
setIsScanning(false);
}
return;
}
};
Expand All @@ -83,20 +97,31 @@ export const AuditHeader = ({
audit?.name
);
if (newName) {
const response = await (
await API.post({
apiName: "auth",
path: "/updateAudit",
options: { body: { id: auditId!, name: newName } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audit", auditId] });
// aria & logging
setAnnounceMessage(`Audit ${audit.name} renamed to ${newName}`, "success");
setIsRenaming(true);
try {
const response = await (
await API.post({
apiName: "auth",
path: "/updateAudit",
options: { body: { id: auditId!, name: newName } },
}).response
).body.json();
//console.log(response);
await queryClient.refetchQueries({ queryKey: ["audit", auditId] });
// aria & logging
setAnnounceMessage(`Audit ${audit.name} renamed to ${newName}`, "success");
} finally {
setIsRenaming(false);
}
return;
}
};

// Show skeleton while audit is loading
if (!audit) {
return <SkeletonAuditHeader />;
}

return (
<div className={styles.AuditHeader}>

Expand All @@ -112,12 +137,16 @@ export const AuditHeader = ({
label="Rename Audit"
icon={<FaPen />}
showLabel={false}
loading={isRenaming}
disabled={isRenaming || isDeleting}
/>
<StyledButton
onClick={deleteAudit}
label="Delete Audit"
icon={<FaTrash />}
showLabel={false}
loading={isDeleting}
disabled={isRenaming || isDeleting}
/>
</div>
)}
Expand All @@ -130,6 +159,7 @@ export const AuditHeader = ({
label="Scan Now"
icon={<GrPowerCycle />}
variant="dark"
loading={isScanning}
/>
)}
<StyledButton
Expand Down
Loading