feat: add notes to relations

docs/source/includes/result_format.md

@@ -182,6 +182,10 @@ You can draw relation arrow between two objects. For example, object detection c
   "type": "relation",
   "to_id": "RQbW3Sj_Zr",
   "from_id": "oid66",
-  "direction": "right"
+  "direction": "right",
+  "labels": ["similar"],
+  "notes": "Optional free text notes about this relationship"
 }]
 ```
+
+The relation object includes `from_id`, `to_id`, and `direction`. It can also include `labels` when using a `<Relations>` tag with `<Relation>` values, and `notes` when annotators add free text to the relationship.

web/libs/editor/src/components/SidePanels/DetailsPanel/Relations.prefix.css

@@ -85,9 +85,11 @@
 
 .relation-meta {
   display: flex;
+  flex-direction: column;
+  gap: 8px;
   padding-left: 64px;
   padding-bottom: 5px;
-  align-items: center;
+  align-items: stretch;
 
   &__title {
     flex: none;
@@ -97,4 +99,25 @@
   &__select {
     flex: 1;
   }
+
+  &__notes {
+    width: 100%;
+    min-height: 48px;
+    padding: 8px;
+    border: 1px solid var(--color-neutral-border);
+    border-radius: 4px;
+    background-color: var(--color-neutral-background);
+    color: var(--color-neutral-content);
+    font: inherit;
+    resize: vertical;
+
+    &:focus {
+      outline: none;
+      border-color: var(--color-primary-border-subtle);
+    }
+
+    &::placeholder {
+      color: var(--color-neutral-content-subtle);
+    }
+  }
 }

web/libs/editor/src/components/SidePanels/DetailsPanel/Relations.tsx

@@ -9,7 +9,7 @@ import {
 } from "@humansignal/icons";
 import { Button, Select } from "@humansignal/ui";
 import { observer } from "mobx-react";
-import { type FC, useCallback, useMemo, useState } from "react";
+import { type ChangeEvent, type FC, useCallback, useMemo, useState } from "react";
 import { cn } from "../../../utils/bem";
 import { wrapArray } from "../../../utils/utilities";
 import { RegionItem } from "./RegionItem";
@@ -91,10 +91,10 @@ const RelationItem: FC<{ relation: any }> = observer(({ relation }) => {
         </div>
         <div className={cn("relations").elem("actions").toClassName()}>
           <div className={cn("relations").elem("action").toClassName()}>
-            {(hovered || relation.showMeta) && relation.hasRelations && (
+            {(hovered || relation.showMeta) && (
               <Button
                 primary={relation.showMeta}
-                aria-label={`${relation.showMeta ? "Hide" : "Show"} Relation Labels`}
+                aria-label={`${relation.showMeta ? "Hide" : "Show"} Relation Details`}
                 type={relation.showMeta ? undefined : "text"}
                 onClick={relation.toggleMeta}
                 style={{ padding: 0 }}
@@ -148,7 +148,8 @@ const RelationItem: FC<{ relation: any }> = observer(({ relation }) => {
 
 const RelationMeta: FC<any> = observer(({ relation }) => {
   const { selectedValues, control } = relation;
-  const { children, choice } = control;
+  const children = control?.children ?? [];
+  const choice = control?.choice;
 
   const selectionMode = useMemo(() => {
     return choice === "multiple";
@@ -162,6 +163,12 @@ const RelationMeta: FC<any> = observer(({ relation }) => {
     },
     [relation],
   );
+  const onNotesChange = useCallback(
+    (event: ChangeEvent<HTMLTextAreaElement>) => {
+      relation.setNotes(event.target.value);
+    },
+    [relation],
+  );
   const options = useMemo(
     () =>
       children.map((c: any) => ({
@@ -173,13 +180,23 @@ const RelationMeta: FC<any> = observer(({ relation }) => {
 
   return (
     <div className={cn("relation-meta").toClassName()}>
-      <Select
-        multiple={selectionMode}
-        style={{ width: "100%" }}
-        placeholder="Select labels"
-        value={selectedValues}
-        onChange={onChange}
-        options={options}
+      {relation.hasRelations && (
+        <Select
+          multiple={selectionMode}
+          style={{ width: "100%" }}
+          placeholder="Select labels"
+          value={selectedValues}
+          onChange={onChange}
+          options={options}
+        />
+      )}
+      <textarea
+        aria-label="Relation note"
+        className={cn("relation-meta").elem("notes").toClassName()}
+        onChange={onNotesChange}
+        placeholder="Add note"
+        rows={2}
+        value={relation.notes}
       />
     </div>
   );

web/libs/editor/src/stores/Annotation/Annotation.js

@@ -1266,6 +1266,7 @@ const _Annotation = types
               `${obj.to_id}#${self.id}`,
               obj.direction,
               obj.labels,
+              obj.notes,
             );
           }
         }

web/libs/editor/src/stores/RelationStore.js

@@ -23,6 +23,9 @@ const Relation = types
 
     // labels
     labels: types.maybeNull(types.array(types.string)),
+
+    // free-text note stored with this relation
+    notes: types.optional(types.string, ""),
   })
   .volatile(() => ({
     showMeta: false,
@@ -101,6 +104,10 @@ const Relation = types
     setRelations(values) {
       self.labels = values;
     },
+
+    setNotes(value) {
+      self.notes = value;
+    },
   }));
 
 const RelationStore = types
@@ -212,18 +219,20 @@ const RelationStore = types
         };
 
         if (r.selectedValues) s.labels = r.selectedValues;
+        if (r.notes) s.notes = r.notes;
 
         return s;
       });
     },
 
-    deserializeRelation(node1, node2, direction, labels) {
+    deserializeRelation(node1, node2, direction, labels, notes) {
       const rl = self.addRelation(node1, node2);
 
       if (!rl) return; // duplicated relation
 
       rl.direction = direction;
       rl.labels = labels;
+      rl.notes = notes ?? "";
     },
 
     toggleConnections() {

web/libs/editor/src/stores/__tests__/RelationStore.test.js

@@ -227,6 +227,15 @@ describe("RelationStore", () => {
       expect(ser[0].labels).toEqual(["parent"]);
     });
 
+    it("serialize includes notes when relation has note text", () => {
+      const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
+      const [r1, r2] = regions;
+      const rl = relationStore.addRelation(r1, r2);
+      rl.setNotes("Needs review before export");
+      const ser = relationStore.serialize();
+      expect(ser[0].notes).toEqual("Needs review before export");
+    });
+
     it("deserializeRelation adds relation with direction and labels", () => {
       const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
       const [r1, r2] = regions;
@@ -237,6 +246,14 @@ describe("RelationStore", () => {
       expect(rl.labels).toEqual(["child"]);
     });
 
+    it("deserializeRelation restores note text", () => {
+      const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
+      const [r1, r2] = regions;
+      relationStore.deserializeRelation(r1, r2, "left", ["child"], "Already reviewed");
+      expect(relationStore.size).toBe(1);
+      expect(relationStore.relations[0].notes).toBe("Already reviewed");
+    });
+
     it("deserializeRelation does nothing when relation already exists", () => {
       const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
       const [r1, r2] = regions;
@@ -438,6 +455,14 @@ describe("Relation (model)", () => {
       expect(rl.labels).toEqual(["parent", "child"]);
     });
 
+    it("setNotes updates note text", () => {
+      const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
+      const [r1, r2] = regions;
+      const rl = relationStore.addRelation(r1, r2);
+      rl.setNotes("Follow up later");
+      expect(rl.notes).toBe("Follow up later");
+    });
+
     it("toggleVisibility toggles visible", () => {
       const { relationStore, regions } = createStoreWithTwoRectRegionsAndRelations();
       const [r1, r2] = regions;

web/libs/editor/tests/integration/e2e/relations/notes.cy.ts

@@ -0,0 +1,165 @@
+import { ImageView, LabelStudio, Relations, Sidebar } from "@humansignal/frontend-test/helpers/LSF";
+
+const config = `
+  <View>
+    <Image name="img" value="$image"></Image>
+    <RectangleLabels name="tag" toName="img">
+      <Label value="Region 1" background="red"></Label>
+      <Label value="Region 2" background="blue"></Label>
+    </RectangleLabels>
+    <Relations>
+      <Relation value="similar"/>
+      <Relation value="different"/>
+    </Relations>
+  </View>
+`;
+
+const configWithoutRelationLabels = `
+  <View>
+    <Image name="img" value="$image"></Image>
+    <RectangleLabels name="tag" toName="img">
+      <Label value="Region 1" background="red"></Label>
+      <Label value="Region 2" background="blue"></Label>
+    </RectangleLabels>
+  </View>
+`;
+
+const image =
+  "https://htx-pub.s3.us-east-1.amazonaws.com/examples/images/nick-owuor-astro-nic-visuals-wDifg5xc9Z4-unsplash.jpg";
+
+const baseResult = [
+  {
+    id: "region1",
+    source: "$image",
+    from_name: "tag",
+    to_name: "img",
+    type: "rectanglelabels",
+    origin: "manual",
+    value: {
+      height: 10,
+      rotation: 0,
+      width: 12,
+      x: 20,
+      y: 20,
+      rectanglelabels: ["Region 1"],
+    },
+  },
+  {
+    id: "region2",
+    source: "$image",
+    from_name: "tag",
+    to_name: "img",
+    type: "rectanglelabels",
+    origin: "manual",
+    value: {
+      height: 10,
+      rotation: 0,
+      width: 12,
+      x: 50,
+      y: 50,
+      rectanglelabels: ["Region 2"],
+    },
+  },
+];
+
+const task = {
+  id: 1,
+  annotations: [
+    {
+      id: 1001,
+      result: baseResult,
+    },
+  ],
+  predictions: [],
+  data: { image },
+};
+
+const openRelationMetadata = () => {
+  cy.get(".lsf-relations__item")
+    .first()
+    .trigger("mouseover")
+    .find('button[aria-label="Show Relation Details"]')
+    .click({ force: true });
+};
+
+describe("Relations: Notes", () => {
+  it("serializes note text added to a relation", () => {
+    LabelStudio.init({ config, task });
+
+    ImageView.waitForImage();
+    Relations.hasRelations(0);
+
+    Sidebar.toggleRegionSelection(0);
+    Relations.toggleCreation();
+    Sidebar.toggleRegionSelection(1);
+    Relations.hasRelations(1);
+
+    openRelationMetadata();
+    cy.get('[aria-label="Relation note"]').type("Needs clinical review before export");
+
+    LabelStudio.serialize().then((result) => {
+      expect(result[2]).to.include({
+        type: "relation",
+        from_id: "region1",
+        to_id: "region2",
+        notes: "Needs clinical review before export",
+      });
+    });
+  });
+
+  it("loads existing relation notes into the metadata panel", () => {
+    LabelStudio.init({
+      config,
+      task: {
+        ...task,
+        annotations: [
+          {
+            ...task.annotations[0],
+            result: [
+              ...baseResult,
+              {
+                from_id: "region1",
+                to_id: "region2",
+                type: "relation",
+                direction: "right",
+                labels: ["similar"],
+                notes: "Already reviewed",
+              },
+            ],
+          },
+        ],
+      },
+    });
+
+    ImageView.waitForImage();
+    Relations.hasRelations(1);
+
+    openRelationMetadata();
+    cy.get('[aria-label="Relation note"]').should("have.value", "Already reviewed");
+  });
+
+  it("allows notes when relation labels are not configured", () => {
+    LabelStudio.init({ config: configWithoutRelationLabels, task });
+
+    ImageView.waitForImage();
+    Relations.hasRelations(0);
+
+    Sidebar.toggleRegionSelection(0);
+    Relations.toggleCreation();
+    Sidebar.toggleRegionSelection(1);
+    Relations.hasRelations(1);
+
+    openRelationMetadata();
+    cy.get('[aria-label="Relation note"]').type("No label taxonomy needed");
+
+    LabelStudio.serialize().then((result) => {
+      expect(result[2]).to.include({
+        type: "relation",
+        from_id: "region1",
+        to_id: "region2",
+        notes: "No label taxonomy needed",
+      });
+      expect(result[2]).not.to.have.property("labels");
+    });
+  });
+});