diff --git a/README.md b/README.md
index 0c1d28c..e24b3cb 100644
--- a/README.md
+++ b/README.md
@@ -11,5 +11,7 @@ Our experimental design involves two phases: one for artists and another for the
We plan to continue this research in a second part through a longitudinal field study.
+Developer documentation for the creator dataset is in [`src/pages/artist/README.md`](src/pages/artist/README.md).
+
1. Kwan, L. Y. -Y., Leung, A. K. -y., & Liou, S. (2018). Culture, creativity, and innovation. Journal of Cross-Cultural Psychology, 49(2), 165–170. https://doi.org/10.1177/0022022117753306s
2. Elisondo, R. (2016). Creativity is Always a Social Process. Creativity. Theories – Research - Applications, 3(2), 2016. 194-210. https://doi.org/10.1515/ctra-2016-0013
diff --git a/server/api/routes/firebaseAPI.ts b/server/api/routes/firebaseAPI.ts
index 5fbc4aa..6f257ac 100644
--- a/server/api/routes/firebaseAPI.ts
+++ b/server/api/routes/firebaseAPI.ts
@@ -11,9 +11,23 @@ const ASSIGNMENT_COLLECTION = "artistAssignment";
router.post("/artist-assignment", async (req, res) => {
try {
- const { sessionId, passageId, prolificPid } = req.body;
- if (!sessionId || !passageId) {
- return res.status(400).json({ error: "Missing sessionId or passageId" });
+ const {
+ sessionId,
+ passageId,
+ tutorialPassageId,
+ passagePoolVersion,
+ prolificPid,
+ } = req.body;
+ if (
+ !sessionId ||
+ !passageId ||
+ !tutorialPassageId ||
+ !passagePoolVersion
+ ) {
+ return res.status(400).json({
+ error:
+ "Missing sessionId, passageId, tutorialPassageId, or passagePoolVersion",
+ });
}
const assignmentRef = db.collection(ASSIGNMENT_COLLECTION).doc(sessionId);
@@ -21,8 +35,40 @@ router.post("/artist-assignment", async (req, res) => {
const existingAssignment = await transaction.get(assignmentRef);
if (existingAssignment.exists) {
const existing = existingAssignment.data()!;
+ const taskPassageId = String(
+ existing.taskPassageId ?? existing.passageId,
+ );
+ const resolvedTutorialPassageId = String(
+ existing.tutorialPassageId ??
+ (tutorialPassageId === taskPassageId
+ ? passageId
+ : tutorialPassageId),
+ );
+ const resolvedPassagePoolVersion = String(
+ existing.passagePoolVersion ?? "legacy-creator-passages",
+ );
+
+ if (
+ !existing.taskPassageId ||
+ !existing.tutorialPassageId ||
+ !existing.passagePoolVersion
+ ) {
+ transaction.set(
+ assignmentRef,
+ {
+ taskPassageId,
+ tutorialPassageId: resolvedTutorialPassageId,
+ passagePoolVersion: resolvedPassagePoolVersion,
+ },
+ { merge: true },
+ );
+ }
+
return {
- passageId: existing.passageId as string,
+ passageId: taskPassageId,
+ taskPassageId,
+ tutorialPassageId: resolvedTutorialPassageId,
+ passagePoolVersion: resolvedPassagePoolVersion,
condition: existing.condition as "LLM" | "NO_AI",
strategy: existing.strategy as string,
};
@@ -36,12 +82,22 @@ router.post("/artist-assignment", async (req, res) => {
sessionId,
prolificPid: prolificPid || null,
passageId: String(passageId),
+ taskPassageId: String(passageId),
+ tutorialPassageId: String(tutorialPassageId),
+ passagePoolVersion: String(passagePoolVersion),
condition,
strategy,
assignedAt: FieldValue.serverTimestamp(),
});
- return { passageId: String(passageId), condition, strategy };
+ return {
+ passageId: String(passageId),
+ taskPassageId: String(passageId),
+ tutorialPassageId: String(tutorialPassageId),
+ passagePoolVersion: String(passagePoolVersion),
+ condition,
+ strategy,
+ };
});
res.json(assignment);
diff --git a/src/components/chatbot/Chatbot.tsx b/src/components/chatbot/Chatbot.tsx
index dc36fd5..5895ffe 100644
--- a/src/components/chatbot/Chatbot.tsx
+++ b/src/components/chatbot/Chatbot.tsx
@@ -10,18 +10,21 @@ import { FiSend } from "react-icons/fi";
import { Button, Textarea } from "@chakra-ui/react";
import { nanoid } from "nanoid";
import type {
- ChatOpening,
+ ChatAvailability,
+ ChatInputActivity,
+ ChatInputSource as ChatInputSourceType,
LlmRequestLog,
Message,
Stage,
} from "../../types";
-import { Role } from "../../types";
+import { ChatInputSource, MessageKind, Role } from "../../types";
import ReactMarkdown from "react-markdown";
import remarkGfm from "remark-gfm";
import { DataContext } from "../../App";
import {
createAssistantMessage,
IDLE_NUDGE_MESSAGES,
+ STAGE_OPENING_MESSAGES,
} from "../../consts/chatMessages";
interface ChatTabProps {
@@ -31,12 +34,14 @@ interface ChatTabProps {
selectedWordIndexes?: number[];
passage: string;
chatReady?: boolean;
- onChatOpened?: (opening: ChatOpening) => void;
+ initialInputActivity?: ChatInputActivity;
+ onChatAvailable?: (availability: ChatAvailability) => void;
+ onInputActivityUpdate?: (activity: ChatInputActivity) => void;
onRequestUpdate?: (request: LlmRequestLog) => void;
}
export const BLACKOUT_ASSISTANT_PROMPT_VERSION =
- "blackout-assistant-2026-08-04-v1";
+ "blackout-assistant-2026-08-05-v2";
/**
* Keep the locator-excerpt convention identical across both stages so users
@@ -50,9 +55,10 @@ Blackout poetry: the poet starts with an existing passage and creates a poem by
Grounding:
- Work only with the passage provided below. Never reference or substitute any other text.
-- When you point to a specific passage word, show it in a short excerpt containing two or three nearby passage words in total when available. Bold only the word you are pointing to. The unbolded words are only a locator to help the user find it; they are not part of the suggestion.
+- When you point to a specific passage word, show it in a short excerpt containing two or three nearby passage words in total when available. Bold only the word you are pointing to and italicize every surrounding locator word. For example: “*nights are* **clear**” or “*sharp,* **glittering** *sunshine*”. The italicized words are only locators to help the user find the bolded word; they are not part of the suggestion.
- Quote passage words exactly as written, keep multiple suggested words in passage order, and point to at most five words in a single response.
- Use bold only for passage words you are pointing to, never for general emphasis.
+- Use italics only for the surrounding locator words in these excerpts, never for general emphasis.
- Never suggest a word that does not appear in the passage.
Style and behavior:
@@ -161,7 +167,9 @@ export default function ChatTab({
stage,
passage,
chatReady = true,
- onChatOpened,
+ initialInputActivity,
+ onChatAvailable,
+ onInputActivityUpdate,
onRequestUpdate,
}: ChatTabProps) {
const context = useContext(DataContext);
@@ -171,7 +179,18 @@ export default function ChatTab({
const chatContainerRef = useRef(null);
const timeoutRef = useRef | null>(null);
const stageStartMessageCountRef = useRef(messages.length);
- const hasLoggedOpeningRef = useRef(false);
+ const hasLoggedAvailabilityRef = useRef(false);
+ const inputRef = useRef("");
+ const hasDraftRef = useRef(false);
+ const inputActivityRef = useRef(
+ initialInputActivity ?? {
+ stage,
+ focusCount: 0,
+ draftStartCount: 0,
+ abandonedDraftCount: 0,
+ hasUnsentDraft: false,
+ },
+ );
const [isLLMLoading, setIsLLMLoading] = useState(false);
const [input, setInput] = useState("");
@@ -205,10 +224,29 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
}, [passage, selectedWordIndexes, stage]);
useEffect(() => {
- if (!chatReady || hasLoggedOpeningRef.current) return;
- hasLoggedOpeningRef.current = true;
- onChatOpened?.({ stage, timestamp: new Date() });
- }, [chatReady, onChatOpened, stage]);
+ if (!chatReady || hasLoggedAvailabilityRef.current) return;
+ hasLoggedAvailabilityRef.current = true;
+
+ const availableAt = new Date();
+ setMessages((previousMessages) => {
+ const alreadyHasOpening = previousMessages.some(
+ (message) =>
+ message.stage === stage &&
+ message.kind === MessageKind.STAGE_OPENING,
+ );
+ return alreadyHasOpening
+ ? previousMessages
+ : [
+ ...previousMessages,
+ createAssistantMessage(
+ STAGE_OPENING_MESSAGES[stage],
+ stage,
+ MessageKind.STAGE_OPENING,
+ ),
+ ];
+ });
+ onChatAvailable?.({ stage, availableAt });
+ }, [chatReady, onChatAvailable, setMessages, stage]);
useEffect(() => {
const element = chatContainerRef.current;
@@ -226,7 +264,11 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
timeoutRef.current = setTimeout(() => {
setMessages((previousMessages) => [
...previousMessages,
- createAssistantMessage(IDLE_NUDGE_MESSAGES[stage]),
+ createAssistantMessage(
+ IDLE_NUDGE_MESSAGES[stage],
+ stage,
+ MessageKind.IDLE_NUDGE,
+ ),
]);
setHasShownIdleNudge(true);
}, 40000);
@@ -245,8 +287,66 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
stage,
]);
- const sendMessage = async (messageContent?: string) => {
- const content = messageContent || input;
+ const publishInputActivity = (activity: ChatInputActivity) => {
+ inputActivityRef.current = activity;
+ onInputActivityUpdate?.(activity);
+ };
+
+ const handleInputFocus = () => {
+ const activity = inputActivityRef.current;
+ publishInputActivity({
+ ...activity,
+ firstFocusedAt: activity.firstFocusedAt ?? new Date(),
+ focusCount: activity.focusCount + 1,
+ });
+ };
+
+ const handleInputChange = (value: string) => {
+ const hadDraft = hasDraftRef.current;
+ const hasDraft = Boolean(value.trim());
+ inputRef.current = value;
+ hasDraftRef.current = hasDraft;
+ setInput(value);
+
+ if (!hadDraft && hasDraft) {
+ const activity = inputActivityRef.current;
+ publishInputActivity({
+ ...activity,
+ firstTypedAt: activity.firstTypedAt ?? new Date(),
+ draftStartCount: activity.draftStartCount + 1,
+ hasUnsentDraft: true,
+ });
+ } else if (hadDraft && !hasDraft) {
+ const activity = inputActivityRef.current;
+ publishInputActivity({
+ ...activity,
+ abandonedDraftCount: activity.abandonedDraftCount + 1,
+ hasUnsentDraft: false,
+ });
+ }
+ };
+
+ const markDraftSubmittedOrReplaced = (
+ inputSource: ChatInputSourceType,
+ ) => {
+ if (!hasDraftRef.current) return;
+
+ hasDraftRef.current = false;
+ const activity = inputActivityRef.current;
+ publishInputActivity({
+ ...activity,
+ abandonedDraftCount:
+ activity.abandonedDraftCount +
+ (inputSource === ChatInputSource.SUGGESTION ? 1 : 0),
+ hasUnsentDraft: false,
+ });
+ };
+
+ const sendMessage = async (
+ messageContent?: string,
+ inputSource: ChatInputSourceType = ChatInputSource.TYPED,
+ ) => {
+ const content = messageContent ?? inputRef.current;
if (!content.trim() || isLLMLoading) return;
if (timeoutRef.current) {
@@ -259,6 +359,9 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
role: Role.ARTIST,
content,
timestamp: new Date(),
+ stage,
+ kind: MessageKind.USER_MESSAGE,
+ inputSource,
};
const requestId = nanoid();
let requestLog: LlmRequestLog = {
@@ -268,6 +371,7 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
userMessageContent: content,
requestedAt: new Date(),
status: "STARTED",
+ inputSource,
systemPrompt: systemMessage.content,
promptVersion: BLACKOUT_ASSISTANT_PROMPT_VERSION,
};
@@ -281,6 +385,8 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
setMarkdownOutput("");
setSendError(null);
setMessages((prev) => [...prev, artistMessage]);
+ markDraftSubmittedOrReplaced(inputSource);
+ inputRef.current = "";
setInput("");
setIsLLMLoading(true);
@@ -320,6 +426,8 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
role: Role.LLM,
content: fullText,
timestamp: new Date(),
+ stage,
+ kind: MessageKind.LLM_RESPONSE,
};
requestLog = {
...requestLog,
@@ -347,9 +455,15 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
setMessages((previousMessages) =>
previousMessages.filter((message) => message.id !== artistMessage.id),
);
- setInput((currentInput) =>
- currentInput.trim() ? currentInput : content,
- );
+ if (!inputRef.current.trim()) {
+ inputRef.current = content;
+ hasDraftRef.current = true;
+ setInput(content);
+ publishInputActivity({
+ ...inputActivityRef.current,
+ hasUnsentDraft: true,
+ });
+ }
} finally {
setIsLLMLoading(false);
}
@@ -363,7 +477,7 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
};
const handlePromptSelection = (prompt: string) => {
- sendMessage(prompt);
+ sendMessage(prompt, ChatInputSource.SUGGESTION);
};
return (
@@ -447,7 +561,8 @@ CURRENT SELECTED WORDS (in passage order): ${selectedWords || "none yet"}`,
>
+ Your final poem will be read by others.
+
{
const navigate = useNavigate();
@@ -40,15 +38,14 @@ const ArtistStep1 = () => {
const llmRequestsRef = useRef(
existingPoem?.llmUsage?.requests ?? [],
);
- const chatOpeningsRef = useRef(
- existingPoem?.llmUsage?.chatOpenings ?? [],
+ const chatAvailabilityRef = useRef(
+ getChatAvailability(existingPoem?.llmUsage),
);
-
- const [sparkMessages, setSparkMessages] = useState(() =>
- isLLM
- ? [createAssistantMessage(STAGE_OPENING_MESSAGES[Stage.SPARK])]
- : [],
+ const inputActivityRef = useRef(
+ existingPoem?.llmUsage?.inputActivity ?? [],
);
+
+ const [sparkMessages, setSparkMessages] = useState([]);
const [sparkNotes, setSparkNotes] = useState("");
// 0 = brainstorm instructions (all users), 1 = LLM assistant info (LLM only), done
@@ -56,13 +53,27 @@ const ArtistStep1 = () => {
const totalPopups = isLLM ? 2 : 1;
const showingPopup = popupStep < totalPopups;
- const handleChatOpened = useCallback((opening: ChatOpening) => {
- const alreadyLogged = chatOpeningsRef.current.some(
- (item) => item.stage === opening.stage,
+ const handleChatAvailable = useCallback((availability: ChatAvailability) => {
+ const alreadyLogged = chatAvailabilityRef.current.some(
+ (item) => item.stage === availability.stage,
);
- if (!alreadyLogged) chatOpeningsRef.current.push(opening);
+ if (!alreadyLogged) chatAvailabilityRef.current.push(availability);
}, []);
+ const handleInputActivityUpdate = useCallback(
+ (activity: ChatInputActivity) => {
+ const existingIndex = inputActivityRef.current.findIndex(
+ (item) => item.stage === activity.stage,
+ );
+ if (existingIndex >= 0) {
+ inputActivityRef.current[existingIndex] = activity;
+ } else {
+ inputActivityRef.current.push(activity);
+ }
+ },
+ [],
+ );
+
const handleRequestUpdate = useCallback((request: LlmRequestLog) => {
const existingIndex = llmRequestsRef.current.findIndex(
(item) => item.id === request.id,
@@ -94,7 +105,8 @@ const ArtistStep1 = () => {
},
},
llmUsage: {
- chatOpenings: chatOpeningsRef.current,
+ chatAvailability: chatAvailabilityRef.current,
+ inputActivity: inputActivityRef.current,
requests: llmRequestsRef.current,
},
};
@@ -182,7 +194,13 @@ const ArtistStep1 = () => {
setMessages={setSparkMessages}
notes={sparkNotes}
setNotes={setSparkNotes}
- onChatOpened={isLLM ? handleChatOpened : undefined}
+ initialInputActivity={inputActivityRef.current.find(
+ (activity) => activity.stage === Stage.SPARK,
+ )}
+ onChatAvailable={isLLM ? handleChatAvailable : undefined}
+ onInputActivityUpdate={
+ isLLM ? handleInputActivityUpdate : undefined
+ }
onRequestUpdate={isLLM ? handleRequestUpdate : undefined}
>
{
{'"' + passage.title + '"'}
- {", " + passage.author + " from The New York Times"}
+
+ {", " +
+ passage.author +
+ (passage.publication ? `, ${passage.publication}` : "")}
+
diff --git a/src/pages/artist/step2/Step2.tsx b/src/pages/artist/step2/Step2.tsx
index 4da710e..dcb8879 100644
--- a/src/pages/artist/step2/Step2.tsx
+++ b/src/pages/artist/step2/Step2.tsx
@@ -5,7 +5,8 @@ import BlackoutPoetry from "../../../components/blackout/Blackout";
import type {
Artist,
ArtistCondition,
- ChatOpening,
+ ChatAvailability,
+ ChatInputActivity,
LlmRequestLog,
Message,
PoemSnapshot,
@@ -14,10 +15,7 @@ import { useContext } from "react";
import { DataContext } from "../../../App";
import { Stage } from "../../../types";
import { Button } from "@chakra-ui/react";
-import {
- createAssistantMessage,
- STAGE_OPENING_MESSAGES,
-} from "../../../consts/chatMessages";
+import { getChatAvailability } from "../../../utils/llmUsage";
const ArtistStep2 = () => {
const navigate = useNavigate();
@@ -40,31 +38,43 @@ const ArtistStep2 = () => {
const llmRequestsRef = useRef(
artistPoem?.llmUsage?.requests ?? [],
);
- const chatOpeningsRef = useRef(
- artistPoem?.llmUsage?.chatOpenings ?? [],
+ const chatAvailabilityRef = useRef(
+ getChatAvailability(artistPoem?.llmUsage),
+ );
+ const inputActivityRef = useRef(
+ artistPoem?.llmUsage?.inputActivity ?? [],
);
const [writeNotes, setWriteNotes] = useState(
artistData?.poem?.sparkNotes || "",
);
- const [writeMessages, setWriteMessages] = useState(() =>
- userType === "LLM"
- ? [
- ...(artistPoem?.sparkConversation ?? []),
- createAssistantMessage(STAGE_OPENING_MESSAGES[Stage.WRITE]),
- ]
- : [...(artistPoem?.sparkConversation ?? [])],
- );
+ const [writeMessages, setWriteMessages] = useState(() => [
+ ...(artistPoem?.sparkConversation ?? []),
+ ]);
const [selectedWordIndexes, setSelectedWordIndexes] = useState([]);
const [poemSnapshots, setPoemSnapshots] = useState([]);
const [showPopup, setShowPopup] = useState(true);
- const handleChatOpened = useCallback((opening: ChatOpening) => {
- const alreadyLogged = chatOpeningsRef.current.some(
- (item) => item.stage === opening.stage,
+ const handleChatAvailable = useCallback((availability: ChatAvailability) => {
+ const alreadyLogged = chatAvailabilityRef.current.some(
+ (item) => item.stage === availability.stage,
);
- if (!alreadyLogged) chatOpeningsRef.current.push(opening);
+ if (!alreadyLogged) chatAvailabilityRef.current.push(availability);
}, []);
+ const handleInputActivityUpdate = useCallback(
+ (activity: ChatInputActivity) => {
+ const existingIndex = inputActivityRef.current.findIndex(
+ (item) => item.stage === activity.stage,
+ );
+ if (existingIndex >= 0) {
+ inputActivityRef.current[existingIndex] = activity;
+ } else {
+ inputActivityRef.current.push(activity);
+ }
+ },
+ [],
+ );
+
const handleRequestUpdate = useCallback((request: LlmRequestLog) => {
const existingIndex = llmRequestsRef.current.findIndex(
(item) => item.id === request.id,
@@ -101,7 +111,8 @@ const ArtistStep2 = () => {
},
},
llmUsage: {
- chatOpenings: chatOpeningsRef.current,
+ chatAvailability: chatAvailabilityRef.current,
+ inputActivity: inputActivityRef.current,
requests: llmRequestsRef.current,
},
};
@@ -170,7 +181,15 @@ const ArtistStep2 = () => {
notes={writeNotes}
setNotes={setWriteNotes}
selectedWordIndexes={selectedWordIndexes}
- onChatOpened={userType === "LLM" ? handleChatOpened : undefined}
+ initialInputActivity={inputActivityRef.current.find(
+ (activity) => activity.stage === Stage.WRITE,
+ )}
+ onChatAvailable={
+ userType === "LLM" ? handleChatAvailable : undefined
+ }
+ onInputActivityUpdate={
+ userType === "LLM" ? handleInputActivityUpdate : undefined
+ }
onRequestUpdate={userType === "LLM" ? handleRequestUpdate : undefined}
>
diff --git a/src/pages/artist/tutorial/Tutorial.tsx b/src/pages/artist/tutorial/Tutorial.tsx
index 4580eba..f91e6b4 100644
--- a/src/pages/artist/tutorial/Tutorial.tsx
+++ b/src/pages/artist/tutorial/Tutorial.tsx
@@ -4,7 +4,7 @@ import { DataContext } from "../../../App";
import FullPageTemplate from "../../../components/shared/pages/fullScrollPage";
import BlackoutPoetry from "../../../components/blackout/Blackout";
import { Button } from "@chakra-ui/react";
-import type { PoemSnapshot } from "../../../types";
+import type { Artist, PoemSnapshot } from "../../../types";
import { Passages } from "../../../consts/passages";
const STEPS = [
@@ -33,8 +33,14 @@ const ArtistTutorial = () => {
throw new Error("Component must be used within a DataContext.Provider");
}
const { userData, addRoleSpecificData } = context;
+ const artistData = userData?.data as Artist;
+ const passage = Passages.find(
+ (candidate) => candidate.id === artistData.assignment?.tutorialPassageId,
+ );
- const passage = Passages.find((p) => p.id === "2")!;
+ if (!passage) {
+ throw new Error("Tutorial passage assignment is missing or invalid");
+ }
const [selectedWordIndexes, setSelectedWordIndexes] = useState
([]);
const [, setPoemSnapshots] = useState([]);
@@ -54,7 +60,7 @@ const ArtistTutorial = () => {
}
prevCountRef.current = count;
- }, [selectedWordIndexes]);
+ }, [selectedWordIndexes, step]);
const handleContinue = () => {
addRoleSpecificData({
@@ -102,7 +108,11 @@ const ArtistTutorial = () => {
{'"' + passage.title + '"'}
- {", " + passage.author}
+
+ {", " +
+ passage.author +
+ (passage.publication ? `, ${passage.publication}` : "")}
+
{/* Continue — only shown on final step */}
diff --git a/src/types.ts b/src/types.ts
index 37dbcdd..5fdf1ad 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -13,6 +13,9 @@ export interface ArtistAssignment {
| "PASSAGE_STRATIFIED_1_TO_1"
| "TEST_OVERRIDE";
passageId: string;
+ tutorialPassageId: string;
+ taskPassageId: string;
+ passagePoolVersion: string;
condition: ArtistCondition;
assignedAt: Date;
}
@@ -32,6 +35,7 @@ export interface ArtistSurvey {
// }
export interface Poem {
+ loggingSchemaVersion: string;
passageId: string; // passageId in Passage.id
passage: Passage;
text: number[]; // this array holds the indexes of each word chosen from the passage
@@ -49,6 +53,9 @@ export interface Message {
role: Role;
content: string;
timestamp: Date;
+ stage: Stage;
+ kind: MessageKind;
+ inputSource?: ChatInputSource;
}
export interface PhaseTiming {
@@ -67,11 +74,41 @@ export interface TaskTiming {
};
}
-export interface ChatOpening {
+export interface ChatAvailability {
+ stage: Stage;
+ availableAt: Date;
+}
+
+export interface LegacyChatOpening {
stage: Stage;
timestamp: Date;
}
+export interface ChatInputActivity {
+ stage: Stage;
+ firstFocusedAt?: Date;
+ focusCount: number;
+ firstTypedAt?: Date;
+ draftStartCount: number;
+ abandonedDraftCount: number;
+ hasUnsentDraft: boolean;
+}
+
+export const ChatInputSource = {
+ TYPED: "TYPED",
+ SUGGESTION: "SUGGESTION",
+} as const;
+export type ChatInputSource =
+ (typeof ChatInputSource)[keyof typeof ChatInputSource];
+
+export const MessageKind = {
+ USER_MESSAGE: "USER_MESSAGE",
+ LLM_RESPONSE: "LLM_RESPONSE",
+ STAGE_OPENING: "STAGE_OPENING",
+ IDLE_NUDGE: "IDLE_NUDGE",
+} as const;
+export type MessageKind = (typeof MessageKind)[keyof typeof MessageKind];
+
export interface LlmRequestLog {
id: string;
stage: Stage;
@@ -82,6 +119,7 @@ export interface LlmRequestLog {
completedAt?: Date;
failedAt?: Date;
status: "STARTED" | "COMPLETED" | "FAILED";
+ inputSource: ChatInputSource;
systemPrompt: string;
promptVersion: string;
model?: string;
@@ -91,8 +129,11 @@ export interface LlmRequestLog {
}
export interface LlmUsage {
- chatOpenings: ChatOpening[];
+ chatAvailability: ChatAvailability[];
+ inputActivity: ChatInputActivity[];
requests: LlmRequestLog[];
+ /** @deprecated Kept only so older saved records remain readable. */
+ chatOpenings?: LegacyChatOpening[];
}
export interface Passage {
@@ -100,6 +141,7 @@ export interface Passage {
text: string;
title: string;
author: string;
+ publication?: string;
}
export const Stage = {
diff --git a/src/utils/artistMetrics.ts b/src/utils/artistMetrics.ts
index 92ba416..957c7a0 100644
--- a/src/utils/artistMetrics.ts
+++ b/src/utils/artistMetrics.ts
@@ -1,8 +1,64 @@
-import type { Poem } from "../types";
+import type { ChatInputActivity, Message, Poem, Stage } from "../types";
+import { ChatInputSource, MessageKind, Stage as StageValue } from "../types";
+import { ARTIST_DATA_LOGGING_VERSION } from "../consts/dataLogging";
+import { getChatAvailability } from "./llmUsage";
const toMillis = (value: Date | string | undefined) =>
value ? new Date(value).getTime() : undefined;
+const elapsedMs = (
+ start: Date | string | undefined,
+ end: Date | string | undefined,
+) => {
+ const startMs = toMillis(start);
+ const endMs = toMillis(end);
+ return startMs !== undefined && endMs !== undefined
+ ? Math.max(0, endMs - startMs)
+ : null;
+};
+
+const getUniqueConversationMessages = (poem: Poem) => {
+ const messagesById = new Map();
+ [...(poem.sparkConversation ?? []), ...(poem.writeConversation ?? [])].forEach(
+ (message) => messagesById.set(message.id, message),
+ );
+ return [...messagesById.values()];
+};
+
+const getStageChatMetrics = (
+ stage: Stage,
+ chatAvailability: ReturnType,
+ inputActivity: ChatInputActivity[],
+ messages: Message[],
+) => {
+ const availability = chatAvailability.find((item) => item.stage === stage);
+ const activity = inputActivity.find((item) => item.stage === stage);
+ const stageMessages = messages.filter((message) => message.stage === stage);
+
+ return {
+ chatAvailable: Boolean(availability),
+ chatFocusCount: activity?.focusCount ?? 0,
+ chatEverTyped: Boolean(activity?.firstTypedAt),
+ chatDraftStartCount: activity?.draftStartCount ?? 0,
+ chatAbandonedDraftCount: activity?.abandonedDraftCount ?? 0,
+ chatEndedWithUnsentDraft: activity?.hasUnsentDraft ?? false,
+ timeFromChatAvailableToFirstFocusMs: elapsedMs(
+ availability?.availableAt,
+ activity?.firstFocusedAt,
+ ),
+ timeFromChatAvailableToFirstTypingMs: elapsedMs(
+ availability?.availableAt,
+ activity?.firstTypedAt,
+ ),
+ stageOpeningShown: stageMessages.some(
+ (message) => message.kind === MessageKind.STAGE_OPENING,
+ ),
+ idleNudgeShown: stageMessages.some(
+ (message) => message.kind === MessageKind.IDLE_NUDGE,
+ ),
+ };
+};
+
export const getFinalPoemText = (poem: Poem) => {
const words = poem.passage.text.split(" ");
return [...new Set(poem.text)]
@@ -28,6 +84,24 @@ export const deriveArtistMetrics = (poem: Poem) => {
const completedRequests = (poem.llmUsage?.requests ?? []).filter(
(request) => request.status === "COMPLETED",
);
+ const requests = poem.llmUsage?.requests ?? [];
+ const hasDetailedChatLogging =
+ poem.loggingSchemaVersion === ARTIST_DATA_LOGGING_VERSION;
+ const chatAvailability = getChatAvailability(poem.llmUsage);
+ const inputActivity = poem.llmUsage?.inputActivity ?? [];
+ const conversationMessages = getUniqueConversationMessages(poem);
+ const sparkChat = getStageChatMetrics(
+ StageValue.SPARK,
+ chatAvailability,
+ inputActivity,
+ conversationMessages,
+ );
+ const writeChat = getStageChatMetrics(
+ StageValue.WRITE,
+ chatAvailability,
+ inputActivity,
+ conversationMessages,
+ );
return {
selectedWordCount: new Set(poem.text).size,
@@ -54,7 +128,85 @@ export const deriveArtistMetrics = (poem: Poem) => {
writeTimeMs: poem.taskTiming?.phases?.write?.durationMs ?? null,
llmUptake: completedRequests.length > 0,
llmTurnCount: completedRequests.length,
- llmAttemptCount: poem.llmUsage?.requests?.length ?? 0,
- chatOpeningCount: poem.llmUsage?.chatOpenings?.length ?? 0,
+ llmAttemptCount: requests.length,
+ llmTypedAttemptCount: hasDetailedChatLogging
+ ? requests.filter(
+ (request) => request.inputSource === ChatInputSource.TYPED,
+ ).length
+ : null,
+ llmSuggestionAttemptCount: hasDetailedChatLogging
+ ? requests.filter(
+ (request) => request.inputSource === ChatInputSource.SUGGESTION,
+ ).length
+ : null,
+ chatAvailableStageCount: new Set(
+ chatAvailability.map((availability) => availability.stage),
+ ).size,
+ chatFocusCount: hasDetailedChatLogging
+ ? sparkChat.chatFocusCount + writeChat.chatFocusCount
+ : null,
+ chatDraftStartCount: hasDetailedChatLogging
+ ? sparkChat.chatDraftStartCount + writeChat.chatDraftStartCount
+ : null,
+ chatAbandonedDraftCount: hasDetailedChatLogging
+ ? sparkChat.chatAbandonedDraftCount +
+ writeChat.chatAbandonedDraftCount
+ : null,
+ sparkChatAvailable: sparkChat.chatAvailable,
+ sparkChatFocusCount: hasDetailedChatLogging
+ ? sparkChat.chatFocusCount
+ : null,
+ sparkChatEverTyped: hasDetailedChatLogging
+ ? sparkChat.chatEverTyped
+ : null,
+ sparkChatDraftStartCount: hasDetailedChatLogging
+ ? sparkChat.chatDraftStartCount
+ : null,
+ sparkChatAbandonedDraftCount: hasDetailedChatLogging
+ ? sparkChat.chatAbandonedDraftCount
+ : null,
+ sparkChatEndedWithUnsentDraft: hasDetailedChatLogging
+ ? sparkChat.chatEndedWithUnsentDraft
+ : null,
+ sparkTimeFromChatAvailableToFirstFocusMs: hasDetailedChatLogging
+ ? sparkChat.timeFromChatAvailableToFirstFocusMs
+ : null,
+ sparkTimeFromChatAvailableToFirstTypingMs: hasDetailedChatLogging
+ ? sparkChat.timeFromChatAvailableToFirstTypingMs
+ : null,
+ sparkStageOpeningShown: hasDetailedChatLogging
+ ? sparkChat.stageOpeningShown
+ : null,
+ sparkIdleNudgeShown: hasDetailedChatLogging
+ ? sparkChat.idleNudgeShown
+ : null,
+ writeChatAvailable: writeChat.chatAvailable,
+ writeChatFocusCount: hasDetailedChatLogging
+ ? writeChat.chatFocusCount
+ : null,
+ writeChatEverTyped: hasDetailedChatLogging
+ ? writeChat.chatEverTyped
+ : null,
+ writeChatDraftStartCount: hasDetailedChatLogging
+ ? writeChat.chatDraftStartCount
+ : null,
+ writeChatAbandonedDraftCount: hasDetailedChatLogging
+ ? writeChat.chatAbandonedDraftCount
+ : null,
+ writeChatEndedWithUnsentDraft: hasDetailedChatLogging
+ ? writeChat.chatEndedWithUnsentDraft
+ : null,
+ writeTimeFromChatAvailableToFirstFocusMs: hasDetailedChatLogging
+ ? writeChat.timeFromChatAvailableToFirstFocusMs
+ : null,
+ writeTimeFromChatAvailableToFirstTypingMs: hasDetailedChatLogging
+ ? writeChat.timeFromChatAvailableToFirstTypingMs
+ : null,
+ writeStageOpeningShown: hasDetailedChatLogging
+ ? writeChat.stageOpeningShown
+ : null,
+ writeIdleNudgeShown: hasDetailedChatLogging
+ ? writeChat.idleNudgeShown
+ : null,
};
};
diff --git a/src/utils/llmUsage.ts b/src/utils/llmUsage.ts
new file mode 100644
index 0000000..239e29e
--- /dev/null
+++ b/src/utils/llmUsage.ts
@@ -0,0 +1,12 @@
+import type { ChatAvailability, LlmUsage } from "../types";
+
+export const getChatAvailability = (
+ llmUsage: LlmUsage | undefined,
+): ChatAvailability[] => {
+ if (llmUsage?.chatAvailability) return llmUsage.chatAvailability;
+
+ return (llmUsage?.chatOpenings ?? []).map((opening) => ({
+ stage: opening.stage,
+ availableAt: opening.timestamp,
+ }));
+};