Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 51 additions & 22 deletions benchmark/convert-locomo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,24 @@ function parseDateTime(dateStr: string): { date: string; time: string } | null {
)
if (!match) return null

let [, hour, min, ampm, day, month, year] = match
const [, hour, min, ampm, day, month, year] = match
let h = Number.parseInt(hour)
if (ampm.toLowerCase() === "pm" && h !== 12) h += 12
if (ampm.toLowerCase() === "am" && h === 12) h = 0

const months: Record<string, string> = {
January: "01", February: "02", March: "03", April: "04",
May: "05", June: "06", July: "07", August: "08",
September: "09", October: "10", November: "11", December: "12",
January: "01",
February: "02",
March: "03",
April: "04",
May: "05",
June: "06",
July: "07",
August: "08",
September: "09",
October: "10",
November: "11",
December: "12",
}

const m = months[month]
Expand Down Expand Up @@ -138,7 +147,10 @@ type: Person
- [role] Conversation participant
- [relationship] Regularly chats with ${speakerB}
`
files.set(`people/${speakerA.toLowerCase().replace(/\s+/g, "-")}.md`, speakerANote)
files.set(
`people/${speakerA.toLowerCase().replace(/\s+/g, "-")}.md`,
speakerANote,
)

const speakerBNote = `---
title: ${speakerB}
Expand All @@ -151,16 +163,19 @@ type: Person
- [role] Conversation participant
- [relationship] Regularly chats with ${speakerA}
`
files.set(`people/${speakerB.toLowerCase().replace(/\s+/g, "-")}.md`, speakerBNote)
files.set(
`people/${speakerB.toLowerCase().replace(/\s+/g, "-")}.md`,
speakerBNote,
)

// Build a MEMORY.md with key facts that accumulate
let memoryLines: string[] = [
`# Long-Term Memory`,
const memoryLines: string[] = [
"# Long-Term Memory",
"",
`## People`,
"## People",
`- ${speakerA} and ${speakerB} are close friends who chat regularly`,
"",
`## Key Events`,
"## Key Events",
]

// Convert each session to a dated note
Expand All @@ -170,7 +185,8 @@ type: Person
const dateTimeStr = c[`${sessionKey}_date_time`]
const parsed = dateTimeStr ? parseDateTime(dateTimeStr) : null

const date = parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const date =
parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const time = parsed?.time || "12:00"

// Get session summary and observations if available
Expand All @@ -184,7 +200,8 @@ type: Person
if (Array.isArray(obs)) {
for (const item of obs) {
const text = Array.isArray(item) ? item[0] : item
if (typeof text === "string") lines.push(`- [${speaker.toLowerCase()}] ${text}`)
if (typeof text === "string")
lines.push(`- [${speaker.toLowerCase()}] ${text}`)
}
}
}
Expand Down Expand Up @@ -213,28 +230,29 @@ date: ${date}
}

// Add conversation
content += `## Conversation\n`
content += "## Conversation\n"
for (const turn of turns) {
const text = turn.text.replace(/\n/g, "\n> ")
content += `**${turn.speaker}:** ${text}\n\n`
}

// Add relations
content += `## Relations\n`
content += "## Relations\n"
content += `- mentions [[${speakerA}]]\n`
content += `- mentions [[${speakerB}]]\n`

// Add to memory summary
if (observation) {
const firstObs = observation.split("\n")[0]?.replace(/^- \[\w+\] /, "") || ""
const firstObs =
observation.split("\n")[0]?.replace(/^- \[\w+\] /, "") || ""
if (firstObs) memoryLines.push(`- [${date}] ${firstObs}`)
}

files.set(`conversations/${date}-session-${sessionNum}.md`, content)
}

// Write MEMORY.md
files.set("MEMORY.md", memoryLines.join("\n") + "\n")
files.set("MEMORY.md", `${memoryLines.join("\n")}\n`)

// Convert QA to benchmark queries
const queries: BenchmarkQuery[] = []
Expand All @@ -253,7 +271,8 @@ date: ${date}
// Find the session's date
const dateTimeStr = c[`session_${sessionNum}_date_time`]
const parsed = dateTimeStr ? parseDateTime(dateTimeStr) : null
const date = parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const date =
parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
groundTruth.add(`conversations/${date}-session-${sessionNum}.md`)
}

Expand All @@ -266,8 +285,14 @@ date: ${date}
query: qa.question,
category,
ground_truth: [...groundTruth],
expected_content: isAdversarial ? undefined : answer.length < 100 ? answer : undefined,
note: isAdversarial ? `Adversarial: correct answer is "${answer}"` : undefined,
expected_content: isAdversarial
? undefined
: answer.length < 100
? answer
: undefined,
note: isAdversarial
? `Adversarial: correct answer is "${answer}"`
: undefined,
})
}

Expand Down Expand Up @@ -303,7 +328,9 @@ async function main() {
const convDir = `corpus-locomo/conv-${idx}`
const outDir = resolve(BENCHMARK_DIR, convDir)

console.log(`\nConverting conversation ${idx} (${conv.conversation.speaker_a} & ${conv.conversation.speaker_b})...`)
console.log(
`\nConverting conversation ${idx} (${conv.conversation.speaker_a} & ${conv.conversation.speaker_b})...`,
)

const { files, queries } = convertConversation(conv, idx)

Expand Down Expand Up @@ -332,8 +359,10 @@ async function main() {
}
}

console.log(`\n✅ Total: ${totalFiles} files, ${totalQueries} queries across ${indices.length} conversations`)
console.log(` Output: benchmark/corpus-locomo/`)
console.log(
`\n✅ Total: ${totalFiles} files, ${totalQueries} queries across ${indices.length} conversations`,
)
console.log(" Output: benchmark/corpus-locomo/")
}

main().catch((err) => {
Expand Down
10 changes: 6 additions & 4 deletions benchmark/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,15 @@ const RESULTS_DIR = resolve(BENCHMARK_DIR, "results")
const CORPUS_SIZE =
process.argv.find((a) => a.startsWith("--corpus="))?.split("=")[1] || "small"
const BM_PROJECT =
process.argv.find((a) => a.startsWith("--project="))?.split("=")[1] || "benchmark"
process.argv.find((a) => a.startsWith("--project="))?.split("=")[1] ||
"benchmark"
const QUERIES_PATH =
process.argv.find((a) => a.startsWith("--queries="))?.split("=")[1] ||
resolve(BENCHMARK_DIR, "queries.json")
const QUERY_LIMIT = Number.parseInt(
process.argv.find((a) => a.startsWith("--limit="))?.split("=")[1] || "0",
) || 0
const QUERY_LIMIT =
Number.parseInt(
process.argv.find((a) => a.startsWith("--limit="))?.split("=")[1] || "0",
) || 0

// ---------------------------------------------------------------------------
// MCP Client
Expand Down
30 changes: 19 additions & 11 deletions index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
import { buildCaptureHandler } from "./hooks/capture.ts"
import { buildRecallHandler } from "./hooks/recall.ts"
import { initLogger, log } from "./logger.ts"
import { CONVERSATION_SCHEMA_CONTENT } from "./schema/conversation-schema.ts"
import { TASK_SCHEMA_CONTENT } from "./schema/task-schema.ts"
import { registerContextTool } from "./tools/build-context.ts"
import { registerDeleteTool } from "./tools/delete-note.ts"
Expand Down Expand Up @@ -103,7 +104,9 @@ export default {
'uv tool install "basic-memory @ git+https://github.com/basicmachines-co/basic-memory.git@main" --force',
{ encoding: "utf-8", timeout: 120_000, stdio: "pipe" },
)
log.info(`basic-memory installed: ${result.trim().split("\n").pop()}`)
log.info(
`basic-memory installed: ${result.trim().split("\n").pop()}`,
)
// Verify it worked
try {
execSync(`command -v ${bmBin}`, { stdio: "ignore" })
Expand All @@ -113,7 +116,7 @@ export default {
"bm installed but not found on PATH. You may need to add uv's bin directory to your PATH (typically ~/.local/bin).",
)
}
} catch (uvErr) {
} catch (_uvErr) {
log.error(
"Cannot auto-install basic-memory: uv not found. " +
"Install uv first (brew install uv, or curl -LsSf https://astral.sh/uv/install.sh | sh), " +
Expand All @@ -130,16 +133,21 @@ export default {
await client.ensureProject(projectPath)
log.debug(`project "${cfg.project}" at ${projectPath}`)

// Seed Task schema if not already present
try {
await client.readNote("schema/Task")
log.debug("Task schema already exists, skipping seed")
} catch {
// Seed schemas if not already present
for (const [name, content] of [
["Task", TASK_SCHEMA_CONTENT],
["Conversation", CONVERSATION_SCHEMA_CONTENT],
] as const) {
try {
await client.writeNote("Task", TASK_SCHEMA_CONTENT, "schema")
log.debug("seeded Task schema note")
} catch (err) {
log.debug("Task schema seed failed (non-fatal)", err)
await client.readNote(`schema/${name}`)
log.debug(`${name} schema already exists, skipping seed`)
} catch {
try {
await client.writeNote(name, content, "schema")
log.debug(`seeded ${name} schema note`)
} catch (err) {
log.debug(`${name} schema seed failed (non-fatal)`, err)
}
}
}

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
"tools/write-note.ts",
"types/openclaw.d.ts",
"schema/task-schema.ts",
"schema/conversation-schema.ts",
"skills/",
"scripts/setup-bm.sh",
"openclaw.plugin.json",
Expand Down
33 changes: 33 additions & 0 deletions schema/conversation-schema.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
/**
* Canonical Conversation schema note content, seeded into new projects on first startup.
* If the schema already exists (user may have customized it), seeding is skipped.
*/
export const CONVERSATION_SCHEMA_CONTENT = `---
title: Conversation
type: schema
entity: Conversation
version: 1
schema:
date: "string, ISO date YYYY-MM-DD"
session_id?: "string, unique session identifier"
channel?: "string, where the conversation happened (webchat, telegram, discord, etc)"
participants?: "array, who was in the conversation"
topic?: "string, brief description of main topic"
summary?: "string, one-paragraph summary of key points"
key_decisions?: "array, decisions made during conversation"
action_items?: "array, things to do as a result"
settings:
validation: warn
---

# Conversation

A record of a conversation session between the agent and user(s).

## Observations
- [convention] Conversation files live in memory/conversations/ with format conversations-YYYY-MM-DD.md
- [convention] Messages are appended as the conversation progresses
- [convention] Summary and key_decisions populated at session end or by memory-reflect
- [convention] Skip routine greetings, heartbeat acks, and tool call details
- [convention] Focus on decisions, actions taken, and key context needed for continuity
`