Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 51 additions & 22 deletions benchmark/convert-locomo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,15 +78,24 @@ function parseDateTime(dateStr: string): { date: string; time: string } | null {
)
if (!match) return null

let [, hour, min, ampm, day, month, year] = match
const [, hour, min, ampm, day, month, year] = match
let h = Number.parseInt(hour)
if (ampm.toLowerCase() === "pm" && h !== 12) h += 12
if (ampm.toLowerCase() === "am" && h === 12) h = 0

const months: Record<string, string> = {
January: "01", February: "02", March: "03", April: "04",
May: "05", June: "06", July: "07", August: "08",
September: "09", October: "10", November: "11", December: "12",
January: "01",
February: "02",
March: "03",
April: "04",
May: "05",
June: "06",
July: "07",
August: "08",
September: "09",
October: "10",
November: "11",
December: "12",
}

const m = months[month]
Expand Down Expand Up @@ -138,7 +147,10 @@ type: Person
- [role] Conversation participant
- [relationship] Regularly chats with ${speakerB}
`
files.set(`people/${speakerA.toLowerCase().replace(/\s+/g, "-")}.md`, speakerANote)
files.set(
`people/${speakerA.toLowerCase().replace(/\s+/g, "-")}.md`,
speakerANote,
)

const speakerBNote = `---
title: ${speakerB}
Expand All @@ -151,16 +163,19 @@ type: Person
- [role] Conversation participant
- [relationship] Regularly chats with ${speakerA}
`
files.set(`people/${speakerB.toLowerCase().replace(/\s+/g, "-")}.md`, speakerBNote)
files.set(
`people/${speakerB.toLowerCase().replace(/\s+/g, "-")}.md`,
speakerBNote,
)

// Build a MEMORY.md with key facts that accumulate
let memoryLines: string[] = [
`# Long-Term Memory`,
const memoryLines: string[] = [
"# Long-Term Memory",
"",
`## People`,
"## People",
`- ${speakerA} and ${speakerB} are close friends who chat regularly`,
"",
`## Key Events`,
"## Key Events",
]

// Convert each session to a dated note
Expand All @@ -170,7 +185,8 @@ type: Person
const dateTimeStr = c[`${sessionKey}_date_time`]
const parsed = dateTimeStr ? parseDateTime(dateTimeStr) : null

const date = parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const date =
parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const time = parsed?.time || "12:00"

// Get session summary and observations if available
Expand All @@ -184,7 +200,8 @@ type: Person
if (Array.isArray(obs)) {
for (const item of obs) {
const text = Array.isArray(item) ? item[0] : item
if (typeof text === "string") lines.push(`- [${speaker.toLowerCase()}] ${text}`)
if (typeof text === "string")
lines.push(`- [${speaker.toLowerCase()}] ${text}`)
}
}
}
Expand Down Expand Up @@ -213,28 +230,29 @@ date: ${date}
}

// Add conversation
content += `## Conversation\n`
content += "## Conversation\n"
for (const turn of turns) {
const text = turn.text.replace(/\n/g, "\n> ")
content += `**${turn.speaker}:** ${text}\n\n`
}

// Add relations
content += `## Relations\n`
content += "## Relations\n"
content += `- mentions [[${speakerA}]]\n`
content += `- mentions [[${speakerB}]]\n`

// Add to memory summary
if (observation) {
const firstObs = observation.split("\n")[0]?.replace(/^- \[\w+\] /, "") || ""
const firstObs =
observation.split("\n")[0]?.replace(/^- \[\w+\] /, "") || ""
if (firstObs) memoryLines.push(`- [${date}] ${firstObs}`)
}

files.set(`conversations/${date}-session-${sessionNum}.md`, content)
}

// Write MEMORY.md
files.set("MEMORY.md", memoryLines.join("\n") + "\n")
files.set("MEMORY.md", `${memoryLines.join("\n")}\n`)

// Convert QA to benchmark queries
const queries: BenchmarkQuery[] = []
Expand All @@ -253,7 +271,8 @@ date: ${date}
// Find the session's date
const dateTimeStr = c[`session_${sessionNum}_date_time`]
const parsed = dateTimeStr ? parseDateTime(dateTimeStr) : null
const date = parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
const date =
parsed?.date || `2023-01-${String(sessionNum).padStart(2, "0")}`
groundTruth.add(`conversations/${date}-session-${sessionNum}.md`)
}

Expand All @@ -266,8 +285,14 @@ date: ${date}
query: qa.question,
category,
ground_truth: [...groundTruth],
expected_content: isAdversarial ? undefined : answer.length < 100 ? answer : undefined,
note: isAdversarial ? `Adversarial: correct answer is "${answer}"` : undefined,
expected_content: isAdversarial
? undefined
: answer.length < 100
? answer
: undefined,
note: isAdversarial
? `Adversarial: correct answer is "${answer}"`
: undefined,
})
}

Expand Down Expand Up @@ -303,7 +328,9 @@ async function main() {
const convDir = `corpus-locomo/conv-${idx}`
const outDir = resolve(BENCHMARK_DIR, convDir)

console.log(`\nConverting conversation ${idx} (${conv.conversation.speaker_a} & ${conv.conversation.speaker_b})...`)
console.log(
`\nConverting conversation ${idx} (${conv.conversation.speaker_a} & ${conv.conversation.speaker_b})...`,
)

const { files, queries } = convertConversation(conv, idx)

Expand Down Expand Up @@ -332,8 +359,10 @@ async function main() {
}
}

console.log(`\n✅ Total: ${totalFiles} files, ${totalQueries} queries across ${indices.length} conversations`)
console.log(` Output: benchmark/corpus-locomo/`)
console.log(
`\n✅ Total: ${totalFiles} files, ${totalQueries} queries across ${indices.length} conversations`,
)
console.log(" Output: benchmark/corpus-locomo/")
}

main().catch((err) => {
Expand Down
10 changes: 6 additions & 4 deletions benchmark/run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -87,13 +87,15 @@ const RESULTS_DIR = resolve(BENCHMARK_DIR, "results")
const CORPUS_SIZE =
process.argv.find((a) => a.startsWith("--corpus="))?.split("=")[1] || "small"
const BM_PROJECT =
process.argv.find((a) => a.startsWith("--project="))?.split("=")[1] || "benchmark"
process.argv.find((a) => a.startsWith("--project="))?.split("=")[1] ||
"benchmark"
const QUERIES_PATH =
process.argv.find((a) => a.startsWith("--queries="))?.split("=")[1] ||
resolve(BENCHMARK_DIR, "queries.json")
const QUERY_LIMIT = Number.parseInt(
process.argv.find((a) => a.startsWith("--limit="))?.split("=")[1] || "0",
) || 0
const QUERY_LIMIT =
Number.parseInt(
process.argv.find((a) => a.startsWith("--limit="))?.split("=")[1] || "0",
) || 0

// ---------------------------------------------------------------------------
// MCP Client
Expand Down
11 changes: 8 additions & 3 deletions bm-client.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -523,18 +523,23 @@ describe("BmClient MCP behavior", () => {
expect(result.file_path).toBe("archive/my-note.md")
})

it("indexConversation does not create fallback note on non-not-found edit errors", async () => {
it("indexConversation falls through to writeNote on any editNote error", async () => {
;(client as any).editNote = jest
.fn()
.mockRejectedValue(new Error("validation failed"))
;(client as any).writeNote = jest.fn()
;(client as any).writeNote = jest.fn().mockResolvedValue({
title: "conversations",
permalink: "conversations",
content: "x",
file_path: "conversations/x.md",
})

await client.indexConversation(
"user message long enough",
"assistant reply long enough",
)

expect((client as any).writeNote).not.toHaveBeenCalled()
expect((client as any).writeNote).toHaveBeenCalledTimes(1)
})

it("indexConversation creates fallback note only on note-not-found errors", async () => {
Expand Down
39 changes: 23 additions & 16 deletions bm-client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ function isRecoverableConnectionError(err: unknown): boolean {
)
}

function isNoteNotFoundError(err: unknown): boolean {
function _isNoteNotFoundError(err: unknown): boolean {
const msg = getErrorMessage(err).toLowerCase()
return (
msg.includes("entity not found") ||
Expand Down Expand Up @@ -830,10 +830,6 @@ export class BmClient {
return payload as unknown as MetadataSearchResult
}

private isNoteNotFoundError(err: unknown): boolean {
return isNoteNotFoundError(err)
}

async indexConversation(
userMessage: string,
assistantResponse: string,
Expand All @@ -855,22 +851,33 @@ export class BmClient {
"---",
].join("\n")

// Try append first — if it fails for ANY reason, fall through to create
try {
await this.editNote(title, "append", entry)
log.debug(`appended conversation to: ${title}`)
return
} catch (err) {
if (!this.isNoteNotFoundError(err)) {
log.error("conversation append failed", err)
return
}
log.debug(`append failed, will create: ${getErrorMessage(err)}`)
}

const content = [`# Conversations ${dateStr}`, "", entry].join("\n")
try {
await this.writeNote(title, content, "conversations")
log.debug(`created conversation note: ${title}`)
} catch (createErr) {
log.error("conversation index failed", createErr)
}
// Create the note with frontmatter and first entry
const content = [
"---",
`title: Conversations ${dateStr}`,
"type: Conversation",
`date: "${dateStr}"`,
"---",
"",
`# Conversations ${dateStr}`,
"",
entry,
].join("\n")

try {
await this.writeNote(title, content, "conversations")
log.debug(`created conversation note: ${title}`)
} catch (err) {
log.error("conversation index failed", err)
}
}

Expand Down
6 changes: 4 additions & 2 deletions index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,9 @@ export default {
'uv tool install "basic-memory @ git+https://github.com/basicmachines-co/basic-memory.git@main" --force',
{ encoding: "utf-8", timeout: 120_000, stdio: "pipe" },
)
log.info(`basic-memory installed: ${result.trim().split("\n").pop()}`)
log.info(
`basic-memory installed: ${result.trim().split("\n").pop()}`,
)
// Verify it worked
try {
execSync(`command -v ${bmBin}`, { stdio: "ignore" })
Expand All @@ -113,7 +115,7 @@ export default {
"bm installed but not found on PATH. You may need to add uv's bin directory to your PATH (typically ~/.local/bin).",
)
}
} catch (uvErr) {
} catch (_uvErr) {
log.error(
"Cannot auto-install basic-memory: uv not found. " +
"Install uv first (brew install uv, or curl -LsSf https://astral.sh/uv/install.sh | sh), " +
Expand Down