From bc9d500e48bb728d3b37fee1a12d5e7744b18fbf Mon Sep 17 00:00:00 2001 From: salmonumbrella <182032677+salmonumbrella@users.noreply.github.com> Date: Sun, 31 May 2026 10:07:23 -0700 Subject: [PATCH] feat(beancount): expose rich directive tables --- .../BeancountLedgerParser.swift | 389 ++++++++++++++++- .../BeancountPlugin.swift | 5 +- .../BeancountPluginDriver.swift | 395 +++++++++++++++++- ...ginMetadataRegistry+RegistryDefaults.swift | 5 +- .../Plugins/BeancountLedgerParserTests.swift | 50 +++ .../Plugins/BeancountPluginDriverTests.swift | 121 +++++- docs/databases/beancount.mdx | 23 +- 7 files changed, 957 insertions(+), 31 deletions(-) diff --git a/Plugins/BeancountDriverPlugin/BeancountLedgerParser.swift b/Plugins/BeancountDriverPlugin/BeancountLedgerParser.swift index 32212367a..323ef6423 100644 --- a/Plugins/BeancountDriverPlugin/BeancountLedgerParser.swift +++ b/Plugins/BeancountDriverPlugin/BeancountLedgerParser.swift @@ -11,6 +11,16 @@ struct BeancountLedger: Sendable { let accounts: [BeancountAccount] let prices: [BeancountPrice] let balances: [BeancountBalance] + let commodities: [BeancountCommodity] + let documents: [BeancountDocument] + let notes: [BeancountNote] + let events: [BeancountEvent] + let pads: [BeancountPad] + let closes: [BeancountClose] + let transactionMetadata: [BeancountTransactionMetadata] + let postingMetadata: [BeancountPostingMetadata] + let transactionTags: [BeancountTransactionTag] + let transactionLinks: [BeancountTransactionLink] let sourceFiles: [URL] let watchedDirectories: [URL] } @@ -23,6 +33,8 @@ struct BeancountTransaction: Sendable { let narration: String? let sourceFile: URL let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } } struct BeancountPosting: Sendable { @@ -34,6 +46,8 @@ struct BeancountPosting: Sendable { let commodity: String? let sourceFile: URL let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } } struct BeancountAccount: Sendable { @@ -42,6 +56,8 @@ struct BeancountAccount: Sendable { let currencies: String? let sourceFile: URL let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } } struct BeancountPrice: Sendable { @@ -52,6 +68,8 @@ struct BeancountPrice: Sendable { let currency: String let sourceFile: URL let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } } struct BeancountBalance: Sendable { @@ -62,6 +80,107 @@ struct BeancountBalance: Sendable { let commodity: String let sourceFile: URL let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountCommodity: Sendable { + let id: Int + let date: String + let commodity: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountDocument: Sendable { + let id: Int + let date: String + let account: String + let filename: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountNote: Sendable { + let id: Int + let date: String + let account: String + let comment: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountEvent: Sendable { + let id: Int + let date: String + let name: String + let value: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountPad: Sendable { + let id: Int + let date: String + let account: String + let sourceAccount: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountClose: Sendable { + let id: Int + let date: String + let account: String + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountTransactionMetadata: Sendable { + let id: Int + let transactionId: Int + let key: String + let value: String? + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountPostingMetadata: Sendable { + let id: Int + let postingId: Int + let transactionId: Int + let key: String + let value: String? + let sourceFile: URL + let line: Int + + var sourceLocation: String { "\(sourceFile.path):\(line)" } +} + +struct BeancountTransactionTag: Sendable { + let id: Int + let transactionId: Int + let tag: String +} + +struct BeancountTransactionLink: Sendable { + let id: Int + let transactionId: Int + let link: String } enum BeancountParserError: LocalizedError { @@ -87,7 +206,18 @@ final class BeancountLedgerParser { private var accountsByName: [String: BeancountAccount] = [:] private var prices: [BeancountPrice] = [] private var balances: [BeancountBalance] = [] + private var commodities: [BeancountCommodity] = [] + private var documents: [BeancountDocument] = [] + private var notes: [BeancountNote] = [] + private var events: [BeancountEvent] = [] + private var pads: [BeancountPad] = [] + private var closes: [BeancountClose] = [] + private var transactionMetadata: [BeancountTransactionMetadata] = [] + private var postingMetadata: [BeancountPostingMetadata] = [] + private var transactionTags: [BeancountTransactionTag] = [] + private var transactionLinks: [BeancountTransactionLink] = [] private var watchedDirectories: Set = [] + private var activeTags: Set = [] func parse(fileURL: URL) throws -> BeancountLedger { visited.removeAll() @@ -98,7 +228,18 @@ final class BeancountLedgerParser { accountsByName.removeAll() prices.removeAll() balances.removeAll() + commodities.removeAll() + documents.removeAll() + notes.removeAll() + events.removeAll() + pads.removeAll() + closes.removeAll() + transactionMetadata.removeAll() + postingMetadata.removeAll() + transactionTags.removeAll() + transactionLinks.removeAll() watchedDirectories.removeAll() + activeTags.removeAll() try parseFile(fileURL.standardizedFileURL) @@ -108,6 +249,16 @@ final class BeancountLedgerParser { accounts: accountsByName.values.sorted { $0.name < $1.name }, prices: prices, balances: balances, + commodities: commodities, + documents: documents, + notes: notes, + events: events, + pads: pads, + closes: closes, + transactionMetadata: transactionMetadata, + postingMetadata: postingMetadata, + transactionTags: transactionTags, + transactionLinks: transactionLinks, sourceFiles: sourceFiles, watchedDirectories: watchedDirectories.sorted { $0.path < $1.path } ) @@ -143,6 +294,15 @@ final class BeancountLedgerParser { guard !trimmed.isEmpty else { continue } + if let tag = parseTagStackDirective(trimmed, keyword: "pushtag") { + activeTags.insert(tag) + continue + } + if let tag = parseTagStackDirective(trimmed, keyword: "poptag") { + activeTags.remove(tag) + continue + } + if let includePath = parseInclude(trimmed) { let includeURLs = try resolveIncludeURLs( includePath, @@ -163,6 +323,18 @@ final class BeancountLedgerParser { parsePrice(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) } else if remainder.hasPrefix("balance ") { parseBalance(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("commodity ") { + parseCommodity(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("document ") { + parseDocument(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("note ") { + parseNote(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("event ") { + parseEvent(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("pad ") { + parsePad(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) + } else if remainder.hasPrefix("close ") { + parseClose(remainder: remainder, date: date, sourceFile: normalized, line: lineNumber) } else if let flag = remainder.first, flag == "*" || flag == "!" { let transactionId = transactions.count + 1 let transaction = parseTransaction( @@ -173,20 +345,64 @@ final class BeancountLedgerParser { line: lineNumber ) transactions.append(transaction) + for tag in activeTags.union(parseMarkers(in: remainder, prefix: "#")).sorted() { + transactionTags.append(BeancountTransactionTag( + id: transactionTags.count + 1, + transactionId: transactionId, + tag: tag + )) + } + for link in parseMarkers(in: remainder, prefix: "^") { + transactionLinks.append(BeancountTransactionLink( + id: transactionLinks.count + 1, + transactionId: transactionId, + link: link + )) + } var postingIndex = index + 1 + var currentPostingId: Int? + var currentPostingIndent = 0 while postingIndex < lines.count { let postingLine = lines[postingIndex] guard postingLine.first?.isWhitespace == true else { break } - if let posting = parsePosting( + let postingLineNumber = postingIndex + 1 + let postingIndent = leadingWhitespaceCount(postingLine) + let postingTrimmed = stripComment(postingLine).trimmingCharacters(in: .whitespaces) + if let metadata = parseMetadata(postingTrimmed) { + if let postingId = currentPostingId, postingIndent > currentPostingIndent { + postingMetadata.append(BeancountPostingMetadata( + id: postingMetadata.count + 1, + postingId: postingId, + transactionId: transactionId, + key: metadata.key, + value: metadata.value, + sourceFile: normalized, + line: postingLineNumber + )) + } else { + transactionMetadata.append(BeancountTransactionMetadata( + id: transactionMetadata.count + 1, + transactionId: transactionId, + key: metadata.key, + value: metadata.value, + sourceFile: normalized, + line: postingLineNumber + )) + currentPostingId = nil + currentPostingIndent = 0 + } + } else if let posting = parsePosting( postingLine, id: postings.count + 1, transactionId: transactionId, date: date, sourceFile: normalized, - line: postingIndex + 1 + line: postingLineNumber ) { postings.append(posting) + currentPostingId = posting.id + currentPostingIndent = postingIndent } postingIndex += 1 } @@ -361,6 +577,85 @@ final class BeancountLedgerParser { )) } + private func parseCommodity(remainder: String, date: String, sourceFile: URL, line: Int) { + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard parts.count >= 2 else { return } + commodities.append(BeancountCommodity( + id: commodities.count + 1, + date: date, + commodity: parts[1], + sourceFile: sourceFile, + line: line + )) + } + + private func parseDocument(remainder: String, date: String, sourceFile: URL, line: Int) { + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard parts.count >= 3 else { return } + let filename = quotedStrings(in: remainder).first ?? parts[2] + documents.append(BeancountDocument( + id: documents.count + 1, + date: date, + account: parts[1], + filename: filename, + sourceFile: sourceFile, + line: line + )) + } + + private func parseNote(remainder: String, date: String, sourceFile: URL, line: Int) { + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard parts.count >= 3 else { return } + let comment = quotedStrings(in: remainder).first ?? parts.dropFirst(2).joined(separator: " ") + notes.append(BeancountNote( + id: notes.count + 1, + date: date, + account: parts[1], + comment: comment, + sourceFile: sourceFile, + line: line + )) + } + + private func parseEvent(remainder: String, date: String, sourceFile: URL, line: Int) { + let quoted = quotedStrings(in: remainder) + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard quoted.count >= 2 || parts.count >= 3 else { return } + events.append(BeancountEvent( + id: events.count + 1, + date: date, + name: quoted.count >= 2 ? quoted[0] : parts[1], + value: quoted.count >= 2 ? quoted[1] : parts.dropFirst(2).joined(separator: " "), + sourceFile: sourceFile, + line: line + )) + } + + private func parsePad(remainder: String, date: String, sourceFile: URL, line: Int) { + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard parts.count >= 3 else { return } + pads.append(BeancountPad( + id: pads.count + 1, + date: date, + account: parts[1], + sourceAccount: parts[2], + sourceFile: sourceFile, + line: line + )) + } + + private func parseClose(remainder: String, date: String, sourceFile: URL, line: Int) { + let parts = remainder.split(whereSeparator: \.isWhitespace).map(String.init) + guard parts.count >= 2 else { return } + closes.append(BeancountClose( + id: closes.count + 1, + date: date, + account: parts[1], + sourceFile: sourceFile, + line: line + )) + } + private func parseTransaction( id: Int, remainder: String, @@ -436,6 +731,96 @@ final class BeancountLedgerParser { return prefix } + private func parseMetadata(_ line: String) -> (key: String, value: String?)? { + guard let firstToken = line.split(whereSeparator: \.isWhitespace).first, + firstToken.hasSuffix(":") else { + return nil + } + let key = String(firstToken.dropLast()) + let pattern = #"^[A-Za-z][A-Za-z0-9_-]*$"# + guard key.range(of: pattern, options: .regularExpression) != nil else { return nil } + + let valueStart = line.index(line.startIndex, offsetBy: firstToken.count) + let rawValue = line[valueStart...].trimmingCharacters(in: .whitespaces) + guard !rawValue.isEmpty else { return (key, nil) } + return (key, unquoted(rawValue)) + } + + private func parseTagStackDirective(_ line: String, keyword: String) -> String? { + guard line.hasPrefix("\(keyword) ") else { return nil } + return parseMarkers(in: line, prefix: "#").first + } + + private func parseMarkers(in line: String, prefix: Character) -> [String] { + let scrubbed = removingQuotedSegments(from: line) + let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-_/.")) + var markers: [String] = [] + var index = scrubbed.startIndex + + while index < scrubbed.endIndex { + guard scrubbed[index] == prefix else { + index = scrubbed.index(after: index) + continue + } + + var markerIndex = scrubbed.index(after: index) + var value = "" + while markerIndex < scrubbed.endIndex { + let character = scrubbed[markerIndex] + guard character.unicodeScalars.allSatisfy({ allowed.contains($0) }) else { break } + value.append(character) + markerIndex = scrubbed.index(after: markerIndex) + } + if !value.isEmpty { + markers.append(value) + } + index = markerIndex + } + + return markers + } + + private func removingQuotedSegments(from line: String) -> String { + var result = "" + var inQuote = false + var isEscaped = false + + for character in line { + if isEscaped { + isEscaped = false + continue + } + if character == "\\" { + isEscaped = true + continue + } + if character == "\"" { + inQuote.toggle() + continue + } + if !inQuote { + result.append(character) + } + } + + return result + } + + private func unquoted(_ value: String) -> String { + guard value.count >= 2, + value.first == "\"", + value.last == "\"" else { + return value + } + let start = value.index(after: value.startIndex) + let end = value.index(before: value.endIndex) + return String(value[start.. Int { + line.prefix { $0.isWhitespace }.count + } + private func quotedStrings(in line: String) -> [String] { var values: [String] = [] var current = "" diff --git a/Plugins/BeancountDriverPlugin/BeancountPlugin.swift b/Plugins/BeancountDriverPlugin/BeancountPlugin.swift index 62e918995..7380edd97 100644 --- a/Plugins/BeancountDriverPlugin/BeancountPlugin.swift +++ b/Plugins/BeancountDriverPlugin/BeancountPlugin.swift @@ -42,7 +42,10 @@ final class BeancountPlugin: NSObject, TableProPlugin, DriverPlugin { ] static let immutableColumns: [String] = [ "id", "transaction_id", "date", "flag", "payee", "narration", - "account", "amount", "commodity", "currency", "source_file", "line" + "posting_id", "account", "source_account", "amount", "commodity", "currency", + "filename", "comment", "name", "value", "key", "tag", "link", + "source_file", "line", "source_location", "column", "end_line", "end_column", + "severity", "phase", "code", "message" ] static let sqlDialect: SQLDialectDescriptor? = SQLDialectDescriptor( diff --git a/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift b/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift index 658b55114..d53f6a760 100644 --- a/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift +++ b/Plugins/BeancountDriverPlugin/BeancountPluginDriver.swift @@ -49,6 +49,24 @@ private struct BeancountSourceSignature: Equatable { let fileSize: UInt64? } +private struct BeancountDiagnostic { + let id: Int + let file: String? + let line: Int? + let column: Int? + let endLine: Int? + let endColumn: Int? + let severity: String? + let phase: String? + let code: String? + let message: String? + + var sourceLocation: String? { + guard let file, let line else { return nil } + return "\(file):\(line)" + } +} + final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { private let config: DriverConnectionConfig private let lock = NSLock() @@ -82,7 +100,7 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { - try Self.load(parsed, into: handle) + try Self.load(parsed, into: handle, ledgerURL: fileURL) } catch { sqlite3_close(handle) throw error @@ -447,7 +465,7 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { } do { - try Self.load(parsed, into: handle) + try Self.load(parsed, into: handle, ledgerURL: ledgerURL) } catch { sqlite3_close(handle) throw error @@ -581,7 +599,85 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { return String(describing: value) } - private static func load(_ ledger: BeancountLedger, into db: OpaquePointer) throws { + private static func validationDiagnostics(for ledgerURL: URL) -> [BeancountDiagnostic] { + do { + let rustledgerPath = try rustledgerExecutablePath() + let process = Process() + process.executableURL = URL(fileURLWithPath: rustledgerPath) + process.arguments = ["check", "-f", "json", ledgerURL.path] + + let stdout = Pipe() + let stderr = Pipe() + process.standardOutput = stdout + process.standardError = stderr + + let outputCollector = PipeDataCollector() + let errorCollector = PipeDataCollector() + let readers = DispatchGroup() + readers.enter() + DispatchQueue.global(qos: .userInitiated).async { + outputCollector.set(stdout.fileHandleForReading.readDataToEndOfFile()) + readers.leave() + } + readers.enter() + DispatchQueue.global(qos: .userInitiated).async { + errorCollector.set(stderr.fileHandleForReading.readDataToEndOfFile()) + readers.leave() + } + + try process.run() + process.waitUntilExit() + readers.wait() + + let output = outputCollector.data + if !output.isEmpty { + return try decodeRustledgerCheckOutput(output) + } + let errorOutput = errorCollector.data + guard !errorOutput.isEmpty else { + return [] + } + return try decodeRustledgerCheckOutput(errorOutput) + } catch { + return [] + } + } + + private static func decodeRustledgerCheckOutput(_ data: Data) throws -> [BeancountDiagnostic] { + let object = try JSONSerialization.jsonObject(with: data) + guard let dictionary = object as? [String: Any], + let rawDiagnostics = dictionary["diagnostics"] as? [[String: Any]] else { + return [] + } + + return rawDiagnostics.enumerated().map { index, raw in + BeancountDiagnostic( + id: index + 1, + file: raw["file"] as? String, + line: intValue(raw["line"]), + column: intValue(raw["column"]), + endLine: intValue(raw["end_line"]), + endColumn: intValue(raw["end_column"]), + severity: raw["severity"] as? String, + phase: raw["phase"] as? String, + code: raw["code"] as? String, + message: raw["message"] as? String + ) + } + } + + private static func intValue(_ value: Any?) -> Int? { + if let number = value as? NSNumber { + return number.intValue + } + if let string = value as? String { + return Int(string) + } + return nil + } + + private static func load(_ ledger: BeancountLedger, into db: OpaquePointer, ledgerURL: URL) throws { + let diagnostics = validationDiagnostics(for: ledgerURL) try exec(db, """ CREATE TABLE transactions ( id INTEGER PRIMARY KEY, @@ -590,7 +686,8 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { payee TEXT, narration TEXT, source_file TEXT NOT NULL, - line INTEGER NOT NULL + line INTEGER NOT NULL, + source_location TEXT NOT NULL ); CREATE TABLE postings ( id INTEGER PRIMARY KEY, @@ -600,14 +697,16 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { amount DECIMAL, commodity TEXT, source_file TEXT NOT NULL, - line INTEGER NOT NULL + line INTEGER NOT NULL, + source_location TEXT NOT NULL ); CREATE TABLE accounts ( name TEXT PRIMARY KEY, open_date DATE NOT NULL, currencies TEXT, source_file TEXT NOT NULL, - line INTEGER NOT NULL + line INTEGER NOT NULL, + source_location TEXT NOT NULL ); CREATE TABLE prices ( id INTEGER PRIMARY KEY, @@ -616,7 +715,8 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { amount DECIMAL NOT NULL, currency TEXT NOT NULL, source_file TEXT NOT NULL, - line INTEGER NOT NULL + line INTEGER NOT NULL, + source_location TEXT NOT NULL ); CREATE TABLE balances ( id INTEGER PRIMARY KEY, @@ -625,7 +725,102 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { amount DECIMAL NOT NULL, commodity TEXT NOT NULL, source_file TEXT NOT NULL, - line INTEGER NOT NULL + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE commodities ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + commodity TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE documents ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + account TEXT NOT NULL, + filename TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE notes ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + account TEXT NOT NULL, + comment TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + name TEXT NOT NULL, + value TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE pads ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + account TEXT NOT NULL, + source_account TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE closes ( + id INTEGER PRIMARY KEY, + date DATE NOT NULL, + account TEXT NOT NULL, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE transaction_metadata ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER NOT NULL, + key TEXT NOT NULL, + value TEXT, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE posting_metadata ( + id INTEGER PRIMARY KEY, + posting_id INTEGER NOT NULL, + transaction_id INTEGER NOT NULL, + key TEXT NOT NULL, + value TEXT, + source_file TEXT NOT NULL, + line INTEGER NOT NULL, + source_location TEXT NOT NULL + ); + CREATE TABLE transaction_tags ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER NOT NULL, + tag TEXT NOT NULL + ); + CREATE TABLE transaction_links ( + id INTEGER PRIMARY KEY, + transaction_id INTEGER NOT NULL, + link TEXT NOT NULL + ); + CREATE TABLE diagnostics ( + id INTEGER PRIMARY KEY, + file TEXT, + line INTEGER, + source_location TEXT, + column INTEGER, + end_line INTEGER, + end_column INTEGER, + severity TEXT, + phase TEXT, + code TEXT, + message TEXT ); CREATE TABLE source_files ( path TEXT PRIMARY KEY @@ -634,8 +829,8 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { for transaction in ledger.transactions { try insert(db, sql: """ - INSERT INTO transactions (id, date, flag, payee, narration, source_file, line) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO transactions (id, date, flag, payee, narration, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, values: [ String(transaction.id), transaction.date, @@ -643,13 +838,16 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { transaction.payee, transaction.narration, transaction.sourceFile.path, - String(transaction.line) + String(transaction.line), + transaction.sourceLocation ]) } for posting in ledger.postings { try insert(db, sql: """ - INSERT INTO postings (id, transaction_id, date, account, amount, commodity, source_file, line) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) + INSERT INTO postings ( + id, transaction_id, date, account, amount, commodity, source_file, line, source_location + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, values: [ String(posting.id), String(posting.transactionId), @@ -658,25 +856,27 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { posting.amount, posting.commodity, posting.sourceFile.path, - String(posting.line) + String(posting.line), + posting.sourceLocation ]) } for account in ledger.accounts { try insert(db, sql: """ - INSERT INTO accounts (name, open_date, currencies, source_file, line) - VALUES (?, ?, ?, ?, ?) + INSERT INTO accounts (name, open_date, currencies, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?) """, values: [ account.name, account.openDate, account.currencies, account.sourceFile.path, - String(account.line) + String(account.line), + account.sourceLocation ]) } for price in ledger.prices { try insert(db, sql: """ - INSERT INTO prices (id, date, commodity, amount, currency, source_file, line) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO prices (id, date, commodity, amount, currency, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, values: [ String(price.id), price.date, @@ -684,13 +884,14 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { price.amount, price.currency, price.sourceFile.path, - String(price.line) + String(price.line), + price.sourceLocation ]) } for balance in ledger.balances { try insert(db, sql: """ - INSERT INTO balances (id, date, account, amount, commodity, source_file, line) - VALUES (?, ?, ?, ?, ?, ?, ?) + INSERT INTO balances (id, date, account, amount, commodity, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) """, values: [ String(balance.id), balance.date, @@ -698,7 +899,155 @@ final class BeancountPluginDriver: PluginDatabaseDriver, @unchecked Sendable { balance.amount, balance.commodity, balance.sourceFile.path, - String(balance.line) + String(balance.line), + balance.sourceLocation + ]) + } + for commodity in ledger.commodities { + try insert(db, sql: """ + INSERT INTO commodities (id, date, commodity, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?) + """, values: [ + String(commodity.id), + commodity.date, + commodity.commodity, + commodity.sourceFile.path, + String(commodity.line), + commodity.sourceLocation + ]) + } + for document in ledger.documents { + try insert(db, sql: """ + INSERT INTO documents (id, date, account, filename, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(document.id), + document.date, + document.account, + document.filename, + document.sourceFile.path, + String(document.line), + document.sourceLocation + ]) + } + for note in ledger.notes { + try insert(db, sql: """ + INSERT INTO notes (id, date, account, comment, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(note.id), + note.date, + note.account, + note.comment, + note.sourceFile.path, + String(note.line), + note.sourceLocation + ]) + } + for event in ledger.events { + try insert(db, sql: """ + INSERT INTO events (id, date, name, value, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(event.id), + event.date, + event.name, + event.value, + event.sourceFile.path, + String(event.line), + event.sourceLocation + ]) + } + for pad in ledger.pads { + try insert(db, sql: """ + INSERT INTO pads (id, date, account, source_account, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(pad.id), + pad.date, + pad.account, + pad.sourceAccount, + pad.sourceFile.path, + String(pad.line), + pad.sourceLocation + ]) + } + for close in ledger.closes { + try insert(db, sql: """ + INSERT INTO closes (id, date, account, source_file, line, source_location) + VALUES (?, ?, ?, ?, ?, ?) + """, values: [ + String(close.id), + close.date, + close.account, + close.sourceFile.path, + String(close.line), + close.sourceLocation + ]) + } + for metadata in ledger.transactionMetadata { + try insert(db, sql: """ + INSERT INTO transaction_metadata ( + id, transaction_id, key, value, source_file, line, source_location + ) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(metadata.id), + String(metadata.transactionId), + metadata.key, + metadata.value, + metadata.sourceFile.path, + String(metadata.line), + metadata.sourceLocation + ]) + } + for metadata in ledger.postingMetadata { + try insert(db, sql: """ + INSERT INTO posting_metadata ( + id, posting_id, transaction_id, key, value, source_file, line, source_location + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(metadata.id), + String(metadata.postingId), + String(metadata.transactionId), + metadata.key, + metadata.value, + metadata.sourceFile.path, + String(metadata.line), + metadata.sourceLocation + ]) + } + for tag in ledger.transactionTags { + try insert(db, sql: """ + INSERT INTO transaction_tags (id, transaction_id, tag) + VALUES (?, ?, ?) + """, values: [String(tag.id), String(tag.transactionId), tag.tag]) + } + for link in ledger.transactionLinks { + try insert(db, sql: """ + INSERT INTO transaction_links (id, transaction_id, link) + VALUES (?, ?, ?) + """, values: [String(link.id), String(link.transactionId), link.link]) + } + for diagnostic in diagnostics { + try insert(db, sql: """ + INSERT INTO diagnostics ( + id, file, line, source_location, column, end_line, end_column, severity, phase, code, message + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, values: [ + String(diagnostic.id), + diagnostic.file, + diagnostic.line.map(String.init), + diagnostic.sourceLocation, + diagnostic.column.map(String.init), + diagnostic.endLine.map(String.init), + diagnostic.endColumn.map(String.init), + diagnostic.severity, + diagnostic.phase, + diagnostic.code, + diagnostic.message ]) } for sourceFile in ledger.sourceFiles { diff --git a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift index 652868123..3b1a65db0 100644 --- a/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift +++ b/TablePro/Core/Plugins/PluginMetadataRegistry+RegistryDefaults.swift @@ -892,7 +892,10 @@ extension PluginMetadataRegistry { defaultPrimaryKeyColumn: nil, immutableColumns: [ "id", "transaction_id", "date", "flag", "payee", "narration", - "account", "amount", "commodity", "currency", "source_file", "line" + "posting_id", "account", "source_account", "amount", "commodity", "currency", + "filename", "comment", "name", "value", "key", "tag", "link", + "source_file", "line", "source_location", "column", "end_line", "end_column", + "severity", "phase", "code", "message" ], systemDatabaseNames: [], systemSchemaNames: [], diff --git a/TableProTests/Plugins/BeancountLedgerParserTests.swift b/TableProTests/Plugins/BeancountLedgerParserTests.swift index 99207b6b0..f57854777 100644 --- a/TableProTests/Plugins/BeancountLedgerParserTests.swift +++ b/TableProTests/Plugins/BeancountLedgerParserTests.swift @@ -113,4 +113,54 @@ struct BeancountLedgerParserTests { #expect(parsed.postings.map(\.account) == ["Assets:Bank:Checking", "Expenses:Food"]) } + + @Test("parses rich directives, metadata, tags, and links") + func parsesRichDirectivesMetadataTagsAndLinks() throws { + let tempDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("beancount-parser-rich-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: tempDirectory, withIntermediateDirectories: true) + defer { + try? FileManager.default.removeItem(at: tempDirectory) + } + + let ledger = tempDirectory.appendingPathComponent("main.beancount") + try """ + pushtag #trip + + 2024-01-01 commodity USD + 2024-01-02 open Assets:Bank:Checking USD + 2024-01-03 close Liabilities:CreditCard + 2024-01-04 pad Assets:Bank:Checking Equity:Opening-Balances + 2024-01-05 document Assets:Bank:Checking "receipts/january.pdf" + 2024-01-06 note Assets:Bank:Checking "Opened checking account" + 2024-01-07 event "location" "Vancouver" + + 2024-01-15 * "Grocery Store" "Weekly shop" #tax ^invoice-123 + receipt: "receipt-123.pdf" + imported: TRUE + Assets:Bank:Checking -100.00 USD + statement_line: "42" + Expenses:Food 100.00 USD + + poptag #trip + """.write(to: ledger, atomically: true, encoding: .utf8) + + let parsed = try BeancountLedgerParser().parse(fileURL: ledger) + + #expect(parsed.commodities.map(\.commodity) == ["USD"]) + #expect(parsed.closes.map(\.account) == ["Liabilities:CreditCard"]) + #expect(parsed.pads.map(\.account) == ["Assets:Bank:Checking"]) + #expect(parsed.pads.map(\.sourceAccount) == ["Equity:Opening-Balances"]) + #expect(parsed.documents.map(\.filename) == ["receipts/january.pdf"]) + #expect(parsed.notes.map(\.comment) == ["Opened checking account"]) + #expect(parsed.events.map(\.name) == ["location"]) + #expect(parsed.events.map(\.value) == ["Vancouver"]) + #expect(parsed.transactionMetadata.map(\.key) == ["receipt", "imported"]) + #expect(parsed.transactionMetadata.map(\.value) == ["receipt-123.pdf", "TRUE"]) + #expect(parsed.postingMetadata.map(\.key) == ["statement_line"]) + #expect(parsed.postingMetadata.map(\.value) == ["42"]) + #expect(parsed.transactionTags.map(\.tag).sorted() == ["tax", "trip"]) + #expect(parsed.transactionLinks.map(\.link) == ["invoice-123"]) + #expect(parsed.transactions.first?.sourceLocation.hasSuffix("main.beancount:11") == true) + } } diff --git a/TableProTests/Plugins/BeancountPluginDriverTests.swift b/TableProTests/Plugins/BeancountPluginDriverTests.swift index f5648ca77..e0125b7ed 100644 --- a/TableProTests/Plugins/BeancountPluginDriverTests.swift +++ b/TableProTests/Plugins/BeancountPluginDriverTests.swift @@ -7,7 +7,7 @@ import Foundation import TableProPluginKit import Testing -@Suite("Beancount plugin driver") +@Suite("Beancount plugin driver", .serialized) struct BeancountPluginDriverTests { @Test("reloads the SQL projection when an included ledger file changes") func reloadsWhenIncludedFileChanges() async throws { @@ -126,6 +126,125 @@ struct BeancountPluginDriverTests { } } + @Test("projects rich directives and source locations into SQL tables") + func projectsRichDirectiveTablesAndSourceLocations() async throws { + let tempDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("beancount-driver-rich-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: tempDirectory, withIntermediateDirectories: true) + defer { + try? FileManager.default.removeItem(at: tempDirectory) + } + + let ledger = tempDirectory.appendingPathComponent("main.beancount") + try """ + 2024-01-01 commodity USD + 2024-01-02 open Assets:Bank:Checking USD + 2024-01-03 close Liabilities:CreditCard + 2024-01-04 pad Assets:Bank:Checking Equity:Opening-Balances + 2024-01-05 document Assets:Bank:Checking "receipts/january.pdf" + 2024-01-06 note Assets:Bank:Checking "Opened checking account" + 2024-01-07 event "location" "Vancouver" + + 2024-01-15 * "Grocery Store" "Weekly shop" #tax ^invoice-123 + receipt: "receipt-123.pdf" + Assets:Bank:Checking -100.00 USD + statement_line: "42" + Expenses:Food 100.00 USD + """.write(to: ledger, atomically: true, encoding: .utf8) + + let driver = BeancountPluginDriver(config: DriverConnectionConfig( + host: "", + port: 0, + username: "", + password: "", + database: ledger.path + )) + try await driver.connect() + defer { + driver.disconnect() + } + + let tables = try await driver.fetchTables(schema: nil) + #expect(tables.map(\.name).contains("commodities")) + #expect(tables.map(\.name).contains("documents")) + #expect(tables.map(\.name).contains("notes")) + #expect(tables.map(\.name).contains("events")) + #expect(tables.map(\.name).contains("pads")) + #expect(tables.map(\.name).contains("closes")) + #expect(tables.map(\.name).contains("transaction_metadata")) + #expect(tables.map(\.name).contains("posting_metadata")) + #expect(tables.map(\.name).contains("transaction_tags")) + #expect(tables.map(\.name).contains("transaction_links")) + #expect(tables.map(\.name).contains("diagnostics")) + + var result = try await driver.execute(query: "SELECT commodity, source_location FROM commodities") + #expect(result.rows.first?[0].asText == "USD") + #expect(result.rows.first?[1].asText?.hasSuffix("main.beancount:1") == true) + + result = try await driver.execute(query: "SELECT account, filename FROM documents") + #expect(result.rows.first?.map(\.asText) == ["Assets:Bank:Checking", "receipts/january.pdf"]) + + result = try await driver.execute(query: "SELECT key, value FROM transaction_metadata") + #expect(result.rows.first?.map(\.asText) == ["receipt", "receipt-123.pdf"]) + + result = try await driver.execute(query: "SELECT tag FROM transaction_tags") + #expect(result.rows.map { $0[0].asText } == ["tax"]) + + result = try await driver.execute(query: "SELECT link FROM transaction_links") + #expect(result.rows.map { $0[0].asText } == ["invoice-123"]) + + result = try await driver.execute(query: "SELECT key, value FROM posting_metadata") + #expect(result.rows.first?.map(\.asText) == ["statement_line", "42"]) + } + + @Test("projects rustledger validation diagnostics") + func projectsRustledgerValidationDiagnostics() async throws { + let rustledger = try #require(Self.bundledRustledgerPath() ?? Self.installedRustledgerPath()) + let tempDirectory = FileManager.default.temporaryDirectory + .appendingPathComponent("beancount-driver-diagnostics-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: tempDirectory, withIntermediateDirectories: true) + defer { + unsetenv("TABLEPRO_RUSTLEDGER_BINARY") + try? FileManager.default.removeItem(at: tempDirectory) + } + + let ledger = tempDirectory.appendingPathComponent("main.beancount") + try """ + 2024-01-01 open Assets:Bank:Checking USD + + 2024-01-02 * "Broken" "Unbalanced" + Assets:Bank:Checking -10.00 USD + Expenses:Food 15.00 USD + """.write(to: ledger, atomically: true, encoding: .utf8) + + setenv("TABLEPRO_RUSTLEDGER_BINARY", rustledger, 1) + + let driver = BeancountPluginDriver(config: DriverConnectionConfig( + host: "", + port: 0, + username: "", + password: "", + database: ledger.path + )) + try await driver.connect() + defer { + driver.disconnect() + } + + let result = try await driver.execute(query: """ + SELECT code, severity, line, source_location, message + FROM diagnostics + ORDER BY code + """) + + #expect(result.rows.map { $0[0].asText } == ["E1001", "E3001"]) + #expect(result.rows.allSatisfy { $0[1].asText == "error" }) + #expect(result.rows.allSatisfy { $0[2].asText == "3" }) + #expect(result.rows.allSatisfy { $0[3].asText?.hasSuffix("main.beancount:3") == true }) + #expect(result.rows.map { $0[4].asText ?? "" }.contains { $0.contains("never opened") }) + #expect(result.rows.map { $0[4].asText ?? "" }.contains { $0.contains("does not balance") }) + } + @Test("executes BQL queries through the rustledger helper") func executesBQLQueriesThroughRustledgerHelper() async throws { let rustledger = try #require(Self.bundledRustledgerPath() ?? Self.installedRustledgerPath()) diff --git a/docs/databases/beancount.mdx b/docs/databases/beancount.mdx index 479cd6fe6..98e4a1f24 100644 --- a/docs/databases/beancount.mdx +++ b/docs/databases/beancount.mdx @@ -5,7 +5,7 @@ description: Open Beancount ledgers with TablePro # Beancount -TablePro opens `.beancount` ledgers as read-only, file-based connections. If the Beancount plugin is not installed yet, TablePro prompts you to download it before opening the ledger. The driver projects transactions, postings, accounts, prices, balances, and source files into SQL tables for browsing and exports. +TablePro opens `.beancount` ledgers as read-only, file-based connections. If the Beancount plugin is not installed yet, TablePro prompts you to download it before opening the ledger. The driver projects transactions, postings, accounts, prices, balances, richer directives, metadata, validation diagnostics, and source files into SQL tables for browsing and exports. The plugin also supports BQL queries through its packaged `rustledger` helper. @@ -38,13 +38,30 @@ See [Connection URL Reference](/databases/connection-urls) for all parameters. | Table | Contents | |-------|----------| -| `transactions` | Transaction date, flag, payee, narration, source file, and line | -| `postings` | Posting account, amount, commodity, source file, and line | +| `transactions` | Transaction date, flag, payee, narration, source file, line, and source location | +| `postings` | Posting account, amount, commodity, source file, line, and source location | | `accounts` | Opened accounts and declared currencies | | `prices` | Price directives | | `balances` | Balance directives | +| `commodities` | Commodity directives | +| `documents` | Document directives and attached filenames | +| `notes` | Account notes | +| `events` | Event name/value directives | +| `pads` | Pad directives and source accounts | +| `closes` | Close directives | +| `transaction_metadata` | Transaction metadata key/value pairs | +| `posting_metadata` | Posting metadata key/value pairs | +| `transaction_tags` | Transaction tags, including active tag stack entries | +| `transaction_links` | Transaction links | +| `diagnostics` | Validation diagnostics from `rustledger check` | | `source_files` | Parsed ledger and include files | +Rows with source positions include `source_file`, `line`, and `source_location` columns. `source_location` is formatted as `path:line` so it can be copied, filtered, or exported directly. + +## Diagnostics + +The `diagnostics` table contains structured validation output from `rustledger check` when the packaged helper is available. It includes file, line, source location, column, severity, phase, diagnostic code, and message columns. If validation is unavailable, the table remains empty and the ledger can still be browsed. + ## Includes The parser follows Beancount `include` directives. Literal includes and glob patterns such as `include "imports/*.beancount"` and `include "imports/**/*.beancount"` are supported.