Fetch OFX files and convert them into Ledger format
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

365 lines
15KB

  1. ;;; ledger-import.el --- Fetch OFX files from bank and push them to Ledger -*- lexical-binding: t; -*-
  2. ;; Copyright (C) 2018 Damien Cassou
  3. ;; Author: Damien Cassou <damien@cassou.me>
  4. ;; Url: https://gitlab.petton.fr/mpdel/libmpdel
  5. ;; Package-requires: ((emacs "25.1") (ledger-mode "3.1.1"))
  6. ;; Version: 1.0.0
  7. ;; This program is free software; you can redistribute it and/or modify
  8. ;; it under the terms of the GNU General Public License as published by
  9. ;; the Free Software Foundation, either version 3 of the License, or
  10. ;; (at your option) any later version.
  11. ;; This program is distributed in the hope that it will be useful,
  12. ;; but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. ;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14. ;; GNU General Public License for more details.
  15. ;; You should have received a copy of the GNU General Public License
  16. ;; along with this program. If not, see <https://www.gnu.org/licenses/>.
  17. ;;; Commentary:
  18. ;; This file contains code to simplify importing new transactions into your
  19. ;; Ledger file. Transactions are fetched using a fetcher (only boobank from the
  20. ;; weboob project is supported for now) and the OFX file format. Then, the OFX
  21. ;; buffers are converted into Ledger format through ledger-autosync.
  22. ;; To use ledger-import, you first have to install and configure boobank, from
  23. ;; the weboob project:
  24. ;;
  25. ;; - http://weboob.org/
  26. ;; - http://weboob.org/applications/boobank
  27. ;;
  28. ;; When you manage to visualize your bank accounts with boobank, you should
  29. ;; configure each in `ledger-import-accounts'. Use `customize-variable' to do
  30. ;; that if you want to. You can check that your configuration works with `M-x
  31. ;; ledger-import-fetch-boobank': after a few tens of seconds, you will get a
  32. ;; buffer with OFX data. If boobank imports transactions that are too old, you
  33. ;; can configure `ledger-import-boobank-import-from-date'.
  34. ;;
  35. ;; To convert an OFX file into Ledger format, ledger-import uses ledger-autosync
  36. ;; that you have to install as well:
  37. ;;
  38. ;; - https://github.com/egh/ledger-autosync
  39. ;;
  40. ;; This doesn't require additional configuration. To test that ledger-autosync
  41. ;; works fine, go back to the buffer containing OFX data (or create a new one),
  42. ;; and type `M-x ledger-import-convert-ofx-to-ledger'. After a few seconds, you
  43. ;; should get your transactions in Ledger format. If you instead get a message
  44. ;; saying that the OFX data did not provide any FID, then you can provide a
  45. ;; random one in `ledger-import-accounts'.
  46. ;;
  47. ;; To fetch transactions from all configured accounts and convert them to Ledger
  48. ;; format, type `M-x ledger-import-all-accounts'. When this is finished, you
  49. ;; can open the result with `M-x ledger-import-pop-to-buffer'.
  50. ;;
  51. ;; If you keep manually modifying the Ledger transactions after they have been
  52. ;; converted, you might prefer to let ledger-import do that for you.
  53. ;; ledger-import gives you 2 ways to rewrite OFX data: either through the
  54. ;; `ledger-import-fetched-hook' or through `ledger-import-ofx-rewrite-rules'.
  55. ;;; Code:
  56. (require 'ledger-mode)
  57. (require 'seq)
  58. (defgroup ledger-import nil
  59. "Fetch OFX files and convert them into Ledger format."
  60. :group 'ledger)
  61. (defcustom ledger-import-accounts nil
  62. "Ledger accounts for which to fetch and convert data."
  63. :group 'ledger-import
  64. :type '(repeat
  65. (group
  66. (string
  67. :tag "Ledger account"
  68. :match ledger-import--non-empty-string-widget-matcher
  69. :doc "Account name (e.g., \"Assets:Current\") as known by Ledger"
  70. :format "%t: %v%h\n")
  71. (radio
  72. :tag "Fetcher"
  73. :value boobank
  74. :doc "Tool to use to get the account's OFX file"
  75. :format "%t: %v%h\n"
  76. (const :tag "Boobank" boobank))
  77. (string
  78. :tag "Boobank account name"
  79. :match ledger-import--non-empty-string-widget-matcher
  80. :doc "Account name as known by boobank"
  81. :format "%t: %v%h\n")
  82. (string
  83. :tag "FID"
  84. :value ""
  85. :doc "Use only if ledger-autosync complains about missing FID"
  86. :format "%t: %v%h\n"))))
  87. (defcustom ledger-import-autosync-command '("ledger-autosync" "--assertions")
  88. "List of strings with ledger-autosync command name and arguments."
  89. :group 'ledger-import
  90. :type '(repeat string))
  91. (defcustom ledger-import-boobank-command '("boobank")
  92. "List of strings with boobank command name and arguments."
  93. :group 'ledger-import
  94. :type '(repeat string))
  95. (defcustom ledger-import-boobank-import-from-date "2019-04-01"
  96. "String representing a date from which to import OFX data with boobank.
  97. The format is YYYY-MM-DD."
  98. :group 'ledger-import
  99. :type '(string
  100. :match (lambda (_ value)
  101. (string-match-p
  102. "[[:digit:]]\\{4\\}-[[:digit:]]\\{2\\}-[[:digit:]]\\{2\\}"
  103. value))))
  104. (defcustom ledger-import-ofx-rewrite-rules nil
  105. "List of (REGEXP . REPLACEMENT) to apply in an OFX buffer."
  106. :group 'ledger-import
  107. :type '(repeat
  108. (cons
  109. (regexp :tag "What to search for" :value "")
  110. (string :tag "What to replace it with" :value ""))))
  111. (defcustom ledger-import-fetched-hook '(ledger-import-ofx-rewrite)
  112. "Hook run when an OFX file is ready to be converted to Ledger format.
  113. The OFX buffer is made current before the hook is run."
  114. :group 'ledger-import
  115. :type 'hook)
  116. (defcustom ledger-import-finished-hook nil
  117. "Hook run when all transactions have been converted to Ledger format.
  118. The `ledger-import-buffer' is made current before the hook is run."
  119. :group 'ledger-import
  120. :type 'hook)
  121. (defun ledger-import-buffer ()
  122. "Return the buffer containing imported transactions."
  123. (get-buffer-create "*ledger sync*"))
  124. (defun ledger-import--finish-import (&optional buffer)
  125. "Cleanup BUFFER and run `ledger-import-finished-hook'.
  126. If BUFFER is nil, use `ledger-import-buffer' instead."
  127. (with-current-buffer (or buffer (ledger-import-buffer))
  128. (ledger-mode)
  129. (ledger-mode-clean-buffer)
  130. (run-hooks 'ledger-import-finished-hook)))
  131. (defun ledger-import--current-ledger-file ()
  132. "Return path to ledger file in current buffer, nil if none."
  133. (when (and (buffer-file-name) (derived-mode-p 'ledger-mode))
  134. (buffer-file-name)))
  135. ;;;###autoload
  136. (defun ledger-import-pop-to-buffer (&optional buffer)
  137. "Make BUFFER visible, `ledger-import-buffer' if nil."
  138. (interactive)
  139. (pop-to-buffer-same-window (or buffer (ledger-import-buffer))))
  140. (defun ledger-import--non-empty-string-widget-matcher (_widget value)
  141. "Return non-nil if VALUE is a non-empty string."
  142. (and (stringp value)
  143. (> (length value) 0)))
  144. (defun ledger-import-account-ledger-name (account)
  145. "Return ACCOUNT's name as known by your Ledger file.
  146. ACCOUNT is a list whose items are defined in `ledger-import-accounts'."
  147. (nth 0 account))
  148. (defun ledger-import-account-fetcher-id (account)
  149. "Return ACCOUNT's identifier as known by the fetcher.
  150. For example, this is the account ID that boobank uses.
  151. ACCOUNT is a list whose items are defined in `ledger-import-accounts'."
  152. (nth 2 account))
  153. (defun ledger-import-account-fid (account)
  154. "Return ACCOUNT's fid, or nil if none is necessary.
  155. This can be useful for ledger-autosync if the OFX data does not provide any.
  156. ACCOUNT is a list whose items are defined in `ledger-import-accounts'."
  157. (let ((fid (nth 3 account)))
  158. (unless (or (null fid) (string= fid ""))
  159. fid)))
  160. (defun ledger-import-choose-account ()
  161. "Ask the user to choose an account among `ledger-import-accounts'."
  162. (let* ((accounts ledger-import-accounts)
  163. (account-name (completing-read "Ledger account: "
  164. (mapcar #'ledger-import-account-ledger-name accounts)
  165. nil
  166. t)))
  167. (seq-find
  168. (lambda (account)
  169. (string= (ledger-import-account-ledger-name account) account-name))
  170. accounts)))
  171. ;;;###autoload
  172. (defun ledger-import-convert-ofx-to-ledger (account in-buffer &optional callback ledger-file)
  173. "Convert ofx data for ACCOUNT in IN-BUFFER to Ledger format.
  174. Display result in `ledger-import-buffer' and execute CALLBACK when done.
  175. `ledger-import-autosync-command' is used to do the conversion.
  176. ACCOUNT is a list whose items are defined in `ledger-import-accounts'.
  177. If LEDGER-FILE is non nil, use transactions from this file to
  178. guess related account names."
  179. (interactive (list (ledger-import-choose-account) (current-buffer) #'ledger-import-pop-to-buffer))
  180. (with-current-buffer in-buffer
  181. (let* ((ledger-name (ledger-import-account-ledger-name account))
  182. (fid (ledger-import-account-fid account))
  183. (file (make-temp-file "ledger-import-" nil ".ledger"))
  184. (command `(,@ledger-import-autosync-command
  185. ,@(when ledger-file `("--ledger" ,ledger-file))
  186. "--account" ,ledger-name
  187. ,@(when fid `("--fid" ,fid))
  188. ,file)))
  189. (write-region nil nil file nil 'no-message)
  190. (make-process
  191. :name "ledger-autosync"
  192. :buffer (ledger-import-buffer)
  193. :command command
  194. :sentinel (lambda (_process event)
  195. (when (and callback (string= event "finished\n"))
  196. (funcall callback))
  197. (when (string-prefix-p "exited abnormally" event)
  198. (pop-to-buffer-same-window (ledger-import-buffer))
  199. (error "There was a problem with ledger-autosync while importing %s" ledger-name)))))))
  200. (defun ledger-import--buffer-has-ofx-data (&optional buffer)
  201. "Return non-nil iff BUFFER, or current buffer, has OFX data."
  202. (with-current-buffer (or buffer (current-buffer))
  203. (save-excursion
  204. (goto-char (point-min))
  205. (and
  206. (looking-at-p "OFXHEADER")
  207. (search-forward "<OFX>")
  208. (search-forward "</OFX>")
  209. (or (= (point) (point-max))
  210. (= (point) (1- (point-max))))))))
  211. (defun ledger-import--buffer-empty-p (&optional buffer)
  212. "Return non-nil if BUFFER, or current buffer, is empty."
  213. (with-current-buffer (or buffer (current-buffer))
  214. (= (point-min) (point-max))))
  215. ;;;###autoload
  216. (defun ledger-import-fetch-boobank (fetcher-account &optional callback retry)
  217. "Use boobank to fetch OFX data for FETCHER-ACCOUNT, a string.
  218. When done, execute CALLBACK with buffer containing OFX data.
  219. RETRY is a number (default 3) indicating the number of times
  220. boobank is executed if it fails. This is because boobank tends
  221. to fail often and restarting usually solves the problem."
  222. (interactive (list (ledger-import-account-fetcher-id (ledger-import-choose-account)) #'ledger-import-pop-to-buffer))
  223. (let ((retry (or retry 3))
  224. (buffer (generate-new-buffer (format "*ledger-import-%s*" fetcher-account)))
  225. (error-buffer (generate-new-buffer (format "*ledger-import-%s <stderr>*" fetcher-account)))
  226. (command `(,@ledger-import-boobank-command
  227. "--formatter=ofx"
  228. "history"
  229. ,fetcher-account
  230. ,ledger-import-boobank-import-from-date)))
  231. (with-current-buffer buffer
  232. (message "Starting boobank for %s" fetcher-account)
  233. (make-process
  234. :name (format "boobank %s" fetcher-account)
  235. :buffer buffer
  236. :stderr error-buffer
  237. :command command
  238. :sentinel (lambda (_process event)
  239. (when (string= event "finished\n")
  240. (if (not (ledger-import--buffer-has-ofx-data buffer))
  241. (ledger-import--fetch-boobank-error retry fetcher-account callback error-buffer)
  242. (if (ledger-import--buffer-empty-p error-buffer)
  243. (kill-buffer error-buffer)
  244. (message "ledger-import: some errors have been logged in %s" error-buffer))
  245. (with-current-buffer buffer (run-hooks 'ledger-import-fetched-hook))
  246. (when callback (funcall callback buffer))))
  247. (when (string-prefix-p "exited abnormally" event)
  248. (ledger-import--fetch-boobank-error retry fetcher-account callback error-buffer)))))))
  249. (defun ledger-import--fetch-boobank-error (retry fetcher-account callback error-buffer)
  250. "Throw an error if RETRY is 0 or try starting boobank again.
  251. FETCHER-ACCOUNT and CALLBACK are the same as in `ledger-import-fetch-boobank'.
  252. ERROR-BUFFER is a buffer containing an error message explaining the problem."
  253. (if (>= retry 0)
  254. (ledger-import-fetch-boobank fetcher-account callback (1- retry))
  255. (pop-to-buffer-same-window error-buffer)
  256. (error "There was a problem with boobank while importing %s" fetcher-account)))
  257. (defun ledger-import-ofx-rewrite ()
  258. "Apply `ledger-import-ofx-rewrite-rules' to current buffer.
  259. The current buffer should be in the OFX format."
  260. (save-match-data
  261. (dolist (pair ledger-import-ofx-rewrite-rules)
  262. (goto-char (point-min))
  263. (while (re-search-forward (car pair) nil t)
  264. (replace-match (cdr pair) t)))))
  265. ;;;###autoload
  266. (defun ledger-import-account (account &optional callback ledger-file)
  267. "Fetch and convert transactions of ACCOUNT.
  268. Write the result in `ledger-import-buffer' and execute CALLBACK when done.
  269. ACCOUNT is a list whose items are defined in
  270. `ledger-import-accounts'. Interactively, user is asked to choose
  271. an account from `ledger-import-accounts'.
  272. If LEDGER-FILE is non nil, use transactions from this file to
  273. guess related account names."
  274. (interactive
  275. (list (ledger-import-choose-account) #'ledger-import--finish-import (ledger-import--current-ledger-file)))
  276. (ledger-import-fetch-boobank
  277. (ledger-import-account-fetcher-id account)
  278. (lambda (ofx-buffer)
  279. (ledger-import-convert-ofx-to-ledger
  280. account
  281. ofx-buffer
  282. (lambda ()
  283. (kill-buffer ofx-buffer)
  284. (when callback
  285. (funcall callback)))
  286. ledger-file))))
  287. (defun ledger-import--accounts (accounts &optional callback ledger-file)
  288. "Import all of ACCOUNTS and put the result in `ledger-import-buffer'.
  289. When done, execute CALLBACK.
  290. ACCOUNTs is a list similar to `ledger-import-accounts'.
  291. If LEDGER-FILE is non nil, use transactions from this file to
  292. guess related account names."
  293. (let ((finished-count 0))
  294. (dolist (account accounts)
  295. (ledger-import-account
  296. account
  297. (lambda ()
  298. (setf finished-count (1+ finished-count))
  299. (when (and (equal finished-count (length accounts))
  300. callback)
  301. (funcall callback)))
  302. ledger-file))))
  303. ;;;###autoload
  304. (defun ledger-import-all-accounts (&optional ledger-file)
  305. "Fetch transactions from all accounts and convert to Ledger format.
  306. Accounts are listed `ledger-import-accounts'.
  307. If LEDGER-FILE is non nil, use transactions from this file to
  308. guess related account names."
  309. (interactive (list (ledger-import--current-ledger-file)))
  310. (with-current-buffer (ledger-import-buffer) (erase-buffer))
  311. (ledger-import--accounts ledger-import-accounts #'ledger-import--finish-import ledger-file))
  312. (provide 'ledger-import)
  313. ;;; ledger-import.el ends here