Move OFX imports from libofx to ofxgo

This makes them native Go code, and will allow for fetching them
directly from financial institutions later.
This commit is contained in:
Aaron Lindsay 2017-05-08 06:01:26 -04:00
parent 32fffdf91b
commit a9cf95dba8
7 changed files with 291 additions and 371 deletions

View File

@ -167,12 +167,11 @@ func ImportGnucash(r io.Reader) (*GnucashImport, error) {
}
}
a.Name = ga.Name
security, ok := securityMap[ga.Commodity.Name]
if ok {
if security, ok := securityMap[ga.Commodity.Name]; ok {
a.SecurityId = security.SecurityId
} else {
return nil, fmt.Errorf("Unable to find security: %s", ga.Commodity.Name)
}
a.SecurityId = security.SecurityId
//TODO find account types
switch ga.Type {

View File

@ -2,11 +2,9 @@ package main
import (
"io"
"io/ioutil"
"log"
"math/big"
"net/http"
"os"
)
/*
@ -15,13 +13,6 @@ import (
func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, accountid int64, importtype string) {
//TODO branch off for different importtype's
// Return Account with this Id
account, err := GetAccount(accountid, user.UserId)
if err != nil {
WriteError(w, 3 /*Invalid Request*/)
return
}
multipartReader, err := r.MultipartReader()
if err != nil {
WriteError(w, 3 /*Invalid Request*/)
@ -40,28 +31,18 @@ func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, ac
return
}
f, err := ioutil.TempFile(tmpDir, user.Username+"_"+account.Name)
if err != nil {
WriteError(w, 999 /*Internal Error*/)
log.Print(err)
return
}
tmpFilename := f.Name()
defer os.Remove(tmpFilename)
_, err = io.Copy(f, part)
f.Close()
if err != nil {
WriteError(w, 999 /*Internal Error*/)
log.Print(err)
return
}
itl, err := ImportOFX(tmpFilename, account)
itl, err := ImportOFX(part)
if err != nil {
//TODO is this necessarily an invalid request (what if it was an error on our end)?
WriteError(w, 3 /*Invalid Request*/)
log.Print(err)
return
}
if len(itl.Accounts) != 1 {
WriteError(w, 3 /*Invalid Request*/)
log.Printf("Found %d accounts when importing OFX, expected 1", len(itl.Accounts))
return
}
@ -72,17 +53,99 @@ func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, ac
return
}
// Return Account with this Id
account, err := GetAccountTx(sqltransaction, accountid, user.UserId)
if err != nil {
sqltransaction.Rollback()
WriteError(w, 3 /*Invalid Request*/)
log.Print(err)
return
}
importedAccount := itl.Accounts[0]
if len(account.ExternalAccountId) > 0 &&
account.ExternalAccountId != importedAccount.ExternalAccountId {
sqltransaction.Rollback()
WriteError(w, 3 /*Invalid Request*/)
log.Printf("OFX import has \"%s\" as ExternalAccountId, but the account being imported to has\"%s\"",
importedAccount.ExternalAccountId,
account.ExternalAccountId)
return
}
if account.Type != importedAccount.Type {
sqltransaction.Rollback()
WriteError(w, 3 /*Invalid Request*/)
log.Printf("Expected %s account, found %s in OFX file", account.Type.String(), importedAccount.Type.String())
return
}
// Find matching existing securities or create new ones for those
// referenced by the OFX import. Also create a map from placeholder import
// SecurityIds to the actual SecurityIDs
var securitymap = make(map[int64]*Security)
for _, ofxsecurity := range itl.Securities {
security, err := ImportGetCreateSecurity(sqltransaction, user, &ofxsecurity)
if err != nil {
sqltransaction.Rollback()
WriteError(w, 999 /*Internal Error*/)
log.Print(err)
return
}
securitymap[ofxsecurity.SecurityId] = security
}
if account.SecurityId != securitymap[importedAccount.SecurityId].SecurityId {
sqltransaction.Rollback()
WriteError(w, 3 /*Invalid Request*/)
log.Printf("OFX import account's SecurityId (%d) does not match this account's (%d)", securitymap[importedAccount.SecurityId].SecurityId, account.SecurityId)
return
}
// TODO Ensure all transactions have at least one split in the account
// we're importing to?
var transactions []Transaction
for _, transaction := range *itl.Transactions {
for _, transaction := range itl.Transactions {
transaction.UserId = user.UserId
transaction.Status = Imported
if !transaction.Valid() {
sqltransaction.Rollback()
WriteError(w, 3 /*Invalid Request*/)
WriteError(w, 999 /*Internal Error*/)
log.Print("Unexpected invalid transaction from OFX import")
return
}
// Ensure that either AccountId or SecurityId is set for this split,
// and fixup the SecurityId to be a valid one for this user's actual
// securities instead of a placeholder from the import
for _, split := range transaction.Splits {
if split.AccountId != -1 {
if split.AccountId != importedAccount.AccountId {
sqltransaction.Rollback()
WriteError(w, 999 /*Internal Error*/)
return
}
split.AccountId = account.AccountId
} else if split.SecurityId != -1 {
if sec, ok := securitymap[split.SecurityId]; ok {
split.SecurityId = sec.SecurityId
} else {
sqltransaction.Rollback()
WriteError(w, 999 /*Internal Error*/)
log.Print("Couldn't find split's SecurityId in map during OFX import")
return
}
} else {
sqltransaction.Rollback()
WriteError(w, 999 /*Internal Error*/)
log.Print("Neither Split.AccountId Split.SecurityId was set during OFX import")
return
}
}
imbalances, err := transaction.GetImbalancesTx(sqltransaction)
if err != nil {
sqltransaction.Rollback()

View File

@ -1,14 +0,0 @@
#include <libofx/libofx.h>
#include "_cgo_export.h"
int ofx_statement_callback(const struct OfxStatementData statement_data, void *data) {
return OFXStatementCallback(statement_data, data);
}
int ofx_account_callback(const struct OfxAccountData account_data, void *data) {
return OFXAccountCallback(account_data, data);
}
int ofx_transaction_callback(const struct OfxTransactionData transaction_data, void *data) {
return OFXTransactionCallback(transaction_data, data);
}

314
libofx.go
View File

@ -1,314 +0,0 @@
package main
//#cgo LDFLAGS: -lofx
//
//#include <stdlib.h>
//
// //The next line disables the definition of static variables to allow for it to
// //be included here (see libofx commit bd24df15531e52a2858f70487443af8b9fa407f4)
//#define OFX_AQUAMANIAC_UGLY_HACK1
//#include <libofx/libofx.h>
//
// typedef int (*ofx_statement_cb_fn) (const struct OfxStatementData, void *);
// extern int ofx_statement_callback(const struct OfxStatementData, void *);
// typedef int (*ofx_account_cb_fn) (const struct OfxAccountData, void *);
// extern int ofx_account_callback(const struct OfxAccountData, void *);
// typedef int (*ofx_transaction_cb_fn) (const struct OfxTransactionData, void *);
// extern int ofx_transaction_callback(const struct OfxTransactionData, void *);
import "C"
import (
"errors"
"math/big"
"time"
"unsafe"
)
type ImportObject struct {
TransactionList OFXImport
Error error
}
type OFXImport struct {
Account *Account
Transactions *[]Transaction
TotalTransactions int64
BeginningBalance string
EndingBalance string
}
func init() {
// Turn off all libofx info/debug messages
C.ofx_PARSER_msg = 0
C.ofx_DEBUG_msg = 0
C.ofx_DEBUG1_msg = 0
C.ofx_DEBUG2_msg = 0
C.ofx_DEBUG3_msg = 0
C.ofx_DEBUG4_msg = 0
C.ofx_DEBUG5_msg = 0
C.ofx_STATUS_msg = 0
C.ofx_INFO_msg = 0
C.ofx_WARNING_msg = 0
C.ofx_ERROR_msg = 0
}
//export OFXStatementCallback
func OFXStatementCallback(statement_data C.struct_OfxStatementData, data unsafe.Pointer) C.int {
// import := (*ImportObject)(data)
return 0
}
//export OFXAccountCallback
func OFXAccountCallback(account_data C.struct_OfxAccountData, data unsafe.Pointer) C.int {
iobj := (*ImportObject)(data)
itl := iobj.TransactionList
if account_data.account_id_valid != 0 {
account_name := C.GoString(&account_data.account_name[0])
account_id := C.GoString(&account_data.account_id[0])
itl.Account.Name = account_name
itl.Account.ExternalAccountId = account_id
} else {
if iobj.Error == nil {
iobj.Error = errors.New("OFX account ID invalid")
}
return 1
}
if account_data.account_type_valid != 0 {
switch account_data.account_type {
case C.OFX_CHECKING, C.OFX_SAVINGS, C.OFX_MONEYMRKT, C.OFX_CMA:
itl.Account.Type = Bank
case C.OFX_CREDITLINE, C.OFX_CREDITCARD:
itl.Account.Type = Liability
case C.OFX_INVESTMENT:
itl.Account.Type = Investment
}
} else {
if iobj.Error == nil {
iobj.Error = errors.New("OFX account type invalid")
}
return 1
}
if account_data.currency_valid != 0 {
currency_name := C.GoString(&account_data.currency[0])
currency, err := GetSecurityByName(currency_name)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
itl.Account.SecurityId = currency.SecurityId
} else {
if iobj.Error == nil {
iobj.Error = errors.New("OFX account currency invalid")
}
return 1
}
return 0
}
//export OFXTransactionCallback
func OFXTransactionCallback(transaction_data C.struct_OfxTransactionData, data unsafe.Pointer) C.int {
iobj := (*ImportObject)(data)
itl := iobj.TransactionList
transaction := new(Transaction)
if transaction_data.name_valid != 0 {
transaction.Description = C.GoString(&transaction_data.name[0])
}
// if transaction_data.reference_number_valid != 0 {
// fmt.Println("reference_number: ", C.GoString(&transaction_data.reference_number[0]))
// }
if transaction_data.date_posted_valid != 0 {
transaction.Date = time.Unix(int64(transaction_data.date_posted), 0)
} else if transaction_data.date_initiated_valid != 0 {
transaction.Date = time.Unix(int64(transaction_data.date_initiated), 0)
}
if transaction_data.fi_id_valid != 0 {
transaction.RemoteId = C.GoString(&transaction_data.fi_id[0])
}
if transaction_data.amount_valid != 0 {
split := new(Split)
r := new(big.Rat)
r.SetFloat64(float64(transaction_data.amount))
security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
split.Amount = r.FloatString(security.Precision)
if transaction_data.memo_valid != 0 {
split.Memo = C.GoString(&transaction_data.memo[0])
}
if transaction_data.check_number_valid != 0 {
split.Number = C.GoString(&transaction_data.check_number[0])
}
split.SecurityId = -1
split.AccountId = itl.Account.AccountId
transaction.Splits = append(transaction.Splits, split)
} else {
if iobj.Error == nil {
iobj.Error = errors.New("OFX transaction amount invalid")
}
return 1
}
var security *Security
var err error
split := new(Split)
units := new(big.Rat)
if transaction_data.units_valid != 0 {
units.SetFloat64(float64(transaction_data.units))
if transaction_data.security_data_valid != 0 {
security_data := transaction_data.security_data_ptr
if security_data.ticker_valid != 0 {
s, err := GetSecurityByName(C.GoString(&security_data.ticker[0]))
if err != nil {
if iobj.Error == nil {
iobj.Error = errors.New("Failed to find OFX transaction security: " + C.GoString(&security_data.ticker[0]))
}
return 1
}
security = s
} else {
if iobj.Error == nil {
iobj.Error = errors.New("OFX security ticker invalid")
}
return 1
}
if security.Type == Stock && security_data.unique_id_valid != 0 && security_data.unique_id_type_valid != 0 && C.GoString(&security_data.unique_id_type[0]) == "CUSIP" {
// Validate the security CUSIP, if possible
if security.AlternateId != C.GoString(&security_data.unique_id[0]) {
if iobj.Error == nil {
iobj.Error = errors.New("OFX transaction security CUSIP failed to validate")
}
return 1
}
}
} else {
security, err = GetSecurity(itl.Account.SecurityId, itl.Account.UserId)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
}
} else {
// Calculate units from other available fields if its not present
// units = - (amount + various fees) / unitprice
units.SetFloat64(float64(transaction_data.amount))
fees := new(big.Rat)
if transaction_data.fees_valid != 0 {
fees.SetFloat64(float64(-transaction_data.fees))
}
if transaction_data.commission_valid != 0 {
commission := new(big.Rat)
commission.SetFloat64(float64(-transaction_data.commission))
fees.Add(fees, commission)
}
units.Add(units, fees)
units.Neg(units)
if transaction_data.unitprice_valid != 0 && transaction_data.unitprice != 0 {
unitprice := new(big.Rat)
unitprice.SetFloat64(float64(transaction_data.unitprice))
units.Quo(units, unitprice)
}
// If 'units' wasn't present, assume we're using the account's security
security, err = GetSecurity(itl.Account.SecurityId, itl.Account.UserId)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
}
split.Amount = units.FloatString(security.Precision)
split.SecurityId = security.SecurityId
split.AccountId = -1
transaction.Splits = append(transaction.Splits, split)
if transaction_data.fees_valid != 0 {
split := new(Split)
r := new(big.Rat)
r.SetFloat64(float64(-transaction_data.fees))
security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
split.Amount = r.FloatString(security.Precision)
split.Memo = "fees"
split.SecurityId = itl.Account.SecurityId
split.AccountId = -1
transaction.Splits = append(transaction.Splits, split)
}
if transaction_data.commission_valid != 0 {
split := new(Split)
r := new(big.Rat)
r.SetFloat64(float64(-transaction_data.commission))
security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId)
if err != nil {
if iobj.Error == nil {
iobj.Error = err
}
return 1
}
split.Amount = r.FloatString(security.Precision)
split.Memo = "commission"
split.SecurityId = itl.Account.SecurityId
split.AccountId = -1
transaction.Splits = append(transaction.Splits, split)
}
// if transaction_data.payee_id_valid != 0 {
// fmt.Println("payee_id: ", C.GoString(&transaction_data.payee_id[0]))
// }
transaction_list := append(*itl.Transactions, *transaction)
iobj.TransactionList.Transactions = &transaction_list
return 0
}
func ImportOFX(filename string, account *Account) (*OFXImport, error) {
var a Account
var t []Transaction
var iobj ImportObject
iobj.TransactionList.Account = &a
iobj.TransactionList.Transactions = &t
a.AccountId = account.AccountId
context := C.libofx_get_new_context()
defer C.libofx_free_context(context)
C.ofx_set_statement_cb(context, C.ofx_statement_cb_fn(C.ofx_statement_callback), unsafe.Pointer(&iobj))
C.ofx_set_account_cb(context, C.ofx_account_cb_fn(C.ofx_account_callback), unsafe.Pointer(&iobj))
C.ofx_set_transaction_cb(context, C.ofx_transaction_cb_fn(C.ofx_transaction_callback), unsafe.Pointer(&iobj))
filename_cstring := C.CString(filename)
defer C.free(unsafe.Pointer(filename_cstring))
C.libofx_proc_file(context, filename_cstring, C.OFX) // unconditionally returns 0.
iobj.TransactionList.TotalTransactions = int64(len(*iobj.TransactionList.Transactions))
if iobj.TransactionList.TotalTransactions == 0 {
return nil, errors.New("No OFX transactions found")
}
if iobj.Error != nil {
return nil, iobj.Error
} else {
return &iobj.TransactionList, nil
}
}

196
ofx.go Normal file
View File

@ -0,0 +1,196 @@
package main
import (
"errors"
"fmt"
"github.com/aclindsa/ofxgo"
"io"
"math/big"
)
type OFXImport struct {
Securities []Security
Accounts []Account
Transactions []Transaction
// Balances map[int64]string // map AccountIDs to ending balances
}
func (i *OFXImport) GetSecurity(ofxsecurityid int64) (*Security, error) {
if ofxsecurityid < 0 || ofxsecurityid > int64(len(i.Securities)) {
return nil, errors.New("OFXImport.GetSecurity: SecurityID out of range")
}
return &i.Securities[ofxsecurityid], nil
}
func (i *OFXImport) GetAddCurrency(isoname string) (*Security, error) {
for _, security := range i.Securities {
if isoname == security.Name && Currency == security.Type {
return &security, nil
}
}
template := FindSecurityTemplate(isoname, Currency)
if template == nil {
return nil, fmt.Errorf("Failed to find Security for \"%s\"", isoname)
}
var security Security = *template
security.SecurityId = int64(len(i.Securities) + 1)
i.Securities = append(i.Securities, security)
return &security, nil
}
func (i *OFXImport) AddTransaction(tran *ofxgo.Transaction, account *Account) error {
var t Transaction
t.Status = Imported
t.Date = tran.DtPosted.UTC()
t.RemoteId = tran.FiTID.String()
// TODO CorrectFiTID/CorrectAction?
// Construct the description from whichever of the descriptive OFX fields are present
if len(tran.Name) > 0 {
t.Description = string(tran.Name)
} else if tran.Payee != nil {
t.Description = string(tran.Payee.Name)
}
if len(tran.Memo) > 0 {
if len(t.Description) > 0 {
t.Description = t.Description + " - " + string(tran.Memo)
} else {
t.Description = string(tran.Memo)
}
}
var s1, s2 Split
if len(tran.ExtdName) > 0 {
s1.Memo = tran.ExtdName.String()
}
if len(tran.CheckNum) > 0 {
s1.Number = tran.CheckNum.String()
} else if len(tran.RefNum) > 0 {
s1.Number = tran.RefNum.String()
}
amt := big.NewRat(0, 1)
// Convert TrnAmt to account's currency if Currency is set
if ok, _ := tran.Currency.Valid(); ok {
amt.Mul(&tran.Currency.CurRate.Rat, &tran.TrnAmt.Rat)
} else {
amt.Set(&tran.TrnAmt.Rat)
}
if account.SecurityId < 1 || account.SecurityId > int64(len(i.Securities)) {
return errors.New("Internal error: security index not found in OFX import\n")
}
security := i.Securities[account.SecurityId-1]
s1.Amount = amt.FloatString(security.Precision)
s2.Amount = amt.Neg(amt).FloatString(security.Precision)
s1.AccountId = account.AccountId
s2.AccountId = -1
s1.SecurityId = -1
s2.SecurityId = security.SecurityId
t.Splits = append(t.Splits, &s1)
t.Splits = append(t.Splits, &s2)
i.Transactions = append(i.Transactions, t)
return nil
}
func (i *OFXImport) importOFXBank(stmt *ofxgo.StatementResponse) error {
security, err := i.GetAddCurrency(stmt.CurDef.String())
if err != nil {
return err
}
account := Account{
AccountId: int64(len(i.Accounts) + 1),
ExternalAccountId: stmt.BankAcctFrom.AcctID.String(),
SecurityId: security.SecurityId,
ParentAccountId: -1,
Type: Bank,
}
for _, tran := range stmt.BankTranList.Transactions {
if err := i.AddTransaction(&tran, &account); err != nil {
return err
}
}
i.Accounts = append(i.Accounts, account)
return nil
}
func (i *OFXImport) importOFXCC(stmt *ofxgo.CCStatementResponse) error {
security, err := i.GetAddCurrency(stmt.CurDef.String())
if err != nil {
return err
}
account := Account{
AccountId: int64(len(i.Accounts) + 1),
ExternalAccountId: stmt.CCAcctFrom.AcctID.String(),
SecurityId: security.SecurityId,
ParentAccountId: -1,
Type: Bank,
}
i.Accounts = append(i.Accounts, account)
for _, tran := range stmt.BankTranList.Transactions {
if err := i.AddTransaction(&tran, &account); err != nil {
return err
}
}
return nil
}
func (i *OFXImport) importOFXInv(stmt *ofxgo.InvStatementResponse) error {
// TODO
return errors.New("unimplemented")
}
func ImportOFX(r io.Reader) (*OFXImport, error) {
var i OFXImport
response, err := ofxgo.ParseResponse(r)
if err != nil {
return nil, fmt.Errorf("Unexpected error parsing OFX response: %s\n", err)
}
if response.Signon.Status.Code != 0 {
meaning, _ := response.Signon.Status.CodeMeaning()
return nil, fmt.Errorf("Nonzero signon status (%d: %s) with message: %s\n", response.Signon.Status.Code, meaning, response.Signon.Status.Message)
}
for _, bank := range response.Bank {
if stmt, ok := bank.(*ofxgo.StatementResponse); ok {
err = i.importOFXBank(stmt)
if err != nil {
return nil, err
}
return &i, nil
}
}
for _, cc := range response.CreditCard {
if stmt, ok := cc.(*ofxgo.CCStatementResponse); ok {
err = i.importOFXCC(stmt)
if err != nil {
return nil, err
}
return &i, nil
}
}
for _, inv := range response.InvStmt {
if stmt, ok := inv.(*ofxgo.InvStatementResponse); ok {
err = i.importOFXInv(stmt)
if err != nil {
return nil, err
}
return &i, nil
}
}
return nil, errors.New("No OFX statement found")
}

View File

@ -3,7 +3,6 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"gopkg.in/gorp.v1"
"log"
"net/http"
@ -234,14 +233,6 @@ func ImportGetCreateSecurity(transaction *gorp.Transaction, user *User, security
return security, nil
}
func GetSecurityByName(name string) (*Security, error) {
return nil, fmt.Errorf("unimplemented")
}
func GetSecurityByNameAndType(name string, _type int64) (*Security, error) {
return nil, fmt.Errorf("unimplemented")
}
func SecurityHandler(w http.ResponseWriter, r *http.Request) {
user, err := GetUserFromSession(r)
if err != nil {

View File

@ -44,8 +44,7 @@ func (s *Split) GetAmount() (*big.Rat, error) {
}
func (s *Split) Valid() bool {
if (s.AccountId == -1 && s.SecurityId == -1) ||
(s.AccountId != -1 && s.SecurityId != -1) {
if (s.AccountId == -1) == (s.SecurityId == -1) {
return false
}
_, err := s.GetAmount()