diff --git a/gnucash.go b/gnucash.go index c7ed50c..7c61298 100644 --- a/gnucash.go +++ b/gnucash.go @@ -167,12 +167,11 @@ func ImportGnucash(r io.Reader) (*GnucashImport, error) { } } a.Name = ga.Name - security, ok := securityMap[ga.Commodity.Name] - if ok { + if security, ok := securityMap[ga.Commodity.Name]; ok { + a.SecurityId = security.SecurityId } else { return nil, fmt.Errorf("Unable to find security: %s", ga.Commodity.Name) } - a.SecurityId = security.SecurityId //TODO find account types switch ga.Type { diff --git a/imports.go b/imports.go index 0d01709..5ad544d 100644 --- a/imports.go +++ b/imports.go @@ -2,11 +2,9 @@ package main import ( "io" - "io/ioutil" "log" "math/big" "net/http" - "os" ) /* @@ -15,13 +13,6 @@ import ( func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, accountid int64, importtype string) { //TODO branch off for different importtype's - // Return Account with this Id - account, err := GetAccount(accountid, user.UserId) - if err != nil { - WriteError(w, 3 /*Invalid Request*/) - return - } - multipartReader, err := r.MultipartReader() if err != nil { WriteError(w, 3 /*Invalid Request*/) @@ -40,28 +31,18 @@ func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, ac return } - f, err := ioutil.TempFile(tmpDir, user.Username+"_"+account.Name) - if err != nil { - WriteError(w, 999 /*Internal Error*/) - log.Print(err) - return - } - tmpFilename := f.Name() - defer os.Remove(tmpFilename) - - _, err = io.Copy(f, part) - f.Close() - if err != nil { - WriteError(w, 999 /*Internal Error*/) - log.Print(err) - return - } - - itl, err := ImportOFX(tmpFilename, account) + itl, err := ImportOFX(part) if err != nil { //TODO is this necessarily an invalid request (what if it was an error on our end)? WriteError(w, 3 /*Invalid Request*/) + log.Print(err) + return + } + + if len(itl.Accounts) != 1 { + WriteError(w, 3 /*Invalid Request*/) + log.Printf("Found %d accounts when importing OFX, expected 1", len(itl.Accounts)) return } @@ -72,17 +53,99 @@ func AccountImportHandler(w http.ResponseWriter, r *http.Request, user *User, ac return } + // Return Account with this Id + account, err := GetAccountTx(sqltransaction, accountid, user.UserId) + if err != nil { + sqltransaction.Rollback() + WriteError(w, 3 /*Invalid Request*/) + log.Print(err) + return + } + + importedAccount := itl.Accounts[0] + + if len(account.ExternalAccountId) > 0 && + account.ExternalAccountId != importedAccount.ExternalAccountId { + sqltransaction.Rollback() + WriteError(w, 3 /*Invalid Request*/) + log.Printf("OFX import has \"%s\" as ExternalAccountId, but the account being imported to has\"%s\"", + importedAccount.ExternalAccountId, + account.ExternalAccountId) + return + } + + if account.Type != importedAccount.Type { + sqltransaction.Rollback() + WriteError(w, 3 /*Invalid Request*/) + log.Printf("Expected %s account, found %s in OFX file", account.Type.String(), importedAccount.Type.String()) + return + } + + // Find matching existing securities or create new ones for those + // referenced by the OFX import. Also create a map from placeholder import + // SecurityIds to the actual SecurityIDs + var securitymap = make(map[int64]*Security) + for _, ofxsecurity := range itl.Securities { + security, err := ImportGetCreateSecurity(sqltransaction, user, &ofxsecurity) + if err != nil { + sqltransaction.Rollback() + WriteError(w, 999 /*Internal Error*/) + log.Print(err) + return + } + securitymap[ofxsecurity.SecurityId] = security + } + + if account.SecurityId != securitymap[importedAccount.SecurityId].SecurityId { + sqltransaction.Rollback() + WriteError(w, 3 /*Invalid Request*/) + log.Printf("OFX import account's SecurityId (%d) does not match this account's (%d)", securitymap[importedAccount.SecurityId].SecurityId, account.SecurityId) + return + } + + // TODO Ensure all transactions have at least one split in the account + // we're importing to? + var transactions []Transaction - for _, transaction := range *itl.Transactions { + for _, transaction := range itl.Transactions { transaction.UserId = user.UserId transaction.Status = Imported if !transaction.Valid() { sqltransaction.Rollback() - WriteError(w, 3 /*Invalid Request*/) + WriteError(w, 999 /*Internal Error*/) + log.Print("Unexpected invalid transaction from OFX import") return } + // Ensure that either AccountId or SecurityId is set for this split, + // and fixup the SecurityId to be a valid one for this user's actual + // securities instead of a placeholder from the import + for _, split := range transaction.Splits { + if split.AccountId != -1 { + if split.AccountId != importedAccount.AccountId { + sqltransaction.Rollback() + WriteError(w, 999 /*Internal Error*/) + return + } + split.AccountId = account.AccountId + } else if split.SecurityId != -1 { + if sec, ok := securitymap[split.SecurityId]; ok { + split.SecurityId = sec.SecurityId + } else { + sqltransaction.Rollback() + WriteError(w, 999 /*Internal Error*/) + log.Print("Couldn't find split's SecurityId in map during OFX import") + return + } + } else { + sqltransaction.Rollback() + WriteError(w, 999 /*Internal Error*/) + log.Print("Neither Split.AccountId Split.SecurityId was set during OFX import") + return + } + } + imbalances, err := transaction.GetImbalancesTx(sqltransaction) if err != nil { sqltransaction.Rollback() diff --git a/libofx.c b/libofx.c deleted file mode 100644 index 3fb2397..0000000 --- a/libofx.c +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include "_cgo_export.h" - -int ofx_statement_callback(const struct OfxStatementData statement_data, void *data) { - return OFXStatementCallback(statement_data, data); -} - -int ofx_account_callback(const struct OfxAccountData account_data, void *data) { - return OFXAccountCallback(account_data, data); -} - -int ofx_transaction_callback(const struct OfxTransactionData transaction_data, void *data) { - return OFXTransactionCallback(transaction_data, data); -} diff --git a/libofx.go b/libofx.go deleted file mode 100644 index 30e9ef4..0000000 --- a/libofx.go +++ /dev/null @@ -1,314 +0,0 @@ -package main - -//#cgo LDFLAGS: -lofx -// -//#include -// -// //The next line disables the definition of static variables to allow for it to -// //be included here (see libofx commit bd24df15531e52a2858f70487443af8b9fa407f4) -//#define OFX_AQUAMANIAC_UGLY_HACK1 -//#include -// -// typedef int (*ofx_statement_cb_fn) (const struct OfxStatementData, void *); -// extern int ofx_statement_callback(const struct OfxStatementData, void *); -// typedef int (*ofx_account_cb_fn) (const struct OfxAccountData, void *); -// extern int ofx_account_callback(const struct OfxAccountData, void *); -// typedef int (*ofx_transaction_cb_fn) (const struct OfxTransactionData, void *); -// extern int ofx_transaction_callback(const struct OfxTransactionData, void *); -import "C" - -import ( - "errors" - "math/big" - "time" - "unsafe" -) - -type ImportObject struct { - TransactionList OFXImport - Error error -} - -type OFXImport struct { - Account *Account - Transactions *[]Transaction - TotalTransactions int64 - BeginningBalance string - EndingBalance string -} - -func init() { - // Turn off all libofx info/debug messages - C.ofx_PARSER_msg = 0 - C.ofx_DEBUG_msg = 0 - C.ofx_DEBUG1_msg = 0 - C.ofx_DEBUG2_msg = 0 - C.ofx_DEBUG3_msg = 0 - C.ofx_DEBUG4_msg = 0 - C.ofx_DEBUG5_msg = 0 - C.ofx_STATUS_msg = 0 - C.ofx_INFO_msg = 0 - C.ofx_WARNING_msg = 0 - C.ofx_ERROR_msg = 0 -} - -//export OFXStatementCallback -func OFXStatementCallback(statement_data C.struct_OfxStatementData, data unsafe.Pointer) C.int { - // import := (*ImportObject)(data) - return 0 -} - -//export OFXAccountCallback -func OFXAccountCallback(account_data C.struct_OfxAccountData, data unsafe.Pointer) C.int { - iobj := (*ImportObject)(data) - itl := iobj.TransactionList - if account_data.account_id_valid != 0 { - account_name := C.GoString(&account_data.account_name[0]) - account_id := C.GoString(&account_data.account_id[0]) - itl.Account.Name = account_name - itl.Account.ExternalAccountId = account_id - } else { - if iobj.Error == nil { - iobj.Error = errors.New("OFX account ID invalid") - } - return 1 - } - if account_data.account_type_valid != 0 { - switch account_data.account_type { - case C.OFX_CHECKING, C.OFX_SAVINGS, C.OFX_MONEYMRKT, C.OFX_CMA: - itl.Account.Type = Bank - case C.OFX_CREDITLINE, C.OFX_CREDITCARD: - itl.Account.Type = Liability - case C.OFX_INVESTMENT: - itl.Account.Type = Investment - } - } else { - if iobj.Error == nil { - iobj.Error = errors.New("OFX account type invalid") - } - return 1 - } - if account_data.currency_valid != 0 { - currency_name := C.GoString(&account_data.currency[0]) - currency, err := GetSecurityByName(currency_name) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - itl.Account.SecurityId = currency.SecurityId - } else { - if iobj.Error == nil { - iobj.Error = errors.New("OFX account currency invalid") - } - return 1 - } - return 0 -} - -//export OFXTransactionCallback -func OFXTransactionCallback(transaction_data C.struct_OfxTransactionData, data unsafe.Pointer) C.int { - iobj := (*ImportObject)(data) - itl := iobj.TransactionList - transaction := new(Transaction) - - if transaction_data.name_valid != 0 { - transaction.Description = C.GoString(&transaction_data.name[0]) - } - // if transaction_data.reference_number_valid != 0 { - // fmt.Println("reference_number: ", C.GoString(&transaction_data.reference_number[0])) - // } - if transaction_data.date_posted_valid != 0 { - transaction.Date = time.Unix(int64(transaction_data.date_posted), 0) - } else if transaction_data.date_initiated_valid != 0 { - transaction.Date = time.Unix(int64(transaction_data.date_initiated), 0) - } - if transaction_data.fi_id_valid != 0 { - transaction.RemoteId = C.GoString(&transaction_data.fi_id[0]) - } - - if transaction_data.amount_valid != 0 { - split := new(Split) - r := new(big.Rat) - r.SetFloat64(float64(transaction_data.amount)) - security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - split.Amount = r.FloatString(security.Precision) - if transaction_data.memo_valid != 0 { - split.Memo = C.GoString(&transaction_data.memo[0]) - } - if transaction_data.check_number_valid != 0 { - split.Number = C.GoString(&transaction_data.check_number[0]) - } - split.SecurityId = -1 - split.AccountId = itl.Account.AccountId - transaction.Splits = append(transaction.Splits, split) - } else { - if iobj.Error == nil { - iobj.Error = errors.New("OFX transaction amount invalid") - } - return 1 - } - - var security *Security - var err error - split := new(Split) - units := new(big.Rat) - - if transaction_data.units_valid != 0 { - units.SetFloat64(float64(transaction_data.units)) - if transaction_data.security_data_valid != 0 { - security_data := transaction_data.security_data_ptr - if security_data.ticker_valid != 0 { - s, err := GetSecurityByName(C.GoString(&security_data.ticker[0])) - if err != nil { - if iobj.Error == nil { - iobj.Error = errors.New("Failed to find OFX transaction security: " + C.GoString(&security_data.ticker[0])) - } - return 1 - } - security = s - } else { - if iobj.Error == nil { - iobj.Error = errors.New("OFX security ticker invalid") - } - return 1 - } - if security.Type == Stock && security_data.unique_id_valid != 0 && security_data.unique_id_type_valid != 0 && C.GoString(&security_data.unique_id_type[0]) == "CUSIP" { - // Validate the security CUSIP, if possible - if security.AlternateId != C.GoString(&security_data.unique_id[0]) { - if iobj.Error == nil { - iobj.Error = errors.New("OFX transaction security CUSIP failed to validate") - } - return 1 - } - } - } else { - security, err = GetSecurity(itl.Account.SecurityId, itl.Account.UserId) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - } - } else { - // Calculate units from other available fields if its not present - // units = - (amount + various fees) / unitprice - units.SetFloat64(float64(transaction_data.amount)) - fees := new(big.Rat) - if transaction_data.fees_valid != 0 { - fees.SetFloat64(float64(-transaction_data.fees)) - } - if transaction_data.commission_valid != 0 { - commission := new(big.Rat) - commission.SetFloat64(float64(-transaction_data.commission)) - fees.Add(fees, commission) - } - units.Add(units, fees) - units.Neg(units) - if transaction_data.unitprice_valid != 0 && transaction_data.unitprice != 0 { - unitprice := new(big.Rat) - unitprice.SetFloat64(float64(transaction_data.unitprice)) - units.Quo(units, unitprice) - } - - // If 'units' wasn't present, assume we're using the account's security - security, err = GetSecurity(itl.Account.SecurityId, itl.Account.UserId) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - } - - split.Amount = units.FloatString(security.Precision) - split.SecurityId = security.SecurityId - split.AccountId = -1 - transaction.Splits = append(transaction.Splits, split) - - if transaction_data.fees_valid != 0 { - split := new(Split) - r := new(big.Rat) - r.SetFloat64(float64(-transaction_data.fees)) - security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - split.Amount = r.FloatString(security.Precision) - split.Memo = "fees" - split.SecurityId = itl.Account.SecurityId - split.AccountId = -1 - transaction.Splits = append(transaction.Splits, split) - } - - if transaction_data.commission_valid != 0 { - split := new(Split) - r := new(big.Rat) - r.SetFloat64(float64(-transaction_data.commission)) - security, err := GetSecurity(itl.Account.SecurityId, itl.Account.UserId) - if err != nil { - if iobj.Error == nil { - iobj.Error = err - } - return 1 - } - split.Amount = r.FloatString(security.Precision) - split.Memo = "commission" - split.SecurityId = itl.Account.SecurityId - split.AccountId = -1 - transaction.Splits = append(transaction.Splits, split) - } - - // if transaction_data.payee_id_valid != 0 { - // fmt.Println("payee_id: ", C.GoString(&transaction_data.payee_id[0])) - // } - - transaction_list := append(*itl.Transactions, *transaction) - iobj.TransactionList.Transactions = &transaction_list - - return 0 -} - -func ImportOFX(filename string, account *Account) (*OFXImport, error) { - var a Account - var t []Transaction - var iobj ImportObject - iobj.TransactionList.Account = &a - iobj.TransactionList.Transactions = &t - - a.AccountId = account.AccountId - - context := C.libofx_get_new_context() - defer C.libofx_free_context(context) - - C.ofx_set_statement_cb(context, C.ofx_statement_cb_fn(C.ofx_statement_callback), unsafe.Pointer(&iobj)) - C.ofx_set_account_cb(context, C.ofx_account_cb_fn(C.ofx_account_callback), unsafe.Pointer(&iobj)) - C.ofx_set_transaction_cb(context, C.ofx_transaction_cb_fn(C.ofx_transaction_callback), unsafe.Pointer(&iobj)) - - filename_cstring := C.CString(filename) - defer C.free(unsafe.Pointer(filename_cstring)) - C.libofx_proc_file(context, filename_cstring, C.OFX) // unconditionally returns 0. - - iobj.TransactionList.TotalTransactions = int64(len(*iobj.TransactionList.Transactions)) - - if iobj.TransactionList.TotalTransactions == 0 { - return nil, errors.New("No OFX transactions found") - } - - if iobj.Error != nil { - return nil, iobj.Error - } else { - return &iobj.TransactionList, nil - } -} diff --git a/ofx.go b/ofx.go new file mode 100644 index 0000000..b072e3f --- /dev/null +++ b/ofx.go @@ -0,0 +1,196 @@ +package main + +import ( + "errors" + "fmt" + "github.com/aclindsa/ofxgo" + "io" + "math/big" +) + +type OFXImport struct { + Securities []Security + Accounts []Account + Transactions []Transaction + // Balances map[int64]string // map AccountIDs to ending balances +} + +func (i *OFXImport) GetSecurity(ofxsecurityid int64) (*Security, error) { + if ofxsecurityid < 0 || ofxsecurityid > int64(len(i.Securities)) { + return nil, errors.New("OFXImport.GetSecurity: SecurityID out of range") + } + return &i.Securities[ofxsecurityid], nil +} + +func (i *OFXImport) GetAddCurrency(isoname string) (*Security, error) { + for _, security := range i.Securities { + if isoname == security.Name && Currency == security.Type { + return &security, nil + } + } + + template := FindSecurityTemplate(isoname, Currency) + if template == nil { + return nil, fmt.Errorf("Failed to find Security for \"%s\"", isoname) + } + var security Security = *template + security.SecurityId = int64(len(i.Securities) + 1) + i.Securities = append(i.Securities, security) + + return &security, nil +} + +func (i *OFXImport) AddTransaction(tran *ofxgo.Transaction, account *Account) error { + var t Transaction + + t.Status = Imported + t.Date = tran.DtPosted.UTC() + t.RemoteId = tran.FiTID.String() + // TODO CorrectFiTID/CorrectAction? + // Construct the description from whichever of the descriptive OFX fields are present + if len(tran.Name) > 0 { + t.Description = string(tran.Name) + } else if tran.Payee != nil { + t.Description = string(tran.Payee.Name) + } + if len(tran.Memo) > 0 { + if len(t.Description) > 0 { + t.Description = t.Description + " - " + string(tran.Memo) + } else { + t.Description = string(tran.Memo) + } + } + + var s1, s2 Split + if len(tran.ExtdName) > 0 { + s1.Memo = tran.ExtdName.String() + } + if len(tran.CheckNum) > 0 { + s1.Number = tran.CheckNum.String() + } else if len(tran.RefNum) > 0 { + s1.Number = tran.RefNum.String() + } + + amt := big.NewRat(0, 1) + // Convert TrnAmt to account's currency if Currency is set + if ok, _ := tran.Currency.Valid(); ok { + amt.Mul(&tran.Currency.CurRate.Rat, &tran.TrnAmt.Rat) + } else { + amt.Set(&tran.TrnAmt.Rat) + } + if account.SecurityId < 1 || account.SecurityId > int64(len(i.Securities)) { + return errors.New("Internal error: security index not found in OFX import\n") + } + security := i.Securities[account.SecurityId-1] + s1.Amount = amt.FloatString(security.Precision) + s2.Amount = amt.Neg(amt).FloatString(security.Precision) + + s1.AccountId = account.AccountId + s2.AccountId = -1 + s1.SecurityId = -1 + s2.SecurityId = security.SecurityId + + t.Splits = append(t.Splits, &s1) + t.Splits = append(t.Splits, &s2) + i.Transactions = append(i.Transactions, t) + + return nil +} + +func (i *OFXImport) importOFXBank(stmt *ofxgo.StatementResponse) error { + security, err := i.GetAddCurrency(stmt.CurDef.String()) + if err != nil { + return err + } + + account := Account{ + AccountId: int64(len(i.Accounts) + 1), + ExternalAccountId: stmt.BankAcctFrom.AcctID.String(), + SecurityId: security.SecurityId, + ParentAccountId: -1, + Type: Bank, + } + + for _, tran := range stmt.BankTranList.Transactions { + if err := i.AddTransaction(&tran, &account); err != nil { + return err + } + } + + i.Accounts = append(i.Accounts, account) + + return nil +} + +func (i *OFXImport) importOFXCC(stmt *ofxgo.CCStatementResponse) error { + security, err := i.GetAddCurrency(stmt.CurDef.String()) + if err != nil { + return err + } + + account := Account{ + AccountId: int64(len(i.Accounts) + 1), + ExternalAccountId: stmt.CCAcctFrom.AcctID.String(), + SecurityId: security.SecurityId, + ParentAccountId: -1, + Type: Bank, + } + i.Accounts = append(i.Accounts, account) + + for _, tran := range stmt.BankTranList.Transactions { + if err := i.AddTransaction(&tran, &account); err != nil { + return err + } + } + + return nil +} + +func (i *OFXImport) importOFXInv(stmt *ofxgo.InvStatementResponse) error { + // TODO + return errors.New("unimplemented") +} + +func ImportOFX(r io.Reader) (*OFXImport, error) { + var i OFXImport + + response, err := ofxgo.ParseResponse(r) + if err != nil { + return nil, fmt.Errorf("Unexpected error parsing OFX response: %s\n", err) + } + + if response.Signon.Status.Code != 0 { + meaning, _ := response.Signon.Status.CodeMeaning() + return nil, fmt.Errorf("Nonzero signon status (%d: %s) with message: %s\n", response.Signon.Status.Code, meaning, response.Signon.Status.Message) + } + + for _, bank := range response.Bank { + if stmt, ok := bank.(*ofxgo.StatementResponse); ok { + err = i.importOFXBank(stmt) + if err != nil { + return nil, err + } + return &i, nil + } + } + for _, cc := range response.CreditCard { + if stmt, ok := cc.(*ofxgo.CCStatementResponse); ok { + err = i.importOFXCC(stmt) + if err != nil { + return nil, err + } + return &i, nil + } + } + for _, inv := range response.InvStmt { + if stmt, ok := inv.(*ofxgo.InvStatementResponse); ok { + err = i.importOFXInv(stmt) + if err != nil { + return nil, err + } + return &i, nil + } + } + + return nil, errors.New("No OFX statement found") +} diff --git a/securities.go b/securities.go index 6e0272f..a8d5f89 100644 --- a/securities.go +++ b/securities.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "errors" - "fmt" "gopkg.in/gorp.v1" "log" "net/http" @@ -234,14 +233,6 @@ func ImportGetCreateSecurity(transaction *gorp.Transaction, user *User, security return security, nil } -func GetSecurityByName(name string) (*Security, error) { - return nil, fmt.Errorf("unimplemented") -} - -func GetSecurityByNameAndType(name string, _type int64) (*Security, error) { - return nil, fmt.Errorf("unimplemented") -} - func SecurityHandler(w http.ResponseWriter, r *http.Request) { user, err := GetUserFromSession(r) if err != nil { diff --git a/transactions.go b/transactions.go index 252b552..5598d7d 100644 --- a/transactions.go +++ b/transactions.go @@ -44,8 +44,7 @@ func (s *Split) GetAmount() (*big.Rat, error) { } func (s *Split) Valid() bool { - if (s.AccountId == -1 && s.SecurityId == -1) || - (s.AccountId != -1 && s.SecurityId != -1) { + if (s.AccountId == -1) == (s.SecurityId == -1) { return false } _, err := s.GetAmount()