Store currency/security values/prices using big.Rat natively

This adds 'shadow' types used only by the store/db internal package whch handle converting these types to their DB-equivalent values. This change should allow reports to be generated significantly faster since it allows a large portion of the computation to be shifted to the database engines.
2025-06-14 13:58:37 -04:00 · 2017-12-12 19:40:38 -05:00
parent 483adb5c56
commit a357d38eee
22 changed files with 695 additions and 201 deletions
--- a/internal/handlers/gnucash.go
+++ b/internal/handlers/gnucash.go
@ -10,7 +10,6 @@ import (
 	"io"
 	"log"
 	"math"
-	"math/big"
 	"net/http"
 	"time"
 )
@ -49,7 +48,7 @@ func (gc *GnucashCommodity) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
 		gc.Precision = template.Precision
 	} else {
 		if gxc.Fraction > 0 {
-			gc.Precision = int(math.Ceil(math.Log10(float64(gxc.Fraction))))
+			gc.Precision = uint64(math.Ceil(math.Log10(float64(gxc.Fraction))))
 		} else {
 			gc.Precision = 0
 		}
@ -178,13 +177,14 @@ func ImportGnucash(r io.Reader) (*GnucashImport, error) {
 		p.CurrencyId = currency.SecurityId
 		p.Date = price.Date.Date.Time

-		var r big.Rat
-		_, ok = r.SetString(price.Value)
-		if ok {
-			p.Value = r.FloatString(currency.Precision)
-		} else {
+		_, ok = p.Value.SetString(price.Value)
+		if !ok {
 			return nil, fmt.Errorf("Can't set price value: %s", price.Value)
 		}
+		if p.Value.Precision() > currency.Precision {
+			// TODO we're possibly losing data here... but do we care?
+			p.Value.Round(currency.Precision)
+		}

 		p.RemoteId = "gnucash:" + price.Id
 		gncimport.Prices = append(gncimport.Prices, p)
@ -293,13 +293,13 @@ func ImportGnucash(r io.Reader) (*GnucashImport, error) {
 			s.Number = gt.Number
 			s.Memo = gs.Memo

-			var r big.Rat
-			_, ok = r.SetString(gs.Amount)
-			if ok {
-				s.Amount = r.FloatString(security.Precision)
-			} else {
+			_, ok = s.Amount.SetString(gs.Amount)
+			if !ok {
 				return nil, fmt.Errorf("Can't set split Amount: %s", gs.Amount)
 			}
+			if s.Amount.Precision() > security.Precision {
+				return nil, fmt.Errorf("Imported price's precision (%d) is greater than the security's (%s)\n", s.Amount.Precision(), security)
+			}

 			t.Splits = append(t.Splits, s)
 		}
@ -356,6 +356,7 @@ func GnucashImportHandler(r *http.Request, context *Context) ResponseWriterWrite
 	}

 	if err != nil {
+		log.Print(err)
 		return NewError(3 /*Invalid Request*/)
 	}

--- a/internal/handlers/imports.go
+++ b/internal/handlers/imports.go
@ -164,7 +164,11 @@ func ofxImportHelper(tx store.Tx, r io.Reader, user *models.User, accountid int6
 					log.Print(err)
 					return NewError(999 /*Internal Error*/)
 				}
-				split.Amount = r.FloatString(security.Precision)
+				split.Amount.Rat = *r
+				if split.Amount.Precision() > security.Precision {
+					log.Printf("Precision on created imbalance-correction split (%d) greater than the underlying security (%s) allows (%d)", split.Amount.Precision(), security, security.Precision)
+					return NewError(999 /*Internal Error*/)
+				}
 				split.SecurityId = -1
 				split.AccountId = imbalanced_account.AccountId
 				transaction.Splits = append(transaction.Splits, split)
--- a/internal/handlers/ofx.go
+++ b/internal/handlers/ofx.go
@ -97,9 +97,12 @@ func (i *OFXImport) AddTransaction(tran *ofxgo.Transaction, account *models.Acco
 	s1.ImportSplitType = models.ImportAccount
 	s2.ImportSplitType = models.ExternalAccount

+	s1.Amount.Rat = *amt
+	s2.Amount.Rat = *amt.Neg(amt)
 	security := i.Securities[account.SecurityId-1]
-	s1.Amount = amt.FloatString(security.Precision)
-	s2.Amount = amt.Neg(amt).FloatString(security.Precision)
+	if s1.Amount.Precision() > security.Precision {
+		return errors.New("Imported transaction amount is too precise for security")
+	}

 	s1.Status = models.Imported
 	s2.Status = models.Imported
@ -262,7 +265,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 			Memo:            memo + "(commission)",
-			Amount:          commission.FloatString(curdef.Precision),
+			Amount:          models.Amount{commission},
 		})
 	}
 	if num := taxes.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -274,7 +277,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 			Memo:            memo + "(taxes)",
-			Amount:          taxes.FloatString(curdef.Precision),
+			Amount:          models.Amount{taxes},
 		})
 	}
 	if num := fees.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -286,7 +289,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 			Memo:            memo + "(fees)",
-			Amount:          fees.FloatString(curdef.Precision),
+			Amount:          models.Amount{fees},
 		})
 	}
 	if num := load.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -298,7 +301,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 			Memo:            memo + "(load)",
-			Amount:          load.FloatString(curdef.Precision),
+			Amount:          models.Amount{load},
 		})
 	}
 	t.Splits = append(t.Splits, &models.Split{
@ -309,7 +312,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	t.Splits = append(t.Splits, &models.Split{
 		// TODO ReversalFiTID?
@ -319,7 +322,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          tradingTotal.FloatString(curdef.Precision),
+		Amount:          models.Amount{tradingTotal},
 	})

 	var units big.Rat
@ -332,7 +335,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})
 	units.Neg(&units)
 	t.Splits = append(t.Splits, &models.Split{
@ -343,7 +346,7 @@ func (i *OFXImport) GetInvBuyTran(buy *ofxgo.InvBuy, curdef *models.Security, ac
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + buy.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})

 	return &t, nil
@ -378,7 +381,7 @@ func (i *OFXImport) GetIncomeTran(income *ofxgo.Income, curdef *models.Security,
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + income.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	total.Neg(&total)
 	t.Splits = append(t.Splits, &models.Split{
@ -389,7 +392,7 @@ func (i *OFXImport) GetIncomeTran(income *ofxgo.Income, curdef *models.Security,
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + income.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})

 	return &t, nil
@ -423,7 +426,7 @@ func (i *OFXImport) GetInvExpenseTran(expense *ofxgo.InvExpense, curdef *models.
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + expense.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	total.Neg(&total)
 	t.Splits = append(t.Splits, &models.Split{
@ -434,7 +437,7 @@ func (i *OFXImport) GetInvExpenseTran(expense *ofxgo.InvExpense, curdef *models.
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + expense.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})

 	return &t, nil
@ -462,7 +465,7 @@ func (i *OFXImport) GetMarginInterestTran(marginint *ofxgo.MarginInterest, curde
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + marginint.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	total.Neg(&total)
 	t.Splits = append(t.Splits, &models.Split{
@ -473,7 +476,7 @@ func (i *OFXImport) GetMarginInterestTran(marginint *ofxgo.MarginInterest, curde
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + marginint.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})

 	return &t, nil
@ -526,7 +529,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 			Memo:            memo + "(commission)",
-			Amount:          commission.FloatString(curdef.Precision),
+			Amount:          models.Amount{commission},
 		})
 	}
 	if num := taxes.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -538,7 +541,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 			Memo:            memo + "(taxes)",
-			Amount:          taxes.FloatString(curdef.Precision),
+			Amount:          models.Amount{taxes},
 		})
 	}
 	if num := fees.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -550,7 +553,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 			Memo:            memo + "(fees)",
-			Amount:          fees.FloatString(curdef.Precision),
+			Amount:          models.Amount{fees},
 		})
 	}
 	if num := load.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -562,7 +565,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 			Memo:            memo + "(load)",
-			Amount:          load.FloatString(curdef.Precision),
+			Amount:          models.Amount{load},
 		})
 	}
 	t.Splits = append(t.Splits, &models.Split{
@ -573,7 +576,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})

 	t.Splits = append(t.Splits, &models.Split{
@ -584,7 +587,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	total.Neg(&total)
 	t.Splits = append(t.Splits, &models.Split{
@ -595,7 +598,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	t.Splits = append(t.Splits, &models.Split{
 		// TODO ReversalFiTID?
@ -605,7 +608,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          tradingTotal.FloatString(curdef.Precision),
+		Amount:          models.Amount{tradingTotal},
 	})

 	var units big.Rat
@ -618,7 +621,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})
 	units.Neg(&units)
 	t.Splits = append(t.Splits, &models.Split{
@ -629,7 +632,7 @@ func (i *OFXImport) GetReinvestTran(reinvest *ofxgo.Reinvest, curdef *models.Sec
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + reinvest.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})

 	return &t, nil
@ -663,7 +666,7 @@ func (i *OFXImport) GetRetOfCapTran(retofcap *ofxgo.RetOfCap, curdef *models.Sec
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + retofcap.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	total.Neg(&total)
 	t.Splits = append(t.Splits, &models.Split{
@ -674,7 +677,7 @@ func (i *OFXImport) GetRetOfCapTran(retofcap *ofxgo.RetOfCap, curdef *models.Sec
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + retofcap.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})

 	return &t, nil
@ -730,7 +733,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 			Memo:            memo + "(commission)",
-			Amount:          commission.FloatString(curdef.Precision),
+			Amount:          models.Amount{commission},
 		})
 	}
 	if num := taxes.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -742,7 +745,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 			Memo:            memo + "(taxes)",
-			Amount:          taxes.FloatString(curdef.Precision),
+			Amount:          models.Amount{taxes},
 		})
 	}
 	if num := fees.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -754,7 +757,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 			Memo:            memo + "(fees)",
-			Amount:          fees.FloatString(curdef.Precision),
+			Amount:          models.Amount{fees},
 		})
 	}
 	if num := load.Num(); !num.IsInt64() || num.Int64() != 0 {
@ -766,7 +769,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 			SecurityId:      curdef.SecurityId,
 			RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 			Memo:            memo + "(load)",
-			Amount:          load.FloatString(curdef.Precision),
+			Amount:          models.Amount{load},
 		})
 	}
 	t.Splits = append(t.Splits, &models.Split{
@ -777,7 +780,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 		SecurityId:      -1,
 		RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          total.FloatString(curdef.Precision),
+		Amount:          models.Amount{total},
 	})
 	t.Splits = append(t.Splits, &models.Split{
 		// TODO ReversalFiTID?
@ -787,7 +790,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 		SecurityId:      curdef.SecurityId,
 		RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          tradingTotal.FloatString(curdef.Precision),
+		Amount:          models.Amount{tradingTotal},
 	})

 	var units big.Rat
@ -800,7 +803,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})
 	units.Neg(&units)
 	t.Splits = append(t.Splits, &models.Split{
@ -811,7 +814,7 @@ func (i *OFXImport) GetInvSellTran(sell *ofxgo.InvSell, curdef *models.Security,
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + sell.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})

 	return &t, nil
@ -842,7 +845,7 @@ func (i *OFXImport) GetTransferTran(transfer *ofxgo.Transfer, account *models.Ac
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + transfer.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})
 	units.Neg(&units)
 	t.Splits = append(t.Splits, &models.Split{
@ -853,7 +856,7 @@ func (i *OFXImport) GetTransferTran(transfer *ofxgo.Transfer, account *models.Ac
 		SecurityId:      security.SecurityId,
 		RemoteId:        "ofx:" + transfer.InvTran.FiTID.String(),
 		Memo:            memo,
-		Amount:          units.FloatString(security.Precision),
+		Amount:          models.Amount{units},
 	})

 	return &t, nil
--- a/internal/handlers/transactions.go
+++ b/internal/handlers/transactions.go
@ -31,9 +31,8 @@ func GetTransactionImbalances(tx store.Tx, t *models.Transaction) (map[int64]big
 			}
 			securityid = account.SecurityId
 		}
-		amount, _ := t.Splits[i].GetAmount()
 		sum := sums[securityid]
-		(&sum).Add(&sum, amount)
+		(&sum).Add(&sum, &t.Splits[i].Amount.Rat)
 		sums[securityid] = sum
 	}
 	return sums, nil