Fix Gnucash security imports

Attempt to find an existing security that matches, trying decreasingly
specific searches, and create the security if nothing close can be
found.

errors.go

@@ -16,7 +16,8 @@ var error_codes = map[int]string{
 	2: "Unauthorized Access",
 	3: "Invalid Request",
 	4: "User Exists",
-	//  5:   "Connection Failed", //client-side error
+	//  5:   "Connection Failed", //reserved for client-side error
+	6:   "Import Error",
 	999: "Internal Error",
 }
 

gnucash.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"log"
@@ -27,19 +28,29 @@ func (gc *GnucashCommodity) UnmarshalXML(d *xml.Decoder, start xml.StartElement)
 		return err
 	}
 
-	gc.Security.Type = Stock // assumed default
-	if gxc.Type == "ISO4217" {
-		gc.Security.Type = Currency
-	}
 	gc.Name = gxc.Name
 	gc.Symbol = gxc.Name
 	gc.Description = gxc.Description
 	gc.AlternateId = gxc.XCode
+
+	gc.Security.Type = Stock // assumed default
+	if gxc.Type == "ISO4217" {
+		gc.Security.Type = Currency
+		// Get the number from our templates for the AlternateId because
+		// Gnucash uses 'id' (our Name) to supply the string ISO4217 code
+		template := FindSecurityTemplate(gxc.Name, Currency)
+		if template == nil {
+			return errors.New("Unable to find security template for Gnucash ISO4217 commodity")
+		}
+		gc.AlternateId = template.AlternateId
+		gc.Precision = template.Precision
+	} else {
 		if gxc.Fraction > 0 {
 			gc.Precision = int(math.Ceil(math.Log10(float64(gxc.Fraction))))
 		} else {
 			gc.Precision = 0
 		}
+	}
 	return nil
 }
 
@@ -252,7 +263,7 @@ func GnucashImportHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// assume there is only one 'part'
+	// Assume there is only one 'part' and it's the one we care about
 	part, err := multipartReader.NextPart()
 	if err != nil {
 		if err == io.EOF {
@@ -281,17 +292,16 @@ func GnucashImportHandler(w http.ResponseWriter, r *http.Request) {
 	// internal IDs
 	securityMap := make(map[int64]int64)
 	for _, security := range gnucashImport.Securities {
-		//TODO FIXME check on AlternateID also, and convert to the case
-		//where users have their own internal securities
-		s, err := GetSecurityByNameAndType(security.Name, security.Type)
+		securityId := security.SecurityId // save off because it could be updated
+		s, err := ImportGetCreateSecurity(sqltransaction, user, &security)
 		if err != nil {
-			//TODO attempt to create security if it doesn't exist
 			sqltransaction.Rollback()
-			WriteError(w, 999 /*Internal Error*/)
+			WriteError(w, 6 /*Import Error*/)
 			log.Print(err)
+			log.Print(security)
 			return
 		}
-		securityMap[security.SecurityId] = s.SecurityId
+		securityMap[securityId] = s.SecurityId
 	}
 
 	// Get/create accounts in the database, building a map from Gnucash account
@@ -349,7 +359,6 @@ func GnucashImportHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			split.AccountId = acctId
-			fmt.Printf("Setting split AccountId to %d\n", acctId)
 		}
 		err := InsertTransactionTx(sqltransaction, &transaction, user)
 		if err != nil {

reports.go

@@ -22,7 +22,7 @@ const (
 	balanceContextKey
 )
 
-const luaTimeoutSeconds time.Duration = 5 // maximum time a lua request can run for
+const luaTimeoutSeconds time.Duration = 30 // maximum time a lua request can run for
 
 type Series struct {
 	Values []float64

securities.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"gopkg.in/gorp.v1"
 	"log"
 	"net/http"
 	"net/url"
@@ -77,6 +78,15 @@ func SearchSecurityTemplates(search string, _type int64, limit int64) []*Securit
 	return results
 }
 
+func FindSecurityTemplate(name string, _type int64) *Security {
+	for _, security := range SecurityTemplates {
+		if name == security.Name && _type == security.Type {
+			return &security
+		}
+	}
+	return nil
+}
+
 func GetSecurity(securityid int64, userid int64) (*Security, error) {
 	var s Security
 
@@ -105,6 +115,14 @@ func InsertSecurity(s *Security) error {
 	return nil
 }
 
+func InsertSecurityTx(transaction *gorp.Transaction, s *Security) error {
+	err := transaction.Insert(s)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 func UpdateSecurity(s *Security) error {
 	transaction, err := DB.Begin()
 	if err != nil {
@@ -163,6 +181,59 @@ func DeleteSecurity(s *Security) error {
 	return nil
 }
 
+func ImportGetCreateSecurity(transaction *gorp.Transaction, user *User, security *Security) (*Security, error) {
+	security.UserId = user.UserId
+	if len(security.AlternateId) == 0 {
+		// Always create a new local security if we can't match on the AlternateId
+		err := InsertSecurityTx(transaction, security)
+		if err != nil {
+			return nil, err
+		}
+		return security, nil
+	}
+
+	var securities []*Security
+
+	_, err := transaction.Select(&securities, "SELECT * from securities where UserId=? AND Type=? AND AlternateId=? AND Precision=?", user.UserId, security.Type, security.AlternateId, security.Precision)
+	if err != nil {
+		return nil, err
+	}
+
+	// First try to find a case insensitive match on the name or symbol
+	upperName := strings.ToUpper(security.Name)
+	upperSymbol := strings.ToUpper(security.Symbol)
+	for _, s := range securities {
+		if (len(s.Name) > 0 && strings.ToUpper(s.Name) == upperName) ||
+			(len(s.Symbol) > 0 && strings.ToUpper(s.Symbol) == upperSymbol) {
+			return s, nil
+		}
+	}
+	//		if strings.Contains(strings.ToUpper(security.Name), upperSearch) ||
+
+	// Try to find a partial string match on the name or symbol
+	for _, s := range securities {
+		sUpperName := strings.ToUpper(s.Name)
+		sUpperSymbol := strings.ToUpper(s.Symbol)
+		if (len(upperName) > 0 && len(s.Name) > 0 && (strings.Contains(upperName, sUpperName) || strings.Contains(sUpperName, upperName))) ||
+			(len(upperSymbol) > 0 && len(s.Symbol) > 0 && (strings.Contains(upperSymbol, sUpperSymbol) || strings.Contains(sUpperSymbol, upperSymbol))) {
+			return s, nil
+		}
+	}
+
+	// Give up and return the first security in the list
+	if len(securities) > 0 {
+		return securities[0], nil
+	}
+
+	// If there wasn't even one security in the list, make a new one
+	err = InsertSecurityTx(transaction, security)
+	if err != nil {
+		return nil, err
+	}
+
+	return security, nil
+}
+
 func GetSecurityByName(name string) (*Security, error) {
 	return nil, fmt.Errorf("unimplemented")
 }