First pass at reorganizing go code into sub-packages

internal/handlers/scripts/gen_cusip_csv.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+QUARTER=2017q1
+
+function get_ticker() {
+	local cusip=$1
+
+	local tmpfile=$tmpdir/curl_tmpfile
+	curl -s -d "sopt=cusip&tickersymbol=${cusip}" http://quantumonline.com/search.cfm > $tmpfile
+	local quantum_name=$(sed -rn 's@<font size="\+1"><center><b>(.+)</b><br></center></font>\s*$@\1@p' $tmpfile | head -n1)
+	local quantum_ticker=$(sed -rn 's@^.*Ticker Symbol: ([A-Z\.0-9\-]+) &nbsp;&nbsp;&nbsp;&nbsp;CUSIP.*$@\1@p' $tmpfile | head -n1)
+
+	if [[ -z $quantum_ticker ]] || [[ -z $quantum_name ]]; then
+		curl -s -d "reqforlookup=REQUESTFORLOOKUP&productid=mmnet&isLoggedIn=mmnet&rows=50&for=stock&by=cusip&criteria=${cusip}&submit=Search" http://quotes.fidelity.com/mmnet/SymLookup.phtml > $tmpfile
+		fidelity_name=$(sed -rn 's@<tr><td height="20" nowrap><font class="smallfont">(.+)</font></td>\s*@\1@p' $tmpfile | sed -r 's/\&amp;/\&/')
+		fidelity_ticker=$(sed -rn 's@\s+<td align="center" width="20%"><font><a href="/webxpress/get_quote\?QUOTE_TYPE=\&SID_VALUE_ID=(.+)">(.+)</a></td>\s*@\1@p' $tmpfile | head -n1)
+		if [[ -z $fidelity_ticker ]] || [[ -z $fidelity_name ]]; then
+			echo $cusip >> $tmpdir/${QUARTER}_bad_cusips.csv
+		else
+			echo "$cusip,$fidelity_ticker,$fidelity_name"
+		fi
+	else
+		echo "$cusip,$quantum_ticker,$quantum_name"
+	fi
+}
+
+tmpdir=$(mktemp -d -p $PWD)
+
+# Get the list of CUSIPs from the SEC and generate a nicer format of it
+wget -q http://www.sec.gov/divisions/investment/13f/13flist${QUARTER}.pdf -O $tmpdir/13flist${QUARTER}.pdf
+pdftotext -layout $tmpdir/13flist${QUARTER}.pdf - > $tmpdir/13flist${QUARTER}.txt
+sed -rn 's/^([A-Z0-9]{6}) ([A-Z0-9]{2}) ([A-Z0-9]) .*$/\1\2\3/p' $tmpdir/13flist${QUARTER}.txt > $tmpdir/${QUARTER}_cusips
+
+# Find tickers and names for all the CUSIPs we can and print them out
+for cusip in $(cat $tmpdir/${QUARTER}_cusips); do
+	get_ticker $cusip
+done
+
+rm -rf $tmpdir

internal/handlers/scripts/gen_security_list.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python
+
+import csv
+from xml.dom import minidom
+import sys
+
+if sys.version_info[0] < 3:
+    from urllib2 import urlopen
+
+    # Allow writing utf-8 to stdout
+    import codecs
+    UTF8Writer = codecs.getwriter('utf8')
+    sys.stdout = UTF8Writer(sys.stdout)
+else:
+    from urllib.request import urlopen
+
+    # This is absent, but also unneeded in python3, so just return the string
+    def unicode(s, encoding):
+        return s
+
+class Security(object):
+    def __init__(self, name, description, number, _type, precision):
+        self.name = name
+        self.description = description
+        self.number = number
+        self.type = _type
+        self.precision = precision
+    def unicode(self):
+        s = """\tSecurity{
+\t\tName: \"%s\",
+\t\tDescription: \"%s\",
+\t\tSymbol: \"%s\",
+\t\tPrecision: %d,
+\t\tType: %s,
+\t\tAlternateId: \"%s\"},\n""" % (self.name, self.description, self.name, self.precision, self.type, str(self.number))
+        try:
+            return unicode(s, 'utf_8')
+        except TypeError:
+            return s
+
+class SecurityList(object):
+    def __init__(self, comment):
+        self.comment = comment
+        self.currencies = {}
+    def add(self, currency):
+        self.currencies[currency.number] = currency
+    def unicode(self):
+        string = "\t// "+self.comment+"\n"
+        for key in sorted(self.currencies.keys()):
+            string += self.currencies[key].unicode()
+        return string
+
+def process_ccyntry(currency_list, node):
+    name = ""
+    nameSet = False
+    number = 0
+    numberSet = False
+    description = ""
+    precision = 0
+    for n in node.childNodes:
+        if n.nodeName == "Ccy":
+            name = n.firstChild.nodeValue
+            nameSet = True
+        elif n.nodeName == "CcyNm":
+            description = n.firstChild.nodeValue
+        elif n.nodeName == "CcyNbr":
+            number = int(n.firstChild.nodeValue)
+            numberSet = True
+        elif n.nodeName == "CcyMnrUnts":
+            if n.firstChild.nodeValue == "N.A.":
+                precision = 0
+            else:
+                precision = int(n.firstChild.nodeValue)
+    if nameSet and numberSet:
+        currency_list.add(Security(name, description, number, "Currency", precision))
+
+def get_currency_list():
+    currency_list = SecurityList("ISO 4217, from http://www.currency-iso.org/en/home/tables/table-a1.html")
+
+    f = urlopen('http://www.currency-iso.org/dam/downloads/lists/list_one.xml')
+    xmldoc = minidom.parse(f)
+    for isonode in xmldoc.childNodes:
+        if isonode.nodeName == "ISO_4217":
+            for ccytblnode in isonode.childNodes:
+                if ccytblnode.nodeName == "CcyTbl":
+                    for ccyntrynode in ccytblnode.childNodes:
+                        if ccyntrynode.nodeName == "CcyNtry":
+                            process_ccyntry(currency_list, ccyntrynode)
+    f.close()
+    return currency_list
+
+def get_cusip_list(filename):
+    cusip_list = SecurityList("")
+    with open(filename) as csvfile:
+        csvreader = csv.reader(csvfile, delimiter=',')
+        for row in csvreader:
+            cusip = row[0]
+            name = row[1]
+            description = ",".join(row[2:])
+            cusip_list.add(Security(name, description, cusip, "Stock", 5))
+    return cusip_list
+
+def main():
+    currency_list = get_currency_list()
+    cusip_list = get_cusip_list('cusip_list.csv')
+
+    print("package handlers\n")
+    print("var SecurityTemplates = []Security{")
+    print(currency_list.unicode())
+    print(cusip_list.unicode())
+    print("}")
+
+if __name__ == "__main__":
+    main()