From 7ffd12b144a19ee8dc9d6597d0163e71961dc4fa Mon Sep 17 00:00:00 2001 From: Aaron Lindsay Date: Fri, 14 Oct 2016 20:50:50 -0400 Subject: [PATCH] Add scripts to generate lists of securities --- Makefile | 8 ++- scripts/gen_cusip_csv.sh | 38 +++++++++++++++ scripts/gen_security_list.py | 95 ++++++++++++++++++++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100755 scripts/gen_cusip_csv.sh create mode 100755 scripts/gen_security_list.py diff --git a/Makefile b/Makefile index 550a295..32d1d02 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ JS_SOURCES = $(wildcard js/*.js) $(wildcard js/*/*.js) -all: static/bundle.js static/react-widgets +all: static/bundle.js static/react-widgets security_templates.go node_modules: npm install @@ -11,4 +11,10 @@ static/bundle.js: $(JS_SOURCES) node_modules static/react-widgets: node_modules/react-widgets/dist node_modules rsync -a node_modules/react-widgets/dist/ static/react-widgets/ +security_templates.go: cusip_list.csv + ./scripts/gen_security_list.py > security_templates.go + +cusip_list.csv: + ./scripts/gen_cusip_csv.sh > cusip_list.csv + .PHONY = all diff --git a/scripts/gen_cusip_csv.sh b/scripts/gen_cusip_csv.sh new file mode 100755 index 0000000..394efb6 --- /dev/null +++ b/scripts/gen_cusip_csv.sh @@ -0,0 +1,38 @@ +#!/bin/bash +QUARTER=2016q3 + +function get_ticker() { + local cusip=$1 + + local tmpfile=$tmpdir/curl_tmpfile + curl -s -d "sopt=cusip&tickersymbol=${cusip}" http://quantumonline.com/search.cfm > $tmpfile + local quantum_name=$(sed -rn 's@
(.+)
\s*$@\1@p' $tmpfile | head -n1) + local quantum_ticker=$(sed -rn 's@^.*Ticker Symbol: ([A-Z\.0-9\-]+)     CUSIP.*$@\1@p' $tmpfile | head -n1) + + if [[ -z $quantum_ticker ]] || [[ -z $quantum_name ]]; then + curl -s -d "reqforlookup=REQUESTFORLOOKUP&productid=mmnet&isLoggedIn=mmnet&rows=50&for=stock&by=cusip&criteria=${cusip}&submit=Search" http://quotes.fidelity.com/mmnet/SymLookup.phtml > $tmpfile + fidelity_name=$(sed -rn 's@(.+)\s*@\1@p' $tmpfile | sed -r 's/\&/\&/') + fidelity_ticker=$(sed -rn 's@\s+(.+)\s*@\1@p' $tmpfile | head -n1) + if [[ -z $fidelity_ticker ]] || [[ -z $fidelity_name ]]; then + echo $cusip >> $tmpdir/${QUARTER}_bad_cusips.csv + else + echo "$cusip,$fidelity_ticker,$fidelity_name" + fi + else + echo "$cusip,$quantum_ticker,$quantum_name" + fi +} + +tmpdir=$(mktemp -d -p $PWD) + +# Get the list of CUSIPs from the SEC and generate a nicer format of it +wget -q http://www.sec.gov/divisions/investment/13f/13flist${QUARTER}.pdf -O $tmpdir/13flist${QUARTER}.pdf +pdftotext -layout $tmpdir/13flist${QUARTER}.pdf - > $tmpdir/13flist${QUARTER}.txt +sed -rn 's/^([A-Z0-9]{6}) ([A-Z0-9]{2}) ([A-Z0-9]) .*$/\1\2\3/p' $tmpdir/13flist${QUARTER}.txt > $tmpdir/${QUARTER}_cusips + +# Find tickers and names for all the CUSIPs we can and print them out +for cusip in $(cat $tmpdir/${QUARTER}_cusips); do + get_ticker $cusip +done + +rm -rf $tmpdir diff --git a/scripts/gen_security_list.py b/scripts/gen_security_list.py new file mode 100755 index 0000000..f907683 --- /dev/null +++ b/scripts/gen_security_list.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python + +import csv +from xml.dom import minidom +from urllib import request + +class Security(object): + def __init__(self, name, description, number, _type, precision): + self.name = name + self.description = description + self.number = number + self.type = _type + self.precision = precision + def __str__(self): + return """\tSecurity{ +\t\tName: \"%s\", +\t\tDescription: \"%s\", +\t\tSymbol: \"%s\", +\t\tPrecision: %d, +\t\tType: %s, +\t\tAlternateId: \"%s\"},\n""" % (self.name, self.description, self.name, self.precision, self.type, str(self.number)) + +class SecurityList(object): + def __init__(self, comment): + self.comment = comment + self.currencies = {} + def add(self, currency): + self.currencies[currency.number] = currency + def __str__(self): + string = "\t// "+self.comment+"\n" + for key in sorted(self.currencies.keys()): + string += str(self.currencies[key]) + return string + +def process_ccyntry(currency_list, node): + name = "" + nameSet = False + number = 0 + numberSet = False + description = "" + precision = 0 + for n in node.childNodes: + if n.nodeName == "Ccy": + name = n.firstChild.nodeValue + nameSet = True + elif n.nodeName == "CcyNm": + description = n.firstChild.nodeValue + elif n.nodeName == "CcyNbr": + number = int(n.firstChild.nodeValue) + numberSet = True + elif n.nodeName == "CcyMnrUnts": + if n.firstChild.nodeValue == "N.A.": + precision = 0 + else: + precision = int(n.firstChild.nodeValue) + if nameSet and numberSet: + currency_list.add(Security(name, description, number, "Currency", precision)) + +def get_currency_list(): + currency_list = SecurityList("ISO 4217, from http://www.currency-iso.org/en/home/tables/table-a1.html") + + with request.urlopen('http://www.currency-iso.org/dam/downloads/lists/list_one.xml') as f: + xmldoc = minidom.parse(f) + for isonode in xmldoc.childNodes: + if isonode.nodeName == "ISO_4217": + for ccytblnode in isonode.childNodes: + if ccytblnode.nodeName == "CcyTbl": + for ccyntrynode in ccytblnode.childNodes: + if ccyntrynode.nodeName == "CcyNtry": + process_ccyntry(currency_list, ccyntrynode) + return currency_list + +def get_cusip_list(filename): + cusip_list = SecurityList("") + with open(filename, newline='') as csvfile: + csvreader = csv.reader(csvfile, delimiter=',') + for row in csvreader: + cusip = row[0] + name = row[1] + description = ",".join(row[2:]) + cusip_list.add(Security(name, description, cusip, "Stock", 5)) + return cusip_list + +def main(): + currency_list = get_currency_list() + cusip_list = get_cusip_list('cusip_list.csv') + + print("package main\n") + print("var SecurityTemplates = []Security{") + print(str(currency_list)) + print(str(cusip_list)) + print("}") + +if __name__ == "__main__": + main()