From 31907e58394484c2447402f630c0a5c93c5b37e7 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 12 May 2013 11:02:04 -0400 Subject: minimum viable demonstration --- README | 10 ++++++ octopart.py | 74 ++++++++++++++++++++++++++++++++++++++ partdb.py | 76 +++++++++++++++++++++++++++++++++++++++ partmom.py | 27 ++++++++++++++ settings.py.example | 9 +++++ start_uwsgi.sh | 4 +++ static/style.css | 37 +++++++++++++++++++ templates/base.html | 14 ++++++++ templates/grid.html | 53 +++++++++++++++++++++++++++ templates/index.html | 8 +++++ xilinx.py | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++ 11 files changed, 412 insertions(+) create mode 100644 README create mode 100644 octopart.py create mode 100644 partdb.py create mode 100755 partmom.py create mode 100644 settings.py.example create mode 100755 start_uwsgi.sh create mode 100644 static/style.css create mode 100644 templates/base.html create mode 100644 templates/grid.html create mode 100644 templates/index.html create mode 100644 xilinx.py diff --git a/README b/README new file mode 100644 index 0000000..f93cc4e --- /dev/null +++ b/README @@ -0,0 +1,10 @@ + +Install: + + python-flask + +Copy ./settings.py.example to ./settings.py and edit the Octopart API key. + +Run: + + ./partmom.py diff --git a/octopart.py b/octopart.py new file mode 100644 index 0000000..106a69f --- /dev/null +++ b/octopart.py @@ -0,0 +1,74 @@ + +import json +import urllib +from decimal import Decimal + +from settings import * + +def fetch_bom(bom): + # inspired by "bom_quote.py" provided at http://octopart.com/api + reply = dict() + queries = [] + for p in bom: + pid = "%s|%s" % p + queries.append({'mpn': p[1], + 'brand': p[0], + 'reference': pid}) + reply[pid] = None + + # do requests in batches of 20 + results = [] + for i in range(0, len(queries), OCTOPART_BATCH_SIZE): + batched_queries = queries[i:i+OCTOPART_BATCH_SIZE] + url = 'http://octopart.com/api/v3/parts/match?queries=%s' \ + % urllib.quote(json.dumps(batched_queries)) + url += '&apikey=%s' % OCTOPART_API_KEY + #print url + data = urllib.urlopen(url).read() + response = json.loads(data) + results.extend(response['results']) + + #print "len(results): %d" % len(results) + + for result in results: + pid = result['reference'] + print "len(items[%s]): %d" % (pid, len(result['items'])) + if len(result['items']) == 0: + reply[pid] = None + else: + reply[pid] = result['items'][0] + return reply + +def price_info(item, quantity=1000): + if not item: + return dict(url=None, css='notfound', price='Not Found') + info = dict(url=item['octopart_url']) + info['css'] = 'unavailable' + info['price'] = 'No Offers' + for offer in item['offers']: + if not offer['is_authorized']: + continue + if not offer['prices'].has_key('USD'): + continue + price = None + for price_pair in offer['prices']['USD']: + if price_pair[0] <= quantity: + if not price or price_pair[1] < price: + price = price_pair[1] + if not price: + print "WARNING: not a price: %s" % price + continue + if not info['price'] or price < info['price']: + info['price'] = Decimal(price) + if offer['in_stock_quantity'] > 0: + info['css'] = 'available' + else: + info['css'] = 'outofstock' + return info + +def url_info(item): + if item: + return item['octopart_url'] + else: + return None + diff --git a/partdb.py b/partdb.py new file mode 100644 index 0000000..379bfd6 --- /dev/null +++ b/partdb.py @@ -0,0 +1,76 @@ + +import os +import json +import urllib +import datetime + +import settings +import octopart + +today = datetime.datetime.utcnow().strftime("%Y%m%d") + +def safe(s): #TODO: this + return s.lower() + +def ensure_dir(path): + if not os.path.isdir(path): + os.mkdir(path) + +def part_path(p): + vendor = safe(p[0]) + mpn = safe(p[1]) + return '/'.join((settings.OCTOPART_CACHE_FOLDER, today, vendor, mpn)) + '.json' + +def read_part(p): + with open(part_path(p), 'r') as f: + part = json.loads(f.read()) + return part + +def write_part(p, data): + pp = part_path(p) + ensure_dir(os.path.dirname(pp)) + with open(part_path(p), 'w') as f: + f.write(json.dumps(data)) + +def check_part(p): + pp = part_path(p) + return os.path.exists(pp) and os.path.isfile(pp) + +def ensure_bom(bom): + # first things first + ensure_dir(settings.OCTOPART_CACHE_FOLDER) + ensure_dir('/'.join((settings.OCTOPART_CACHE_FOLDER, today))) + + fetch_list = [] + for p in bom: + if not check_part(p): + fetch_list.append(( safe(p[0]), safe(p[1]) )) + print "Will fetch part: %s" % str(p) + + if len(fetch_list): + results = octopart.fetch_bom(fetch_list) + for p in fetch_list: + pid = "%s|%s" % p + if results[pid]: + write_part(p, results[pid]) + else: + write_part(p, dict()) + print "Part not found: %s" % str(p) + +def part_url(p): + return octopart.url_info(read_part(p)) + +def best_price_info(bom): + best = None + for p in bom: + info = octopart.price_info(read_part(p)) + if not best: + best = info + elif not best['price'] and info['price']: + best = info + elif info['price'] and info['price'] < best['price']: + best = info + if type(best['price']) not in (str, unicode): + best['price'] = "$%.2f" % best['price'] + return best + diff --git a/partmom.py b/partmom.py new file mode 100755 index 0000000..b78281d --- /dev/null +++ b/partmom.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +from flask import Flask, request, session, g, redirect, url_for, \ + abort, render_template +from werkzeug import secure_filename +import os + +from settings import * +import xilinx + +app = Flask(__name__) +app.config.from_object(__name__) + +@app.route('/', methods=['GET']) +def index(): + return render_template('index.html') + +@app.route('/xilinx/spartan6', methods=['GET']) +def xilinx_spartan6(): + return render_template('grid.html', grid=xilinx.spartan6_grid) + +@app.route('/xilinx/zynq7000', methods=['GET']) +def xilinx_zynq7000(): + return render_template('grid.html', grid=xilinx.zynq7000_grid) + +if __name__ == '__main__': + app.run(debug=DEBUG, port=PORT, host=HOST) diff --git a/settings.py.example b/settings.py.example new file mode 100644 index 0000000..f953c3a --- /dev/null +++ b/settings.py.example @@ -0,0 +1,9 @@ + +# copy this file to "settings.py", then edit the below +PORT = 8080 +HOST = '127.0.0.1' +DEBUG = True + +OCTOPART_CACHE_FOLDER = './cache' +OCTOPART_API_KEY = 'CHANGEME' +OCTOPART_BATCH_SIZE = 20 diff --git a/start_uwsgi.sh b/start_uwsgi.sh new file mode 100755 index 0000000..9f82bf5 --- /dev/null +++ b/start_uwsgi.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env sh + +mkdir -p cache +uwsgi_python -s /tmp/uwsgi_partmon.sock -w partmom:app --processes 4 -C a+rw diff --git a/static/style.css b/static/style.css new file mode 100644 index 0000000..4b9f896 --- /dev/null +++ b/static/style.css @@ -0,0 +1,37 @@ +/* partmom default stylesheet */ +body { + margin: 25px; + font-family: helvetica; + } +h1 { + border-bottom: 2px solid; + } +.error { + padding: 15px; + background-color: yellow; + border: 4px solid orange; + color:red; + } +table { + border: 1px solid black; + border-collapse: collapse; + border-spacing:2px; + } +th, td { + border-width: 1px; + border-style: inset; + border-color: gray; + padding: 2px; + text-align: center; + } + +td.outofstock a { + color: #000; + font-size: 1.2em; + text-decoration: none; + } +td.available a { + color: #000; + font-size: 1.2em; + text-decoration: none; + } diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..779181a --- /dev/null +++ b/templates/base.html @@ -0,0 +1,14 @@ + + + + + {% block title %}partmom{% endblock %} + + {% block head %}{% endblock %} + + +
+{% block content %}{% endblock %} +
+ + diff --git a/templates/grid.html b/templates/grid.html new file mode 100644 index 0000000..3a4c71e --- /dev/null +++ b/templates/grid.html @@ -0,0 +1,53 @@ +{% extends "base.html" %} +{% block title %}partmom: {{ grid.vendor }} {{ grid.familyname }}{% endblock %} +{% block content %} +

{{ grid.vendor }} {{ grid.familyname }}

+ +{% for spec in grid.shared_specs %} + {{ spec }}: {{ grid.shared_specs[spec] }}
+{% endfor %} +
+ + +{% for row in grid.data_table %} + + {% for cell in row %} + + {% endfor %} + +{% endfor %} + + + +{% for cell in grid.price_row %} + +{% endfor %} + + +{% for row in grid.package_table %} + + {% for cell in row %} + + {% endfor %} + +{% endfor %} + + + +{% for cell in grid.suffix_row %} + +{% endfor %} + + +
{{ cell[1] }}
Price: + {% if cell.url %}{% endif %} + {{ cell.price }} + {% if cell.url %}{% endif %} +
{{ cell }}
All Parts: + {% for part in cell %} + {% if part.url %}{{ part.suffix }} + {% else %}{{ part.suffix }} + {% endif %}
+ {% endfor %} +
+{% endblock %} diff --git a/templates/index.html b/templates/index.html new file mode 100644 index 0000000..cc04491 --- /dev/null +++ b/templates/index.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} +{% block content %} +Part families: + +{% endblock %} diff --git a/xilinx.py b/xilinx.py new file mode 100644 index 0000000..ece0db7 --- /dev/null +++ b/xilinx.py @@ -0,0 +1,100 @@ + +import partdb +import csv + +def load_csv(path): + table = [] + with open(path, 'r') as f: + csv_reader = csv.reader(f) + for row in csv_reader: + table.append(row) + return table + +def process_csv(data_path, shared_path, speed_grades, temp_grade): + raw_specs = load_csv(shared_path) + raw_data = load_csv(data_path) + split_row = 0 + + # find row split between data and pins + for i in range(len(raw_data)): + if raw_data[i][0] == "###": + split_row = i + break + + package_table = raw_data[split_row+1:] + prefix_list = raw_data[0][1:] + + # need to infill sparse prefix_lists (eg, xilinx zynq) + last = None + for i in range(len(prefix_list)): + if not prefix_list[i]: + prefix_list[i] = last + else: + last = prefix_list[i] + + suffix_row = [list() for i in range(len(prefix_list))] + bom = [] + for row_num in range(len(package_table)): + package = package_table[row_num][0] + for cell_num in range(len(package_table[row_num][1:])): + cell = package_table[row_num][cell_num+1] + if cell: + for speed_grade in speed_grades: + suffix = speed_grade + package + temp_grade + mpn = prefix_list[cell_num] + suffix + bom.append(('Xilinx', mpn)) + suffix_row[cell_num].append(dict(mpn=mpn, suffix=suffix)) + + shared_specs = dict() + for row in raw_specs: + shared_specs[row[0]] = row[1] + + data_table = [] + span = 1 + for raw_row in raw_data[:split_row]: + row = [[1, raw_row[0]], ] + for raw_cell in raw_row[1:]: + if raw_cell: + row.append([1, raw_cell]) + else: + row[-1][0] += 1 + data_table.append(row) + + partdb.ensure_bom(bom) + price_row = [None for i in range(len(prefix_list))] + for i in range(len(prefix_list)): + bom = [('Xilinx', suf['mpn']) for suf in suffix_row[i]] + price_row[i] = partdb.best_price_info(bom) + + # attach URLs to suffix row entries + for cell_num in range(len(suffix_row)): + cell = suffix_row[cell_num] + for part_num in range(len(cell)): + part = cell[part_num] + p = ('Xilinx', part['mpn']) + part['url'] = partdb.part_url(p) + cell[part_num] = part + suffix_row[cell_num] = cell + + return dict(shared_specs=shared_specs, + data_table=data_table, + price_row=price_row, + package_table=package_table, + suffix_row=suffix_row) + +today = partdb.today +spartan6_grid = process_csv( + 'xilinx_data/spartan6.csv', + 'xilinx_data/spartan6_shared.csv', + speed_grades=['-2', '-3'], + temp_grade='C') +spartan6_grid['vendor'] = "Xilinx" +spartan6_grid['familyname'] = "Spartan6" +zynq7000_grid = process_csv( + 'xilinx_data/zynq7000.csv', + 'xilinx_data/zynq7000_shared.csv', + speed_grades=['-1'], + temp_grade='C') +zynq7000_grid['vendor'] = "Xilinx" +zynq7000_grid['familyname'] = "Zynq7000" + -- cgit v1.2.3