finish the gsmgarena hierachy display

Author: xtutran

hierarchy-data/data.ipynb

@@ -15,6 +15,17 @@
     "import urllib"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from starbase import Connection\n"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 146,
@@ -167,10 +178,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 156,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 12,
+   "metadata": {},
    "outputs": [],
    "source": [
     "import happybase\n",
@@ -180,7 +189,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 157,
+   "execution_count": 13,
    "metadata": {
     "collapsed": true
    },
@@ -191,7 +200,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 158,
+   "execution_count": 14,
    "metadata": {},
    "outputs": [
     {
@@ -200,7 +209,7 @@
        "['phone_specs']"
       ]
      },
-     "execution_count": 158,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -222,10 +231,8 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 168,
-   "metadata": {
-    "collapsed": true
-   },
+   "execution_count": 8,
+   "metadata": {},
    "outputs": [],
    "source": [
     "connection.close()"

hierarchy-data/mysite/.idea/misc.xml

@@ -18,19 +18,18 @@
     <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.6/js/bootstrap.min.js"
             integrity="sha384-0mSbJDEHialfmuBBQP6A4Qrprq5OVfW37PRR3j5ELqxss1yVqOtnepnHVP9aJ7xS"
             crossorigin="anonymous"></script>
-
     {% load static %}
     <link rel="stylesheet" href="{% static 'css/specs.css' %}" type="text/css"/>
-    {% load static %}
+    <!--{% load static %}-->
     <script src="{% static 'js/colResizable-1.5.min.js' %}"></script>
-    {% load static %}
+    <!--{% load static %}-->
     <script src="{% static 'js/fuzzyset.js' %}"></script>
-    {% load static %}
+    <!--{% load static %}-->
     <script src="{% static 'js/fuzzy.js' %}"></script>
-    {% load static %}
+    <!--{% load static %}-->
     <script src="{% static 'js/quantities.js' %}"></script>
-    {% load static %}
-    <script src="{% static 'js/jquery-highlight-min1.js' %}"></script>
+    <!--{% load static %}-->
+    <script src="{% static 'js/jquery.highlight-5.js' %}"></script>
     <script type="text/javascript">
         var d3Regex = /\b([0-9]+(\.[0-9]+)?)[\s]*x[\s]*\b([0-9]+(\.[0-9]+)?)[\s]*x[\s]*\b([0-9]+(\.[0-9]+)?)[\s]*([\w]+)/g
         var d2Regex = /\b([0-9]+(\.[0-9]+)?)[\s]* x[\s]*\b([0-9]+(\.[0-9]+)?)[\s]*([\w]+)/g;
@@ -403,6 +402,8 @@
     </center>
 </div>
 <br/>
+
+{% load custom_tag %}
 {% if phone_data %}
 <table class="table table-hover" id="specs" border="1" width="100%">
     <tr class="danger">
@@ -423,34 +424,50 @@ <h2 class="htitle">{{ spec.0 }}</h2>
                     {% if sub != '' %}
                     <li class="s_lv_1">
                         <strong>{{ sub }}: </strong>
-                        <ul>
-                            {% for subsub, v in value.items %}
-                            {% if subsub != '' %}
-                            <li class="s_lv_2">
-                                <strong>{{ subsub }}: </strong>
+                        {% if value.items != '' %}
+                            <ul>
+                            {% split value as values %}
+                            {% for v in values%}
+                                <!--<div class="hiddenRow">-->
+                                    <!--<div class="accordion-body collapse packageDetails1" id="accordion3">-->
+                                        <li class="s_lv_3">
+                                            <input type="checkbox" class="phone" value="{{ v }}"> {{ v }}
+                                        </li>
+                                    <!--</div>-->
+                                <!--</div>-->
+                            {% endfor %}
+                            </ul>
+                        {% else %}
+                            <ul>
+                                {% for subsub, v in value.items %}
+                                {% if subsub != '' %}
+                                <li class="s_lv_2">
+                                    <strong>{{ subsub }}: </strong>
+                                    <div class="hiddenRow">
+                                        <ul >
+                                            <div class="accordion-body collapse packageDetails1" id="accordion1">
+                                                <li class="s_lv_3">
+                                                    <input type="checkbox" class="phone" value="{{ v.0 }}"> {{ v.0 }}
+                                                </li>
+                                            </div >
+                                        </ul>
+                                    </div>
+                                </li>
+                                {% else %}
+                                {% for d in v %}
                                 <div class="hiddenRow">
-                                    <ul >
-                                        <div class="accordion-body collapse packageDetails1" id="accordion1">
-                                            <li class="s_lv_3">
-                                                <input type="checkbox" class="phone" value="{{ v.0 }}"> {{ v.0 }}
-                                            </li>
-                                        </div >
-                                    </ul>
-                                </div>
-                            </li>
-                            {% else %}
-                            {% for d in v %}
-                            <div class="hiddenRow">
-                                <div class="accordion-body collapse packageDetails1" id="accordion2">
-                                    <li class="s_lv_3">
-                                        <input type="checkbox" class="phone" value="{{ d }}"> {{ d }}
-                                    </li>
+                                    <div class="accordion-body collapse packageDetails1" id="accordion2">
+                                        <li class="s_lv_3">
+                                            <input type="checkbox" class="phone" value="{{ d }}"> {{ d }}
+                                        </li>
+                                    </div>
                                 </div>
-                            </div>
-                            {% endfor %}
-                            {% endif %}
-                            {% endfor %}
-                        </ul>
+                                {% endfor %}
+                                {% endif %}
+                                {% endfor %}
+                            </ul>
+                        {% endif %}
+
                     </li>
                     {% else %}
                     {% for subsub, v in value.items %}

hierarchy-data/mysite/.idea/modules.xml

@@ -0,0 +1,7 @@
+from django.conf import settings
+
+for a in settings.INSTALLED_APPS:
+    try:
+        __path__.extend(__import__(a + '.templatetags', {}, {}, ['']).__path__)
+    except ImportError:
+        pass

hierarchy-data/mysite/.idea/mysite.iml

@@ -0,0 +1,11 @@
+from django import template
+
+register = template.Library()
+
+
+@register.simple_tag
+def split(value):
+    return value.split('^^^')
+
+
+# register.tag('split', split)

hierarchy-data/mysite/.idea/workspace.xml

@@ -5,6 +5,6 @@
 urlpatterns = [
     url(r'^phone?$', views.index, name='index'),
     url(r'^phone/list/$', views.listing, name='list'),
-    # url(r'^phone/detail/$', views.detail, name='detail'),
+    url(r'^phone/detail/$', views.detail, name='detail'),
     # url(r'^freebase/matching/$', views.freebase, name='freebase')
 ]

hierarchy-data/webapp/db.sqlite3 renamed to hierarchy-data/mysite/db.sqlite3

@@ -0,0 +1,122 @@
+from django.shortcuts import render
+from django.http import HttpResponse, HttpResponseBadRequest
+from django.template import loader
+import happybase
+import pandas as pd
+import sys,os
+# import extract
+from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
+
+# pool = happybase.ConnectionPool(size=3, host='192.168.1.240')
+global_schema = {}# extract.read_global_schema("integration/rules/global_schema.txt")
+
+
+# Create your views here.
+def index(request):
+    return HttpResponse("Welcome to Data Integration site!")
+
+
+def paginate(phones, page):
+    paginator = Paginator(phones, 25)  # Show 25 contacts per page
+    try:
+        phones = paginator.page(page)
+    except PageNotAnInteger:
+        # If page is not an integer, deliver first page.
+        phones = paginator.page(1)
+    except EmptyPage:
+        # If page is out of range (e.g. 9999), deliver last page of results.
+        phones = paginator.page(paginator.num_pages)
+    return phones
+
+
+def _get_specs(table, column, row_prefix, limit):
+    # with pool.connection() as connection:
+    phones = []
+    exit_code = 0
+    if column not in table.families():
+        return phones, exit_code
+
+    try:
+        for key, data in table.scan(filter=b'FirstKeyOnlyFilter()',
+                                    row_prefix=bytes(row_prefix), limit=limit, columns=[column]):
+            phones.append((key.decode('utf-8'), key.decode('utf-8')))
+    except IOError and AssertionError:
+        exit_code = 1
+    return phones, exit_code
+
+
+def _get_spec(table, column, row_key):
+    try:
+        row = table.row(bytes(row_key), columns=[column])
+    except IOError and AssertionError:
+        return {}, 1
+
+    spec = pd.DataFrame(row, index=[0]).T.reset_index()
+    # spec.columns = ['feature', 'value']
+    features = spec['index'].apply(lambda x: pd.Series(x.replace('{}:'.format(column.lower()), '').split(':')))
+    feature_cols = map(lambda x: 'level_{}'.format(x), features.columns)
+    features.columns = feature_cols
+    features['value'] = spec[0]
+
+    while len(feature_cols) != 1:
+        features = features.groupby(feature_cols[:-1]).\
+            apply(lambda x: dict(zip(x[feature_cols[-1]], x.value))).\
+            reset_index().\
+            rename(columns={0: 'value'})
+        feature_cols = features.columns[:-1]
+
+    return map(lambda x: tuple(x), features.values), 0
+
+
+def listing(request):
+    parameters = request.GET
+    tables = parameters.getlist('table') if 'table' in parameters else ['phonearena', 'gsmgarena']
+    row_prefix = parameters['prefix'] if 'prefix' in parameters else ''
+    try:
+        limit = int(parameters.get('limit'))
+        limit = limit if (limit <= 300) and (limit > 0) else 300
+    except ValueError:
+        limit = 100
+    except TypeError:
+        limit = 100
+    page = request.GET.get('page')
+
+    pool = happybase.ConnectionPool(size=3, host='192.168.56.101')
+    with pool.connection() as connection:
+        table = connection.table('phone_specs')
+        phone_data = {}
+        for column in tables:
+            phones, exit_code = _get_specs(table, column.lower(), row_prefix=row_prefix.upper(), limit=limit)
+            if exit_code == 0:
+                phone_data[column] = paginate(phones, page)
+                phone_data[column] = phones
+
+        return render(request, 'list.html', {'phones': phone_data,
+                                         'prefix': row_prefix, 'limit': limit})
+
+
+def detail(request):
+    print request.POST
+    if 'phone' in request.POST:
+        params = request.POST.getlist('phone')
+        pool = happybase.ConnectionPool(size=3, host='192.168.56.101')
+        with pool.connection() as connection:
+            table = connection.table('phone_specs')
+            phone_data = {}
+            for param in params:
+                tokens = param.split('###')
+                column = tokens[0]
+                row_key = tokens[1]
+                data, exit_code = _get_spec(table, column, row_key)
+                phone_data[param] = sorted(data, key=sortfn)
+
+            return render(request, 'detail.html', {'phone_data': phone_data})
+    else:
+        return HttpResponseBadRequest('Bad request')
+
+
+def sortfn(item):
+    if len(global_schema) == 0:
+        return item[0]
+    else:
+        return global_schema[item[0].lower()]