前言

最近想升级一下电脑硬件，又不是很了解当前电脑硬件的价格趋势。登录一些比价网站，商品太多了，找起来好麻烦。一些高级的功能还要收费。干脆自己写一个吧。想要的功能很简单。

Django 平台

models.py

from django.db import models

# Create your models here.

#状态
NO        = 0
YES       = 1

STATUS_CHOICE = {
    NO:     '否',
    YES:    '是',
}
class MallInfo(models.Model):
    '''商品信息'''
    sid = models.AutoField(primary_key=True, verbose_name='编号')
    status_choices = ((k, v) for k,v in STATUS_CHOICE.items())
    sku = models.CharField(max_length=32, verbose_name='商品id')
    name = models.CharField(max_length=255, verbose_name='商品名', default='xxxx')
    src = models.CharField(max_length=20, verbose_name='来源', default='jd')
    url = models.CharField(max_length=255, verbose_name='链接', default='xxxx')
    addtime =  models.DateTimeField(auto_now_add=True, verbose_name='添加时间')
    status = models.SmallIntegerField(default=NO, choices=status_choices, verbose_name='是否监控')

    # admin显示订单的id
    def __str__(self):
        return self.name

    class Meta:
        db_table = 'mall_info'
        verbose_name = '商品信息'
        verbose_name_plural = '商品信息'

class PriceInfo(models.Model):
    '''历史价格'''
    sid = models.AutoField(primary_key=True, verbose_name='编号')
    sku = models.CharField(max_length=32, verbose_name='商品id')
    price = models.IntegerField(default=0, verbose_name='价格')
    note = models.CharField(max_length=255, default='无', verbose_name='活动')
    addtime = models.DateTimeField(auto_now_add=True, verbose_name='日期')

    # admin显示订单的id
    def __str__(self):
        return str(self.sid)

    class Meta:
        db_table = 'price_info'
        verbose_name = '商品价格'
        verbose_name_plural = '商品价格'

admin.py

from django.contrib import admin
from mall.models import  MallInfo, PriceInfo
from django.utils.html import format_html
from bs4 import BeautifulSoup
import requests

# Register your models here.
class MallInfoAdmin(admin.ModelAdmin):
    list_display = ['sku', 'name_and_url', 'show_history', 'src', 'addtime', 'status']
    #进入编辑的字段
    list_display_links = ['sku']

    #只读字段
    readonly_fields = ['url']

    #标题带链接
    def name_and_url(self, obj):
        return format_html("<a href='{url}' target='_blank'>{name}</a>", url=obj.url, name=obj.name)

    #字段描述
    name_and_url.short_description = "商品名称"

    def show_history(self, obj):
        return format_html("<a href='/mall/echarts/?sku={sku}' target='_blank'>历史价格</a>", sku=obj.sku)

    show_history.short_description = "历史价格"

    #重写保存函数
    def save_model(self, request, obj, form, change):
        def get_name_by_sku():
            """根据sku获取名称"""
            url = 'https://item.jd.com/%s.html' % (obj.sku)
            try:
                resp = requests.get(url)
                soup = BeautifulSoup(resp.text, 'html.parser')
                name = soup.find(class_='sku-name')
                return name.text.strip()
            except Exception as e:
                print(e)
            return '0000'

        if not change:  # 新增记录
            obj.name = get_name_by_sku()
            obj.url = 'https://item.jd.com/%s.html' % (obj.sku)
        super(MallInfoAdmin, self).save_model(request, obj, form, change)

class PriceInfoAdmin(admin.ModelAdmin):
    list_display = ['sku', 'price', 'note', 'addtime']

    #只读字段
    readonly_fields = ['sku', 'price', 'note', 'addtime']

admin.site.register(MallInfo, MallInfoAdmin)

#admin.site.register(PriceInfo, PriceInfoAdmin)

views.py

from django.shortcuts import render
from mall.models import MallInfo, PriceInfo
from django.http import HttpResponse
from django.template import Template, Context
from rest_framework.views import APIView

import json

from pyecharts.charts import Bar
from pyecharts import options as opts

from .utils import *

# Create your views here.

JsonResponse = json_response
JsonError = json_error

def get_skus_need_monitor(request):
    """
    获取需要监控价格的商品列表
    """
    resp = {}
    skus = []
    malls = MallInfo.objects.filter(status=1)

    if malls:
        for mall in malls:
            skus.append(mall.sku)

    return JsonResponse(skus)

def bar_base(sku) -> Bar:
    """
    图表制作
    """
    x_date = []
    y_price = []

    mall = MallInfo.objects.filter(sku=sku).last()
    prices = PriceInfo.objects.filter(sku=sku).order_by('-sid')[:15]

    if prices:
        for price in prices[::-1]:
            x_date.append(price.addtime.strftime('%Y-%m-%d'))
            y_price.append(price.price)


    b = (Bar()
        .add_xaxis(x_date)
        .add_yaxis("sku:{}".format(sku),
                y_price,
                )
        .set_global_opts(title_opts=opts.TitleOpts(title="商品历史价格曲线", subtitle="{}".format(mall.name)),
                xaxis_opts=opts.AxisOpts(axislabel_opts={"interval":"0","rotate":45}),
                datazoom_opts=[opts.DataZoomOpts(), opts.DataZoomOpts(type_="inside")],
                )
        .dump_options_with_quotes()
    )

    return b

class get_mall_price_api(APIView):
    """
    历史价格获取价格数据api
    """
    def get(self, request, *args, **kwargs):
        sku = request.GET.get("sku")
        return JsonResponse(json.loads(bar_base(sku)))

class echartsView(APIView):
    """
    历史价格渲染页面
    """
    def get(self, request, *args, **kwargs):
        sku = request.GET.get("sku")
        p_max =  PriceInfo.objects.filter(sku=sku).order_by('-price')[0].price
        p_min =  PriceInfo.objects.filter(sku=sku).order_by('price')[0].price
        p_now =  PriceInfo.objects.filter(sku=sku).last().price
        n_note = PriceInfo.objects.filter(sku=sku).last().note

        #多重查询条件
        l_note = PriceInfo.objects.filter(sku=sku).exclude(note='无').order_by('-sid')

        t = Template(open("./templates/echarts.html").read())
        c = Context({'sku': sku, 'p_max': p_max, 'p_min': p_min, 'n_note': n_note, 'l_note':l_note, 'p_now':p_now})
        html = t.render(c)
        return HttpResponse(html)

echarts.html

<!DOCTYPE html>
<html>
<head>
    <meta charset="UTF-8">
    <title>商品历史价格曲线</title>
    <script src="https://cdn.bootcss.com/jquery/3.0.0/jquery.min.js"></script>
    <script type="text/javascript" src="https://assets.pyecharts.org/assets/echarts.min.js"></script>
</head>
<body>
<div style="position:absolute; top:10; left:10; width:1200px; height:95%;">
    <div id="line" style="width:1100px;height:93%"></div>
    <div style="margin-top:20px">
        <span>历史高价: <a style='color:red'>{{p_max}}</a></span>
        <span>历史低价: <a style='color:green'>{{p_min}}</a> </span>
        <span>当前价格: <a style='color:orange'>{{p_now}}</a> </span>
        <span>最新活动: <a style='color:blue'>{{n_note}}</a> </span>
    </div>
</div>
<div style="margin-left:1020px; height:865px; overflow:auto;">
    <table style="maxHeight:1200px;" border="1" cellpadding="3" cellspacing="0">
        <caption>  历史活动  </caption>
            <tr>
                <td>日期</td><td>内容</td>
            <tr>
            {% for i in l_note %}
                <tr>
                        <td>{{ i.addtime }}</td> <td> {{ i.note }}</td>
                </tr>
            {% endfor %}
        </table>

</div>
<script>
    var chart = echarts.init(document.getElementById('line'), 'white', {renderer: 'canvas'});

    $(
        function () {
            fetchData(chart);
        }
    );

    function fetchData() {
        $.ajax({
            type: "GET",
            url: "/mall/getprices/?sku={{sku}}",
            dataType: 'json',
            success: function (result) {
                chart.setOption(result.data);
            }
        });
    }
</script>
</body>
</html>

feadper 价格采集

crawl_mall_spider.py

import feapder
from items import price_info_item
from feapder.utils import tools
import requests, json, datetime

class MallSpider(feapder.BaseParser):
    def start_requests(self):
        resp = requests.get('http://x.x.x.x:801/api/mall/getskus/')
        for sku in json.loads(resp.text)['data']:
            url = 'https://item.jd.com/%s.html' % (sku)
            yield feapder.Request(url, render=True)

    def parse(self, request, response):
        r_list = response.xpath('//div[@class="summary-price-wrap"]')
        #解析结果
        for r in r_list:
            price = r.xpath('//span[@class="p-price"]/span[2]/text()').extract_first()
            note = r.xpath('//div[@class="J-prom"]/div[last()]/em[last()]/text()').extract_first()

            price_item = price_info_item.PriceInfoItem()
            price_item.sku = request.url[20:-5]
            price_item.price = price.lstrip().rstrip()
            if note:
                price_item.note = note.lstrip().rstrip()
            else:
                price_item.note = '无'

            price_item.addtime = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")

            yield price_item