socketref,再见!高德

https://github.com/adoggie

  C++博客 :: 首页 :: 联系 :: 聚合  :: 管理
  246 Posts :: 4 Stories :: 312 Comments :: 0 Trackbacks

常用链接

留言簿(54)

我参与的团队

搜索

  •  

最新评论

阅读排行榜

评论排行榜

几个性能问题: 
  1. 商品图像信息加入pgsql, 字段类型: model.BinaryField  (django 1.6) 
  2. django1.6提供数据库长连接功能  
CONN_MAX_AGE 参数,可加快访问速度
  3. gevent.spawn() 创建所有task,将导致进程被直接Kill掉,因为内存疯长,所涉及的资源达到顶端将被OS杀死,包括:内存overload,file handle...
      使用 gevent.pool.Pool控制并发数量
   4. postgresql.conf 修改max_connections参数到300并发数
  5. 导入数据时,可先去除Table的索引,加快插入速度 
  
6 . django关闭DEBUG模式(会导致内存泄露不释放) 





#
--coding:utf-8--


'''
    yixun_crawler - 易迅网站商品信息收集器
    author: scott
    date:

    目前仅考虑单点设备登录

    lxml 的xpath还存在部分问题(还是自己没完全领会呢?)

    source-code and db-sample:
           http://114.215.178.29/static/projects/crawler/
'''

import imp
imp.load_source('init','../init_script.py')
import gevent
import os,os.path,sys,struct,time,traceback,signal,threading,copy,base64,datetime

# from django.db import connection
#
 from django.db.models import Sum
from django.db import transaction

import yixun.models as  yixun

from bson.objectid import ObjectId
import lxml.etree as etree

import urllib2,urlparse

dbfile = 'goods.txt'

fdbfile = open(dbfile,'wb')

class ResourceItem:
    def __init__(self,text,href=None,tag=None,parent=None):
        self.text = text
        self.tag=tag
        self.href=href
        self.children=[]
        self.parent = parent


def scrape_page(url,pageIndex,cat1,cat2,cat3):
    print 'scrape_page:',url

    req = urllib2.urlopen(url)
    data = req.read()
    # savefile(data)
    html = etree.HTML(data.decode('utf-8'))

    #page size

    curPage = 0
    r = html.xpath('//*[@id="list"]/div[5]/div[2]/span/b/text()')
    if not r: return False
    curPage = r[0]
    r = html.xpath('//*[@id="list"]/div[5]/div[2]/span/text()')
    if not r : return False
    pageNum = int(r[0][1:])
    print pageNum,curPage

    #有一种情况,传入大于总page数量的值,server会返回第一个page

    if pageIndex > pageNum:
        return False


    #检索品牌
    goods = html.xpath(u"//div[@class='mod_goods']")
    if not goods:
        print 'skipped..'
        return False
    for g in goods:
        for e in g.getchildren():
            if e.get('class') ==  'mod_goods_info':    #一下search动作用xpath无法实现,所以只好挨个查找
                name = ''
                price =None
                link = ''
                for  p in e.getchildren():
                    if p.get('class')=='mod_goods_tit':
                        a= p.getchildren()[0]
                        name =  a.text.encode('utf-8')
                        link = a.get('href')

                    if p.get('class')=='mod_goods_price':
                        price = p.getchildren()[0].getchildren()[1].text.encode('utf-8')
                if name and price and link:
                    # print name , price ,link
                    text = "%s || %s || %s || %s || %s || %s\n"%(cat1,cat2,cat3,name,price,link.strip())

                    print text

                    gitem = yixun.GoodsItem()
                    gitem.cat1 = cat1
                    gitem.cat2 = cat2
                    gitem.cat3 = cat3
                    gitem.name = name
                    gitem.cat5 =  link

                    try:
                        gitem.price = float(price)
                    except:
                        pass
                    gitem.save()

                    # fdbfile.write(text)
                    # fdbfile.flush()


    return True

    # ss= p.xpath('..//dd/a')

'''
http://searchex.yixun.com/705740t705741-1-/?YTAG=2.1738456040037
http://searchex.yixun.com/html?path=705740t705741&area=1&sort=0&show=0&page=2&size=40&pf=0&as=0&charset=utf-8&YTAG=2.1738456040037#list
http://searchex.yixun.com/html?path=705740t705741&area=1&sort=0&show=0&page=1&size=40&pf=0&as=0&charset=utf-8&YTAG=2.1738456040037#list
'''
def scrape_cat(cat,yPageId,yPageLevel,tag,cat1,cat2,cat3):
    try:
        print cat.href
        #parse url
        url = cat.href
        fs =  urlparse.urlparse(url)
        path,qs=fs[2],fs[4]
        cat_idx =  path[1:].split('-')[0]
        # tag = qs.split('=')[1]
        tag = "%s.%s%s"%(yPageLevel,yPageId,tag)
        #make path url
        for page in range(1,500):
            url = "http://searchex.yixun.com/html?path=%s&area=1&sort=0&show=0&page=%s&size=40&pf=0&as=0&charset=utf-8&YTAG=%s#list"%(cat_idx,page,tag)
            if not scrape_page(url,page,cat1,cat2,cat3):
                break

        return


    except:
        traceback.print_exc()
        # print 'page is null,skipped..'

def savefile(d,filename='sample.html'):
    f = open(filename,'w')
    f.write(d)
    f.close()

def test():
    try:
        url = 'http://searchex.yixun.com/705740t705741-1-/?YTAG=2.1738456040037'
        fs =  urlparse.urlparse(url)
        path,qs=fs[2],fs[4]
        cat_idx =  path[1:].split('-')[0]
        tag = qs.split('=')[1]
        print cat_idx,tag

        return

        all_url = 'http://searchex.yixun.com/html?YTAG=3.705766287001&path=705882t705893'
        req = urllib2.urlsplit(all_url)
        html = req.read()
        savefile(html)

        dom = etree.HTML(html.decode('utf-8'))
        p = dom.xpath(u"//div[@title='品牌']")[0]
        ss= p.xpath('..//dd/a')
        print ss[0].text.encode('utf-8')

    except:
        traceback.print_exc()

def craw_start():
    import re
    try:
        all_url = 'http://searchex.yixun.com/?YTAG=2.1738456090000'
        req = urllib2.urlopen(all_url)
        html = req.read()

        # group = re.search("window\.yPageId ='(.*?)'",html)
        yPageId = re.search("window\.yPageId\s*=\s*'(\d+?)'",html).group(1)
        yPageLevel = re.search("window\.yPageLevel\s*=\s*'(\d+?)'",html).group(1)
        print yPageId,yPageLevel

        dom = etree.HTML(html.decode('gb2312'))
        all_cats=[]
        cat1_list = dom.xpath("//div[@class='m_classbox']")
        for cat in cat1_list:
            cat1_text = cat.xpath('h3/text()')[0]
            cat1_e = ResourceItem(cat1_text)
            all_cats.append(cat1_e)
            print cat1_e.text.encode('utf-8')
            div = cat.xpath("div")[0]
            for dl in  div.xpath('dl'):
                cat2 = dl.xpath('dt/a')[0]
                cat2_e = ResourceItem(cat2.text,href=cat2.attrib['href'],tag=cat2.attrib['ytag'],parent=cat1_e)
                cat1_e.children.append(cat2_e)
                print ' '*4,cat1_e.text.encode('utf-8'),cat2_e.href,cat2_e.tag
                for cat3 in dl.xpath('dd/a'):
                    cat3_e = ResourceItem(cat3.text,href=cat3.attrib['href'],tag=cat3.attrib['ytag'],parent=cat2_e)
                    cat2_e.children.append(cat3_e)
                    print ' '*8,cat3_e.text.encode('utf-8'),cat3_e.href,cat3_e.tag
        tasks =[]
        for e1 in all_cats:
            print '-'*1,e1.text.encode('utf-8')
            for e2 in e1.children:
                print '  '*2    ,e2.text.encode('utf-8')
                for e3 in e2.children:
                    print '  '*4,e3.text.encode('utf-8')
                    task = gevent.spawn(scrape_cat,e3,yPageId,yPageLevel,e2.tag,e1.text.encode('utf-8'),e2.text.encode('utf-8'),e3.text.encode('utf-8'))
                    tasks.append(task)
                    # scrape_cat(e3,yPageId,yPageLevel,e2.tag,e1.text.encode('utf-8'),e2.text.encode('utf-8'),e3.text.encode('utf-8'))
                    # return
        gevent.joinall(tasks)
    except:
        traceback.print_exc()


if __name__ == '__main__':
    craw_start()
    # test()
    pass
posted on 2014-05-20 11:33 放屁阿狗 阅读(17156) 评论(0)  编辑 收藏 引用

只有注册用户登录后才能发表评论。
网站导航: 博客园   IT新闻   BlogJava   知识库   博问   管理