一、爬取电影信息

http://www.imdb.cn/nowplaying/{num}    #页面规则

http://www.imdb.cn/title/tt{num}    #某部电影信息

获取电影url和title

新建项目

scrapy startproject imdb


修改items.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# -*- coding: utf-8 -*-
 
# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html
 
import scrapy
 
 
class ImdbItem(Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    #url = scrapy.Field()    #url
    #title = scrapy.Field()  #影片名
 
    video_title = Field()
    video_rating = Field()
    video_name = Field()
    video_alias = Field()
    video_director = Field()
    video_actor = Field()
    video_length = Field()
    video_language = Field()
    video_year = Field()
    video_type = Field()
    video_color = Field()
    video_area = Field()
    video_voice = Field()
    video_summary = Field()
    video_url = Field()

在spiders目录下新建爬虫文件moive.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
# -*- coding: utf-8 -*-
 
from scrapy.spiders import CrawlSpider, Request, Rule
from imdb.items import ImdbItem
from scrapy.linkextractor import LinkExtractor
 
class ImdbSpider(CrawlSpider):
    name = 'imdb'
    allowed_domains = ['www.imdb.cn']
    rules = (
        Rule(LinkExtractor(allow=r"/title/tt\d+$"), callback="parse_imdb", follow=True),
    )
 
    def start_requests(self):
        for in range(120):
            url = "http://www.imdb.cn/nowplaying/" + str(i)
            yield Request(url=url, callback=self.parse)
 
    def parse_imdb(self, response):
        item = ImdbItem()
        try:
            item['video_title'= "".join(response.xpath('//*[@class="fk-3"]/div[@class="hdd"]/h3/text()').extract())
            item['video_rating'= "".join(
                response.xpath('//*[@class="fk-3"]/div[@class="hdd"]/span/i/text()').extract())
            content = response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li').extract()
            for in range(0len(content)):
                if "片名" in content[i]:
                    if == 0:
                        item['video_name'= "".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[1]/a/text()').extract())
                if "别名" in content[i]:
                    if == 1:
                        item['video_alias'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[2]/a/text()').extract())
                if "导演" in content[i]:
                    if == 1:
                        item['video_director'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[2]/a/text()').extract())
                    elif == 2:
                        item['video_director'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[3]/a/text()').extract())
                if "主演" in content[i]:
                    if == 2:
                        item['video_actor'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[3]/a/text()').extract())
                    if == 3:
                        item['video_actor'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[4]/a/text()').extract())
                if "上映时间" in content[i]:
                    if == 4:
                        item['video_year'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[5]/a[1]/text()').extract())
                        = response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[5]/a').extract()
                        length = len(a) - 1
                        try:
                            item['video_color'= "".join(
                                response.xpath(
                                    '//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[5]/a/text()').extract()[length])
                        except Exception as e:
                            item['video_color'= ""
                        try:
                            type = "|".join(
                                response.xpath(
                                    '//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[5]/a/text()').extract()[1:length])
                            maohao = type.split(":")
                            if len(maohao) > 0:
                                item['video_type'= maohao[0]
                            else:
                                item['video_type'= ""
                        except Exception as e:
                            item['video_type'= ""
                    if == 5:
                        item['video_year'= "".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a[1]/text()').extract())
                        = response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a').extract()
                        length = len(a) - 1
                        try:
                            item['video_color'= "".join(
                                response.xpath(
                                    '//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a/text()').extract()[length])
                        except Exception as e:
                            item['video_color'= ""
                        try:
                            type = "|".join(
                                response.xpath(
                                    '//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a/text()').extract()[1:length])
                            maohao = type.split(":")
                            if len(maohao) > 0:
                                item['video_type'= maohao[0]
                            else:
                                item['video_type'= ""
                        except Exception as e:
                            item['video_type'= ""
 
                if "国家" in content[i]:
                    if == 5:
                        item['video_area'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a[1]/text()').extract())
                        item['video_voice'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[6]/a[2]/text()').extract())
                    if == 6:
                        item['video_area'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[7]/a[1]/text()').extract())
                        item['video_voice'= "|".join(
                            response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[7]/a[2]/text()').extract())
            item['video_length'= "".join(
                response.xpath(
                    '//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[@class="nolink"]/text()').extract()).replace(
                "&nbsp", "")
            item['video_language'= "".join(
                response.xpath('//*[@class="fk-3"]/div[@class="bdd clear"]/ul/li[@class="nolink"]/a/text()').extract())
            item['video_summary'= "".join(
                response.xpath(
                    '//*[@class="fk-4 clear"]/div[@class="bdd clear"]/i/text()').extract()).lstrip().rstrip().replace(
                "<br>", "")
            item['video_url'= response.url
            yield item
        except Exception as error:
            log(error)

在spiders目录下新建run.py启动文件

vim run.py

# coding:utf-8

from scrapy import cmdline

cmdline.execute("scrapy crawl imdb".split())



二、有限深度爬取

新建项目

scrapy startproject douban


scrapy中,我们在settings.py设置深度使用DEPTH_LIMIT,例如:DEPTH_LIMIT = 5,该深度是相对于初始请求url的深度

修改settings.py

DEPTH_LIMIT = 4

#豆瓣有反爬虫机制,因此设置延时DOWNLOAD_DELAY

DOWNLOAD_DELAY = 2

USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'    #设置代理




items.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
from scrapy import Item, Field
 
# 音乐
class MusicItem(Item):
    music_name = Field()
    music_alias = Field()
    music_singer = Field()
    music_time = Field()
    music_rating = Field()
    music_votes = Field()
    music_tags = Field()
    music_url = Field()
 
class MusicReviewItem(Item):
    review_title = Field()
    review_content = Field()
    review_author = Field()
    review_music = Field()
    review_time = Field()
    review_url = Field()


爬虫文件music.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# --*-- coding: utf-8 --*--
 
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from douban.items import MusicItem, MusicReviewItem
from scrapy import log
 
class ReviewSpider(CrawlSpider):
    name = 'review'
    allowed_domains = ['music.douban.com']
    start_urls = ['https://music.douban.com/subject/1406522/']
    rules = (
        Rule(LinkExtractor(allow=r"/subject/\d+/reviews$")),
        Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time$")),
        Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time\&start=\d+$")),
        Rule(LinkExtractor(allow=r"/review/\d+/$"), callback="parse_review", follow=True),
    )
 
    def parse_review(self,response):
        try:
            item = MusicReviewItem()
            item['review_title'= "".join(response.xpath('//*[@property="v:summary"]/text()').extract())
            content = "".join(
                response.xpath('//*[@id="link-report"]/div[@property="v:description"]/text()').extract()
            )
            item['review_content'= content.lstrip().rstrip().replace('\n'," ")
            item['review_author'= "".join(response.xpath('//*[@property="v:reviewer"]/text()').extract())
            item['review_music'= "".join(response.xpath('//*[@class="main-hd"]/a[2]/text()').extract())
            item['review_time'= "".join(response.xpath('//*[@class="main-hd"]/p/text()').extract())
            item['review_url'= response.url
            yield item
        except Exception as error:
            log(error)


启动命令文件run.py

1
2
3
# --*-- coding: utf-8 --*--
from scrapy import cmdline
cmdline.execute("scrapy crawl review -o review.json".split())

-o 参数导出结果到review.json文件



多个爬虫组合

我们现在有这么个需求,既要爬取音乐详情又要爬取乐评,既要爬取电影详情又要爬取影评,这个要怎么搞,难道是每一个需求就要创建一个项目么,如果按这种方式,我们就要创建四个项目,分别来爬取音乐、乐评、电影、影评,显然这么做的话,代码不仅有很多重合的部分,而且还不容易维护爬虫


新建项目

scrapy startproject multi


修改settings.py

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'
DOWNLOAD_DELAY = 2
 
修改items.py
from scrapy import Item, Field
 
# 音乐
class MusicItem(Item):
    music_name = Field()
    music_alias = Field()
    music_singer = Field()
    music_time = Field()
    music_rating = Field()
    music_votes = Field()
    music_tags = Field()
    music_url = Field()
 
# 乐评
class MusicReviewItem(Item):
    review_title = Field()
    review_content = Field()
    review_author = Field()
    review_music = Field()
    review_time = Field()
    review_url = Field()
 
# 电影
class VideoItem(Item):
    video_name = Field()
    video_alias = Field()
    video_actor = Field()
    video_year = Field()
    video_time = Field()
    video_rating = Field()
    video_votes = Field()
    video_tags = Field()
    video_url = Field()
    video_director = Field()
    video_type = Field()
    video_bigtype = Field()
    video_area = Field()
    video_language = Field()
    video_length = Field()
    video_writer = Field()
    video_desc = Field()
    video_episodes = Field()
 
 
# 影评
class VideoReviewItem(Item):
    review_title = Field()
    review_content = Field()
    review_author = Field()
    review_video = Field()
    review_time = Field()
    review_url = Field()
     
     
spiders目录下新建两个爬虫文件
 
videospider.py
# --*-- coding: utf-8 --*--
 
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from multi.items import VideoItem, VideoReviewItem
from scrapy import log
 
import re
 
AREA = re.compile(r"制片国家/地区:</span> (.+?)<br>")
ALIAS = re.compile(r"又名:</span> (.+?)<br>")
LANGUAGE = re.compile(r"语言:</span> (.+?)<br>")
EPISODES = re.compile(r"集数:</span> (.+?)<br>")
LENGTH = re.compile(r"单集片长:</span> (.+?)<br>")
 
class VideoSpider(CrawlSpider):
    name = 'video'
    allowed_domains = ['movie.douban.com']
    start_urls = [
        'https://movie.douban.com/tag/',
        'https://movie.douban.com/tag/?view=cloud'
    ]
    rules = (Rule(LinkExtractor(allow=r"/tag/((\d+)|([\u4e00-\u9fa5]+)|(\w+))$")),
             Rule(LinkExtractor(allow=r"/tag/((\d+)|([\u4e00-\u9fa5]+)|(\w+))\?start=\d+\&type=T$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/reviews$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?start=\d+$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/$"), callback="parse_video", follow=True),
             Rule(LinkExtractor(allow=r"/review/\d+/$"), callback="parse_review", follow=True),
             )
 
    def parse_video(self, response):
        item = VideoItem()
        try:
            item["video_url"= response.url
            item["video_name"= ''.join(
                response.xpath('//*[@id="content"]/h1/span[@property="v:itemreviewed"]/text()').extract()
            )
            try:
                item["video_year"= ''.join(
                    response.xpath('//*[@id="content"]/h1/span[@class="year"]/text()').extract()).replace(
                    "(", "").replace(")", ""
                )
            except Exception as e:
                print('Exception:', e)
                item['video_year'= ''
 
            introduction = response.xpath('//*[@id="link-report"]/span[@property="v:summary"]/text()').extract()
            if introduction:
                item["video_desc"= ''.join(introduction).strip().replace("\r\n"" ")
            else:
                item["video_desc"= ''.join(
                    response.xpath('//*[@id="link-report"]/span/text()').extract()).strip().replace("\r\n"" ")
 
            item["video_director"= "|".join(
                response.xpath('//*[@id="info"]/span/span/a[@rel="v:directedBy"]/text()').extract())
            item["video_writer"= "|".join(
                response.xpath('//*[@id="info"]/span[2]/span[2]/a/text()').extract())
 
            item["video_actor"= "|".join(response.xpath("//a[@rel='v:starring']/text()").extract())
 
            item["video_type"= "|".join(response.xpath('//*[@id="info"]/span[@property="v:genre"]/text()').extract())
 
            = "".join(response.xpath("//div[@id='info']").extract())
            = AREA.search(S)
            if is not None:
                item["video_area"= "|".join([area.strip() for area in M.group(1).split("/")])
            else:
                item['video_area'= ''
 
            = "".join(response.xpath("//div[@id='info']").extract())
            AL = ALIAS.search(A)
            if AL is not None:
                item["video_alias"= "|".join([alias.strip() for alias in AL.group(1).split("/")])
            else:
                item["video_alias"= ""
 
            video_info = "".join(response.xpath("//div[@id='info']").extract())
            language = LANGUAGE.search(video_info)
            episodes = EPISODES.search(video_info)
            length = LENGTH.search(video_info)
 
            if language is not None:
                item["video_language"= "|".join([language.strip() for language in language.group(1).split("/")])
            else:
                item['video_language'= ''
            if length is not None:
                item["video_length"= "|".join([runtime.strip() for runtime in length.group(1).split("/")])
            else:
                item["video_length"= "".join(
                    response.xpath('//*[@id="info"]/span[@property="v:runtime"]/text()').extract())
 
            item['video_time'= "/".join(
                response.xpath('//*[@id="info"]/span[@property="v:initialReleaseDate"]/text()').extract())
            if episodes is not None:
                item['video_bigtype'= "电视剧"
                item["video_episodes"= "|".join([episodes.strip() for episodes in episodes.group(1).split("/")])
            else:
                item['video_bigtype'= "电影"
                item['video_episodes'= ''
            item['video_tags'= "|".join(
                response.xpath('//*[@class="tags"]/div[@class="tags-body"]/a/text()').extract())
 
            try:
                item['video_rating'= "".join(response.xpath(
                    '//*[@class="rating_self clearfix"]/strong/text()').extract())
                item['video_votes'= "".join(response.xpath(
                    '//*[@class="rating_self clearfix"]/div/div[@class="rating_sum"]/a/span/text()').extract())
            except Exception as error:
                item['video_rating'= '0'
                item['video_votes'= '0'
                log(error)
 
            yield item
 
        except Exception as error:
            log(error)
 
 
    def parse_review(self, response):
        try:
            item = VideoReviewItem()
            item['review_title'= "".join(response.xpath('//*[@property="v:summary"]/text()').extract())
            content = "".join(
                response.xpath('//*[@id="link-report"]/div[@property="v:description"]/text()').extract())
            item['review_content'= content.lstrip().rstrip().replace("\n"" ")
            item['review_author'= "".join(response.xpath('//*[@property = "v:reviewer"]/text()').extract())
            item['review_video'= "".join(response.xpath('//*[@class="main-hd"]/a[2]/text()').extract())
            item['review_time'= "".join(response.xpath('//*[@class="main-hd"]/p/text()').extract())
            item['review_url'= response.url
            yield item
        except Exception as error:
            log(error)
             
 新建musicspider.py
  
 # --*-- coding: utf-8 --*--
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from multi.items import MusicItem, MusicReviewItem
from scrapy import log
import re
 
class MusicSpider(CrawlSpider):
    name = "music"
    allowed_domains = ['music.douban.com']
    start_urls = [
        'https://music.douban.com/tag/',
        'https://music.douban.com/tag/?view=cloud'
    ]
 
    rules = (Rule(LinkExtractor(allow=r"/tag/((\d+)|([\u4e00-\u9fa5]+)|(\w+))$")),
             Rule(LinkExtractor(allow=r"/tag/((\d+)|([\u4e00-\u9fa5]+)|(\w+))\?start=\d+\&type=T$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/reviews\?sort=time\&start=\d+$")),
             Rule(LinkExtractor(allow=r"/subject/\d+/$"), callback="parse_music", follow=True),
             Rule(LinkExtractor(allow=r"/review/\d+/$"), callback="parse_review", follow=True),
             )
 
    def parse_music(self,response):
        item = MusicItem()
        try:
            item['music_name'= response.xpath('//*[@id="wrapper"]/h1/span/text()').extract()[0]
            content = "".join(response.xpath('//*[@id="info"]').extract())
            info = response.xpath('//*[@id="info"]/span').extract()
            item['music_alias'= ""
            item['music_singer'= ""
            item['music_time'= ""
            for in range(0len(info)):
                if "又名" in info[i]:
                    if == 0:
                        item['music_alias'= response.xpath('//*[@id="info"]/text()').extract()[1].replace("\xa0", "").replace("\n", "").rstrip()
                    elif == 1:
                        item['music_alias'= response.xpath('//*[@id="info"]/text()').extract()[2].replace("\xa0", "").replace("\n", "").rstrip()
                    elif == 2:
                        item['music_alias'= response.xpath('//*[@id="info"]/text()').extract()[3].replace("\xa0", "").replace("\n", "").rstrip()
                    else:
                        item['music_alias'= ""
 
                if "表演者" in info[i]:
                    if == 0:
                        item['music_singer'= "|".join(response.xpath('//*[@id="info"]/span[1]/span/a/text()').extract())
                    elif == 1:
                        item['music_singer'= "|".join(
                            response.xpath('//*[@id="info"]/span[2]/span/a/text()').extract())
 
                    elif == 2:
                        item['music_singer'= "|".join(
                            response.xpath('//*[@id="info"]/span[3]/span/a/text()').extract())
 
                    else:
                        item['music_singer'= ""
                if "发行时间" in info[i]:
                    nbsp = re.findall(r"<span class=\"pl\">发行时间:</span>(.*?)<br>", content, re.S)
                    item['music_time'= "".join(nbsp).replace("\xa0", "").replace("\n", "").replace(" ", "")
 
            try:
                item['music_rating'= "".join(response.xpath('//*[@class="rating_self clearfix"]/strong/text()').extract())
                item['music_votes'= "".join(response.xpath('//*[@class="rating_self clearfix"]/div/div[@class="rating_sum"]/a/span/text()').extract())
            except Exception as error:
                item['music_rating'= '0'
                item['music_votes'= '0'
                log(error)
            item['music_tags'= "|".join(response.xpath('//*[@id="db-tags-section"]/div/a/text()').extract())
            item['music_url'= response.url
            yield item
        except Exception as error:
            log(error)
 
    def parse_review(self, response):
        try:
            item = MusicReviewItem()
            item['review_title'= "".join(response.xpath('//*[@property="v:summary"]/text()').extract())
            content = "".join(
                response.xpath('//*[@id="link-report"]/div[@property="v:description"]/text()').extract()
            )
            item['review_content'= content.lstrip().rstrip().replace("\n"" ")
            item['review_author'= "".join(response.xpath('//*[@property = "v:reviewer"]/text()').extract())
            item['review_music'= "".join(response.xpath('//*[@class="main-hd"]/a[2]/text()').extract())
            item['review_time'= "".join(response.xpath('//*[@class="main-hd"]/p/text()').extract())
            item['review_url'= response.url
            yield item
        except Exception as error:
            log(error)
             
 新建启动文件run.py
 # --*-- coding: utf-8 --*--
from scrapy import cmdline
cmdline.execute("scrapy crawl music".split())
cmdline.execute("scrapy crawl video".split())