Swoole实践：如何使用协程构建高性能爬虫

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

<?php

use SwooleCoroutineHttpClient;

class Spider

{

    private $urls = array();

    private $queue;

    private $maxDepth = 3; // 最大爬取深度

    private $currDepth = 0; // 当前爬取深度

    private $startTime;

    private $endTime;

    private $concurrency = 10; // 并发数

    private $httpClient;

    public function __construct($urls)

    {

        $this->urls = $urls;

        $this->queue = new SplQueue();

        $this->httpClient = new Client('127.0.0.1', 80);

    }

    public function run()

    {

        $this->startTime = microtime(true);

        foreach ($this->urls as $url) {

            $this->queue->enqueue($url);

        }

        while (!$this->queue->isEmpty() && $this->currDepth <= $this->maxDepth) {

            $this->processUrls();

            $this->currDepth++;

        }

        $this->endTime = microtime(true);

        echo "爬取完成，用时：" . ($this->endTime - $this->startTime) . "s

";

    }

    private function processUrls()

    {

        $n = min($this->concurrency, $this->queue->count());

        $array = array();

        for ($i = 0; $i < $n; $i++) {

            $url = $this->queue->dequeue();

            $array[] = $this->httpClient->get($url);

        }

        // 等待所有请求结束

        foreach ($array as $httpResponse) {

            $html = $httpResponse->body;

            $this->parseHtml($html);

        }

    }

    private function parseHtml($html)

    {

        // 解析页面

        // ...

        // 处理并存储数据

        // ...

        // 将页面中的URL添加到队列中

        // ...

    }

}