为plumemo博客添砖铺瓦(后台篇)

功能背景

看了别人的博客,在加上自己也非常喜欢动漫然后就有了这个功能

废话不多说直接上效果

没错就是一个简单bilibili爬虫友情提示: 爬虫学的好,监狱进的早数据玩的溜,牢饭吃个够

下面我们看下目录结构

核心代码

我就直接贴出了部分代码实现起来也是比较简单的后面有需要的我在都发出来


 @Autowired
    private SpiderPipeline spiderPipeline;

    //@Autowired
    //private ConfigBeanProp configBeanProp;



     private Site site = new Site().setRetryTimes(3).setSleepTime(100);



    @Override
    public void process(Page page) {
        String rawText = page.getRawText();
        Map<String,Object> map = (Map<String, Object>) JSON.parse(rawText);
        Map<String,Object> item = (Map<String, Object>) map.get("data");
        float total = Float.parseFloat(item.get("total")+"");
        int number = (int)Math.ceil(total / 15f);
        for (int i = 1;i<=number;i++){
            List<Map<String,Object>> list = (List<Map<String, Object>>) item.get("list");
            page.addTargetRequest("你的URL");
            page.putField("biliBiliSpider",list);
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    //@Scheduled(cron = "0 15 10 L * ?")
    @Scheduled(initialDelay = 1000, fixedDelay = 100 * 1000)
    private void getUrlByBiliBili() {
        String url = "你的URL";
        //String url = String.format(this.configBeanProp.getUrl(), String.valueOf(1));
        Spider.create(new SpiderProcessor())
                    .addUrl(url)
                    .addPipeline(this.spiderPipeline)
                    .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                    //  .thread(5)
                    .run();

    }