百度文库下载器,Python在线百度文库爬虫

功用展现

图片 1

初叶界面.png

图片 2

雕塑课感想_百度文库_and_四周小站.png

图片 3

取得资源.png

图片 4

来得文本.png

有关ppt的爬取,小编也写出来了,不过我的私房服务器带宽太小,导致前者等待时间过长,就把它注释了…有趣味的可以看此前写过的那篇小说百度文库下载器,文章底部提供了编译好的可执行文件(协理ppt,txt,pdf,word格式的下载),可以下载到当地玩一玩…

骨干源码:

import os
import re
import json
import requests
from lxml import etree
import sys
from fangyuanxiaozhan.settings import BASE_DIR


# import tkinter

# 创建文库基类
class BaiduWK(object):
    def __init__(self, url):
        self.title = None
        self.url = url
        self.docType = None
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'}
        self.get_response_content(self.url)
        self.get_doc_type_and_title()

    def get_response_content(self, url):
        try:
            response = requests.get(url, headers=self.headers)
            return response.content
        except Exception as e:
            print(e)
            pass

    def get_doc_type_and_title(self):
        # 获取源码
        source_html = self.get_response_content(self.url)
        # 解析源码
        content = source_html.decode('gbk')
        # 获取文档类型
        self.docType = re.findall(r"docType.*?\:.*?\'(.*?)\'\,", content)[0]
        # 获取文档标题
        self.title = re.findall(r"title.*?\:.*?\'(.*?)\'\,", content)[0]


# 创建获取txt的类
class BDWKTXT(BaiduWK):
    def __init__(self, url):
        super().__init__(url)
        self.docId = None
        pass

    def get_txt(self, url):
        # 获取源码
        source_html = self.get_response_content(url)
        content = source_html.decode("gbk")
        # 获取docId
        self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]
        # 拼接请求url
        token_url = "https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id=" + self.docId
        # 再次请求
        first_json = self.get_response_content(token_url).decode()
        str_first_json = re.match(r'.*?\((\{.*?\})\).*', first_json).group(1)
        # print(str_first_json)
        the_first_json = json.loads(str_first_json)
        md5sum = the_first_json["md5sum"]
        rn = the_first_json["docInfo"]["totalPageNum"]
        rsign = the_first_json["rsign"]
        # print(md5sum,"-->",rn)
        # 请求目标url
        target_url = "https://wkretype.bdimg.com/retype/text/" + self.docId + "?" + md5sum + "&callback=cb" + "&pn=1&rn=" + rn + "&type=txt" + "&rsign=" + rsign

        # https://wkretype.bdimg.com/retype/text/de53bafaf705cc1755270982?md5sum=b89f055e765e6f73db57525bcfc3c2d2&sign=7b385f9cf7&callback=cb&pn=1&rn=12&type=txt

        sec_json = self.get_response_content(target_url).decode()
        # print(type(sec_json),"-->")
        str_sec_json = re.match(r'.*?\(\[(.*)\]\)$', sec_json).group(1)
        str_sec_json += ","
        str_json_list = str_sec_json.split('},')
        result_txt = ""
        # 截取尾部空格
        str_json_list = str_json_list[:-1]
        for str_json in str_json_list:
            str_json += "}"
            pure_dic = json.loads(str_json)
            pure_txt = pure_dic["parags"][0]["c"].strip()
            result_txt += pure_txt

        # 创建文件目录
        try:
            # path = "." + os.sep + self.docType
            path = BASE_DIR + "/static/spider_file/bdwk" + os.sep + self.docType
            print(type(path),"path-->",path)

            os.makedirs(path)
        except Exception as e:
            # print("文件夹%s已存在"%(path))
            pass
        # 创建文件,保存信息
        try:
            print(os.path)
            file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt"
            with open(file_name, 'w', encoding='utf-8') as f:
                f.write(result_txt)
                print("已经保存为:", self.title + '.txt')
        except Exception as e:
            # print(e)
            pass


# 创建获取word的类
class BDWKDOC(BaiduWK):
    def __init__(self, url):
        super().__init__(url)
        # 保存数据来源url
        self.pure_addr_list = list()

    # 获取数据来源url
    def get_pure_addr_list(self):
        # 获取页面源码
        source_html = self.get_response_content(self.url).decode('gbk')
        # 从源码中批量提取数据url
        all_addr = re.findall(r'wkbos\.bdimg\.com.*?json.*?expire.*?\}', source_html)
        pure_addr_list = list()
        # 获取文档标题
        self.title = etree.HTML(source_html).xpath("//title/text()")[0].strip()
        # 净化数据来源url列表
        for addr in all_addr:
            addr = "https://" + addr.replace("\\\\\\/", "/")
            addr = addr[:-5]
            pure_addr_list.append(addr)
        # 将处理好的url列表保存为全局属性
        self.pure_addr_list = pure_addr_list

        return pure_addr_list

    # 从数据来源的url列表中提取数据
    def get_json_content(self, url_list):
        content = ''
        result = ''
        sum = len(url_list)
        i = 1
        for pure_addr in url_list:
            print("正在下载第%d条数据, 剩余%d条" % (i, sum - i))
            i += 1
            try:
                # 获取json数据
                content = self.get_response_content(pure_addr).decode()
                # 处理json数据
                content = re.match(r'.*?\((.*)\)$', content).group(1)

                # 将json数据中需要的内容提取出来
                all_body_info = json.loads(content)["body"]
                # 遍历获取所有信息,并将信息拼接到一起
                for body_info in all_body_info:
                    try:
                        result = result + body_info["c"].strip()
                        # print(">>",result)
                    except Exception as e:
                        print(e)
                        pass

            except Exception as e:
                print(e)
                pass
        # 创建文件目录
        try:
            path = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType
            # print(type(path),"path-->",path)

            os.makedirs(path)
        except Exception as e:
            # print("文件夹%s已存在"%(path))
            pass
        # 创建文件,保存信息
        try:
            file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt"
            with open(file_name, 'w', encoding='utf-8') as f:
                f.write(result)
                print("已经保存为:", self.title + '.txt')
        except Exception as e:
            print(e)


# 创建获取ppt的类
class BDWKPPT(BaiduWK):
    def __init__(self, url):
        self.all_img_url = list()
        super().__init__(url)

    # 获取json数据保存文件
    def get_ppt_json_info(self):
        # 获取源文件
        ppt_source_html = self.get_response_content(self.url)
        # 解析源文件
        content = ppt_source_html.decode('gbk')
        # 测试
        with open("test.html", "w") as f:
            f.write(content)

        # 获取文档Id
        self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]
        # 拼接请求json的接口
        source_json_url = 'https://wenku.baidu.com/browse/getbcsurl?doc_id=%s&type=ppt&callback=zhaozhao' % self.docId
        # 获取字符串类型的json数据
        str_source_json = self.get_response_content(source_json_url).decode()
        # 处理字符串类型的json数据,使其成为标准格式
        pure_str_source_json = re.match(r'.*?\((.*?)\)', str_source_json).group(1)
        # 将字符串json转为可处理的正式json
        source_json = json.loads(pure_str_source_json)

        # 遍历字典中的数据类型list
        for j in source_json['list']:
            # 创建临时列表
            temp_num_url = list()
            # 将url和page拼接到列表中
            temp_num_url.append(j["zoom"])
            temp_num_url.append(j["page"])
            # 将列表信息添加到全局变量中
            self.all_img_url.append(temp_num_url)
        # 建立文件夹
        try:
            os.makedirs(BASE_DIR+"/static/spider_file/bdwk/ppt/%s" % (self.title))
        except Exception as e:
            pass

        for img_url in self.all_img_url:
            print(BASE_DIR+"正在获取第%d页资源(剩余%d页)" % (img_url[1], len(self.all_img_url) - img_url[1]))
            data = self.get_response_content(img_url[0])
            path = BASE_DIR+"/static/spider_file/bdwk/ppt/%s/%s" % (self.title, str(img_url[1]) + '.jpg')
            with open(path, 'wb') as f:
                f.write(data)

        print("写入完毕")


# 运行主程序
def main(url):
    try:
        # url = input("请输入资源所在的网址:")
        docType = BaiduWK(url).docType
    except:
        print("您输入的url,有误请重新输入!")
        os.exit()
    print("类型为", "-->", docType)

    if docType == "ppt2":
        ppt = BDWKPPT(url)
        print("您将要获取的演示文稿(ppt)名称为:", ppt.title)
        ppt.get_ppt_json_info()
        data = dict()
        src = "/static/spider_file/bdwk/ppt/%s/" % (ppt.title)
        title_name = ppt.title
        data["src"] = src
        data["title_name"] = title_name
        return data

    elif docType == "doc":
        word = BDWKDOC(url)
        print("您将要获取的文档(word)名称为", word.title)
        pure_addr_list = word.get_pure_addr_list()
        word.get_json_content(pure_addr_list)
        data = dict()
        src = "/static/spider_file/bdwk/doc/%s.txt" % (word.title)
        title_name = word.title
        data["src"] = src
        data["title_name"] = title_name+".txt"
        return data

    elif docType == "pdf":
        pdf = BDWKDOC(url)
        print("您将要获取的PDF名称为:", pdf.title)
        pure_addr_list = pdf.get_pure_addr_list()
        pdf.get_json_content(pure_addr_list)
        data = dict()
        src = "/static/spider_file/bdwk/pdf/%s.txt" % (pdf.title)
        title_name = pdf.title
        data["src"] = src
        data["title_name"] = title_name+".txt"
        return data


    elif docType == "txt":
        txt = BDWKTXT(url)
        print("您将要下载的文本文档(txt)名称为:", txt.title)
        txt.get_txt(url)
        data = dict()
        src = "/static/spider_file/bdwk/txt/%s.txt" % (txt.title)
        title_name = txt.title
        data["src"] = src
        data["title_name"] = title_name+".txt"
        return data

    else:
        other = BDWKPPT(url)
        print("暂不支持下载%s类型" % (other.docType))
        data = dict()
        data["src"] = ""
        data["title_name"] = "暂不支持%s类型"%(other.docType)
        return data

下载器的数据来源

浅析资源四处页面的源码,获取请求资源的接口,用requests库请求资源,然后手动落成文件的拼接规则,最终把公文内容输出到剧本同级目录下的文本夹中

百度文库,文字类型的资源(如txt, doc,
pdf),是足以在前端源码分析获得到的,如若能根据规则合理的提取这一个文字资源,就可以完毕免下载券获取资源.

小编完毕了百度文库的在线版爬虫,只需提供文库资源页面的url,就足以赢得相应的丛书资源(免下载券)

ppt类型文档

图片 5

ppt.png

在线测试地方:

http://zhaozhaoli.vicp.io/spider/bdwk

word类型文档

图片 6

word.png

windows平台运行

图片 7

doc.gif

txt文档

图片 8

image.png

贯彻源码:

import os
import re
import json
import requests
from lxml import etree

# 创建文库基类
class BaiduWK(object):
    def __init__(self, url):
        self.title = None
        self.url = url
        self.docType = None
        self.headers = {'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'}
        self.get_response_content(self.url)
        self.get_doc_type_and_title()

    def get_response_content(self, url):
        try:
            response = requests.get(url, headers = self.headers)
            return response.content
        except Exception as e:
            print(e)
            pass
    def get_doc_type_and_title(self):
        # 获取源码
        source_html = self.get_response_content(self.url)
        # 解析源码
        content = source_html.decode('gbk')
        # 获取文档类型
        self.docType = re.findall(r"docType.*?\:.*?\'(.*?)\'\,", content)[0]
        # 获取文档标题
        self.title = re.findall(r"title.*?\:.*?\'(.*?)\'\,", content)[0]

# 创建获取txt的类
class BDWKTXT(BaiduWK):
    def __init__(self, url):
        super().__init__(url)
        self.docId = None
        pass

    def get_txt(self, url):
        # 获取源码
        source_html = self.get_response_content(url)
        content = source_html.decode("gbk")
        # 获取docId
        self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]
        # 拼接请求url
        token_url =  "https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id=" + self.docId
        # 再次请求
        first_json = self.get_response_content(token_url).decode()
        str_first_json = re.match(r'.*?\((\{.*?\})\).*', first_json).group(1)
        # print(str_first_json)
        the_first_json = json.loads(str_first_json)
        md5sum = the_first_json["md5sum"]
        rn = the_first_json["docInfo"]["totalPageNum"]
        rsign = the_first_json["rsign"]
        # print(md5sum,"-->",rn)
        # 请求目标url
        target_url = "https://wkretype.bdimg.com/retype/text/" + self.docId + "?" + md5sum + "&callback=cb" + "&pn=1&rn=" + rn + "&type=txt" +"&rsign=" + rsign

        # https://wkretype.bdimg.com/retype/text/de53bafaf705cc1755270982?md5sum=b89f055e765e6f73db57525bcfc3c2d2&sign=7b385f9cf7&callback=cb&pn=1&rn=12&type=txt

        sec_json = self.get_response_content(target_url).decode()
        # print(type(sec_json),"-->")
        str_sec_json = re.match(r'.*?\(\[(.*)\]\)$', sec_json).group(1)
        str_sec_json += ","
        str_json_list = str_sec_json.split('},')
        result_txt = ""
        # 截取尾部空格
        str_json_list = str_json_list[:-1]
        for str_json in str_json_list:
            str_json += "}"
            pure_dic = json.loads(str_json)
            pure_txt = pure_dic["parags"][0]["c"].strip()
            result_txt += pure_txt

        # 创建文件目录
        try:
            path = "." + os.sep + self.docType
            # print(type(path),"path-->",path)

            os.makedirs(path)
        except Exception as e:
            # print("文件夹%s已存在"%(path))
            pass
        # 创建文件,保存信息
        try:
            file_name = "." + os.sep +self.docType + os.sep + self.title + ".txt"
            with open(file_name, 'w') as f:
                f.write(result_txt)
                print("已经保存为:",self.title + '.txt')
        except Exception as e:
            # print(e)
            pass



# 创建获取word的类
class BDWKDOC(BaiduWK):
    def __init__(self, url):
        super().__init__(url)
        # 保存数据来源url
        self.pure_addr_list = list()

    # 获取数据来源url
    def get_pure_addr_list(self):
        # 获取页面源码
        source_html = self.get_response_content(self.url).decode('gbk')
        # 从源码中批量提取数据url
        all_addr = re.findall(r'wkbos\.bdimg\.com.*?json.*?expire.*?\}',source_html)
        pure_addr_list = list()
        # 获取文档标题
        self.title = etree.HTML(source_html).xpath("//title/text()")[0].strip()
        # 净化数据来源url列表
        for addr in all_addr:
            addr = "https://" + addr.replace("\\\\\\/", "/")
            addr = addr[:-5]
            pure_addr_list.append(addr)
        # 将处理好的url列表保存为全局属性
        self.pure_addr_list = pure_addr_list

        return pure_addr_list
    # 从数据来源的url列表中提取数据
    def get_json_content(self, url_list):
        content = ''
        result = ''
        sum = len(url_list)
        i = 1
        for pure_addr in url_list:
            print("正在下载第%d条数据, 剩余%d条"%(i, sum-i))
            i += 1
            try:
                # 获取json数据
                content = self.get_response_content(pure_addr).decode()
                # 处理json数据
                content = re.match(r'.*?\((.*)\)$', content).group(1)

                # 将json数据中需要的内容提取出来
                all_body_info = json.loads(content)["body"]
                # 遍历获取所有信息,并将信息拼接到一起
                for body_info in all_body_info:
                    try:
                        result = result + body_info["c"].strip()
                    except Exception as e:
                        # print(e)
                        pass

            except Exception as e:
                # print(e)
                pass
        # 创建文件目录
        try:
            path = "." + os.sep + self.docType
            # print(type(path),"path-->",path)

            os.makedirs(path)
        except Exception as e:
            # print("文件夹%s已存在"%(path))
            pass
        # 创建文件,保存信息
        try:
            file_name = "." + os.sep +self.docType + os.sep + self.title + ".txt"
            with open(file_name, 'w') as f:
                f.write(result)
                print("已经保存为:",self.title + '.txt')
        except Exception as e:
            # print(e)
            pass


# 创建获取ppt的类
class BDWKPPT(BaiduWK):
    def __init__(self, url):
        self.all_img_url = list()
        super().__init__(url)

    # 获取json数据保存文件
    def get_ppt_json_info(self):
        # 获取源文件
        ppt_source_html = self.get_response_content(self.url)
        # 解析源文件
        content = ppt_source_html.decode('gbk')
        # print("-->",len(content))
        # 测试
        with open("test.html", "w") as f:
            f.write(content)

        # 获取文档Id
        self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0]
        # 拼接请求json的接口
        source_json_url = 'https://wenku.baidu.com/browse/getbcsurl?doc_id=%s&type=ppt&callback=zhaozhao'%self.docId
        # 获取字符串类型的json数据
        str_source_json = self.get_response_content(source_json_url).decode()
        # 处理字符串类型的json数据,使其成为标准格式
        pure_str_source_json = re.match(r'.*?\((.*?)\)', str_source_json).group(1)
        # 将字符串json转为可处理的正式json
        source_json = json.loads(pure_str_source_json)

        # 遍历字典中的数据类型list
        for j in source_json['list']:
            # 创建临时列表
            temp_num_url = list()
            # 将url和page拼接到列表中
            temp_num_url.append(j["zoom"])
            temp_num_url.append(j["page"])
            # 将列表信息添加到全局变量中
            self.all_img_url.append(temp_num_url)

        # 建立文件夹
        try:
            os.makedirs("./ppt/%s"%(self.title))
        except Exception as e:
            # print("---->>",e)
            pass

        for img_url in self.all_img_url:
            # print(img_url)
            print("正在获取第%d页资源(剩余%d页)"%(img_url[1], len(self.all_img_url)-img_url[1]))
            data = self.get_response_content(img_url[0])
            path = "./ppt/%s/%s"%(self.title, str(img_url[1])+'.jpg')
            with open (path, 'wb') as f:
                f.write(data)

        print("写入完毕")



# 运行主程序
def main():
    try:
        url = input("请输入资源所在的网址:")
        docType = BaiduWK(url).docType
    except:
        print("您输入的url,有误请重新输入!")
        os.exit()
    print("类型为","-->",docType)

    if docType == "ppt":

        ppt = BDWKPPT(url)
        print("您将要获取的演示文稿(ppt)名称为:", ppt.title)
        ppt.get_ppt_json_info()

    elif docType == "doc":
        word = BDWKDOC(url)
        print("您将要获取的文档(word)名称为", word.title)
        pure_addr_list = word.get_pure_addr_list()
        word.get_json_content(pure_addr_list)

    elif docType == "pdf":
        pdf = BDWKDOC(url)
        print("您将要获取的PDF名称为:", pdf.title)
        pure_addr_list = pdf.get_pure_addr_list()
        pdf.get_json_content(pure_addr_list)

    elif docType == "txt":

        txt = BDWKTXT(url)
        print("您将要下载的文本文档(txt)名称为:", txt.title)
        txt.get_txt(url)

    else:
        other = BDWKPPT(url)
        print("暂不支持下载%s类型"%(other.docType))
        pass


if __name__ == '__main__':
    main()

自己把那一个本子编译成了exe文件,windows用户从上面的资源帖子里按小说名自取:
http://www.jianshu.com/p/4f28e1ae08b1

  • 百度文库有一对急需下载券,才能下载的材料
  • 但实际上文库是允许大家预览的,不过不容许大家复制内容
  • 咱俩只是须求中间的文字内容,对情节的样式没有何样须要

下载器已毕的效应:

1.比照输入的网址,自动判断文档类型,并将下载好的资源位居相应的公文夹中

图片 9

机动分类.png

  1. 将ppt类型的文档自动转换为图片,并按原来的逐一命名保存

    图片 10

    image.png

    3.pdf,word.txt类型的多少总体免除格式,以txt格式保存文本

    图片 11

    image.png

兑现效益

图片 12

下载word与pdf.png

图片 13

下载ppt.png

图片 14

下载txt.png

相关文章