ui爬虫工具-未完成

抓取页面的一块ui，将属于ui的html、css分离出来，需要配合浏览器机器人抓取html

const TinyCss=require('./utils/TinyCss')
var getCssText = require("./utils/getCssText");
var getText = require("./utils/getText");
var setText = require("./utils/setText");
//ui 爬虫
async function init() {
    const htmlText=await getText('./src/test.html');
    const cssText=await getText('./src/test.css');
    // const cssText=await getCssText('https://cloud.baidu.com/product/bcd/search.html?keyword=%E5%85%AB%E6%88%92%E7%AE%97%E5%91%BD','html')
    // console.log(htmlText)
    // console.log(cssText);
    const app=new TinyCss([htmlText],cssText);
    const css=app.getTinyAst('.note-list-wrapper').toString();
    // console.log(css);
    setText('./src/testmin.css',css);

}
init();

TinyCss.js

//TinyCss.js
const Api=require('./Api');
//解析成语法树
const compiler = require('vue-template-compiler');
const postcss  = require('postcss');
const querySelectorList=require('./querySelectorList')

//构建出一个css语法树和多个html语法书，分析css的使用率。
class TinyCss{
    constructor(htmlTextArr,cssText){

        //多个html书法树
        this.htmlTextArr=htmlTextArr;

        //一个css书法树
        this.cssAst=postcss.parse(cssText);
        this.cssList=Api.depthSearch(this.cssAst,'nodes').filter(function (node) {
            return node.type==='rule'&&!/keyframes/.test(node.parent.name);
        })

        //输出的部分
        this.bigMap=null;
        this.map=null;
        this.data=null;
        this.emptyCss=null;
        this.emptyKeyFrames=null;


    }

    //移除数组中的子元素
    removeObj(item,arr){
        for(let i=0;i<arr.length;i++){
            if(arr[i]===item){
                arr.splice(i,1)
                break;
            }
        }
    }
    //获取矩阵数据
    getBigMap(){
        if(this.bigMap){
            return this.bigMap;
        }
        let map=[];
        for(let i=0;i<this.htmlTextArr.length;i++){
            const htmlAst=compiler.compile(this.htmlTextArr[i]).ast;

            const ccRect=new querySelectorList(htmlAst,this.cssList);
            const rect=ccRect.analysis();
            map.push(rect)
        }
        this.bigMap=map;
        return map;
    }
    //获取小数据，矩阵数据
    getMap(){
        if(this.map){
            return this.map;
        }
        let map=[];
        for(let i=0;i<this.htmlTextArr.length;i++){
            const htmlText=this.htmlTextArr[i];
            const htmlAst=compiler.compile(htmlText).ast;
            const ccRect=new querySelectorList(htmlAst,this.cssList);
            const arr=ccRect.analysis().map(function (item) {
                return item.reduce((x,y)=>x+y);
            });
            for(let j=0;j<arr.length;j++){
                if(!map[j])map[j]=[];
                map[j].push(arr[j])
            }
        }
        this.map=map;
        return map;
    }
    getUiMap(selector){
        if(this.uiMap){
            return this.uiMap;
        }
        let map=[];
        for(let i=0;i<this.htmlTextArr.length;i++){
            const htmlText=this.htmlTextArr[i];
            const htmlAst=compiler.compile(htmlText).ast;
            const ccRect=new querySelectorList(htmlAst,this.cssList);
            const uiArr=ccRect.querySelectorAndChild(selector)
            const arr=ccRect.analysis().map(function (item) {
                let index=0;
                for(let k=0;k<item.length;k++){
                    if(item[k]===1&&uiArr[k]===1){
                        index++;
                    }
                }
                return index;
            });
            for(let j=0;j<arr.length;j++){
                if(!map[j])map[j]=[];
                map[j].push(arr[j])
            }
        }
        this.uiMap=map;
        return map;
    }
    //移除无用的css
    getEmptyCss(selector){
        if(this.emptyCss){
            return this.emptyCss;
        }
        const cssList=this.cssList;
        const data=[];
        const map=selector?this.getUiMap(selector):this.getMap();
        for(let i=0;i<map.length;i++){
            //存在比0大的就是用到的，都是0就是无用的css
            if(map[i].every(function (n) {
                return n===0
            })){
                //从ast中移除节点
                this.removeObj(cssList[i],cssList[i].parent.nodes);
                data.push(cssList[i].selector);
            }
        }

        this.emptyCss=data;
        return data;
    }
    //移除空的动画
    getEmptyKeyFrames(){
        if(this.emptyKeyFrames){
            return this.emptyKeyFrames;
        }
        const keyframesList=Api.depthSearch(this.cssAst,'nodes').filter(function (node) {
            return node.type==='atrule'&&/keyframes/.test(node.name);
        })
        const vals=Api.depthSearch(this.cssAst,'nodes').filter(function (node) {
            return node.type==='decl'&&/animation/.test(node.prop);
        })
        const delArr=keyframesList.filter(function (node) {
            return !vals.some(function (node2) {
                return node2.value.split(' ').indexOf(node.params)>-1
            })
        })
        const emptyKeyFrames=[];
        delArr.forEach( (node) =>{
            //从ast中移除节点
            this.removeObj(node,node.parent.nodes);
            emptyKeyFrames.push('@'+node.name+' '+node.params)
        })
        this.emptyKeyFrames=emptyKeyFrames;
        return emptyKeyFrames;
    }
    //移除注释
    removeComment(){
        const commentArr=Api.depthSearch(this.cssAst,'nodes').filter(function (node) {
            return node.type==='comment';
        })
        commentArr.forEach((node)=>{
            this.removeObj(node,node.parent.nodes);
        })
    }
    getTinyAst(selector){
        this.getEmptyCss(selector);
        this.getEmptyKeyFrames();
        this.removeComment();

        return this.cssAst;
    }
}
module.exports=TinyCss;

querySelectorList.js

//querySelectorList.js
const Api=require('./Api');
//命中规则

/*css rule矩阵，3*6
行对应selector['.id','.class1','.class2']
列对应html节点 ['body','body div','body div div','body div p','body div span','body div span a']
[
    [0,0,0,0,1,0],
    [0,0,0,0,1,0],
    [0,0,0,0,1,0]
]
*/
class querySelectorList{

    constructor(htmlAst,cssList){

        //记录selector查找历史
        this.selectotCache={};

        //构建html语法树和矩阵bitmap
        this.htmlAst=htmlAst;
        this.htmlList=Api.depthSearch(this.htmlAst).filter(function (node) {
            return node.type===1;
        })

        //构建css语法树和矩阵bitmap
        this.cssList=cssList;
    }
    //分析
    analysis(){
        const cssList=this.cssList;
        const map=[]
        for(let i=0;i<cssList.length;i++){
            map[i]=this.querySelector(cssList[i].selector);
        }
        return map;
    }
    //获取选择器和它得子元素
    querySelectorAndChild(selector){
        const arr=this.querySelector(selector);
        for(let i=0;i<arr.length;i++){
            if(arr[i]===1){
                const cLen=Api.depthSearch(this.htmlList[arr[i]]).filter(function (node) {
                    return node.type===1;
                }).length;
                for(let k=1;k<cLen;k++){
                    i++;
                    arr[i]=1;
                }
            }
        }
        return arr;
    }
    //可能是多选择器
    querySelector(selector){
        if(/,/.test(selector)){
            const arr=selector.split(',');
            const data=[];
            for(let i=0;i<arr.length;i++){
                const item=this.queryOneSelector(arr[i]);
                for(let k=0;k<item.length;k++){
                    if(item[k]===1){
                        data[k]=1;
                    }else{
                        data[k]=0;
                    }
                }
            }
            return data;
        }else{
            return this.queryOneSelector(selector)
        }
    }
    //查询css_rule，返回［array astNode］
    queryOneSelector(selector){
        selector=selector.trim();//去掉左右空格

        //解析css rule
        const selectorArr=[]
        selector.replace(/(.+?)([ >~+]+(?!d)(?! *:)|$)/ig,function (m,p1,p2) {
            selectorArr.push(p1,p2);
        })
        // console.log(selectorArr)
        this.selectorArr=selectorArr;
        // console.log(selectorArr)
        //设置缓存

        let preSelector='';
        for(let i=0;i<selectorArr.length;i=i+2){
            const exec=selectorArr[i-1]||'';
            const curSelector=selectorArr[i];

            this.setSelectotCache(preSelector,exec,curSelector);
            preSelector=preSelector+exec+curSelector
        }
        const arr=new Array(this.htmlList.length).fill(0);
        // if(/ ::/.test(selector))
        // console.log(selector,selectorArr)
        this.selectotCache[selector].forEach( (node) =>{
            arr[this.htmlList.indexOf(node)]=1;
        })
        return arr;
    }
    //记录selector查询html语法树
    setSelectotCache(preSelector,exec,curSelector){

        const nextSelector=preSelector+exec+curSelector;
        //已有缓存
        if(this.selectotCache[nextSelector]){return;}
        if(!preSelector&&!exec){
            this.selectotCache[curSelector]=this.breadthHit(curSelector,this.htmlAst)
            return;
        }
        const arr=this.selectotCache[preSelector];

        this.selectotCache[nextSelector]=[];
        if(/^ +$/.test(exec)){
            arr.forEach((node)=>{
                this.selectotCache[nextSelector]=this.selectotCache[nextSelector].concat(this.breadthHit(curSelector,node));
            })
        }else if(/^ *> *$/.test(exec)){
            arr.forEach((node)=>{
                this.selectotCache[nextSelector]=this.selectotCache[nextSelector].concat(this.childHit(curSelector,node));
            })
        }else if(/^ *+ *$/.test(exec)){
            arr.forEach((node)=>{
                this.selectotCache[nextSelector]=this.selectotCache[nextSelector].concat(this.sublingHit(curSelector,node));
            })
        }else if(/^ *~ *$/.test(exec)){
            arr.forEach((node)=>{
                this.selectotCache[nextSelector]=this.selectotCache[nextSelector].concat(this.sublingsHit(curSelector,node));
            })
        }else{
            console.log('exec异常:'+exec)
        }

    }
    //css_rule:element+element
    sublingHit(tag,astNode){
        if(!astNode.parent){
            return [astNode].filter( (node) =>{
                return this.hitNode(tag,node);
            })
        }
        return Api.nextSublingSearch(astNode,astNode.parent).filter( (node) =>{
            return this.hitNode(tag,node);
        })
    }
    //css_rule:element~element
    sublingsHit(tag,astNode){
        return Api.nextSublingsSearch(astNode,astNode.parent).filter(function (node) {
            return this.hitNode(tag,node);
        })
    }
    //css_rule:element element
    breadthHit(tag,astNode){
        return Api.breadthSearch(astNode).filter( (node)=> {
            return node.type===1&&this.hitNode(tag,node);
        })
    }
    //css_rule:element>element
    childHit(tag,astNode){
        return Api.childSearch(astNode).filter( (node)=> {
            return node.type===1&&this.hitNode(tag,node);
        })
    }
    //tag是否命中ast节点，返回true、false
    hitNode(selector,astNode) {

        //分割字符串 (tag)、(id、class)(val)
        if(selector==='*'){
            return true;
        }else if(/:root/.test(selector)){
            return astNode.tag==='html';
        }else{
            const arr=[];
            //tag
            if(/(^[a-z]+)/i.test(selector)){
                const tag=RegExp.$1;
                arr.push(astNode.tag===tag)
            }
            //class
            if(/.([w-]+)/.test(selector)){
                const val=RegExp.$1;
                arr.push(astNode.attrsMap.class&&astNode.attrsMap.class.split(' ').indexOf(val)>-1);
            }
            //id
            if(/#(w+)/.test(selector)){
                const val=RegExp.$1;
                arr.push(astNode.attrsMap.id===val);
            }
            //属性
            if(/[([w-]+)(~=|=||=)?(w+)?]/.test(selector)){
                const key=RegExp.$1;
                const exec=RegExp.$2;
                const val=RegExp.$3;
                // console.log(selector,'属性选择器,只判断是否存在属性')
                arr.push(astNode.attrsMap.hasOwnProperty(key));
            }
            //伪类选择器
            if(/(:.+)/.test(selector)){
                const key=RegExp.$1;
                // console.log(selector,'解析->',selector.replace(/:.+$/,''))
                arr.push(true)
                // arr.push(astNode.attrsMap.id===val);
            }
            if(arr.length==0){
                // console.log(this.selectorArr)
                console.log(selector,this.selectorArr,'css 解析异常')
            }
            return arr.every((item)=>item);
        }



    }
}
module.exports=querySelectorList;