Init commit with workable stuff

comichandler.js

@@ -0,0 +1,142 @@
+var $ = require("cheerio");
+var request = require('request');
+var agent = require("socks5-http-client/lib/Agent");
+var fs = require('fs');
+var ListHandler = require('./listhandler.js');
+
+require('./config.js');
+
+
+var ComicHandler = {
+    NowIndex:0,
+    ListTree:[],
+    start:function(fromComicID){
+        if(!this.ListTree.length) this.ListTree = ListLib.tree;
+        if(fromComicID){
+            for(var i in this.ListTree){
+                if(this.ListTree[i]==fromComicID);
+                this.NowIndex = i;
+                break;
+            }
+        }
+        if(this.NowIndex==this.ListTree.length){
+            console.log('Comic Download Complete');
+        }else{
+            var comicID = this.ListTree[this.NowIndex];
+            new ComicSaver(comicID);
+            this.NowIndex++;
+        }
+    }
+}
+
+var ComicSaver = function(comicID){
+    var ep = 0;
+    var nowEp = 0;
+    var imgLength = 0;
+    var imgDownloaded = 0;
+    var retryTimes = {};
+
+    var comicOBJ = ListLib[comicID];
+    if(!comicOBJ){
+        console.log('ComicID '+comicID+' wrong!');
+        return;
+    }
+
+    console.log('[BEGIN] Start Download Comic [ '+comicID+' ]');
+    console.log('Creating directory for [ '+comicOBJ.name+' ]');
+    var comicPath = './save/'+comicOBJ.id+'-'+comicOBJ.name+'/';
+    if(!fs.existsSync(comicPath)){
+        fs.mkdirSync(comicPath);
+    }
+
+    var options = {
+        url:ApiURL.comicEp.replace('{ComicID}',comicOBJ.id)
+    }
+
+    console.log('Analysing......');
+
+    request(options,function(error,response,body){
+        if(!error && body){
+            var data = JSON.parse(body);
+            ep = data['ep_count'];
+            nowEp = 1;
+            console.log('Analysie Success!');
+            console.log('Comic [ '+comicOBJ.id+' ] has [ '+ep+' ] EP!!!');
+            downloadEP();
+        }else{
+            console.log('Comic [ '+comicOBJ.id+' ] analysie error!!!');
+        }
+    });
+
+    function downloadEP(){
+        if(nowEp>ep){
+            finished();
+            return;
+        }
+        var opt = {
+            url:ApiURL.comic.replace('{ComicID}',comicOBJ.id).replace('{EP}',nowEp)
+        }
+        console.log('Comic [ '+comicOBJ.id+' ] start download!!!');
+        var epPath = comicPath+nowEp+'/';
+        if(!fs.existsSync(epPath)){
+            fs.mkdirSync(epPath);
+        }
+        request(opt,function(error,response,body){
+            if(!error && body){
+                var data = JSON.parse(body);
+                imgLength = data.length;
+                console.log('Comic [ '+comicOBJ.id+' ] EP [ '+nowEp+' ] has [ '+imgLength+' ] images!!!');
+                console.log('Comic [ '+comicOBJ.id+' ] EP [ '+nowEp+' ] images start download!!!');
+                for(var i in data){
+                    retryTimes[data[i]['url']] = 0;
+                    saveImg(data[i]['url']);
+                }
+            }else{
+                console.log(error);
+                console.log(body);
+                console.log('Comic [ '+comicOBJ.id+' ] download error!!!');
+            }
+        });
+    }
+
+    function saveImg(imgSrc){
+        if(!imgSrc){
+            console.log('Comic [ '+comicOBJ.id+' ] image download error!!!');
+            console.log('Image [ '+imgSrc+' ] error');
+        }
+        var splitURL = imgSrc.split('/');
+        var opt = {
+            url:imgSrc,
+            encoding:'binary'
+        }
+        request(opt,function(error,response,body){
+            if(!error && body){
+                var data = body;
+                var name = splitURL[splitURL.length-1];
+                var savePath = comicPath+nowEp+'/'+name;
+                fs.writeFileSync(savePath,data,'binary');
+                imgDownloaded++;
+                if(imgDownloaded==(imgLength-1)){
+                    nowEp++;
+                    downloadEP();
+                }
+            }else{
+                if(retryTimes[imgSrc]==3){
+                    console.log('Comic [ '+comicOBJ.id+' ] image download error!!!');
+                    console.log('Image [ '+imgSrc+' ] error');
+                }else{
+                    retryTimes[imgSrc]++;
+                    saveImg(imgSrc);
+                    console.log('Image [ '+imgSrc+' ] redownload!');
+                }
+                
+            }
+        });
+    }
+
+    function finished(){
+        console.log('[FINISH] Comic [ '+comicOBJ.id+' ] All File Downloaded!!!');
+    }
+}
+
+module.exports = ComicHandler;

config.js

@@ -0,0 +1,10 @@
+global.ApiURL = {
+    category:"http://picaman.picacomic.com/api/categories",
+    list:"http://picaman.picacomic.com/api/categories/{CID}/page/{PAGE}/comics",
+    comicEp:"http://picaman.picacomic.com/api/comics/{ComicID}",
+    comic:"http://picaman.picacomic.com/api/comics/{ComicID}/ep/{EP}"
+}
+
+global.CategoryLib = null;
+global.ListLib = null;
+global.ComicLib = null;

listhandler.js

@@ -0,0 +1,65 @@
+var $ = require("cheerio");
+var request = require('request');
+var agent = require("socks5-http-client/lib/Agent");
+var fs = require('fs');
+
+require('./config.js');
+
+var ListHandler = function(CategoryID,StartPage,callback){
+    var ListData = {};
+    var PAGE = StartPage?StartPage:1;
+    var CID = CategoryID?CategoryID:1;
+    var total = 0;
+    var ListTree = [];
+    getlist();
+
+    function getlist(){
+        var url = ApiURL.list.replace('{CID}',CID).replace('{PAGE}',PAGE);
+        var options = {
+            url:url
+            // agentClass:agent,
+            // agentOption:{
+            //     socksHost:'127.0.0.1',
+            //     socksPort:1080
+            // }
+        };
+        request(options,function(error,response,body){
+            if(!error && body){
+                var oldData = JSON.parse(body);
+                if(oldData.length>0){
+                    for(var i in oldData){
+                        var ListObj = oldData[i];
+                        ListData[ListObj.id] = {
+                            id:ListObj.id,
+                            name:ListObj.name,
+                            total:ListObj.total_page
+                        };
+                        ListTree.push(ListObj.id);
+                        total++;
+                    }
+                    console.log('PAGE [ '+PAGE+' ] DONE');
+                    PAGE++;
+                    getlist();
+                }else{
+                    PAGE--;
+                    ListData['total'] = total;
+                    ListData['tree'] = ListTree;
+                    saveTemp();
+                }
+            }else{
+                console.log('List read error: ',CID,PAGE);
+            }
+        });
+    }
+
+    function saveTemp(){
+        fs.writeFileSync('./temp/Category_'+CID+'.json',JSON.stringify(ListData),'UTF-8');
+        ListLib = ListData;
+        console.log('Category '+CID+' with '+PAGE+' pages DONE!');
+        if(callback){
+            callback.call(this);
+        }
+    }
+};
+
+module.exports = ListHandler;

package.json

@@ -0,0 +1,20 @@
+{
+  "name": "webSniffer",
+  "version": "1.0.0",
+  "description": "User action tracer for chh",
+  "main": "sniffer.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "fs-extra":"latest",
+    "cheerio":"latest",
+    "request":"latest",
+    "socks5-http-client":"latest"
+  },
+  "engines": {
+    "node": ">=0.12.5"
+  },
+  "author": "HonorLee",
+  "license": "ISC"
+}

sniffer.js

@@ -0,0 +1,72 @@
+var $ = require("cheerio");
+var request = require('request');
+var agent = require("socks5-http-client/lib/Agent");
+var fs = require('fs');
+var ListHandler = require('./listhandler.js');
+var ComicHandler = require('./comichandler.js');
+
+require('./config.js');
+
+
+var loadCategory = 12;
+
+var CheckLibsTimer = setInterval(checkLibs,1000);
+setInterval(function(){},1000000);
+// getCategories(getList);
+function getCategories(callback){
+    var options = {
+        url:ApiURL.category
+        // agentClass:agent,
+        // agentOption:{
+        //     socksHost:'127.0.0.1',
+        //     socksPort:1080
+        // }
+    };
+    request(options,function(error,response,body){
+        if(!error && body){
+            var oldData = JSON.parse(body);
+            for(var i in oldData){
+                var ListObj = oldData[i];
+                CategoryLib[ListObj.id] = {
+                    id:ListObj.id,
+                    name:ListObj.name,
+                    length:ListObj.all_comics
+                };
+            }
+            fs.writeFileSync('./temp/Category.json',JSON.stringify(CategoryLib),'UTF-8');
+            if(callback){
+                callback.call(this);
+            }
+        }
+    });
+}
+
+function getList(){
+    new ListHandler(12,1);
+}
+
+
+getTempLib();
+function getTempLib(){
+    var data;
+    if(fs.existsSync('./temp/Category.json')){
+        data = fs.readFileSync('./temp/Category.json');
+        CategoryLib = JSON.parse(data);
+    }else{
+        getCategories();
+    }
+    if(fs.existsSync('./temp/Category_'+loadCategory+'.json')){
+        data = fs.readFileSync('./temp/Category_'+loadCategory+'.json');
+        ListLib = JSON.parse(data);
+    }else{
+        new ListHandler(loadCategory);
+    }
+}
+
+function checkLibs(){
+    if(CategoryLib && ListLib){
+        clearInterval(CheckLibsTimer);
+        ComicHandler.start();
+    }
+}
+