Initial Commit
This commit is contained in:
commit
c61c5de12c
5 changed files with 553 additions and 0 deletions
1
.gitignore
vendored
Normal file
1
.gitignore
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
node_modules
|
||||
114
index.js
Executable file
114
index.js
Executable file
|
|
@ -0,0 +1,114 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const yargs = require('yargs');
|
||||
const { spawnSync } = require('child_process');
|
||||
const {
|
||||
scrapeAnimevibeDownloadPage,
|
||||
scrapeVidCDNDownloadPage,
|
||||
scrapeAnimevibeSeriesPage,
|
||||
} = require('./scrape.js');
|
||||
|
||||
const argv = yargs
|
||||
.command('download <urls...>', 'Download one or more episodes', yargs =>
|
||||
yargs.positional('urls', {
|
||||
describe: 'List of urls to download from',
|
||||
})
|
||||
.option('start-episode', {
|
||||
alias: 's',
|
||||
type: 'number',
|
||||
describe: 'First episode to download. Default is current episode',
|
||||
})
|
||||
.option('end-episode', {
|
||||
alias: 'e',
|
||||
type: 'number',
|
||||
describe: 'Last episode to download. Default is current episode',
|
||||
})
|
||||
.option('all-episodes', {
|
||||
alias: 'a',
|
||||
type: 'boolean',
|
||||
describe: 'Download all available episodes',
|
||||
})
|
||||
.option('sleep', {
|
||||
alias: 'S',
|
||||
type: 'number',
|
||||
describe: 'Seconds to sleep after each download',
|
||||
}),
|
||||
download
|
||||
)
|
||||
.help().alias('help', 'h')
|
||||
.demandCommand()
|
||||
.strict()
|
||||
.argv;
|
||||
|
||||
async function download(options) {
|
||||
for (const url of options.urls) {
|
||||
const info = await scrapeAnimevibeSeriesPage(url);
|
||||
|
||||
let startEpisode;
|
||||
let endEpisode;
|
||||
if (options.allEpisodes) {
|
||||
startEpisode = 1;
|
||||
endEpisode = info.episodeCount;
|
||||
} else {
|
||||
startEpisode = options.startEpisode || info.currentEpisodeNumber || 1;
|
||||
endEpisode = options.endEpisode || info.currentEpisodeNumber || 1;
|
||||
}
|
||||
|
||||
for (let i = startEpisode; i <= endEpisode; i++) {
|
||||
console.log(`Downloading '${info.titles.english}' episode ${i}`);
|
||||
const downloadInfo = await getDownloadOptions(info.downloadId, i);
|
||||
const download = getBestDownload(downloadInfo.downloads);
|
||||
|
||||
const filename = getOutputFilename(info, download, pad(i, 4));
|
||||
console.log(`Saving to ${filename}`);
|
||||
fs.mkdirSync(path.dirname(filename), { recursive: true });
|
||||
spawnSync('wget', ['--output-document', filename, download.url], { stdio: 'inherit' });
|
||||
|
||||
if (i < endEpisode && options.sleep) {
|
||||
console.log(`Sleeping for ${options.sleep} seconds`);
|
||||
await sleep(options.sleep*1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function getOutputFilename(seriesInfo, download, episode) {
|
||||
const basename = `${seriesInfo.titles.english}-${seriesInfo.titles.japanese}`;
|
||||
const filename = seriesInfo.type.startsWith('Movie')
|
||||
? `${basename}.${download.format}`
|
||||
: `${basename}/${episode}.${download.format}`;
|
||||
return filename
|
||||
.replace(/ /g, '_')
|
||||
.replace(/[^\/\w\d-_\.]/g, '');
|
||||
}
|
||||
|
||||
function pad(n, width, z) {
|
||||
z = z || '0';
|
||||
n = n + '';
|
||||
return n.length >= width ? n : new Array(width - n.length + 1).join(z) + n;
|
||||
}
|
||||
|
||||
function getBestDownload(downloads) {
|
||||
downloads.sort(flip(comparing(d => d.quality)));
|
||||
return downloads[0];
|
||||
}
|
||||
|
||||
async function getDownloadOptions(downloadId, episode) {
|
||||
const url = `https://animevibe.tv/anime-downloader/?id=${downloadId}&episode=${episode}`;
|
||||
const mirrors = await scrapeAnimevibeDownloadPage(url);
|
||||
return await scrapeVidCDNDownloadPage(mirrors.vidcdn);
|
||||
}
|
||||
|
||||
function sleep(millis) {
|
||||
return new Promise(resolve => setTimeout(resolve, millis));
|
||||
}
|
||||
|
||||
function flip(f) {
|
||||
return (a, b) => f(b, a);
|
||||
}
|
||||
|
||||
function comparing(key) {
|
||||
return (a, b) => key(a) > key(b) ? 1 : -1;
|
||||
}
|
||||
274
package-lock.json
generated
Executable file
274
package-lock.json
generated
Executable file
|
|
@ -0,0 +1,274 @@
|
|||
{
|
||||
"name": "animevibe-cli",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 1,
|
||||
"requires": true,
|
||||
"dependencies": {
|
||||
"@types/color-name": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@types/color-name/-/color-name-1.1.1.tgz",
|
||||
"integrity": "sha512-rr+OQyAjxze7GgWrSaJwydHStIhHq2lvY3BOC2Mj7KnzI7XK0Uw1TOOdI9lDoajEbSWLiYgoo4f1R51erQfhPQ=="
|
||||
},
|
||||
"@types/node": {
|
||||
"version": "14.11.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/node/-/node-14.11.2.tgz",
|
||||
"integrity": "sha512-jiE3QIxJ8JLNcb1Ps6rDbysDhN4xa8DJJvuC9prr6w+1tIh+QAbYyNF3tyiZNLDBIuBCf4KEcV2UvQm/V60xfA=="
|
||||
},
|
||||
"ansi-regex": {
|
||||
"version": "5.0.0",
|
||||
"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz",
|
||||
"integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg=="
|
||||
},
|
||||
"ansi-styles": {
|
||||
"version": "4.2.1",
|
||||
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
|
||||
"integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
|
||||
"requires": {
|
||||
"@types/color-name": "1.1.1",
|
||||
"color-convert": "2.0.1"
|
||||
}
|
||||
},
|
||||
"boolbase": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
|
||||
"integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
|
||||
},
|
||||
"cheerio": {
|
||||
"version": "1.0.0-rc.3",
|
||||
"resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
|
||||
"integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
|
||||
"requires": {
|
||||
"css-select": "1.2.0",
|
||||
"dom-serializer": "0.1.1",
|
||||
"entities": "1.1.2",
|
||||
"htmlparser2": "3.10.1",
|
||||
"lodash": "4.17.20",
|
||||
"parse5": "3.0.3"
|
||||
}
|
||||
},
|
||||
"cliui": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.1.tgz",
|
||||
"integrity": "sha512-rcvHOWyGyid6I1WjT/3NatKj2kDt9OdSHSXpyLXaMWFbKpGACNW8pRhhdPUq9MWUOdwn8Rz9AVETjF4105rZZQ==",
|
||||
"requires": {
|
||||
"string-width": "4.2.0",
|
||||
"strip-ansi": "6.0.0",
|
||||
"wrap-ansi": "7.0.0"
|
||||
}
|
||||
},
|
||||
"color-convert": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
|
||||
"integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
|
||||
"requires": {
|
||||
"color-name": "1.1.4"
|
||||
}
|
||||
},
|
||||
"color-name": {
|
||||
"version": "1.1.4",
|
||||
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
|
||||
"integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA=="
|
||||
},
|
||||
"css-select": {
|
||||
"version": "1.2.0",
|
||||
"resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
|
||||
"integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
|
||||
"requires": {
|
||||
"boolbase": "1.0.0",
|
||||
"css-what": "2.1.3",
|
||||
"domutils": "1.5.1",
|
||||
"nth-check": "1.0.2"
|
||||
}
|
||||
},
|
||||
"css-what": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
|
||||
"integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
|
||||
},
|
||||
"dom-serializer": {
|
||||
"version": "0.1.1",
|
||||
"resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
|
||||
"integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
|
||||
"requires": {
|
||||
"domelementtype": "1.3.1",
|
||||
"entities": "1.1.2"
|
||||
}
|
||||
},
|
||||
"domelementtype": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
|
||||
"integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w=="
|
||||
},
|
||||
"domhandler": {
|
||||
"version": "2.4.2",
|
||||
"resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
|
||||
"integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
|
||||
"requires": {
|
||||
"domelementtype": "1.3.1"
|
||||
}
|
||||
},
|
||||
"domutils": {
|
||||
"version": "1.5.1",
|
||||
"resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
|
||||
"integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
|
||||
"requires": {
|
||||
"dom-serializer": "0.1.1",
|
||||
"domelementtype": "1.3.1"
|
||||
}
|
||||
},
|
||||
"emoji-regex": {
|
||||
"version": "8.0.0",
|
||||
"resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz",
|
||||
"integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A=="
|
||||
},
|
||||
"entities": {
|
||||
"version": "1.1.2",
|
||||
"resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
|
||||
"integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w=="
|
||||
},
|
||||
"escalade": {
|
||||
"version": "3.1.0",
|
||||
"resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.0.tgz",
|
||||
"integrity": "sha512-mAk+hPSO8fLDkhV7V0dXazH5pDc6MrjBTPyD3VeKzxnVFjH1MIxbCdqGZB9O8+EwWakZs3ZCbDS4IpRt79V1ig=="
|
||||
},
|
||||
"get-caller-file": {
|
||||
"version": "2.0.5",
|
||||
"resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz",
|
||||
"integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="
|
||||
},
|
||||
"htmlparser2": {
|
||||
"version": "3.10.1",
|
||||
"resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
|
||||
"integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
|
||||
"requires": {
|
||||
"domelementtype": "1.3.1",
|
||||
"domhandler": "2.4.2",
|
||||
"domutils": "1.5.1",
|
||||
"entities": "1.1.2",
|
||||
"inherits": "2.0.4",
|
||||
"readable-stream": "3.6.0"
|
||||
}
|
||||
},
|
||||
"inherits": {
|
||||
"version": "2.0.4",
|
||||
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
|
||||
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ=="
|
||||
},
|
||||
"is-fullwidth-code-point": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz",
|
||||
"integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg=="
|
||||
},
|
||||
"lodash": {
|
||||
"version": "4.17.20",
|
||||
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
|
||||
"integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
|
||||
},
|
||||
"node-fetch": {
|
||||
"version": "2.6.1",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz",
|
||||
"integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw=="
|
||||
},
|
||||
"nth-check": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
|
||||
"integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
|
||||
"requires": {
|
||||
"boolbase": "1.0.0"
|
||||
}
|
||||
},
|
||||
"parse5": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
|
||||
"integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
|
||||
"requires": {
|
||||
"@types/node": "14.11.2"
|
||||
}
|
||||
},
|
||||
"readable-stream": {
|
||||
"version": "3.6.0",
|
||||
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
|
||||
"integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==",
|
||||
"requires": {
|
||||
"inherits": "2.0.4",
|
||||
"string_decoder": "1.3.0",
|
||||
"util-deprecate": "1.0.2"
|
||||
}
|
||||
},
|
||||
"require-directory": {
|
||||
"version": "2.1.1",
|
||||
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
|
||||
"integrity": "sha1-jGStX9MNqxyXbiNE/+f3kqam30I="
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.2.1",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
|
||||
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
|
||||
},
|
||||
"string-width": {
|
||||
"version": "4.2.0",
|
||||
"resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.0.tgz",
|
||||
"integrity": "sha512-zUz5JD+tgqtuDjMhwIg5uFVV3dtqZ9yQJlZVfq4I01/K5Paj5UHj7VyrQOJvzawSVlKpObApbfD0Ed6yJc+1eg==",
|
||||
"requires": {
|
||||
"emoji-regex": "8.0.0",
|
||||
"is-fullwidth-code-point": "3.0.0",
|
||||
"strip-ansi": "6.0.0"
|
||||
}
|
||||
},
|
||||
"string_decoder": {
|
||||
"version": "1.3.0",
|
||||
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
|
||||
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
|
||||
"requires": {
|
||||
"safe-buffer": "5.2.1"
|
||||
}
|
||||
},
|
||||
"strip-ansi": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz",
|
||||
"integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==",
|
||||
"requires": {
|
||||
"ansi-regex": "5.0.0"
|
||||
}
|
||||
},
|
||||
"util-deprecate": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
|
||||
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
|
||||
},
|
||||
"wrap-ansi": {
|
||||
"version": "7.0.0",
|
||||
"resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz",
|
||||
"integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==",
|
||||
"requires": {
|
||||
"ansi-styles": "4.2.1",
|
||||
"string-width": "4.2.0",
|
||||
"strip-ansi": "6.0.0"
|
||||
}
|
||||
},
|
||||
"y18n": {
|
||||
"version": "5.0.1",
|
||||
"resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.1.tgz",
|
||||
"integrity": "sha512-/jJ831jEs4vGDbYPQp4yGKDYPSCCEQ45uZWJHE1AoYBzqdZi8+LDWas0z4HrmJXmKdpFsTiowSHXdxyFhpmdMg=="
|
||||
},
|
||||
"yargs": {
|
||||
"version": "16.0.3",
|
||||
"resolved": "https://registry.npmjs.org/yargs/-/yargs-16.0.3.tgz",
|
||||
"integrity": "sha512-6+nLw8xa9uK1BOEOykaiYAJVh6/CjxWXK/q9b5FpRgNslt8s22F2xMBqVIKgCRjNgGvGPBy8Vog7WN7yh4amtA==",
|
||||
"requires": {
|
||||
"cliui": "7.0.1",
|
||||
"escalade": "3.1.0",
|
||||
"get-caller-file": "2.0.5",
|
||||
"require-directory": "2.1.1",
|
||||
"string-width": "4.2.0",
|
||||
"y18n": "5.0.1",
|
||||
"yargs-parser": "20.2.0"
|
||||
}
|
||||
},
|
||||
"yargs-parser": {
|
||||
"version": "20.2.0",
|
||||
"resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.0.tgz",
|
||||
"integrity": "sha512-2agPoRFPoIcFzOIp6656gcvsg2ohtscpw2OINr/q46+Sq41xz2OYLqx5HRHabmFU1OARIPAYH5uteICE7mn/5A=="
|
||||
}
|
||||
}
|
||||
}
|
||||
20
package.json
Executable file
20
package.json
Executable file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"name": "animevibe-cli",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"main": "index.js",
|
||||
"bin": {
|
||||
"animevibe": "./index.js"
|
||||
},
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"keywords": [],
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"dependencies": {
|
||||
"cheerio": "^1.0.0-rc.3",
|
||||
"node-fetch": "^2.6.1",
|
||||
"yargs": "^16.0.3"
|
||||
}
|
||||
}
|
||||
144
scrape.js
Executable file
144
scrape.js
Executable file
|
|
@ -0,0 +1,144 @@
|
|||
const fetch = require('node-fetch');
|
||||
const cheerio = require('cheerio');
|
||||
|
||||
function makeUrlScrape(scrape, defaultUrl) {
|
||||
return async (url=defaultUrl) => {
|
||||
const res = await fetch(url);
|
||||
const $ = cheerio.load(await res.text());
|
||||
return { url, ...scrape($, { url }) };
|
||||
};
|
||||
}
|
||||
|
||||
const scrapeAnimevibeDownloadPage = makeUrlScrape($ => {
|
||||
const entries = $('.alert > p').map((_, p) => {
|
||||
const key = $(p).text().split(' ')[0].toLowerCase();
|
||||
const href = $(p).next('a').attr('href');
|
||||
return [[key, href]];
|
||||
}).get();
|
||||
return Object.fromEntries(entries);
|
||||
});
|
||||
|
||||
const scrapeVidCDNDownloadPage = makeUrlScrape($ => {
|
||||
const downloads = scrapeDownloads($);
|
||||
const info = scrapeInfo($);
|
||||
return { ...info, downloads };
|
||||
});
|
||||
|
||||
function scrapeDownloads($) {
|
||||
return $('div.mirror_link').first()
|
||||
.find('.dowload > a').map((_, a) => {
|
||||
const info = parseDownload($(a).text());
|
||||
const url = $(a).attr('href');
|
||||
return { ...info, url };
|
||||
}).get();
|
||||
}
|
||||
|
||||
function scrapeInfo($) {
|
||||
const keys = ['filename', 'filesize', 'duration', 'resolution'];
|
||||
const entries = $('.sumer_l > ul > li > span').map((i, span) => {
|
||||
const key = keys[i];
|
||||
const value = $(span).text();
|
||||
return [[key, value]];
|
||||
}).get();
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
|
||||
function parseDownload(text) {
|
||||
const regex = /Download\s+\((?<quality>[\w\d]+)P\s+-\s+(?<format>[\w\d]+)\)/;
|
||||
const { groups } = regex.exec(text);
|
||||
return groups;
|
||||
}
|
||||
|
||||
const scrapeAnimevibeSeriesPage = makeUrlScrape(($, { url }) => {
|
||||
const seriesId = url.split('/')[4];
|
||||
const downloadId = scrapeDownloadId($);
|
||||
const currentEpisodeNumber = scrapeEpisodeNumber($);
|
||||
|
||||
const infoDiv = $('#blogShort');
|
||||
const englishTitle = infoDiv.find('h5.title-av-search-res').text();
|
||||
const thumbnailUrl = infoDiv.find('#thumb-rsz').attr('data-bg');
|
||||
|
||||
const info = scrapeInfoDiv(infoDiv, $);
|
||||
const episodeCount = parseInt(info['Number of Episodes'].split(' ', 1));
|
||||
const views = parseInt(info['Views'].split(' ', 1));
|
||||
const summary = info['Summary'];
|
||||
const gernes = info['Genre'].split(', ');
|
||||
const otherTitles = parseTitles(info['Alternate Titles']);
|
||||
const myAnimeListScore = parseFloat(info['[MyAnimeList] Score']);
|
||||
const type = info['Type'];
|
||||
const status = info['Status'];
|
||||
const dates = parseDates(info['Date']);
|
||||
|
||||
return {
|
||||
seriesId,
|
||||
downloadId,
|
||||
currentEpisodeNumber,
|
||||
episodeCount,
|
||||
titles: { english: englishTitle, ...otherTitles },
|
||||
thumbnailUrl,
|
||||
type,
|
||||
...dates,
|
||||
status,
|
||||
gernes,
|
||||
summary,
|
||||
myAnimeListScore,
|
||||
views,
|
||||
};
|
||||
});
|
||||
|
||||
function scrapeEpisodeNumber($) {
|
||||
const button = $('.current-episode-button');
|
||||
return button.length ? parseInt(button.text()) : null;
|
||||
}
|
||||
|
||||
function scrapeDownloadId($) {
|
||||
const href = $('.download-av > a:nth-child(1)').attr('href');
|
||||
return new URL(href).searchParams.get('id');
|
||||
}
|
||||
|
||||
function parseTitles(text) {
|
||||
const regex = /(?<nativeJapanese>[^,]*), (?<japanese>[^,]*), (?<jsonParsableAbbreviations>\[.*\])/;
|
||||
const { groups } = regex.exec(text);
|
||||
const json = groups.jsonParsableAbbreviations.replace(/'/g, '"');
|
||||
return {
|
||||
nativeJapanese: groups.nativeJapanese,
|
||||
japanese: groups.japanese,
|
||||
abbreviations: JSON.parse(json),
|
||||
};
|
||||
}
|
||||
|
||||
function parseDates(text) {
|
||||
const regex = /(?<releaseDate>\w{3} \d{1,2}, \d{4})( to (\?|(?<finishedDate>\w{3} \d{1,2}, \d{4})))?/;
|
||||
const { groups } = regex.exec(text);
|
||||
const releaseDate = groups.releaseDate;
|
||||
const finishedDate = groups.finishedDate || null;
|
||||
return { releaseDate, finishedDate };
|
||||
}
|
||||
|
||||
function scrapeInfoDiv(infoDiv, $) {
|
||||
const entries = infoDiv.find('h6.excerpt-anime-info').map((_, h6) => {
|
||||
const [key, value] = $(h6).text().split(/(^[^:]+): /).slice(1);
|
||||
return [[key, value]];
|
||||
}).get();
|
||||
|
||||
return Object.fromEntries(entries);
|
||||
}
|
||||
|
||||
async function scrapeMp4UploadVideoFileUrl(url) {
|
||||
const id = url.split('/').pop();
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
body: new URLSearchParams({
|
||||
id, op: 'download2'
|
||||
}),
|
||||
redirect: 'manual',
|
||||
});
|
||||
return res.headers.get('location');
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
scrapeVidCDNDownloadPage,
|
||||
scrapeAnimevibeSeriesPage,
|
||||
scrapeAnimevibeDownloadPage,
|
||||
scrapeMp4UploadVideoFileUrl,
|
||||
};
|
||||
Reference in a new issue