Yet Another Youtube Down Loader

⌈⌋ ⎇ branch:  yaydl


Check-in [cb81ca4cea]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:yaydl 0.2.6: Switched from reqwest to ureq to reduce the number of dependencies and (potentially) speed up the compilation process.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA3-256: cb81ca4cea6031f7dacb19d0751ed03affb5e13d9a65c8cabb76ffb13c4cc9c1
User & Date: Cthulhux 2020-11-19 15:45:08
Context
2020-11-19
15:48
Style check-in: 607bb4dcad user: Cthulhux tags: trunk
15:45
yaydl 0.2.6: Switched from reqwest to ureq to reduce the number of dependencies and (potentially) speed up the compilation process. check-in: cb81ca4cea user: Cthulhux tags: trunk
2020-11-18
12:16
Enabled LTO check-in: 1d94b8e5b7 user: Cthulhux tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Cargo.toml.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20


21
22
23
24
[package]
name = "yaydl"
description = "yet another youtube (and more) down loader"
version = "0.2.5"
authors = ["Cthulhux <git@tuxproject.de>"]
edition = "2018"
license = "CDDL-1.0"
repository = "https://code.rosaelefanten.org/yaydl"
categories = ["command-line-utilities"]
keywords = ["youtube", "vimeo"]

[dependencies]
anyhow = "1.0"
clap = "3.0.0-beta.2"
indicatif = "0.15.0"
inventory = "0.1.9"
qstring = "0.7.2"
regex = "1.4.1"
reqwest = { version = "0.10.8", features = ["blocking"] }
serde_json = "1.0"


urlencoding = "1.1.1"

[profile.release]
lto = true



|














<

>
>




1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18

19
20
21
22
23
24
25
[package]
name = "yaydl"
description = "yet another youtube (and more) down loader"
version = "0.2.6"
authors = ["Cthulhux <git@tuxproject.de>"]
edition = "2018"
license = "CDDL-1.0"
repository = "https://code.rosaelefanten.org/yaydl"
categories = ["command-line-utilities"]
keywords = ["youtube", "vimeo"]

[dependencies]
anyhow = "1.0"
clap = "3.0.0-beta.2"
indicatif = "0.15.0"
inventory = "0.1.9"
qstring = "0.7.2"
regex = "1.4.1"

serde_json = "1.0"
ureq = { version = "1.5", default_features = false, features = ["native-tls"] }
url = "2.2.0"
urlencoding = "1.1.1"

[profile.release]
lto = true

Changes to src/handlers/vimeo.rs.

26
27
28
29
30
31
32

33
34
35
36
37
38
39
40
static mut VIDEO_TITLE: String = String::new();

unsafe fn get_video_info(url: &str) -> Result<Value> {
    if VIDEO_INFO.is_empty() {
        // We need to fetch the video information first.
        // Those are hidden behing a config file defined in the page source code.
        // Search for: window.vimeo.clip_page_config.player = {"config_url":"(.+?)"

        let body = reqwest::blocking::get(url)?.text()?;
        let re =
            Regex::new("window.vimeo.clip_page_config.player = .\"config_url\":\"(?P<URL>.+?)\"")
                .unwrap();
        let search = re.captures(&body).unwrap();

        // While we're grepping the source code: Vimeo also hides
        // the video title here.







>
|







26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
static mut VIDEO_TITLE: String = String::new();

unsafe fn get_video_info(url: &str) -> Result<Value> {
    if VIDEO_INFO.is_empty() {
        // We need to fetch the video information first.
        // Those are hidden behing a config file defined in the page source code.
        // Search for: window.vimeo.clip_page_config.player = {"config_url":"(.+?)"
        let req = ureq::get(url).call();
        let body = req.into_string()?;
        let re =
            Regex::new("window.vimeo.clip_page_config.player = .\"config_url\":\"(?P<URL>.+?)\"")
                .unwrap();
        let search = re.captures(&body).unwrap();

        // While we're grepping the source code: Vimeo also hides
        // the video title here.
49
50
51
52
53
54
55
56

57
58
59
60
61
62
63
        let video_info_url = search
            .name("URL")
            .map_or("", |u| u.as_str())
            .replace("\\", "");

        // The "config_url" body is a JSON structure.
        // Grab and store it:
        let config_body = reqwest::blocking::get(&video_info_url)?.text()?;

        VIDEO_INFO = config_body;
    }

    // Return it:
    let v: Value = serde_json::from_str(&VIDEO_INFO)?;
    Ok(v)
}







|
>







50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
        let video_info_url = search
            .name("URL")
            .map_or("", |u| u.as_str())
            .replace("\\", "");

        // The "config_url" body is a JSON structure.
        // Grab and store it:
        let config_req = ureq::get(&video_info_url).call();
        let config_body = config_req.into_string()?;
        VIDEO_INFO = config_body;
    }

    // Return it:
    let v: Value = serde_json::from_str(&VIDEO_INFO)?;
    Ok(v)
}

Changes to src/handlers/youtube.rs.

30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
unsafe fn get_video_info(id: &str) -> Result<Value> {
    if VIDEO_INFO.is_empty() {
        // We need to fetch the video information first.
        let video_url = format!(
            "https://www.youtube.com/get_video_info?video_id={}&el=embedded&ps=default",
            id
        );
        let res = reqwest::blocking::get(video_url.as_str())?;
        let body = res.text()?;

        // Try to find the encoded JSON object in the response.
        let qs = QString::from(body.as_str());
        let json = qs.get("player_response").unwrap_or("");

        VIDEO_INFO = json.replace("+", " ");
    }







|
|







30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
unsafe fn get_video_info(id: &str) -> Result<Value> {
    if VIDEO_INFO.is_empty() {
        // We need to fetch the video information first.
        let video_url = format!(
            "https://www.youtube.com/get_video_info?video_id={}&el=embedded&ps=default",
            id
        );
        let req = ureq::get(&video_url).call();
        let body = req.into_string()?;

        // Try to find the encoded JSON object in the response.
        let qs = QString::from(body.as_str());
        let json = qs.get("player_response").unwrap_or("");

        VIDEO_INFO = json.replace("+", " ");
    }

Changes to src/main.rs.

15
16
17
18
19
20
21
22
23
24
25
26
27

28
29
30
31
32
33
34

// Yet Another Youtube Down Loader
// - main.rs file -

use anyhow::Result;
use clap::{App, Arg};
use indicatif::{ProgressBar, ProgressStyle};
use reqwest::{blocking::Client, header, Url};
use std::{
    fs,
    io::{self, copy, Read},
    path::{Path, PathBuf},
};


mod definitions;
mod ffmpeg;
mod handlers;

struct DownloadProgress<R> {
    inner: R,







<





>







15
16
17
18
19
20
21

22
23
24
25
26
27
28
29
30
31
32
33
34

// Yet Another Youtube Down Loader
// - main.rs file -

use anyhow::Result;
use clap::{App, Arg};
use indicatif::{ProgressBar, ProgressStyle};

use std::{
    fs,
    io::{self, copy, Read},
    path::{Path, PathBuf},
};
use url::Url;

mod definitions;
mod ffmpeg;
mod handlers;

struct DownloadProgress<R> {
    inner: R,
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59

60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

82
83
84
85

86
87
88
89
90
91
92
93
94
95
            n
        })
    }
}

fn download(url: &str, filename: &str) -> Result<()> {
    let url = Url::parse(url)?;
    let client = Client::new();

    // Find the video size:
    let total_size = {
        let resp = client.head(url.as_str()).send()?;
        if resp.status().is_success() {
            resp.headers()
                .get(header::CONTENT_LENGTH)
                .and_then(|ct_len| ct_len.to_str().ok())
                .and_then(|ct_len| ct_len.parse().ok())
                .unwrap_or(0)

        } else {
            return Err(anyhow::Error::msg(format!(
                "Couldn't download URL: {}. Error: {:?}",
                url,
                resp.status(),
            )));
        }
    };

    let mut request = client.get(url.as_str());

    // Display a progress bar:
    let pb = ProgressBar::new(total_size);
    pb.set_style(ProgressStyle::default_bar()
                 .template("{spinner:.green} [{elapsed_precise}] [{bar:40.green/blue}] {bytes}/{total_bytes} ({eta})")
                 .progress_chars("#>-"));

    let file = Path::new(filename);

    if file.exists() {
        // Continue the file:
        let size = file.metadata()?.len() - 1;

        request = request.header(header::RANGE, format!("bytes={}-", size));
        pb.inc(size);
    }


    let mut source = DownloadProgress {
        progress_bar: pb,
        inner: request.send()?,
    };

    let mut dest = fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(&file)?;








|



<
|
|
<
<
<
|
>









|












>
|



>


|







42
43
44
45
46
47
48
49
50
51
52

53
54



55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
            n
        })
    }
}

fn download(url: &str, filename: &str) -> Result<()> {
    let url = Url::parse(url)?;
    let resp = ureq::get(url.as_str()).call();

    // Find the video size:
    let total_size = {

        if resp.ok() {
            resp.header("Content-Length")



                .unwrap_or("0")
                .parse::<u64>()?
        } else {
            return Err(anyhow::Error::msg(format!(
                "Couldn't download URL: {}. Error: {:?}",
                url,
                resp.status(),
            )));
        }
    };

    let mut request = ureq::get(url.as_str());

    // Display a progress bar:
    let pb = ProgressBar::new(total_size);
    pb.set_style(ProgressStyle::default_bar()
                 .template("{spinner:.green} [{elapsed_precise}] [{bar:40.green/blue}] {bytes}/{total_bytes} ({eta})")
                 .progress_chars("#>-"));

    let file = Path::new(filename);

    if file.exists() {
        // Continue the file:
        let size = file.metadata()?.len() - 1;
        // Override the range:
        request = ureq::get(url.as_str()).set("Range", &format!("bytes={}-", size)).to_owned();
        pb.inc(size);
    }

    let resp = request.call();
    let mut source = DownloadProgress {
        progress_bar: pb,
        inner: resp.into_reader(),
    };

    let mut dest = fs::OpenOptions::new()
        .create(true)
        .append(true)
        .open(&file)?;