Yet Another Youtube Down Loader

⌈⌋ ⎇ branch:  yaydl


Check-in [1e34f21ef5]

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:yaydl 0.12.1: xhamster uses the agent as well ; simplified code
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | release-0.12.1
Files: files | file ages | folders
SHA3-256: 1e34f21ef5fc86f707f4aaafc62bbada67c2a1eab86855a3115da679d5ebdf92
User & Date: Cthulhux 2023-02-27 14:59:06
Context
2023-03-23
01:59
unbroke build check-in: c6723756b0 user: Cthulhux tags: trunk
2023-02-27
14:59
yaydl 0.12.1: xhamster uses the agent as well ; simplified code check-in: 1e34f21ef5 user: Cthulhux tags: trunk, release-0.12.1
14:32
yaydl 0.12.0: experimental proxy support; improved youtube regex for /shorts/. check-in: 93d75ae530 user: Cthulhux tags: trunk, release-0.12.0
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to Cargo.toml. 

1
2
3
4
5
6
7
8
9
10
11

[package]
name = "yaydl"
description = "yet another youtube (and more) down loader"
version = "0.12.0"
authors = ["Cthulhux <git@tuxproject.de>"]
edition = "2021"
license = "CDDL-1.0"
repository = "https://code.rosaelefanten.org/yaydl"
categories = ["command-line-utilities"]
keywords = ["youtube", "downloading", "video"]





|








1
2
3
4
5
6
7
8
9
10
11

[package]
name = "yaydl"
description = "yet another youtube (and more) down loader"
version = "0.12.1"
authors = ["Cthulhux <git@tuxproject.de>"]
edition = "2021"
license = "CDDL-1.0"
repository = "https://code.rosaelefanten.org/yaydl"
categories = ["command-line-utilities"]
keywords = ["youtube", "downloading", "video"]
︙︙

Changes to src/download.rs.

︙︙
43
44
45
46
47
48
49
50
51
52
53
54
55
56


57
58
59
60
61
62
63
64
65
66
67
68
69

pub fn download_from_playlist(url: &str, filename: &str, verbose: bool) -> Result<()> {
    // Download the playlist file into the temporary directory:
    if verbose {
        println!("{}", "Found a playlist. Fetching ...");
    }

    let mut url = Url::parse(url)?;
    let mut playlist_text = "".to_string();
    let mut agent = ureq::agent();

    if let Some(env_proxy) = env_proxy::for_url(&url).host_port() {
        // Use a proxy:
        let proxy = ureq::Proxy::new(format!("{}:{}", env_proxy.0, env_proxy.1));
        agent = ureq::AgentBuilder::new().proxy(proxy.unwrap()).build();


        let request = agent.get(url.as_str());
        playlist_text = request.call()?.into_string()?;
    } else {
        let request = ureq::get(url.as_str());
        playlist_text = request.call()?.into_string()?;
    }
    if verbose {
        println!("{}", "Parsing ...");
    }

    // Parse the playlist:
    let playlist = m3u8_rs::parse_media_playlist(&playlist_text.as_bytes())
        .finish()








<






>
>
|
|
<
<
<
|








43
44
45
46
47
48
49

50
51
52
53
54
55
56
57
58
59



60
61
62
63
64
65
66
67

pub fn download_from_playlist(url: &str, filename: &str, verbose: bool) -> Result<()> {
    // Download the playlist file into the temporary directory:
    if verbose {
        println!("{}", "Found a playlist. Fetching ...");
    }

    let mut url = Url::parse(url)?;

    let mut agent = ureq::agent();

    if let Some(env_proxy) = env_proxy::for_url(&url).host_port() {
        // Use a proxy:
        let proxy = ureq::Proxy::new(format!("{}:{}", env_proxy.0, env_proxy.1));
        agent = ureq::AgentBuilder::new().proxy(proxy.unwrap()).build();
    }

    let request = agent.get(url.as_str());
    let playlist_text = request.call()?.into_string()?;




    if verbose {
        println!("{}", "Parsing ...");
    }

    // Parse the playlist:
    let playlist = m3u8_rs::parse_media_playlist(&playlist_text.as_bytes())
        .finish()
︙︙
136
137
138
139
140
141
142

143
144
145
146
147
148
149
150


    let file = Path::new(filename);

    if file.exists() {
        // Continue the file:
        let size = file.metadata()?.len() - 1;
        // Override the range:

        request = ureq::get(url.as_str())
            .set("Range", &format!("bytes={}-", size))
            .to_owned();
        pb.inc(size);
    }

    let resp = request.call()?;
    let mut source = DownloadProgress {








>
|








134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149


    let file = Path::new(filename);

    if file.exists() {
        // Continue the file:
        let size = file.metadata()?.len() - 1;
        // Override the range:
        request = agent
            .get(url.as_str())
            .set("Range", &format!("bytes={}-", size))
            .to_owned();
        pb.inc(size);
    }

    let resp = request.call()?;
    let mut source = DownloadProgress {
︙︙

Changes to src/handlers/xhamster.rs.

︙︙
27
28
29
30
31
32
33

34






35

36
37
38
39
40
41
42
43

use crate::VIDEO;

fn get_video_info(video: &mut VIDEO, url: &str) -> Result<bool> {
    if video.info.is_empty() {
        // We need to fetch the video information first.
        // It will contain the whole body for now.
        let local_url = url.to_owned();








        video.info.push_str(

            ureq::get(&local_url)
                .call()
                .expect("Could not go to the url")
                .into_string()
                .expect("Could not read the site source")
                .as_str(),
        );
    }








>

>
>
>
>
>
>

>
|








27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51

use crate::VIDEO;

fn get_video_info(video: &mut VIDEO, url: &str) -> Result<bool> {
    if video.info.is_empty() {
        // We need to fetch the video information first.
        // It will contain the whole body for now.
        let local_url = url.to_owned();
        let mut agent = ureq::agent();

        if let Some(env_proxy) = env_proxy::for_url(&url).host_port() {
            // Use a proxy:
            let proxy = ureq::Proxy::new(format!("{}:{}", env_proxy.0, env_proxy.1));
            agent = ureq::AgentBuilder::new().proxy(proxy.unwrap()).build();
        }

        video.info.push_str(
            agent
                .get(&local_url)
                .call()
                .expect("Could not go to the url")
                .into_string()
                .expect("Could not read the site source")
                .as_str(),
        );
    }
︙︙
82
83
84
85
86
87
88







89
90
91
92
93
94
95
96
97
98
99
100
101
102
103

        video: &'a mut VIDEO,
        url: &'a str,
        _webdriver_port: u16,
        _onlyaudio: bool,
    ) -> Result<String> {
        let _not_used = get_video_info(video, url)?;
        let video_info_html = Html::parse_document(video.info.as_str());








        // Find the playlist first:
        let url_selector = Selector::parse(r#"link[rel="preload"][as="fetch"]"#).unwrap();
        let url_elem = video_info_html.select(&url_selector).next().unwrap();
        let url_contents = url_elem.value().attr("href").unwrap();

        let mut playlist_url = Url::parse(url_contents)?;
        let request = ureq::get(playlist_url.as_str());
        let playlist_text = request.call()?.into_string()?;

        // Parse the playlist:
        let playlist = m3u8_rs::parse_media_playlist(&playlist_text.as_bytes())
            .finish()
            .unwrap();









>
>
>
>
>
>
>







|








90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

        video: &'a mut VIDEO,
        url: &'a str,
        _webdriver_port: u16,
        _onlyaudio: bool,
    ) -> Result<String> {
        let _not_used = get_video_info(video, url)?;
        let video_info_html = Html::parse_document(video.info.as_str());
        let mut agent = ureq::agent();

        if let Some(env_proxy) = env_proxy::for_url(&url).host_port() {
            // Use a proxy:
            let proxy = ureq::Proxy::new(format!("{}:{}", env_proxy.0, env_proxy.1));
            agent = ureq::AgentBuilder::new().proxy(proxy.unwrap()).build();
        }

        // Find the playlist first:
        let url_selector = Selector::parse(r#"link[rel="preload"][as="fetch"]"#).unwrap();
        let url_elem = video_info_html.select(&url_selector).next().unwrap();
        let url_contents = url_elem.value().attr("href").unwrap();

        let mut playlist_url = Url::parse(url_contents)?;
        let request = agent.get(playlist_url.as_str());
        let playlist_text = request.call()?.into_string()?;

        // Parse the playlist:
        let playlist = m3u8_rs::parse_media_playlist(&playlist_text.as_bytes())
            .finish()
            .unwrap();
︙︙