ssbu_lokrez/lokrez/resources.py

import io
import os
import pathlib
import sys
import urllib
import zipfile

import requests

# -----------------------------------------------------------------------------
def download_file(url, with_progressbar = True, proxy = None):

    r = requests.get(
            url,
            stream = with_progressbar,
            proxies = { "https": proxy, "http": proxy },
            )

    if not with_progressbar:
        return io.BytesIO(r.content)

    total = r.headers.get("content-length")

    f = io.BytesIO()

    if total is not None:
        downloaded = 0
        total = int(total)
        for data in r.iter_content(
                chunk_size = max(int(total/1000), 1024*1024),
                ):
            f.write(data)
            downloaded += len(data)
            done = int(50*downloaded/total)
            sys.stdout.write( "\r[{}{}] ({:02d}%)".format(
                "█" * done,
                " " * (50-done),
                done*2,
                ) )
            sys.stdout.flush()
        sys.stdout.write("\n")
    else:
        f = write(r.content)

    return f

# -----------------------------------------------------------------------------
def download_res(
        dstdir,
        game = None,
        source = None,
        store_raw = False,
        proxy = None,
        log = None,
        ):
    """TODO: Docstring for download_res_pplus.
    :returns: TODO

    """
    if not game:
        return

    # Select default source if needed
    if not source:
        if game.GAME.name in [ "pplus", "melee" ]:
            source = "smashlyon"
        elif game.GAME.name == "ssbu":
            source = "spriters"

        log.info("Using default source {}".format(source))

    if source not in ["smashlyon", "spriters"]:
        raise NotImplementedError(
                "The only working sources are 'smashlyon' and 'spriters'",
                )

    gamedir = dstdir / game.GAME.name

    try:
        gamedir.mkdir(parents=True)
    except FileExistsError:
        log.debug("Game directory already exist")

    # A cache to save time
    cache = {}

    # Iter over each character
    for character in game.EVERYONE:
        log.warning("Downloading images for {}...".format(character.name))

        chardir = gamedir /  character.name

        if not store_raw:
            try:
                chardir.mkdir(parents=True)
            except FileExistsError:
                log.info(
                        "Directory already exists for {}" \
                                .format(character.name)
                        )

                try:
                    next(chardir.iterdir())
                    log.warning(
                            "Directory not empty for {}, skipping" \
                                    .format(character.name)
                            )
                    continue
                except StopIteration:
                    log.info(
                            "Directory empty, downloading",
                            )

        # Download urls & write image files
        for url_nb, url in enumerate(character.res_urls[source]):

            # If we have the file in cache, just get it
            if url in cache and cache[url] is not None:
                log.debug("Found url '{}' in cache".format(url))
                f = cache[url]

            else:
                try:
                    f = download_file(url, proxy = proxy)

                except Exception as e:
                    try:
                        # Try the download a 2nd time
                        log.warning("Download failed ({}), retrying".format(e))
                        f = download_file(url, proxy = proxy)
                    except Exception as e:
                        log.error("Download failed({})".format(e))
                        log.debug(e, exc_info = True)
                        continue

                # We save the file in cache if it's the second time we need
                # to download it.
                if url in cache:
                    log.debug("Saving url '{}' in cache".format(url))
                    cache[url] = f
                else:
                    log.debug("Marking url '{}' in cache".format(url))
                    cache[url] = None


            # if store_raw: we just save the raw zip file
            if store_raw:
                outfile_name = pathlib.Path(
                        urllib.parse.urlparse(url).path
                        ) \
                                .name
                with open(str(gamedir/outfile_name), "wb") as outfile:
                    outfile.write(f.getbuffer())

                # Add symlink for readablity
                os.symlink(
                        str(outfile_name),
                        str( gamedir/
                        "{charname}.{nb}.zip".format(
                            charname = character.name,
                            nb = url_nb+1,
                            )),
                        )

                continue

            # otherwise: get the characters pictures and write them in the
            # outdir
            with zipfile.ZipFile(f) as zf:
                for source_file in zf.filelist:

                    if source_file.is_dir():
                        continue

                    source_filename = source_file.filename

                    if "No Gamma Fix" in source_filename:
                        continue

                    if not any(c in source_filename for c in character.codenames):
                        continue

                    target_filename = pathlib.Path(source_filename).name

                    if target_filename in ["","Tag.txt"]:
                        continue

                    target_filename = pathlib.Path(source_filename).name

                    for codename in character.codenames:
                        target_filename = target_filename.replace(
                                codename,
                                character.name,
                                )

                    log.debug("Writing file '{}'".format(target_filename))

                    target_filename = chardir / target_filename

                    with open(str(target_filename), "wb") as tf:
                        tf.write(zf.read(source_filename))

# =============================================================================
if __name__ == '__main__':

    import argparse
    import logging
    import tempfile

    logging.basicConfig(
            level = logging.DEBUG,
            format = "%(message)s",
            )

    parser = argparse.ArgumentParser()

    parser.add_argument(
            "dstdir",
            default = None,
            help = "directory where to store the downloaded resources " \
                   "(default to a temporary file)",
                   )

    args = parser.parse_args()

    if args.dstdir is None:
        args.dstdir = tempfile.mkdtemp()
        logging.warning(
                "Storing in temporary directory : {}".format(args.dstdir)
                )

    download_res_ssbu(
            dstdir = args.dstdir,
            log = logging,
            )