Skip to content

download

canari_ml.hydra.download

canari_ml.hydra.download.logger = logging.getLogger(__name__) module-attribute

canari_ml.hydra.download.main(cfg)

Download ERA5 reanalysis data using AWS downloader in download-toolbox.

Processes configuration settings, sets up the download parameters and downloads daily ERA5 data from AWS mirror.

Parameters:

Name Type Description Default
cfg DictConfig

Hydra configuration parameters.

 required
Source code in src/canari_ml/hydra/download.py 
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
@hydra.main(
    version_base=None,
    config_path=str(Path(__file__).parent / "../conf"),
    config_name="download",
)
def main(cfg: DictConfig) -> None:
    """Download ERA5 reanalysis data using AWS downloader in `download-toolbox`.

    Processes configuration settings, sets up the download parameters
    and downloads daily ERA5 data from AWS mirror.

    Args:
        cfg: Hydra configuration parameters.
    """
    from canari_ml.hydra.utils import print_omega_config
    print_omega_config(cfg)

    logger.info("AWS Data Downloading")

    vars = [cfg.vars] if isinstance(cfg.vars, str) else cfg.vars
    levels = [cfg.levels] if isinstance(cfg.levels, int) else cfg.levels

    var_names = []
    var_levels = []
    for var_name, var_level in zip(vars, levels):
        var_names.append(var_name)
        if not var_level:
            var_levels.append(None)
        elif isinstance(var_level, int):
            var_levels.append([var_level])
        else:
            var_levels.append([int(level) for level in var_level.split("|")])

    download_daily(
        var_names=var_names,
        var_levels=var_levels,
        start_dates=cfg.dates.start,
        end_dates=cfg.dates.end,
        hemisphere=cfg.hemisphere,
        frequency=cfg.frequency,
        output_group_by=cfg.output_group_by,
        config_path=cfg.paths.download.config_file,
        overwrite=cfg.overwrite_config,
        delete_cache=cfg.delete_cache,
        cache_only=cfg.cache_only,
        compress=0,
        workers=cfg.workers,
    )