diff --git a/results/tec_256/afno_backbone_tec_256/d6p_test1/hyperparams.yaml b/results/tec_256/afno_backbone_tec_256/d6p_test1/hyperparams.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e3f27acfe9db8fe93f2020050629a980872e3ec6 --- /dev/null +++ b/results/tec_256/afno_backbone_tec_256/d6p_test1/hyperparams.yaml @@ -0,0 +1,69 @@ +log_to_wandb: 'False' +lr: '0.0005' +batch_size: '8' +patch_size: '2' +depth: '6' +img_size: '[128, 256]' +max_epochs: '1500' +scheduler: CosineAnnealingLR +in_channels: '[ 0 1 2 3 4 5 6 7 8 9 10 11 12]' +out_channels: '[ 0 1 2 3 4 5 6 7 8 9 10 11 12]' +prediction_length: '100' +orography: 'False' +orography_path: None +exp_dir: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256 +train_data_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train +valid_data_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test +inf_data_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/out_of_sample +time_means_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/time_means.npy +global_means_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_means.npy +global_stds_path: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_stds.npy +loss: l2 +num_data_workers: '4' +dt: '1' +n_history: '0' +prediction_type: iterative +n_initial_conditions: '5' +ics_type: default +save_raw_forecasts: 'True' +save_channel: 'False' +masked_acc: 'False' +maskpath: None +perturb: 'False' +add_grid: 'False' +N_grid_channels: '0' +gridtype: sinusoidal +roll: 'False' +num_blocks: '8' +nettype: afno +width: '56' +modes: '32' +target: default +normalization: zscore +log_to_screen: 'True' +save_checkpoint: 'True' +enable_nhwc: 'False' +optimizer_type: FusedAdam +crop_size_x: None +crop_size_y: None +two_step_training: 'False' +plot_animations: 'False' +add_noise: 'False' +noise_std: '0' +epsilon_factor: '0' +world_size: '1' +experiment_dir: + /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1 +checkpoint_path: + /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/ckpt.tar +best_checkpoint_path: + /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/best_ckpt.tar +resuming: 'False' +local_rank: '0' +enable_amp: 'True' +name: afno_backbone_tec_256_d6p_test1 +group: era5_precipafno_backbone_tec_256 +project: ERA5_precip +entity: flowgan +N_in_channels: '13' +N_out_channels: '13' diff --git a/results/tec_256/afno_backbone_tec_256/d6p_test1/out.log b/results/tec_256/afno_backbone_tec_256/d6p_test1/out.log new file mode 100644 index 0000000000000000000000000000000000000000..0992be295312b619c4ee8f5cde371ada1546b940 --- /dev/null +++ b/results/tec_256/afno_backbone_tec_256/d6p_test1/out.log @@ -0,0 +1,297 @@ +2024-02-17 17:39:59,479 - root - INFO - --------------- Versions --------------- +2024-02-17 17:39:59,483 - root - INFO - git branch: b'* master' +2024-02-17 17:39:59,486 - root - INFO - git hash: b'5402c07844da9944017e0e5fb5f0530abb33711e' +2024-02-17 17:39:59,486 - root - INFO - Torch: 2.1.2+cu118 +2024-02-17 17:39:59,486 - root - INFO - ---------------------------------------- +2024-02-17 17:39:59,486 - root - INFO - ------------------ Configuration ------------------ +2024-02-17 17:39:59,486 - root - INFO - Configuration file: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/config/AFNO.yaml +2024-02-17 17:39:59,486 - root - INFO - Configuration name: afno_backbone_tec_256 +2024-02-17 17:39:59,486 - root - INFO - log_to_wandb False +2024-02-17 17:39:59,486 - root - INFO - lr 0.0005 +2024-02-17 17:39:59,486 - root - INFO - batch_size 8 +2024-02-17 17:39:59,486 - root - INFO - patch_size 2 +2024-02-17 17:39:59,486 - root - INFO - depth 6 +2024-02-17 17:39:59,486 - root - INFO - img_size [128, 256] +2024-02-17 17:39:59,486 - root - INFO - max_epochs 1500 +2024-02-17 17:39:59,486 - root - INFO - scheduler CosineAnnealingLR +2024-02-17 17:39:59,487 - root - INFO - in_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:39:59,487 - root - INFO - out_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:39:59,487 - root - INFO - prediction_length 100 +2024-02-17 17:39:59,487 - root - INFO - orography False +2024-02-17 17:39:59,487 - root - INFO - orography_path None +2024-02-17 17:39:59,487 - root - INFO - exp_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256 +2024-02-17 17:39:59,487 - root - INFO - train_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train +2024-02-17 17:39:59,487 - root - INFO - valid_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test +2024-02-17 17:39:59,487 - root - INFO - inf_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/out_of_sample +2024-02-17 17:39:59,487 - root - INFO - time_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/time_means.npy +2024-02-17 17:39:59,487 - root - INFO - global_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_means.npy +2024-02-17 17:39:59,487 - root - INFO - global_stds_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_stds.npy +2024-02-17 17:39:59,487 - root - INFO - loss l2 +2024-02-17 17:39:59,487 - root - INFO - num_data_workers 4 +2024-02-17 17:39:59,487 - root - INFO - dt 1 +2024-02-17 17:39:59,487 - root - INFO - n_history 0 +2024-02-17 17:39:59,487 - root - INFO - prediction_type iterative +2024-02-17 17:39:59,487 - root - INFO - n_initial_conditions 5 +2024-02-17 17:39:59,487 - root - INFO - ics_type default +2024-02-17 17:39:59,487 - root - INFO - save_raw_forecasts True +2024-02-17 17:39:59,487 - root - INFO - save_channel False +2024-02-17 17:39:59,487 - root - INFO - masked_acc False +2024-02-17 17:39:59,487 - root - INFO - maskpath None +2024-02-17 17:39:59,487 - root - INFO - perturb False +2024-02-17 17:39:59,487 - root - INFO - add_grid False +2024-02-17 17:39:59,487 - root - INFO - N_grid_channels 0 +2024-02-17 17:39:59,487 - root - INFO - gridtype sinusoidal +2024-02-17 17:39:59,487 - root - INFO - roll False +2024-02-17 17:39:59,487 - root - INFO - num_blocks 8 +2024-02-17 17:39:59,487 - root - INFO - nettype afno +2024-02-17 17:39:59,487 - root - INFO - width 56 +2024-02-17 17:39:59,487 - root - INFO - modes 32 +2024-02-17 17:39:59,487 - root - INFO - target default +2024-02-17 17:39:59,487 - root - INFO - normalization zscore +2024-02-17 17:39:59,487 - root - INFO - log_to_screen True +2024-02-17 17:39:59,487 - root - INFO - save_checkpoint True +2024-02-17 17:39:59,487 - root - INFO - enable_nhwc False +2024-02-17 17:39:59,487 - root - INFO - optimizer_type FusedAdam +2024-02-17 17:39:59,488 - root - INFO - crop_size_x None +2024-02-17 17:39:59,488 - root - INFO - crop_size_y None +2024-02-17 17:39:59,488 - root - INFO - two_step_training False +2024-02-17 17:39:59,488 - root - INFO - plot_animations False +2024-02-17 17:39:59,488 - root - INFO - add_noise False +2024-02-17 17:39:59,488 - root - INFO - noise_std 0 +2024-02-17 17:39:59,488 - root - INFO - epsilon_factor 0 +2024-02-17 17:39:59,488 - root - INFO - world_size 1 +2024-02-17 17:39:59,488 - root - INFO - experiment_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1 +2024-02-17 17:39:59,488 - root - INFO - checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/ckpt.tar +2024-02-17 17:39:59,488 - root - INFO - best_checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/best_ckpt.tar +2024-02-17 17:39:59,488 - root - INFO - resuming False +2024-02-17 17:39:59,488 - root - INFO - local_rank 0 +2024-02-17 17:39:59,488 - root - INFO - enable_amp True +2024-02-17 17:39:59,488 - root - INFO - name afno_backbone_tec_256_d6p_test1 +2024-02-17 17:39:59,488 - root - INFO - group era5_precipafno_backbone_tec_256 +2024-02-17 17:39:59,488 - root - INFO - project ERA5_precip +2024-02-17 17:39:59,488 - root - INFO - entity flowgan +2024-02-17 17:39:59,488 - root - INFO - --------------------------------------------------- +2024-02-17 17:39:59,492 - root - INFO - rank 0, begin data loader init +2024-02-17 17:39:59,493 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train/2005.h5 +2024-02-17 17:39:59,494 - root - INFO - Number of samples per year: 336 +2024-02-17 17:39:59,494 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train. Number of examples: 2352. Image Shape: 128 x 256 x 13 +2024-02-17 17:39:59,494 - root - INFO - Delta t: 6 hours +2024-02-17 17:39:59,494 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:39:59,494 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test/2008.h5 +2024-02-17 17:39:59,494 - root - INFO - Number of samples per year: 337 +2024-02-17 17:39:59,495 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test. Number of examples: 674. Image Shape: 128 x 256 x 13 +2024-02-17 17:39:59,495 - root - INFO - Delta t: 6 hours +2024-02-17 17:39:59,495 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:39:59,495 - root - INFO - rank 0, data loader initialized +2024-02-17 17:39:59,940 - root - INFO - Number of trainable model parameters: 36514560 +2024-02-17 17:39:59,940 - root - INFO - Starting Training Loop... +2024-02-17 17:44:54,003 - root - INFO - --------------- Versions --------------- +2024-02-17 17:44:54,011 - root - INFO - git branch: b'* master' +2024-02-17 17:44:54,014 - root - INFO - git hash: b'5402c07844da9944017e0e5fb5f0530abb33711e' +2024-02-17 17:44:54,014 - root - INFO - Torch: 2.1.2+cu118 +2024-02-17 17:44:54,014 - root - INFO - ---------------------------------------- +2024-02-17 17:44:54,014 - root - INFO - ------------------ Configuration ------------------ +2024-02-17 17:44:54,014 - root - INFO - Configuration file: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/config/AFNO.yaml +2024-02-17 17:44:54,014 - root - INFO - Configuration name: afno_backbone_tec_256 +2024-02-17 17:44:54,014 - root - INFO - log_to_wandb False +2024-02-17 17:44:54,014 - root - INFO - lr 0.0005 +2024-02-17 17:44:54,015 - root - INFO - batch_size 8 +2024-02-17 17:44:54,015 - root - INFO - patch_size 2 +2024-02-17 17:44:54,015 - root - INFO - depth 6 +2024-02-17 17:44:54,015 - root - INFO - img_size [128, 256] +2024-02-17 17:44:54,015 - root - INFO - max_epochs 1500 +2024-02-17 17:44:54,015 - root - INFO - scheduler CosineAnnealingLR +2024-02-17 17:44:54,015 - root - INFO - in_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:44:54,015 - root - INFO - out_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:44:54,015 - root - INFO - prediction_length 100 +2024-02-17 17:44:54,015 - root - INFO - orography False +2024-02-17 17:44:54,015 - root - INFO - orography_path None +2024-02-17 17:44:54,015 - root - INFO - exp_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256 +2024-02-17 17:44:54,015 - root - INFO - train_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train +2024-02-17 17:44:54,015 - root - INFO - valid_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test +2024-02-17 17:44:54,015 - root - INFO - inf_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/out_of_sample +2024-02-17 17:44:54,015 - root - INFO - time_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/time_means.npy +2024-02-17 17:44:54,015 - root - INFO - global_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_means.npy +2024-02-17 17:44:54,015 - root - INFO - global_stds_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_stds.npy +2024-02-17 17:44:54,015 - root - INFO - loss l2 +2024-02-17 17:44:54,015 - root - INFO - num_data_workers 4 +2024-02-17 17:44:54,015 - root - INFO - dt 1 +2024-02-17 17:44:54,015 - root - INFO - n_history 0 +2024-02-17 17:44:54,015 - root - INFO - prediction_type iterative +2024-02-17 17:44:54,015 - root - INFO - n_initial_conditions 5 +2024-02-17 17:44:54,015 - root - INFO - ics_type default +2024-02-17 17:44:54,015 - root - INFO - save_raw_forecasts True +2024-02-17 17:44:54,015 - root - INFO - save_channel False +2024-02-17 17:44:54,015 - root - INFO - masked_acc False +2024-02-17 17:44:54,015 - root - INFO - maskpath None +2024-02-17 17:44:54,015 - root - INFO - perturb False +2024-02-17 17:44:54,015 - root - INFO - add_grid False +2024-02-17 17:44:54,015 - root - INFO - N_grid_channels 0 +2024-02-17 17:44:54,015 - root - INFO - gridtype sinusoidal +2024-02-17 17:44:54,015 - root - INFO - roll False +2024-02-17 17:44:54,015 - root - INFO - num_blocks 8 +2024-02-17 17:44:54,015 - root - INFO - nettype afno +2024-02-17 17:44:54,015 - root - INFO - width 56 +2024-02-17 17:44:54,015 - root - INFO - modes 32 +2024-02-17 17:44:54,015 - root - INFO - target default +2024-02-17 17:44:54,016 - root - INFO - normalization zscore +2024-02-17 17:44:54,016 - root - INFO - log_to_screen True +2024-02-17 17:44:54,016 - root - INFO - save_checkpoint True +2024-02-17 17:44:54,016 - root - INFO - enable_nhwc False +2024-02-17 17:44:54,016 - root - INFO - optimizer_type FusedAdam +2024-02-17 17:44:54,016 - root - INFO - crop_size_x None +2024-02-17 17:44:54,016 - root - INFO - crop_size_y None +2024-02-17 17:44:54,016 - root - INFO - two_step_training False +2024-02-17 17:44:54,016 - root - INFO - plot_animations False +2024-02-17 17:44:54,016 - root - INFO - add_noise False +2024-02-17 17:44:54,016 - root - INFO - noise_std 0 +2024-02-17 17:44:54,016 - root - INFO - epsilon_factor 0 +2024-02-17 17:44:54,016 - root - INFO - world_size 1 +2024-02-17 17:44:54,016 - root - INFO - experiment_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1 +2024-02-17 17:44:54,016 - root - INFO - checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/ckpt.tar +2024-02-17 17:44:54,016 - root - INFO - best_checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/best_ckpt.tar +2024-02-17 17:44:54,016 - root - INFO - resuming False +2024-02-17 17:44:54,016 - root - INFO - local_rank 0 +2024-02-17 17:44:54,016 - root - INFO - enable_amp True +2024-02-17 17:44:54,016 - root - INFO - name afno_backbone_tec_256_d6p_test1 +2024-02-17 17:44:54,016 - root - INFO - group era5_precipafno_backbone_tec_256 +2024-02-17 17:44:54,016 - root - INFO - project ERA5_precip +2024-02-17 17:44:54,016 - root - INFO - entity flowgan +2024-02-17 17:44:54,016 - root - INFO - --------------------------------------------------- +2024-02-17 17:44:54,020 - root - INFO - rank 0, begin data loader init +2024-02-17 17:44:54,020 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train/2005.h5 +2024-02-17 17:44:54,021 - root - INFO - Number of samples per year: 336 +2024-02-17 17:44:54,021 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train. Number of examples: 2352. Image Shape: 128 x 256 x 13 +2024-02-17 17:44:54,021 - root - INFO - Delta t: 6 hours +2024-02-17 17:44:54,021 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:44:54,021 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test/2008.h5 +2024-02-17 17:44:54,021 - root - INFO - Number of samples per year: 337 +2024-02-17 17:44:54,021 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test. Number of examples: 674. Image Shape: 128 x 256 x 13 +2024-02-17 17:44:54,021 - root - INFO - Delta t: 6 hours +2024-02-17 17:44:54,021 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:44:54,022 - root - INFO - rank 0, data loader initialized +2024-02-17 17:44:54,481 - root - INFO - Number of trainable model parameters: 36514560 +2024-02-17 17:44:54,481 - root - INFO - Starting Training Loop... +2024-02-17 17:49:21,197 - root - INFO - --------------- Versions --------------- +2024-02-17 17:49:21,202 - root - INFO - git branch: b'* master' +2024-02-17 17:49:21,205 - root - INFO - git hash: b'5402c07844da9944017e0e5fb5f0530abb33711e' +2024-02-17 17:49:21,205 - root - INFO - Torch: 2.1.2+cu118 +2024-02-17 17:49:21,205 - root - INFO - ---------------------------------------- +2024-02-17 17:49:21,206 - root - INFO - ------------------ Configuration ------------------ +2024-02-17 17:49:21,206 - root - INFO - Configuration file: /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/config/AFNO.yaml +2024-02-17 17:49:21,206 - root - INFO - Configuration name: afno_backbone_tec_256 +2024-02-17 17:49:21,206 - root - INFO - log_to_wandb False +2024-02-17 17:49:21,206 - root - INFO - lr 0.0005 +2024-02-17 17:49:21,206 - root - INFO - batch_size 8 +2024-02-17 17:49:21,206 - root - INFO - patch_size 2 +2024-02-17 17:49:21,206 - root - INFO - depth 6 +2024-02-17 17:49:21,206 - root - INFO - img_size [128, 256] +2024-02-17 17:49:21,206 - root - INFO - max_epochs 1500 +2024-02-17 17:49:21,206 - root - INFO - scheduler CosineAnnealingLR +2024-02-17 17:49:21,206 - root - INFO - in_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:49:21,206 - root - INFO - out_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] +2024-02-17 17:49:21,206 - root - INFO - prediction_length 100 +2024-02-17 17:49:21,206 - root - INFO - orography False +2024-02-17 17:49:21,206 - root - INFO - orography_path None +2024-02-17 17:49:21,206 - root - INFO - exp_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256 +2024-02-17 17:49:21,206 - root - INFO - train_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train +2024-02-17 17:49:21,206 - root - INFO - valid_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test +2024-02-17 17:49:21,206 - root - INFO - inf_data_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/out_of_sample +2024-02-17 17:49:21,206 - root - INFO - time_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/time_means.npy +2024-02-17 17:49:21,206 - root - INFO - global_means_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_means.npy +2024-02-17 17:49:21,206 - root - INFO - global_stds_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/global_stds.npy +2024-02-17 17:49:21,206 - root - INFO - loss l2 +2024-02-17 17:49:21,206 - root - INFO - num_data_workers 4 +2024-02-17 17:49:21,206 - root - INFO - dt 1 +2024-02-17 17:49:21,206 - root - INFO - n_history 0 +2024-02-17 17:49:21,206 - root - INFO - prediction_type iterative +2024-02-17 17:49:21,206 - root - INFO - n_initial_conditions 5 +2024-02-17 17:49:21,206 - root - INFO - ics_type default +2024-02-17 17:49:21,206 - root - INFO - save_raw_forecasts True +2024-02-17 17:49:21,206 - root - INFO - save_channel False +2024-02-17 17:49:21,206 - root - INFO - masked_acc False +2024-02-17 17:49:21,206 - root - INFO - maskpath None +2024-02-17 17:49:21,206 - root - INFO - perturb False +2024-02-17 17:49:21,207 - root - INFO - add_grid False +2024-02-17 17:49:21,207 - root - INFO - N_grid_channels 0 +2024-02-17 17:49:21,207 - root - INFO - gridtype sinusoidal +2024-02-17 17:49:21,207 - root - INFO - roll False +2024-02-17 17:49:21,207 - root - INFO - num_blocks 8 +2024-02-17 17:49:21,207 - root - INFO - nettype afno +2024-02-17 17:49:21,207 - root - INFO - width 56 +2024-02-17 17:49:21,207 - root - INFO - modes 32 +2024-02-17 17:49:21,207 - root - INFO - target default +2024-02-17 17:49:21,207 - root - INFO - normalization zscore +2024-02-17 17:49:21,207 - root - INFO - log_to_screen True +2024-02-17 17:49:21,207 - root - INFO - save_checkpoint True +2024-02-17 17:49:21,207 - root - INFO - enable_nhwc False +2024-02-17 17:49:21,207 - root - INFO - optimizer_type FusedAdam +2024-02-17 17:49:21,207 - root - INFO - crop_size_x None +2024-02-17 17:49:21,207 - root - INFO - crop_size_y None +2024-02-17 17:49:21,207 - root - INFO - two_step_training False +2024-02-17 17:49:21,207 - root - INFO - plot_animations False +2024-02-17 17:49:21,207 - root - INFO - add_noise False +2024-02-17 17:49:21,207 - root - INFO - noise_std 0 +2024-02-17 17:49:21,207 - root - INFO - epsilon_factor 0 +2024-02-17 17:49:21,207 - root - INFO - world_size 1 +2024-02-17 17:49:21,207 - root - INFO - experiment_dir /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1 +2024-02-17 17:49:21,207 - root - INFO - checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/ckpt.tar +2024-02-17 17:49:21,207 - root - INFO - best_checkpoint_path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/results/tec_256/afno_backbone_tec_256/d6p_test1/training_checkpoints/best_ckpt.tar +2024-02-17 17:49:21,207 - root - INFO - resuming False +2024-02-17 17:49:21,207 - root - INFO - local_rank 0 +2024-02-17 17:49:21,207 - root - INFO - enable_amp True +2024-02-17 17:49:21,207 - root - INFO - name afno_backbone_tec_256_d6p_test1 +2024-02-17 17:49:21,207 - root - INFO - group era5_precipafno_backbone_tec_256 +2024-02-17 17:49:21,207 - root - INFO - project ERA5_precip +2024-02-17 17:49:21,207 - root - INFO - entity flowgan +2024-02-17 17:49:21,208 - root - INFO - --------------------------------------------------- +2024-02-17 17:49:21,212 - root - INFO - rank 0, begin data loader init +2024-02-17 17:49:21,213 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train/2005.h5 +2024-02-17 17:49:21,214 - root - INFO - Number of samples per year: 336 +2024-02-17 17:49:21,214 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/train. Number of examples: 2352. Image Shape: 128 x 256 x 13 +2024-02-17 17:49:21,214 - root - INFO - Delta t: 6 hours +2024-02-17 17:49:21,214 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:49:21,215 - root - INFO - Getting file stats from /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test/2008.h5 +2024-02-17 17:49:21,215 - root - INFO - Number of samples per year: 337 +2024-02-17 17:49:21,215 - root - INFO - Found data at path /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/Data_TEC_256x512/test. Number of examples: 674. Image Shape: 128 x 256 x 13 +2024-02-17 17:49:21,215 - root - INFO - Delta t: 6 hours +2024-02-17 17:49:21,215 - root - INFO - Including 0 hours of past history in training at a frequency of 6 hours +2024-02-17 17:49:21,215 - root - INFO - rank 0, data loader initialized +2024-02-17 17:49:21,692 - root - INFO - Number of trainable model parameters: 36514560 +2024-02-17 17:49:21,692 - root - INFO - Starting Training Loop... +2024-02-17 17:51:33,324 - root - INFO - Time taken for epoch 1 is 131.6311058998108 sec +2024-02-17 17:51:33,324 - root - INFO - Train loss: 0.2586471140384674. Valid loss: 0.39238470792770386 +2024-02-17 17:53:45,529 - root - INFO - Time taken for epoch 2 is 132.20482349395752 sec +2024-02-17 17:53:45,529 - root - INFO - Train loss: 0.25994643568992615. Valid loss: 0.39973366260528564 +2024-02-17 17:55:57,840 - root - INFO - Time taken for epoch 3 is 132.31037664413452 sec +2024-02-17 17:55:57,840 - root - INFO - Train loss: 0.25442761182785034. Valid loss: 0.3699217438697815 +2024-02-17 17:58:11,041 - root - INFO - Time taken for epoch 4 is 133.20036244392395 sec +2024-02-17 17:58:11,041 - root - INFO - Train loss: 0.25081461668014526. Valid loss: 0.3364608585834503 +2024-02-17 18:00:24,743 - root - INFO - Time taken for epoch 5 is 133.7016966342926 sec +2024-02-17 18:00:24,743 - root - INFO - Train loss: 0.248686745762825. Valid loss: 0.335431843996048 +2024-02-17 18:02:40,919 - root - INFO - Time taken for epoch 6 is 136.17567682266235 sec +2024-02-17 18:02:40,919 - root - INFO - Train loss: 0.2504323720932007. Valid loss: 0.3266797661781311 +2024-02-17 18:04:57,059 - root - INFO - Time taken for epoch 7 is 136.1393678188324 sec +2024-02-17 18:04:57,059 - root - INFO - Train loss: 0.24679289758205414. Valid loss: 0.3245602250099182 +2024-02-17 18:07:13,195 - root - INFO - Time taken for epoch 8 is 136.13509464263916 sec +2024-02-17 18:07:13,195 - root - INFO - Train loss: 0.24331507086753845. Valid loss: 0.3170415759086609 +2024-02-17 18:09:29,810 - root - INFO - Time taken for epoch 9 is 136.61501479148865 sec +2024-02-17 18:09:29,811 - root - INFO - Train loss: 0.2444596290588379. Valid loss: 0.3148426413536072 +2024-02-17 18:11:46,133 - root - INFO - Time taken for epoch 10 is 136.3226912021637 sec +2024-02-17 18:11:46,134 - root - INFO - Train loss: 0.24120047688484192. Valid loss: 0.2968638837337494 +2024-02-17 18:14:02,651 - root - INFO - Time taken for epoch 11 is 136.51737570762634 sec +2024-02-17 18:14:02,652 - root - INFO - Train loss: 0.23906859755516052. Valid loss: 0.2953372895717621 +2024-02-17 18:16:18,134 - root - INFO - Time taken for epoch 12 is 135.48209857940674 sec +2024-02-17 18:16:18,134 - root - INFO - Train loss: 0.23819145560264587. Valid loss: 0.300484299659729 +2024-02-17 18:18:36,583 - root - INFO - Time taken for epoch 13 is 138.4484360218048 sec +2024-02-17 18:18:36,583 - root - INFO - Train loss: 0.233670175075531. Valid loss: 0.3222668170928955 +2024-02-17 18:20:54,404 - root - INFO - Time taken for epoch 14 is 137.82066202163696 sec +2024-02-17 18:20:54,405 - root - INFO - Train loss: 0.22833436727523804. Valid loss: 0.35155221819877625 +2024-02-17 18:23:09,741 - root - INFO - Time taken for epoch 15 is 135.33607172966003 sec +2024-02-17 18:23:09,741 - root - INFO - Train loss: 0.22580155730247498. Valid loss: 0.3962205648422241 +2024-02-17 18:25:25,871 - root - INFO - Time taken for epoch 16 is 136.12991642951965 sec +2024-02-17 18:25:25,872 - root - INFO - Train loss: 0.22372101247310638. Valid loss: 0.4145030975341797 +2024-02-17 18:27:46,402 - root - INFO - Time taken for epoch 17 is 140.53029203414917 sec +2024-02-17 18:27:46,403 - root - INFO - Train loss: 0.2227291464805603. Valid loss: 0.4282892346382141 +2024-02-17 18:30:04,505 - root - INFO - Time taken for epoch 18 is 138.1018943786621 sec +2024-02-17 18:30:04,505 - root - INFO - Train loss: 0.22263231873512268. Valid loss: 0.4343600869178772