update

data_ljkj/AFNO.yaml

@@ -64,8 +64,8 @@ afno_backbone_ljkj: &LJKJ
   <<: *FULL_FIELD
   log_to_wandb: !!bool False
   lr: 5E-4
-  batch_size: 8
-  patch_size: 4
+  batch_size: 4
+  patch_size: 2
   depth : 6 # default 12
   img_size: [192, 288]
   max_epochs: 1500

data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2/hyperparams.yaml

 log_to_wandb: 'False'
 lr: '0.0005'
-batch_size: '8'
-patch_size: '4'
+batch_size: '4'
+patch_size: '2'
 depth: '6'
 img_size: '[192, 288]'
 max_epochs: '1500'
 scheduler: CosineAnnealingLR
-in_channels: '[ 0  1  2  3  4  5  6  7  8  9 10 11 12]'
-out_channels: '[ 0  1  2  3  4  5  6  7  8  9 10 11 12]'
+in_channels: "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22
+  23\n 24 25 26 27 28 29]"
+out_channels: "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21
+  22 23\n 24 25 26 27 28 29]"
 prediction_length: '100'
 orography: 'False'
 orography_path: None
@@ -19,6 +21,7 @@ time_means_path: ./time_means.npy
 global_means_path: ./global_means.npy
 global_stds_path: ./global_stds.npy
 loss: l2
+retrain: 'False'
 num_data_workers: '4'
 dt: '1'
 n_history: '0'
@@ -52,7 +55,8 @@ add_noise: 'False'
 noise_std: '0'
 epsilon_factor: '0'
 world_size: '1'
-experiment_dir: /home/ess/cxt/work/FourCastNetTEC/data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2
+experiment_dir: 
+  /home/cxt/work/fourcastnet_TEC/FourCastNetTEC/data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2
 checkpoint_path: ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/ckpt.tar
 best_checkpoint_path: ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/best_ckpt.tar
 resuming: 'False'
@@ -62,5 +66,5 @@ name: afno_backbone_ljkj_d6p2
 group: era5_precipafno_backbone_ljkj
 project: ERA5_precip
 entity: flowgan
-N_in_channels: '13'
-N_out_channels: '13'
+N_in_channels: '30'
+N_out_channels: '30'

data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2/out.log

-2024-02-19 21:39:12,490 - root - INFO - --------------- Versions ---------------
-2024-02-19 21:39:12,518 - root - INFO - git branch: b'* master'
-2024-02-19 21:39:12,538 - root - INFO - git hash: b'7b87da9222c1ec166fb9b9804e38288691d68bf4'
-2024-02-19 21:39:12,538 - root - INFO - Torch: 1.13.1+cu117
-2024-02-19 21:39:12,538 - root - INFO - ----------------------------------------
-2024-02-19 21:39:12,538 - root - INFO - ------------------ Configuration ------------------
-2024-02-19 21:39:12,538 - root - INFO - Configuration file: /home/ess/cxt/work/FourCastNetTEC/data_ljkj/AFNO.yaml
-2024-02-19 21:39:12,538 - root - INFO - Configuration name: afno_backbone_ljkj
-2024-02-19 21:39:12,538 - root - INFO - log_to_wandb False
-2024-02-19 21:39:12,538 - root - INFO - lr 0.0005
-2024-02-19 21:39:12,538 - root - INFO - batch_size 8
-2024-02-19 21:39:12,538 - root - INFO - patch_size 4
-2024-02-19 21:39:12,538 - root - INFO - depth 6
-2024-02-19 21:39:12,538 - root - INFO - img_size [192, 288]
-2024-02-19 21:39:12,538 - root - INFO - max_epochs 1500
-2024-02-19 21:39:12,538 - root - INFO - scheduler CosineAnnealingLR
-2024-02-19 21:39:12,538 - root - INFO - in_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
-2024-02-19 21:39:12,538 - root - INFO - out_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
-2024-02-19 21:39:12,538 - root - INFO - prediction_length 100
-2024-02-19 21:39:12,539 - root - INFO - orography False
-2024-02-19 21:39:12,539 - root - INFO - orography_path None
-2024-02-19 21:39:12,539 - root - INFO - exp_dir ./results/tec_256
-2024-02-19 21:39:12,539 - root - INFO - train_data_path ./train
-2024-02-19 21:39:12,539 - root - INFO - valid_data_path ./test
-2024-02-19 21:39:12,539 - root - INFO - inf_data_path ./out_of_sample
-2024-02-19 21:39:12,539 - root - INFO - time_means_path ./time_means.npy
-2024-02-19 21:39:12,539 - root - INFO - global_means_path ./global_means.npy
-2024-02-19 21:39:12,539 - root - INFO - global_stds_path ./global_stds.npy
-2024-02-19 21:39:12,539 - root - INFO - loss l2
-2024-02-19 21:39:12,539 - root - INFO - num_data_workers 4
-2024-02-19 21:39:12,539 - root - INFO - dt 1
-2024-02-19 21:39:12,539 - root - INFO - n_history 0
-2024-02-19 21:39:12,539 - root - INFO - prediction_type iterative
-2024-02-19 21:39:12,539 - root - INFO - n_initial_conditions 5
-2024-02-19 21:39:12,539 - root - INFO - ics_type default
-2024-02-19 21:39:12,539 - root - INFO - save_raw_forecasts True
-2024-02-19 21:39:12,539 - root - INFO - save_channel False
-2024-02-19 21:39:12,539 - root - INFO - masked_acc False
-2024-02-19 21:39:12,539 - root - INFO - maskpath None
-2024-02-19 21:39:12,539 - root - INFO - perturb False
-2024-02-19 21:39:12,539 - root - INFO - add_grid False
-2024-02-19 21:39:12,539 - root - INFO - N_grid_channels 0
-2024-02-19 21:39:12,539 - root - INFO - gridtype sinusoidal
-2024-02-19 21:39:12,539 - root - INFO - roll False
-2024-02-19 21:39:12,539 - root - INFO - num_blocks 8
-2024-02-19 21:39:12,539 - root - INFO - nettype afno
-2024-02-19 21:39:12,539 - root - INFO - width 56
-2024-02-19 21:39:12,539 - root - INFO - modes 32
-2024-02-19 21:39:12,539 - root - INFO - target default
-2024-02-19 21:39:12,539 - root - INFO - normalization zscore
-2024-02-19 21:39:12,539 - root - INFO - log_to_screen True
-2024-02-19 21:39:12,539 - root - INFO - save_checkpoint True
-2024-02-19 21:39:12,539 - root - INFO - enable_nhwc False
-2024-02-19 21:39:12,540 - root - INFO - optimizer_type FusedAdam
-2024-02-19 21:39:12,540 - root - INFO - crop_size_x None
-2024-02-19 21:39:12,540 - root - INFO - crop_size_y None
-2024-02-19 21:39:12,540 - root - INFO - two_step_training False
-2024-02-19 21:39:12,540 - root - INFO - plot_animations False
-2024-02-19 21:39:12,540 - root - INFO - add_noise False
-2024-02-19 21:39:12,540 - root - INFO - noise_std 0
-2024-02-19 21:39:12,540 - root - INFO - epsilon_factor 0
-2024-02-19 21:39:12,540 - root - INFO - world_size 1
-2024-02-19 21:39:12,540 - root - INFO - experiment_dir /home/ess/cxt/work/FourCastNetTEC/data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2
-2024-02-19 21:39:12,541 - root - INFO - checkpoint_path ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/ckpt.tar
-2024-02-19 21:39:12,541 - root - INFO - best_checkpoint_path ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/best_ckpt.tar
-2024-02-19 21:39:12,542 - root - INFO - resuming False
-2024-02-19 21:39:12,542 - root - INFO - local_rank 0
-2024-02-19 21:39:12,542 - root - INFO - enable_amp True
-2024-02-19 21:39:12,542 - root - INFO - name afno_backbone_ljkj_d6p2
-2024-02-19 21:39:12,542 - root - INFO - group era5_precipafno_backbone_ljkj
-2024-02-19 21:39:12,542 - root - INFO - project ERA5_precip
-2024-02-19 21:39:12,542 - root - INFO - entity flowgan
-2024-02-19 21:39:12,542 - root - INFO - ---------------------------------------------------
-2024-02-19 21:39:12,549 - root - INFO - rank 0, begin data loader init
-2024-02-19 21:41:26,053 - root - INFO - --------------- Versions ---------------
-2024-02-19 21:41:26,076 - root - INFO - git branch: b'* master'
-2024-02-19 21:41:26,086 - root - INFO - git hash: b'7b87da9222c1ec166fb9b9804e38288691d68bf4'
-2024-02-19 21:41:26,087 - root - INFO - Torch: 1.13.1+cu117
-2024-02-19 21:41:26,087 - root - INFO - ----------------------------------------
-2024-02-19 21:41:26,087 - root - INFO - ------------------ Configuration ------------------
-2024-02-19 21:41:26,087 - root - INFO - Configuration file: /home/ess/cxt/work/FourCastNetTEC/data_ljkj/AFNO.yaml
-2024-02-19 21:41:26,087 - root - INFO - Configuration name: afno_backbone_ljkj
-2024-02-19 21:41:26,087 - root - INFO - log_to_wandb False
-2024-02-19 21:41:26,087 - root - INFO - lr 0.0005
-2024-02-19 21:41:26,087 - root - INFO - batch_size 8
-2024-02-19 21:41:26,088 - root - INFO - patch_size 4
-2024-02-19 21:41:26,088 - root - INFO - depth 6
-2024-02-19 21:41:26,088 - root - INFO - img_size [192, 288]
-2024-02-19 21:41:26,088 - root - INFO - max_epochs 1500
-2024-02-19 21:41:26,088 - root - INFO - scheduler CosineAnnealingLR
-2024-02-19 21:41:26,088 - root - INFO - in_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
-2024-02-19 21:41:26,088 - root - INFO - out_channels [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
-2024-02-19 21:41:26,088 - root - INFO - prediction_length 100
-2024-02-19 21:41:26,088 - root - INFO - orography False
-2024-02-19 21:41:26,089 - root - INFO - orography_path None
-2024-02-19 21:41:26,089 - root - INFO - exp_dir ./results/tec_256
-2024-02-19 21:41:26,089 - root - INFO - train_data_path ./train
-2024-02-19 21:41:26,089 - root - INFO - valid_data_path ./test
-2024-02-19 21:41:26,089 - root - INFO - inf_data_path ./out_of_sample
-2024-02-19 21:41:26,089 - root - INFO - time_means_path ./time_means.npy
-2024-02-19 21:41:26,089 - root - INFO - global_means_path ./global_means.npy
-2024-02-19 21:41:26,089 - root - INFO - global_stds_path ./global_stds.npy
-2024-02-19 21:41:26,089 - root - INFO - loss l2
-2024-02-19 21:41:26,089 - root - INFO - num_data_workers 4
-2024-02-19 21:41:26,090 - root - INFO - dt 1
-2024-02-19 21:41:26,090 - root - INFO - n_history 0
-2024-02-19 21:41:26,090 - root - INFO - prediction_type iterative
-2024-02-19 21:41:26,090 - root - INFO - n_initial_conditions 5
-2024-02-19 21:41:26,090 - root - INFO - ics_type default
-2024-02-19 21:41:26,090 - root - INFO - save_raw_forecasts True
-2024-02-19 21:41:26,090 - root - INFO - save_channel False
-2024-02-19 21:41:26,090 - root - INFO - masked_acc False
-2024-02-19 21:41:26,090 - root - INFO - maskpath None
-2024-02-19 21:41:26,090 - root - INFO - perturb False
-2024-02-19 21:41:26,091 - root - INFO - add_grid False
-2024-02-19 21:41:26,091 - root - INFO - N_grid_channels 0
-2024-02-19 21:41:26,091 - root - INFO - gridtype sinusoidal
-2024-02-19 21:41:26,091 - root - INFO - roll False
-2024-02-19 21:41:26,091 - root - INFO - num_blocks 8
-2024-02-19 21:41:26,091 - root - INFO - nettype afno
-2024-02-19 21:41:26,091 - root - INFO - width 56
-2024-02-19 21:41:26,091 - root - INFO - modes 32
-2024-02-19 21:41:26,091 - root - INFO - target default
-2024-02-19 21:41:26,091 - root - INFO - normalization zscore
-2024-02-19 21:41:26,092 - root - INFO - log_to_screen True
-2024-02-19 21:41:26,092 - root - INFO - save_checkpoint True
-2024-02-19 21:41:26,092 - root - INFO - enable_nhwc False
-2024-02-19 21:41:26,092 - root - INFO - optimizer_type FusedAdam
-2024-02-19 21:41:26,092 - root - INFO - crop_size_x None
-2024-02-19 21:41:26,092 - root - INFO - crop_size_y None
-2024-02-19 21:41:26,092 - root - INFO - two_step_training False
-2024-02-19 21:41:26,093 - root - INFO - plot_animations False
-2024-02-19 21:41:26,093 - root - INFO - add_noise False
-2024-02-19 21:41:26,093 - root - INFO - noise_std 0
-2024-02-19 21:41:26,093 - root - INFO - epsilon_factor 0
-2024-02-19 21:41:26,093 - root - INFO - world_size 1
-2024-02-19 21:41:26,093 - root - INFO - experiment_dir /home/ess/cxt/work/FourCastNetTEC/data_ljkj/results/tec_256/afno_backbone_ljkj/d6p2
-2024-02-19 21:41:26,093 - root - INFO - checkpoint_path ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/ckpt.tar
-2024-02-19 21:41:26,093 - root - INFO - best_checkpoint_path ./results/tec_256/afno_backbone_ljkj/d6p2/training_checkpoints/best_ckpt.tar
-2024-02-19 21:41:26,093 - root - INFO - resuming False
-2024-02-19 21:41:26,094 - root - INFO - local_rank 0
-2024-02-19 21:41:26,094 - root - INFO - enable_amp True
-2024-02-19 21:41:26,094 - root - INFO - name afno_backbone_ljkj_d6p2
-2024-02-19 21:41:26,094 - root - INFO - group era5_precipafno_backbone_ljkj
-2024-02-19 21:41:26,094 - root - INFO - project ERA5_precip
-2024-02-19 21:41:26,094 - root - INFO - entity flowgan
-2024-02-19 21:41:26,094 - root - INFO - ---------------------------------------------------
-2024-02-19 21:41:26,115 - root - INFO - rank 0, begin data loader init

data_ljkj_6h/AFNO.yaml

@@ -65,7 +65,7 @@ afno_backbone_ljkj: &LJKJ
   log_to_wandb: !!bool False
   lr: 5E-4
   batch_size: 8
-  patch_size: 4
+  patch_size: 2
   depth : 6 # default 12
   img_size: [192, 288]
   max_epochs: 1500
@@ -86,4 +86,4 @@ afno_backbone_ljkj: &LJKJ
 afno_backbone_ustc:
   <<: *LJKJ
   batch_size: 32
-  depth: 12
\ No newline at end of file
+  depth: 6
\ No newline at end of file

data_ljkj_6h/submit_batch.sh

@@ -6,7 +6,7 @@
 
 config_file=./AFNO.yaml
 config='afno_backbone_ustc'
-run_num='d12p4'
+run_num='d6p2'
 
 export HDF5_USE_FILE_LOCKING=FALSE
 export NCCL_NET_GDR_LEVEL=PHB