update

8129a24b · xuetaowave · ee72b125 · 8129a24b · 8129a24b · 8129a24b
Commit 8129a24b authored 1 year ago by xuetaowave
--- a/data_ljkj/calculate_mean_std.ipynb
+++ b/data_ljkj/calculate_mean_std.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "initial_id",
+   "metadata": {
+    "collapsed": true,
+    "ExecuteTime": {
+     "end_time": "2024-02-19T08:47:23.852497Z",
+     "start_time": "2024-02-19T08:47:23.704697700Z"
+    }
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import h5py"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "<KeysViewHDF5 ['fields']>"
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data_load = h5py.File('train/2010.h5', 'r')\n",
+    "data_load.keys()"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-19T08:47:23.854510700Z",
+     "start_time": "2024-02-19T08:47:23.850494900Z"
+    }
+   },
+   "id": "152ab0a0c8eecc72",
+   "execution_count": 2
+  },
+  {
+   "cell_type": "code",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(1, 30, 1, 1)\n"
+     ]
+    }
+   ],
+   "source": [
+    "data = data_load['fields']\n",
+    "global_mean = np.mean(data, axis=(0, 2, 3))[None, :, None, None]\n",
+    "print(global_mean.shape)\n",
+    "np.save('global_means.npy', global_mean)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-19T08:47:30.076812300Z",
+     "start_time": "2024-02-19T08:47:23.905421200Z"
+    }
+   },
+   "id": "f25578284e85faed",
+   "execution_count": 3
+  },
+  {
+   "cell_type": "code",
+   "outputs": [],
+   "source": [
+    "global_stds = np.std(data, axis=(0, 2, 3))[None, :, None, None]\n",
+    "np.save('global_stds.npy', global_stds)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-19T08:47:39.515216200Z",
+     "start_time": "2024-02-19T08:47:30.072271100Z"
+    }
+   },
+   "id": "c70926d3068ad7ac",
+   "execution_count": 4
+  },
+  {
+   "cell_type": "code",
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "(1, 30, 192, 288)\n"
+     ]
+    }
+   ],
+   "source": [
+    "time_means = np.mean(data, axis=0)[None]\n",
+    "print(time_means.shape)\n",
+    "np.save('time_means.npy', time_means)"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-02-19T08:47:42.876707800Z",
+     "start_time": "2024-02-19T08:47:39.517222100Z"
+    }
+   },
+   "id": "e5d84c8e9cb760d8",
+   "execution_count": 5
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+%% Cell type:code id:initial_id tags:
+``` python
+import numpy as np
+import h5py
+```
+%% Cell type:code id:152ab0a0c8eecc72 tags:
+``` python
+data_load = h5py.File('train/2010.h5', 'r')
+data_load.keys()
+```
+%% Output
+<KeysViewHDF5 ['fields']>
+%% Cell type:code id:f25578284e85faed tags:
+``` python
+data = data_load['fields']
+global_mean = np.mean(data, axis=(0, 2, 3))[None, :, None, None]
+print(global_mean.shape)
+np.save('global_means.npy', global_mean)
+```
+%% Output
+    (1, 30, 1, 1)
+%% Cell type:code id:c70926d3068ad7ac tags:
+``` python
+global_stds = np.std(data, axis=(0, 2, 3))[None, :, None, None]
+np.save('global_stds.npy', global_stds)
+```
+%% Cell type:code id:e5d84c8e9cb760d8 tags:
+``` python
+time_means = np.mean(data, axis=0)[None]
+print(time_means.shape)
+np.save('time_means.npy', time_means)
+```
+%% Output
+    (1, 30, 192, 288)
--- a/data_ljkj/calculate_mean_std.py
+++ b/data_ljkj/calculate_mean_std.py
+#!/usr/bin/env python
+# coding: utf-8
+# In[1]:
+import numpy as np
+import h5py
+# In[2]:
+data_load = h5py.File('train/2010.h5', 'r')
+data_load.keys()
+# In[3]:
+data = data_load['fields']
+global_mean = np.mean(data, axis=(0, 2, 3))[None, :, None, None]
+print(global_mean.shape)
+np.save('global_means.npy', global_mean)
+# In[4]:
+global_stds = np.std(data, axis=(0, 2, 3))[None, :, None, None]
+np.save('global_stds.npy', global_stds)
+# In[5]:
+time_means = np.mean(data, axis=0)[None]
+print(time_means.shape)
+np.save('time_means.npy', time_means)
--- a/data_ljkj/collect_data.ipynb
+++ b/data_ljkj/collect_data.ipynb
--- a/data_ljkj/create_dataset.ipynb
+++ b/data_ljkj/create_dataset.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np \n",
+    "import h5py "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(['fields', 'date'],\n",
+       " (16992, 192, 288),\n",
+       " (16992, 192, 288),\n",
+       " (16992, 192, 288),\n",
+       " (16992, 192, 288),\n",
+       " (16992, 192, 288))"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data_load_D = np.load('Density.npz')\n",
+    "D = data_load_D['fields']\n",
+    "\n",
+    "data_load_P = np.load('P.npz')\n",
+    "P = data_load_P['fields']\n",
+    "\n",
+    "data_load_T = np.load('T.npz')\n",
+    "T = data_load_T['fields']\n",
+    "\n",
+    "data_load_U = np.load('U.npz')\n",
+    "U = data_load_U['fields']\n",
+    "\n",
+    "data_load_V = np.load('V.npz')\n",
+    "V = data_load_V['fields']\n",
+    "\n",
+    "list(data_load_D.keys()), D.shape, P.shape, T.shape, U.shape, V.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "data = np.hstack((D.reshape((-1, 6)+D.shape[1:]), P.reshape((-1, 6)+P.shape[1:]), T.reshape((-1, 6)+T.shape[1:]), U.reshape((-1, 6)+U.shape[1:]), V.reshape((-1, 6)+V.shape[1:])))\n",
+    "train_mask = data.shape[0]//6*5"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/mnt/data/work/FourCastNet/data_ljkj\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "print(os.getcwd())\n",
+    "os.makedirs('train', exist_ok=True)\n",
+    "os.makedirs('test', exist_ok=True)\n",
+    "os.makedirs('out_of_sample', exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import h5py\n",
+    "with h5py.File('train/2010.h5', 'w') as f:\n",
+    "    f.create_dataset(\"fields\", data=data[:train_mask])\n",
+    "    \n",
+    "with h5py.File('test/2010.h5', 'w') as f:\n",
+    "    f.create_dataset(\"fields\", data=data[train_mask:])\n",
+    "    \n",
+    "with h5py.File('out_of_sample/2010.h5', 'w') as f:\n",
+    "    f.create_dataset(\"fields\", data=data[train_mask:])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "castnet",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.18"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
+%% Cell type:code id: tags:
+``` python
+import numpy as np
+import h5py
+```
+%% Cell type:code id: tags:
+``` python
+data_load_D = np.load('Density.npz')
+D = data_load_D['fields']
+data_load_P = np.load('P.npz')
+P = data_load_P['fields']
+data_load_T = np.load('T.npz')
+T = data_load_T['fields']
+data_load_U = np.load('U.npz')
+U = data_load_U['fields']
+data_load_V = np.load('V.npz')
+V = data_load_V['fields']
+list(data_load_D.keys()), D.shape, P.shape, T.shape, U.shape, V.shape
+```
+%% Output
+    (['fields', 'date'],
+     (16992, 192, 288),
+     (16992, 192, 288),
+     (16992, 192, 288),
+     (16992, 192, 288),
+     (16992, 192, 288))
+%% Cell type:code id: tags:
+``` python
+data = np.hstack((D.reshape((-1, 6)+D.shape[1:]), P.reshape((-1, 6)+P.shape[1:]), T.reshape((-1, 6)+T.shape[1:]), U.reshape((-1, 6)+U.shape[1:]), V.reshape((-1, 6)+V.shape[1:])))
+train_mask = data.shape[0]//6*5
+```
+%% Cell type:code id: tags:
+``` python
+import os
+print(os.getcwd())
+os.makedirs('train', exist_ok=True)
+os.makedirs('test', exist_ok=True)
+os.makedirs('out_of_sample', exist_ok=True)
+```
+%% Output
+    /mnt/data/work/FourCastNet/data_ljkj
+%% Cell type:code id: tags:
+``` python
+import h5py
+with h5py.File('train/2010.h5', 'w') as f:
+    f.create_dataset("fields", data=data[:train_mask])
+with h5py.File('test/2010.h5', 'w') as f:
+    f.create_dataset("fields", data=data[train_mask:])
+with h5py.File('out_of_sample/2010.h5', 'w') as f:
+    f.create_dataset("fields", data=data[train_mask:])
+```
+%% Cell type:code id: tags:
+``` python
+```