容器的md5值: a724bbd839cf7faacfd31a75c9c476ce

使用方法：
(注：“/path/to”表示举例路径，现网需要根据实际情况调整，建议部署在共享路径中，后续章节凡是遇到安装路径的命令，都以现网实际规划的安装路径为准进行替换，不再单独说明。)

一、安装装singularity
yum -y install libatomic libstdc++ libstdc++-devel libseccomp-devel glib2-devel gcc squashfs-tools tar
cd /path/to/go_build 
wget https://go.dev/dl/go1.19.linux-arm64.tar.gz --no-ch 
tar -C /path/to/go_install -xf go1.19.linux-arm64.tar.gz 
export PATH=/path/to/go_install/go/bin:$PATH
source ~/.bashrc

cd /path/to/singularity_build 
wget https://github.com/sylabs/singularity/releases/download/v3.10.2/singularity-ce-3.10.2.tar.gz --no-ch 
tar -xf singularity-ce-3.10.2.tar.gz 
cd singularity-ce-3.10.2 
./mconfig --prefix=/path/to/singularity_install
make -C ./builddir 
make -C ./builddir install 
export PATH=/path/to/singularity_install/bin:$PATH

二、安装hmpi
cd /path/to/bs_build
wget https://mirrors.huaweicloud.com/kunpeng/archive/compiler/bisheng_compiler/BiShengCompiler-3.2.0-aarch64-linux-mini.tar.gz
tar xzvf BiShengCompiler-3.2.0-aarch64-linux-mini.tar.gz -C /path/to/bs_install --strip-components=1
export PATH=/path/to/bs_install/bin:$PATH
export LD_LIBRARY_PATH=/path/to/bs_install/lib:/path/to/bs_install/lib/clang/15.0.4/lib:$LD_LIBRARY_PATH
export INCLUDE=/path/to/bs_install/include:/path/to/bs_install/lib/clang/15.0.4/include:$INCLUDE
export CC=clang CXX=clang++ FC=flang

yum install -y perl-Data-Dumper autoconf automake libtool binutils flex cmake
cd /path/to/hucx_build
wget https://gitee.com/kunpengcompute/hucx/repository/archive/v2.3.0-huawei.tar.gz -f hucx-2.3.0-huawei.tar.gz
tar xf hucx-2.3.0-huawei.tar.gz
cd hucx-v2.3.0-huawei
./autogen.sh
./contrib/configure-opt --prefix=/path/to/hucx_install --disable-numa --enable-mt CFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" CXXFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion"
make -j
make -j install
export PATH=/path/to/hucx_install/bin:$PATH
export LD_LIBRARY_PATH=/path/to/hucx_install/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=/path/to/hucx_install/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/path/to/hucx_install/include:$CPLUS_INCLUDE_PATH
export CFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" CXXFLAGS="-Wno-unused-but-set-variable -Wno-error=int-conversion" FFLAGS="-Wno-error=int-conversion"

cd /path/to/xucg_build
wget https://gitee.com/kunpengcompute/xucg/repository/archive/v2.3.0-huawei.tar.gz -f xucg-2.3.0-huawei.tar.gz
tar xf xucg-2.3.0-huawei.tar.gz
cd xucg-v2.3.0-huawei
mkdir build && cd build
cmake .. -DCMAKE_INSTALL_PREFIX=/path/to/xucg_install -DCMAKE_BUILD_TYPE=Release -DUCG_BUILD_WITH_UCX=/path/to/hucx_install -DUCG_ENABLE_MT=ON -DUCG_BUILD_TESTS=OFF -DCMAKE_C_COMPILER=$CC -DCMAKE_CXX_COMPILER=$CXX -DCMAKE_Fortran_COMPILER=$FC
make -j
make -j install
export PATH=/path/to/xucg_install/bin:$PATH
export LD_LIBRARY_PATH=/path/to/xucg_install/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=/path/to/xucg_install/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/path/to/xucg_install/include:$CPLUS_INCLUDE_PATH

cd /path/to/hmpi_build
wget https://gitee.com/kunpengcompute/hmpi/repository/archive/v2.3.0-huawei.tar.gz -f hmpi-2.3.0-huawei.tar.gz
tar xf hmpi-2.3.0-huawei.tar.gz
cd hmpi-v2.3.0-huawei/
./autogen.pl
./configure --prefix=/path/to/hmpi_install --with-platform=contrib/platform/mellanox/optimized --enable-mpi1-compatibility --with-ucx=/path/to/hucx_install --with-ucg=/path/to/xucg_install
make -j
make -j install
export PATH=/path/to/hmpi_install/bin:$PATH
export LD_LIBRARY_PATH=/path/to/hmpi_install/lib:$LD_LIBRARY_PATH
export C_INCLUDE_PATH=/path/to/hmpi_install/include:$C_INCLUDE_PATH
export CPLUS_INCLUDE_PATH=/path/to/hmpi_install/include:$CPLUS_INCLUDE_PATH

三、获取容器sif
在本页面下载CESM_2.1.3_SVE_v1.sif文件，并上传到服务器目录/path/to/CESM

四、获取输入数据
在本页面下载cesm.inputdata.tgz文件，上传到服务器，解压得到的cesm目录放到目录/path/to/CESM/share下

五、运行容器
1、单节点运行：

CURRENTDIR=$( pwd )
CESM_DIR=/path/to/CESM
CASE_NAME=f19_g16_1node
CASE_DIR=/hpcrunner/CESM-release-cesm2.1.3/cesm/scratch/$CASE_NAME

rm $CURRENTDIR/run -rf

singularity exec -B $CURRENTDIR:/mnt --pwd $CASE_DIR/run $CESM_DIR/CESM-2.1.3-sve.sif cp -rf $CASE_DIR/run /mnt

if [ ! -d $CURRENTDIR/run/timing/checkpoints ]; then
    mkdir -p $CURRENTDIR/run/timing/checkpoints
fi

time -p mpirun -np 128 --mca pml ucx --mca btl ^vader,tcp,openib,uct --allow-run-as-root -hostfile $HOSTFILE -x PATH -x LD_LIBRARY_PATH --bind-to core singularity exec -B $CESM_DIR/share/cesm/inputdata:/hpcrunner/CESM-release-cesm2.1.3/cesm/inputdata,$CURRENTDIR/run:$CASE_DIR/run --pwd $CASE_DIR/run $CESM_DIR/CESM-2.1.3-sve.sif $CASE_DIR/bld/cesm.exe

（说明：-B：绑定本地输入数据目录及输出目录，exec命令进入容器为只读模式，不可在容器内对文件进行读写操作。注意单节点CASE_NAME为f19_g16_1node，2节点CASE_NAME为f19_g16_2nodes。引用输出文件及日志在当前目录的run目录下，通过查看cpl.log中的应用起止时间以评估应用性能。）

2、多节点运行：

CURRENTDIR=$( pwd )
CESM_DIR=/path/to/CESM
CASE_NAME=f19_g16_2nodes
CASE_DIR=/hpcrunner/CESM-release-cesm2.1.3/cesm/scratch/$CASE_NAME

rm $CURRENTDIR/run -rf

singularity exec -B $CURRENTDIR:/mnt --pwd $CASE_DIR/run $CESM_DIR/CESM-2.1.3-sve.sif cp -rf $CASE_DIR/run /mnt

if [ ! -d $CURRENTDIR/run/timing/checkpoints ]; then
    mkdir -p $CURRENTDIR/run/timing/checkpoints
fi

time -p mpirun -np 256 -N 128 --mca pml ucx --mca btl ^vader,tcp,openib,uct --allow-run-as-root -hostfile $HOSTFILE -x PATH -x LD_LIBRARY_PATH --bind-to core singularity exec -B $CESM_DIR/share/cesm/inputdata:/hpcrunner/CESM-release-cesm2.1.3/cesm/inputdata,$CURRENTDIR/run:$CASE_DIR/run --pwd $CASE_DIR/run $CESM_DIR/CESM-2.1.3-sve.sif $CASE_DIR/bld/cesm.exe