From 1ca935ecbd1b5cf1beafc5701565e22d154b19a4 Mon Sep 17 00:00:00 2001 From: wb-yz775491 Date: Wed, 8 Jan 2025 10:54:53 +0800 Subject: [PATCH 1/5] Updated Docker image --- examples/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/readme.md b/examples/readme.md index bf936cf21..09d5af9de 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -42,8 +42,8 @@ docker exec -it bash (2) `python=3.8.10` + `tenserflow=2.10.0` -docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.8.1 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.8.1 docker exec -it bash ``` From da5b80dc0730b800b5da5f7c32a9d909b3143b14 Mon Sep 17 00:00:00 2001 From: wb-yz775491 Date: Thu, 16 Jan 2025 13:48:37 +0800 Subject: [PATCH 2/5] Revert "Updated Docker image" This reverts commit 1ca935ecbd1b5cf1beafc5701565e22d154b19a4. --- examples/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/readme.md b/examples/readme.md index 09d5af9de..bf936cf21 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -42,8 +42,8 @@ docker exec -it bash (2) `python=3.8.10` + `tenserflow=2.10.0` -docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.8.1 -docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.8.1 +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 +docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.10-0.7.4 docker exec -it bash ``` From 2199c0416d6f69970f6a7c3f5458aaa376594ef8 Mon Sep 17 00:00:00 2001 From: wb-yz775491 Date: Wed, 30 Apr 2025 15:38:25 +0800 Subject: [PATCH 3/5] =?UTF-8?q?docs:=E4=BF=AE=E6=AD=A3=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E6=8B=BC=E5=86=99=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将 'tenserflow' 修正为 'tensorflow'- 更新示例环境配置的描述 --- docs/source/quick_start/local_tutorial.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/quick_start/local_tutorial.md b/docs/source/quick_start/local_tutorial.md index adb4bb972..83f1d915c 100644 --- a/docs/source/quick_start/local_tutorial.md +++ b/docs/source/quick_start/local_tutorial.md @@ -10,7 +10,7 @@ 温馨提示：**在搭载Apple M系列芯片的MacBook上必须使用TensorFlow 2.5或更高版本**，安装方法请查看TF官方文档。 -Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` +Demo实验中使用的环境为 `python=3.6.8` + `tensorflow=1.12.0` ```bash conda create -n py36_tf12 python=3.6.8 @@ -45,7 +45,7 @@ python setup.py install #### Docker镜像启动 -Docker的环境为`python=3.6.9` + `tenserflow=1.15.5` +Docker的环境为`python=3.6.9` + `tensorflow=1.15.5` ##### 方法一：拉取已上传的镜像（推荐） From dbff71ec4021153d0c5c780b5e597efe8540adbc Mon Sep 17 00:00:00 2001 From: wb-yz775491 Date: Wed, 30 Apr 2025 15:38:38 +0800 Subject: [PATCH 4/5] =?UTF-8?q?docs:=E4=BF=AE=E6=AD=A3=E6=96=87=E6=A1=A3?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E6=8B=BC=E5=86=99=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将 'tenserflow' 修正为 'tensorflow'- 更新示例环境配置的描述 --- examples/readme.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/readme.md b/examples/readme.md index a587cc991..dc2122d2e 100644 --- a/examples/readme.md +++ b/examples/readme.md @@ -10,7 +10,7 @@ ## 本地Anaconda安装 -Demo实验中使用的环境为 `python=3.6.8` + `tenserflow=1.12.0` +Demo实验中使用的环境为 `python=3.6.8` + `tensorflow=1.12.0` ```bash conda create -n py36_tf12 python=3.6.8 @@ -35,13 +35,13 @@ git clone https://github.com/alibaba/EasyRec.git cd EasyRec -- Docker环境可选 -(1) `python=3.6.9` + `tenserflow=1.15.5` +(1) `python=3.6.9` + `tensorflow=1.15.5` docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15-0.8.5 docker exec -it bash -(2) `python=3.8.10` + `tenserflow=2.12.0` +(2) `python=3.8.10` + `tensorflow=2.12.0` docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 docker run -td --network host -v /local_path/EasyRec:/docker_path/EasyRec mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12-0.8.5 @@ -55,11 +55,11 @@ git clone https://github.com/alibaba/EasyRec.git cd EasyRec -- Docker环境可选 -(1) `python=3.6.9` + `tenserflow=1.15.5` +(1) `python=3.6.9` + `tensorflow=1.15.5` bash scripts/build_docker_tf115.sh sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py36-tf1.15- -(2) `python=3.8.10` + `tenserflow=2.12.0` +(2) `python=3.8.10` + `tensorflow=2.12.0` bash scripts/build_docker_tf212.sh sudo docker run -td --network host -v /local_path:/docker_path mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easyrec/easyrec:py38-tf2.12- From 2fa02d7d3dd5c9bf819d60cd2e04382563d19c43 Mon Sep 17 00:00:00 2001 From: wb-yz775491 Date: Tue, 16 Dec 2025 16:02:04 +0800 Subject: [PATCH 5/5] fix__get_grad_norm --- easy_rec/python/compat/optimizers.py | 78 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/easy_rec/python/compat/optimizers.py b/easy_rec/python/compat/optimizers.py index d31a4cd41..0fc98c984 100644 --- a/easy_rec/python/compat/optimizers.py +++ b/easy_rec/python/compat/optimizers.py @@ -362,51 +362,53 @@ def optimize_loss(loss, # clip_ops.global_norm(list(zip(*gradients))[0])) # Optionally clip gradients by global norm. - if isinstance(clip_gradients, float): - # gradients = _clip_gradients_by_norm(gradients, clip_gradients) - sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel) - summary.scalar('global_norm/sparse_grad', sparse_norm) - summary.scalar('global_norm/dense_grad', dense_norm) - summary.scalar('global_norm/gradient_norm', grad_norm) - grads = [x[0] for x in gradients] - vars = [x[1] for x in gradients] - clipped_grads, _ = clip_ops.clip_by_global_norm( - grads, clip_gradients, use_norm=grad_norm) - gradients = list(zip(clipped_grads, vars)) - elif callable(clip_gradients): - gradients = clip_gradients(gradients) - elif clip_gradients is not None: - raise ValueError('Unknown type %s for clip_gradients' % - type(clip_gradients)) + if not embedding_parallel: + if isinstance(clip_gradients, float): + # gradients = _clip_gradients_by_norm(gradients, clip_gradients) + sparse_norm, dense_norm, grad_norm = _get_grad_norm( + gradients, embedding_parallel) + summary.scalar('global_norm/sparse_grad', sparse_norm) + summary.scalar('global_norm/dense_grad', dense_norm) + summary.scalar('global_norm/gradient_norm', grad_norm) + grads = [x[0] for x in gradients] + vars = [x[1] for x in gradients] + clipped_grads, _ = clip_ops.clip_by_global_norm( + grads, clip_gradients, use_norm=grad_norm) + gradients = list(zip(clipped_grads, vars)) + elif callable(clip_gradients): + gradients = clip_gradients(gradients) + elif clip_gradients is not None: + raise ValueError('Unknown type %s for clip_gradients' % + type(clip_gradients)) # Add scalar summary for loss. if 'loss' in summaries: summary.scalar('loss', loss) # Add histograms for variables, gradients and gradient norms. + + for gradient, variable in gradients: + if isinstance(gradient, indexed_slices.IndexedSlices): + grad_values = gradient.values + else: + grad_values = gradient + + if grad_values is not None: + var_name = variable.name.replace(':', '_') + if 'gradients' in summaries: + summary.histogram('gradients/%s' % var_name, grad_values) + if 'gradient_norm' in summaries: + summary.scalar('gradient_norm/%s' % var_name, + clip_ops.global_norm([grad_values])) + if not embedding_parallel: - for gradient, variable in gradients: - if isinstance(gradient, indexed_slices.IndexedSlices): - grad_values = gradient.values - else: - grad_values = gradient - - if grad_values is not None: - var_name = variable.name.replace(':', '_') - if 'gradients' in summaries: - summary.histogram('gradients/%s' % var_name, grad_values) - if 'gradient_norm' in summaries: - summary.scalar('gradient_norm/%s' % var_name, - clip_ops.global_norm([grad_values])) - - if clip_gradients is not None and ('global_gradient_norm' in summaries or - 'gradient_norm' in summaries): - sparse_norm, dense_norm, grad_norm = _get_grad_norm( - gradients, embedding_parallel) - summary.scalar('global_norm/clipped_sparse_grad', sparse_norm) - summary.scalar('global_norm/clipped_dense_grad', dense_norm) - summary.scalar('global_norm/clipped_gradient_norm', grad_norm) + if clip_gradients is not None and ('global_gradient_norm' in summaries or + 'gradient_norm' in summaries): + sparse_norm, dense_norm, grad_norm = _get_grad_norm( + gradients, embedding_parallel) + summary.scalar('global_norm/clipped_sparse_grad', sparse_norm) + summary.scalar('global_norm/clipped_dense_grad', dense_norm) + summary.scalar('global_norm/clipped_gradient_norm', grad_norm) # Create gradient updates. def _apply_grad():