add image blur mapper

configs/config_all.yaml

@@ -55,6 +55,10 @@ process:
       caption_num: 1                                          # how many candidate captions to generate for each image
       keep_candidate_mode: 'random_any'                       # retain strategy for the generated $caption_num$ candidates. should be in ["random_any", "similar_one_simhash", "all"].
       keep_original_sample: true                              # whether to keep the original sample. If it's set to False, there will be only generated captions in the final datasets and the original captions will be removed. It's True in default.
+  - image_blur_mapper:                                      # mapper to blur images.
+      p: 0.2                                                  # probability of the image being blured
+      blur_type: 'gaussian'                                   # type of blur kernel, including ['mean', 'box', 'gaussian']
+      radius: 2                                               # radius of blur kernel
   - nlpaug_en_mapper:                                       # simply augment texts in English based on the nlpaug library
       sequential: false                                       # whether combine all augmentation methods to a sequence. If it's True, a sample will be augmented by all opened augmentation methods sequentially. If it's False, each opened augmentation method would generate its augmented samples independently.
       aug_num: 1                                              # number of augmented samples to be generated. If `sequential` is True, there will be total aug_num augmented samples generated. If it's False, there will be (aug_num * #opened_aug_method) augmented samples generated.

data_juicer/ops/mapper/__init__.py

@@ -2,10 +2,11 @@
 from . import (chinese_convert_mapper, clean_copyright_mapper,
                clean_email_mapper, clean_html_mapper, clean_ip_mapper,
                clean_links_mapper, expand_macro_mapper, fix_unicode_mapper,
-               generate_caption_mapper, nlpaug_en_mapper, nlpcda_zh_mapper,
-               punctuation_normalization_mapper, remove_bibliography_mapper,
-               remove_comments_mapper, remove_header_mapper,
-               remove_long_words_mapper, remove_non_chinese_character_mapper,
+               generate_caption_mapper, image_blur_mapper, nlpaug_en_mapper,
+               nlpcda_zh_mapper, punctuation_normalization_mapper,
+               remove_bibliography_mapper, remove_comments_mapper,
+               remove_header_mapper, remove_long_words_mapper,
+               remove_non_chinese_character_mapper,
                remove_repeat_sentences_mapper, remove_specific_chars_mapper,
                remove_table_text_mapper,
                remove_words_with_incorrect_substrings_mapper,

data_juicer/ops/mapper/image_blur_mapper.py

@@ -0,0 +1,80 @@
+import os
+
+import numpy as np
+
+from data_juicer.utils.constant import Fields
+from data_juicer.utils.mm_utils import load_data_with_context, load_image
+
+from ..base_op import OPERATORS, Mapper
+from ..op_fusion import LOADED_IMAGES
+
+
+@OPERATORS.register_module('image_blur_mapper')
+@LOADED_IMAGES.register_module('image_blur_mapper')
+class ImageBlurMapper(Mapper):
+    """Mapper to blur images.
+    """
+
+    def __init__(self,
+                 p: float = 0.2,
+                 blur_type: str = 'gaussian',
+                 radius: float = 2,
+                 *args,
+                 **kwargs):
+        """
+        Initialization method.
+
+        :param p: Probability of the image being blured.
+        :param blur_type: Type of blur kernel, including
+        ['mean', 'box', 'gaussian'].
+        :param radius: Radius of blur kernel.
+        :param args: extra args
+        :param kwargs: extra args
+        """
+        super().__init__(*args, **kwargs)
+        if blur_type not in ['mean', 'box', 'gaussian']:
+            raise ValueError(
+                f'Blur_type [{blur_type}] is not supported. '
+                f'Can only be one of ["mean", "box", "gaussian"]. ')
+        if radius < 0:
+            raise ValueError('Radius must be >= 0. ')
+
+        self.p = p
+
+        from PIL import ImageFilter
+        if blur_type == 'mean':
+            self.blur = ImageFilter.BLUR
+        elif blur_type == 'box':
+            self.blur = ImageFilter.BoxBlur(radius)
+        else:
+            self.blur = ImageFilter.GaussianBlur(radius)
+
+    def process(self, sample, context=False):
+        # there is no image in this sample
+        if self.image_key not in sample or not sample[self.image_key]:
+            return sample
+
+        # load images
+        loaded_image_keys = sample[self.image_key]
+        sample, images = load_data_with_context(sample, context,
+                                                loaded_image_keys, load_image)
+
+        for index, value in enumerate(loaded_image_keys):
+            if self.p < np.random.rand():
+                continue
+            else:
+                blured_image_key = os.path.join(
+                    os.path.dirname(value),
+                    '_blured.'.join(os.path.basename(value).split('.')))
+                if not os.path.exists(
+                        blured_image_key) or blured_image_key not in images:
+                    blured_image = images[value].convert('RGB').filter(
+                        self.blur)
+                    images[blured_image_key] = blured_image
+                    blured_image.save(blured_image_key)
+                    if context:
+                        sample[Fields.context][blured_image_key] = blured_image
+                loaded_image_keys[index] = blured_image_key
+
+        sample[self.image_key] = loaded_image_keys
+        return sample

docs/Operators.md

@@ -10,7 +10,7 @@ The operators in Data-Juicer are categorized into 5 types.
 | Type                              | Number | Description                                     |
 |-----------------------------------|:------:|-------------------------------------------------|
 | [ Formatter ]( #formatter )       |   7    | Discovers, loads, and canonicalizes source data |
-| [ Mapper ]( #mapper )             |   24   | Edits and transforms samples                    |
+| [ Mapper ]( #mapper )             |   25   | Edits and transforms samples                    |
 | [ Filter ]( #filter )             |   27   | Filters out low-quality samples                 |
 | [ Deduplicator ]( #deduplicator ) |   4    | Detects and removes duplicate samples           |
 | [ Selector ]( #selector )         |   2    | Selects top samples based on ranking            |
@@ -56,7 +56,8 @@ All the specific operators are listed below, each featured with several capabili
 | clean_links_mapper                                  | General, Code      | en, zh | Removes links, such as those starting with http or ftp                                                         |
 | expand_macro_mapper                                 | LaTeX              | en, zh | Expands macros usually defined at the top of TeX documents                                                     |
 | fix_unicode_mapper                                  | General            | en, zh | Fixes broken Unicodes (by [ftfy](https://ftfy.readthedocs.io/))                                                |
-| generate_caption_mapper                             | Multimodal         |  -     | generate samples whose captions are generated based on another model (such as blip2) and the figure within the original sample. | 
+| generate_caption_mapper                             | Multimodal         |  -     | generate samples whose captions are generated based on another model (such as blip2) and the figure within the original sample |
+| image_blur_mapper                                   | Multimodal         |  -     | Blur images |
 | nlpaug_en_mapper                                    | General            | en     | Simply augments texts in English based on the `nlpaug` library                                                 | 
 | nlpcda_zh_mapper                                    | General            | zh     | Simply augments texts in Chinese based on the `nlpcda` library                                                 | 
 | punctuation_normalization_mapper                    | General            | en, zh | Normalizes various Unicode punctuations to their ASCII equivalents                                             |

docs/Operators_ZH.md

@@ -9,7 +9,7 @@ Data-Juicer 中的算子分为以下 5 种类型。
 | 类型                                | 数量 | 描述            |
 |------------------------------------|:--:|---------------|
 | [ Formatter ]( #formatter )        |  7 | 发现、加载、规范化原始数据 |
-| [ Mapper ]( #mapper )              | 24 | 对数据样本进行编辑和转换  |
+| [ Mapper ]( #mapper )              | 25 | 对数据样本进行编辑和转换  |
 | [ Filter ]( #filter )              | 27 | 过滤低质量样本       |
 | [ Deduplicator ]( #deduplicator )  |  4 | 识别、删除重复样本     |
 | [ Selector ]( #selector )          |  2 | 基于排序选取高质量样本   |
@@ -55,6 +55,7 @@ Data-Juicer 中的算子分为以下 5 种类型。
 | expand_macro_mapper                                 | LaTeX                 | en, zh    | 扩展通常在 TeX 文档顶部定义的宏                                     |
 | fix_unicode_mapper                                  | General               | en, zh    | 修复损坏的 Unicode（借助 [ftfy](https://ftfy.readthedocs.io/)） |
 | generate_caption_mapper                             | Multimodal            |  -    | 生成样本，其标题是根据另一个辅助模型（例如 blip2）和原始样本中的图形生成的。                                             |
+| image_blur_mapper                                   | Multimodal            |  -        | 对图像进行模糊处理                                              ｜
 | nlpaug_en_mapper                                    | General               | en        | 使用`nlpaug`库对英语文本进行简单增强                                 | 
 | nlpcda_zh_mapper                                    | General               | zh        | 使用`nlpcda`库对中文文本进行简单增强                                 | 
 | punctuation_normalization_mapper                    | General               | en, zh    | 将各种 Unicode 标点符号标准化为其 ASCII 等效项                        |

tests/ops/mapper/test_image_blur_mapper.py

@@ -0,0 +1,124 @@
+import os
+import unittest
+import numpy as np
+
+from datasets import Dataset
+from data_juicer.utils.mm_utils import load_image
+
+from data_juicer.ops.mapper.image_blur_mapper import ImageBlurMapper
+
+
+class ImageBlurMapperTest(unittest.TestCase):
+
+    data_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),
+                             '..', 'data')
+    img1_path = os.path.join(data_path, 'img1.png')
+    img2_path = os.path.join(data_path, 'img2.jpg')
+    img3_path = os.path.join(data_path, 'img3.jpg')
+
+    def _get_blured_img_path(self, path):
+        return os.path.join(os.path.dirname(path), '_blured.'.join(os.path.basename(path).split('.')))
+
+    def _get_blur_kernel(self, blur_type = 'gaussian', radius = 2):
+        from PIL import ImageFilter
+        if blur_type == 'mean':
+            return ImageFilter.BLUR
+        elif blur_type == 'box':
+            return ImageFilter.BoxBlur(radius)
+        else:
+            return ImageFilter.GaussianBlur(radius)
+
+    def _run_image_blur_mapper(self, op, source_list, target_list, blur_kernel):
+        dataset = Dataset.from_list(source_list)
+        dataset = dataset.map(op.process)
+        res_list = dataset.to_list()
+        self.assertEqual(res_list, target_list)
+        for source, res in zip(source_list, res_list):
+            for s_path, r_path in zip(source[op.image_key], res[op.image_key]):
+                s_img = load_image(s_path).convert('RGB').filter(blur_kernel)
+                t_path = 'temp4test' + os.path.splitext(s_path)[-1]
+                s_img.save(t_path)
+                t_img = np.array(load_image(t_path))
+                r_img = np.array(load_image(r_path))
+                os.remove(t_path)
+                np.testing.assert_array_equal(t_img, r_img)
+
+    def test(self):
+        ds_list = [{
+            'images': [self.img1_path]
+        }, {
+            'images': [self.img2_path]
+        }, {
+            'images': [self.img3_path]
+        }]
+        tgt_list = [{
+            'images': [self._get_blured_img_path(self.img1_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img2_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img3_path)]
+        }]
+        op = ImageBlurMapper(p = 1, blur_type = 'gaussian', radius = 2)
+        blur_kernel = self._get_blur_kernel('gaussian', 2)
+        self._run_image_blur_mapper(op, ds_list, tgt_list, blur_kernel)
+
+    def test_blur_type(self):
+        ds_list = [{
+            'images': [self.img2_path]
+        }, {
+            'images': [self.img3_path]
+        }, {
+            'images': [self.img1_path]
+        }]
+        tgt_list = [{
+            'images': [self._get_blured_img_path(self.img2_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img3_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img1_path)]
+        }]
+        op = ImageBlurMapper(p = 1, blur_type = 'box', radius = 2)
+        blur_kernel = self._get_blur_kernel('box', 2)
+        self._run_image_blur_mapper(op, ds_list, tgt_list, blur_kernel)
+
+    def test_radius(self):
+        ds_list = [{
+            'images': [self.img3_path]
+        }, {
+            'images': [self.img2_path]
+        }, {
+            'images': [self.img1_path]
+        }]
+        tgt_list = [{
+            'images': [self._get_blured_img_path(self.img3_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img2_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img1_path)]
+        }]
+        op = ImageBlurMapper(p = 1, blur_type = 'gaussian', radius = 5)
+        blur_kernel = self._get_blur_kernel('gaussian', 5)
+        self._run_image_blur_mapper(op, ds_list, tgt_list, blur_kernel)
+
+    def test_multi_img(self):
+        ds_list = [{
+            'images': [self.img1_path, self.img2_path, self.img3_path]
+        }, {
+            'images': [self.img2_path]
+        }, {
+            'images': [self.img3_path, self.img1_path]
+        }]
+        tgt_list = [{
+            'images': [self._get_blured_img_path(self.img1_path), self._get_blured_img_path(self.img2_path), self._get_blured_img_path(self.img3_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img2_path)]
+        }, {
+            'images': [self._get_blured_img_path(self.img3_path), self._get_blured_img_path(self.img1_path)]
+        }]
+        op = ImageBlurMapper(p = 1, blur_type = 'gaussian', radius = 2)
+        blur_kernel = self._get_blur_kernel('gaussian', 2)
+        self._run_image_blur_mapper(op, ds_list, tgt_list, blur_kernel)
+
+
+if __name__ == '__main__':
+    unittest.main()