User.java@Data@EqualsAndHashCode(callSuper = false)@Accessors(chain = true)@ApiModel(value=https://www.isolves.com/it/cxkf/yy/JAVA/2020-06-09/"User对象", description="")public class User implements Serializable {private static final long serialVersionUID = 1L;@TableId(value = "id", type = IdType.AUTO)private Long id;@ApiModelProperty(value = "姓名")private String name;@ApiModelProperty(value = "年龄")private Integer age;@ApiModelProperty(value = "邮箱")private String email;public User(String name) {this.name = name;}/*** 因为会在List中判断user是否存在, 所以需要重写equals和hashCode方法* @param o* @return*/@Overridepublic boolean equals(Object o) {if (this == o) return true;if (o == null || getClass() != o.getClass()) return false;User user = (User) o;return Objects.equals(name, user.name);}@Overridepublic int hashCode() {return Objects.hash(name);}}HashUtils.javapublic class HashUtils {public static int hash(String data) {return data.hashCode() & Integer.MAX_VALUE;}}核心逻辑/** * <p> *服务实现类 * </p> * * @author 当我遇上你 * @since 2020-05-19 */@Slf4j@Servicepublic class UserServiceImpl extends ServiceImpl<UserMapper, User> implements UserService {@Autowiredprivate RedisTemplate redisTemplate;/*** 1. bitmap判断是否存在* 2. 内存中数据是否重复* 3. redis和mysql批量插入* 4. 数据库中是否插入失败* @param list* @return*/@Overridepublic JSONObject importBatch(List<User> list) {if (CollectionUtils.isEmpty(list)) {throw new NullPointerException("数据为空");}CopyOnWriteArrayList<User> importFailList = new CopyOnWriteArrayList<>();CopyOnWriteArrayList<User> importSuccessList = new CopyOnWriteArrayList<>();list.stream().forEach(user -> {Boolean exist = redisTemplate.opsForValue().getBit("user", HashUtils.hash(user.getName()));if (exist) {log.error("Redis中name={}的用户已存在", user.getName());// 数据已存在，数据放入失败集合importFailList.add(user);return;}if (importSuccessList.contains(user)) {log.error("内存中name={}的用户已存在", user.getName());importFailList.add(user);return;}importSuccessList.add(user);});if (!CollectionUtils.isEmpty(importSuccessList)) {try {// 批量插入数据库this.saveBatch(importSuccessList);} catch (Exception e) {log.error("MySQL写入冲突:{}", e.getMessage());Iterator<User> iterator = importSuccessList.iterator();while (iterator.hasNext()) {User user = iterator.next();if (user.getId() == null) {log.error("MySQL中name={}的用户已存在", user.getName());importFailList.add(user);importSuccessList.remove(user);}}}// 将导入成功的数据批量写入bitmapredisTemplate.executePipelined(new RedisCallback<String>() {@Overridepublic String doInRedis(RedisConnection redisConnection) throws DataAccessException {importSuccessList.stream().forEach(user -> {redisConnection.setBit("user".getBytes(), HashUtils.hash(user.getName()), true);});return null;}});}JSONObject result = new JSONObject();result.put("success", importSuccessList);result.put("failure", importFailList);return result;}}场景测试@Slf4j@SpringBootTestclass UserServiceImplTest {@AutowiredUserService userService;/*** 模拟内存中存在重复数据** 2020-05-19 15:18:10.468 ERROR 6612 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: 内存中name=张三的用户已存在* 2020-05-19 15:18:10.475WARN 6612 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@304e1e4e] was not registered for synchronization because DataSource is not transactional* 2020-05-19 15:18:10.533INFO 6612 --- [main] com.alibaba.druid.pool.DruidDataSource: {dataSource-1} inited* 2020-05-19 15:18:10.794INFO 6612 --- [main] c.i.d.m.m.s.impl.UserServiceImplTest: {"success":[{"id":1,"name":"张三"}],"failure":[{"name":"张三"}]}*/@Testvoid importBatch1() {User user1 = new User("张三");User user2 = new User("张三");List<User> userList = Arrays.asList(user1, user2);JSONObject result = userService.importBatch(userList);log.info(result.toJSONString());}/*** 模拟Redis中存在重复数据** 2020-05-19 15:18:40.700 ERROR 13352 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: Redis中name=张三的用户已存在* 2020-05-19 15:18:40.708WARN 13352 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@60251ddb] was not registered for synchronization because DataSource is not transactional* 2020-05-19 15:18:40.768INFO 13352 --- [main] com.alibaba.druid.pool.DruidDataSource: {dataSource-1} inited* 2020-05-19 15:18:41.043INFO 13352 --- [main] c.i.d.m.m.s.impl.UserServiceImplTest: {"success":[{"id":2,"name":"李四"}],"failure":[{"name":"张三"}]}*/@Testvoid importBatch2() {User user1 = new User("张三");User user2 = new User("李四");List<User> userList = Arrays.asList(user1, user2);JSONObject result = userService.importBatch(userList);log.info(result.toJSONString());}/*** 手动在MySQL中添加1条数据, 模拟MySQL中存在重复数据** 2020-05-19 15:19:22.337 ERROR 14128 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: Redis中name=张三的用户已存在* 2020-05-19 15:19:22.339 ERROR 14128 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: Redis中name=李四的用户已存在* 2020-05-19 15:19:22.347WARN 14128 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: SqlSession [org.apache.ibatis.session.defaults.DefaultSqlSession@69fe8c75] was not registered for synchronization because DataSource is not transactional* 2020-05-19 15:19:22.405INFO 14128 --- [main] com.alibaba.druid.pool.DruidDataSource: {dataSource-1} inited* 2020-05-19 15:19:22.609 ERROR 14128 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: MySQL写入冲突:cn.idea360.demo.modules.mp.mapper.UserMapper.insert (batch index #1) failed. Cause: java.sql.BatchUpdateException: Duplicate entry '王五' for key 'name'* ; Duplicate entry '王五' for key 'name'; nested exception is java.sql.BatchUpdateException: Duplicate entry '王五' for key 'name'* 2020-05-19 15:19:22.609 ERROR 14128 --- [main] c.i.d.m.mp.service.impl.UserServiceImpl: MySQL中name=王五的用户已存在* 2020-05-19 15:19:22.697INFO 14128 --- [main] c.i.d.m.m.s.impl.UserServiceImplTest: {"success":[],"failure":[{"name":"张三"},{"name":"李四"},{"name":"王五"}]}*/@Testvoid importBatch3() {User user1 = new User("张三");User user2 = new User("李四");User user3 = new User("王五");List<User> userList = Arrays.asList(user1, user2, user3);JSONObject result = userService.importBatch(userList);log.info(result.toJSONString());}}