Asynchronous Download

1, Background

• At present, the system is slow to download large files, slow to export, a large number of interfaces occupy server bandwidth and other problems, which seriously affect the user experience. Based on this background, the asynchronous download function is developed and implemented.

2, Project structure

• Brain map thinking

3, Environmental preparation

• maven dependency

<!-- zip encryption -->
<dependency>
    <groupId>net.lingala.zip4j</groupId>
    <artifactId>zip4j</artifactId>
    <version>2.5.2</version>
</dependency>

<!-- Google Image Compression -->
<dependency>
    <groupId>net.coobird</groupId>
    <artifactId>thumbnailator</artifactId>
    <version>0.4.11</version>
</dependency>

4, Specific implementation

• Asynchronous download source code

	  /**
     * Asynchronous Download: download files to new folder -- > compress folder -- > delete folder
     *
     * @author mc
     * @date 2020-05-18 17:50
     **/
    @ResponseBody
    @PostMapping("/asyDown")
    @ApiOperation(value = "Asynchronous Download", notes = "Download files to new folder-->Compress folder-->remove folders")
    public Result<String> asynchronousDownload(@RequestBody List<String> list) throws InterruptedException{
        if (CollectionUtils.isEmpty(list)) {
            return Result.genFailResult("Get file download address is empty!");
        }
        // Output a new folder for each request example: 2020052052021
        String folder = DateUtil.dateToStr(new Date(), DateUtil.TIMEFORMAT3) + RandomUtil.randomNumbers(2);
        // Production server file directory
        String[] filePath = {path + folder + "/"};
        File file = new File(filePath[0]);
        // Get oss download token
        String token = ossUtil.getOssToken();
        list.forEach(x -> {
            // Bulk download with thread pool
            threadPoolExecutor.execute(() -> {
                log.warn("Start execution --> {}", Thread.currentThread().getName());
                downloadFile(token,x,file,filePath);
            });
        });
        // Compress folder (randomly produce 4-digit password) -- > zip file
        String pw = RandomUtil.randomString(4);
        String zipPath = filePath[0].substring(0, filePath[0].length() - 1) + ".zip";
        while(true){
            // All subthreads end file compression
            if(threadPoolExecutor.getActiveCount() == 0){
                FileUtils.encryptZip(file, zipPath, pw);
                // remove folders
                FileUtils.delete(file);
                break;
            }
            Thread.sleep(1000);
        }
        return Result.genSuccessResult("Saved successfully!", pw);
    }

/**
 * Circular download file body
 * @param token Request oss token
 * @param x     Download address
 * @param file  Folder directory
 * @param filePath File directory
 */
private void downloadFile(String token, String x, File file, String[] filePath) {
    try {
        // Download pictures from oss server to Linux server
        com.haier.rrswl.common.result.Result<byte[]> ossResult = ossUtil.downLoadFile(token, x);
        if (!ossResult.getFlag()) {
            return;
        }
        // Redefining file name example: 17213943.jpg
        String fileName = DateUtil.dateToStr(new Date(), DateUtil.TIMEFORMATCl) + RandomUtil.randomNumbers(2) + x.substring(x.lastIndexOf("."));
        // Determine whether the folder exists. If the folder does not exist, create a new folder
        FileUtil.mkdir(file);
        // Write to file (note that the file name must be added after the file save path)
        FileOutputStream fileOut = new FileOutputStream(filePath[0] + fileName);
        BufferedOutputStream bos = new BufferedOutputStream(fileOut);
        // Save file
        bos.write(ossResult.getData());
        fileOut.close();
        bos.close();
    } catch (Exception e) {
        e.printStackTrace();
    }
}

• Zip encryption compression

		/**
     * Compress the files under the specified path to the specified zip file and encrypt with the specified password. If the password is empty, encryption protection will not be performed
     *
     * @param file     Folders to be compressed
     * @param pathFile Compressed path + filename
     * @param passWord Encrypted password
     * @author mc
     * @date 2020-05-20 17:21
     **/
    public static void encryptZip(File file, String pathFile, String passWord) {
        try {
            ZipFile zipFile = new ZipFile(pathFile);
            ZipParameters parameters = new ZipParameters();
            // Set encryption if password is not empty
            if (StringUtils.isNotBlank(passWord)) {
                // Set password
                zipFile.setPassword(passWord.toCharArray());
                // Encryption method
                parameters.setEncryptionMethod(EncryptionMethod.ZIP_STANDARD);
                parameters.setEncryptFiles(true);
            }
            zipFile.addFolder(file, parameters);
        } catch (ZipException e) {
            log.error(e.getMessage());
            throw new RuntimeException("failed to compress file, specific reason" + 	e.getMessage(), e);
        }
    }

• Folder recursive delete

/**
 * Empty Folder 
 *
 * @param directory folder
 * @return boolean
 */
private static boolean clean(File directory) throws IORuntimeException {
    if (directory == null || !directory.exists() || !directory.isDirectory()) {
        return true;
    }
    // Get all subdirectories
    final File[] files = directory.listFiles();
    if (null == files) {
        return true;
    }
    // Traverse all child files
    for (File childFile : files) {
        // If there is an error in deleting a task, it will fail to delete this task
        if (!childFile.delete()) {
            return false;
        }
    }
    return true;
}

/**
 * Delete file / a file delete failure will terminate the delete operation
 *
 * @param file File object
 * @return boolean
 */
public static boolean delete(File file) throws IORuntimeException {
    // If the file does not exist or has been deleted, return true here to indicate the deletion is successful
    if (file == null || !file.exists()) {
        return true;
    }
    if (file.isDirectory()) {
        // Empty all files and directories in the directory
        if (!clean(file)) {
            return false;
        }
    }
    // Delete files or empty directories
    return file.delete();
}

• Picture stream compression

/**
 * Compress pictures to scale 
 * @author mc
 * @date 2020-05-21 09:21 
 * @param picByte Picture input stream
 * @param suffix Image suffix default jpg
 * @return Picture output stream
 **/
private static byte[] decompressPicByte(byte[] picByte, String suffix) {
    ByteArrayInputStream inputStream = new ByteArrayInputStream(picByte);
    Thumbnails.Builder<? extends InputStream> builder = Thumbnails.of(inputStream).scale(0.5);
    try {
        BufferedImage bufferedImage = builder.asBufferedImage();
        ByteArrayOutputStream bao = new ByteArrayOutputStream();
        ImageIO.write(bufferedImage, suffix, bao);
        return bao.toByteArray();
    } catch (IOException e) {
        log.error(e.getMessage());
    }
    return picByte;
}

• Zero-copy

5, Server

• Nginx configuration

    server {
        listen       80;
        server_name  local;
        add_header 'Access-Control-Allow-Origin' *;
        add_header 'Access-Control-Allow-Credentials' true;

        location / {
            root /Document storage address/;
        }

          #error_page  404              /404.html;
    }

• Cronntab scheduled task

# Delete files 2 hours ago
find /File address/ -type f -mmin +120 -exec rm {} ;

# Create a shell script
touch remove.sh
vim remove.sh
chmod 777 remove.sh

# Edit crontab script once an hour
crontab -e
0 */1 * * *  sh /shell Path address/remove.sh > /dev/null 2>&1

# View a list of crontab scripts
crontab -l

5, Precautions

• Package path specification: ` com. Company name. Department name. Project name

  The package path must be changed! It must be changed! Don't rename another project!

• Profile encoding must be UTF-8