About big file upload

thinking

Use js to read the file selected in the form form, calculate the md5 value of the file, upload the md5 value to the server, and check whether the file has been uploaded (similar to the second pass function)
If the file has not been uploaded, cut it into 1MB blocks according to its size. If it is smaller than 1MB, do not cut it
Using ajax to submit the cut block asynchronously and upload it to the server (one block, one request, non blocking, multi threading)
When all the blocks are uploaded, a request to merge files is initiated, and the server merges the previously uploaded file blocks, which means the upload is completed.

Realization

JS computing file MD5 uses spark-md5.js, which is said to use the fastest MD5 algorithm in the world.

js slice the file and upload the slice using ajax:

let size = file.size; //Get file size
const shardSize = 1024 * 1024; // Block size 1MB
let shardCount = Math.ceil(size/shardSize); //Number of blocks that can be cut

for(let i = 0; i < shardCount; i++){
  let start = i * shardSize,
      end = Math.min(size, start + shardSize);
  let form = new FormData();
  form.append('file', file.slice(start, end));  //Slice with slice method
  form.append('size', end - start);
  form.append('name', name);
  form.append('total', shardCount);
  form.append('md5', file_md5); //File md5 value
  form.append('index', i);  //The first few blocks

  $.ajax({
    url: 'upload.php?type=shard',
    type: "POST",
    data: form,
    // async: false, / / whether to upload asynchronously, true by default
    processData: false, //It's important to tell jquery not to process the form
    contentType: false, //It is important to specify false to form the correct content type
    success: function (res) {
      // Successful callback
    }
  }
}

Save slice in php

$path = __DIR__ . '/uploads';
$file = $_FILES['file'];
$total = $_POST['total'];
$index = $_POST['index'];
$size = $_POST['size'];
$dst_file = $path . '/' . $name . '-' . $total . ':' . $index;  // File name of slice file storage 
if ($file["error"] > 0) {
    echo json_encode(['code'=>400, 'msg'=>$file["error"]]);die;
} else {
    $res = move_uploaded_file($file['tmp_name'], $dst_file);
    if ($res) {
        file_put_contents($dst_file . '.info', $size);  // The slice is uploaded successfully. Write a file to save its size. Subsequent consolidation is used to verify the file size
        echo json_encode(['code'=>200, 'msg'=>'shard ok']);die;
    } else {
        echo json_encode(['code'=>400, 'msg'=>'shard move_uploaded_file error']);die;
    }
}

php side merge

function mergeFile($name, $total, &$msg)
{
    // Verify that the slice files are uploaded and complete
    for ($i = 0; $i < $total; $i++) { 
        if (!file_exists($name . '-' . $total . ':' . $i . '.info') || !file_exists($name . '-' . $total . ':' . $i)) {
            $msg = "shard error $i";
            return false;
        } else if (filesize($name . '-' . $total . ':' . $i) != file_get_contents($name . '-' . $total . ':' . $i . '.info')) {
            $msg = "shard size error $i";
            return false;
        }
    }
    @unlink($name);
    if (file_exists($name . '.lock')) {   //Lock to prevent other processes from writing files, causing file damage
        $msg = 'on lock';
        return false;
    }
    touch($name . '.lock');
    $file = fopen($name, 'a+');
    for ($i = 0; $i < $total; $i++) {   //Write files in slice order
        $shardFile = fopen($name . '-' . $total . ':' . $i, 'r');
        $shardData = fread($shardFile, filesize($name . '-' . $total . ':' . $i));
        fwrite($file, $shardData);
        fclose($shardFile);
        unlink($name . '-' . $total . ':' . $i); 
        unlink($name . '-' . $total . ':' . $i . '.info');
    }
    fclose($file);
    unlink($name . '.lock');
    return true;
}

I also wrote a demo, the portal

The following is the rendering of this demo: