前端大文件上传 - 总结（Vue3 + hook + Web Worker实现，通过多个Worker线程大大提高Hash计算的速度）

流程图

选择文件

文件分片&计算文件hash值

代码实现：

// 大文件切片上传，worker.js
import sparkmd5 from 'spark-md5';
const defaultchunksize = 1024 * 1024 * 50; // 50mb
// const defaultchunksize = 1024 * 1024 * 1; // 1mb

self.onmessage = (e) => {
	const { file, chunksize = defaultchunksize } = e.data;

	let blobslice = file.prototype.slice || file.prototype.mozslice || file.prototype.webkitslice,
		chunks = math.ceil(file.size / chunksize),
		currentchunk = 0,
		spark = new sparkmd5.arraybuffer(),
		filechunkhashlist = [],
		filechunklist = [],
		filereader = new filereader();

	loadnext();

	function loadnext() {
		let start = currentchunk * chunksize,
			end = ((start + chunksize) >= file.size) ? file.size : start + chunksize;

		let chunk = blobslice.call(file, start, end);
		filechunklist.push(chunk);
		filereader.readasarraybuffer(chunk);
	}

	function getchunkhash(e) {
		const chunkspark = new sparkmd5.arraybuffer();
		chunkspark.append(e.target.result);
		filechunkhashlist.push(chunkspark.end());
	}

	// 处理每一块的分片
	filereader.onload = function (e) {
		spark.append(e.target.result); // append array buffer
		currentchunk++;

		getchunkhash(e)

		if (currentchunk < chunks) {
			loadnext();
		} else {
			// 计算完成后，返回结果
			self.postmessage({
				filemd5: spark.end(),
				filechunklist,
				filechunkhashlist,
			});
			filereader.abort();
			filereader = null;
		}
	}

	// 读取失败
	filereader.onerror = function () {
		self.postmessage({
			error: 'oops, something went wrong.'
		});
	}
};

async function handlecutfile() {
	// vite中使用worker.js脚本
	const worker = new worker(new url('@/workers/cutfile.js', import.meta.url), {
		type: 'module',
	})
	
	worker.postmessage({ file: file.value.raw })
	worker.onmessage = (e) => {
		handlecutsuccess(e.data)
		worker.terminate()
	}
}

文件上传

检查文件是否上传过

async function checkfile() {
	// 这个接口要配置防响应拦截
	const params = {
		filename: filename.value,
		file_hash: filemd5.value,
		total_chunks: chunktotal.value,
	}
	const data = await checkfilefn(params)
	// 已经上传过返回已上传的分块ids：chunk_upload[]
	if (data.code === 0) {
		return data.data
	}

	if (data.code === 1) {
		// 空间不足
		modal.msgerror(t('sample.notenoughspace'))
		return false
	}

	modal.msgerror(data.msg)
	return false
}

async function uploadfile() {
	const data = await checkfile()
	if (!data) return

	const { chunk_upload, upload_id } = data
	uploadid.value = upload_id

	if (chunk_upload.length === 0) {
		// 上传整个文件
	}
	
	if (chunk_upload.length !== chunktotal.value) {
		// 上传未上传的分片 - 断点续传
	}
	// 上传完成 - 秒传（可能需要发起合并请求）
}

处理所有分片的请求 & 进度的获取

// 通过请求池的方式上传文件
async function handleuploadrequest(uploadedchunks = []) {
	const requestlist = []
	for (let i = 0; i < filechunklist.value.length; i++) {
		if (uploadedchunks.indexof(i + 1) === -1) {
			requestlist.push(uploadchunk(filechunklist.value[i], i, filemd5.value))
		}
	}

	// 这里控制同时请求的接口数量
	await processrequests(requestlist)
}

// 处理分片文件的上传
function uploadchunk(chunk, index, filemd5) {
	const params = {
		chunk_id: index + 1,
		file_hash: filemd5,
		upload_id: uploadid.value,
		chunk_hash: filechunkhashlist.value[index],
	}
	const formdata = new formdata()
	formdata.append('file_chunk', chunk)
	return {
		url: uploadurl,
		method: 'post',
		timeout: 5 * 60 * 1000,
		data: formdata,
		params,
		skipinterceptor: true,
		headers: {
			// 以前的人写了避免重复提交
			repeatsubmit: false,
		},
		onuploadprogress: (progressevent) => {
			chunkuploadedsize[index] = progressevent.loaded

			// 计算uploadedsize的总和
			const size = object.values(chunkuploadedsize).reduce((total, item) => {
				return total + item
			}, 0)

			// 计算总的上传进度
			uploadprogress = ((size + uploadedsize) / filesize.value) * 100

			store.dispatch('upload/setprogress', uploadprogress)
		}
	}
}

async function processrequests(requestlist) {
	// 同时发起三个请求
	maplimit(requestlist, 3, async (req) => {
		await request(req)
	}, async (err) => {
		if (err) {
			console.log('err: ', err)
			return false
		}
		// 全部发送成功应该发起合并请求
		await mergefilefn({ upload_id: uploadid.value })
		return true
	})
}

完整代码

这里通过 hook 封装了这一块代码。代码中并没有处理文件上传完成，但没有发起合并请求的情况，因为后端没返回这种情况，所以这里没写；建议与后端进行沟通要求考虑全部情况。

import request from '@/utils/request'
import modal from '@/plugins/modal'
import { maplimit } from 'async'
import { usei18n } from "vue-i18n"

const defaultchunksize = 1024 * 1024 * 50; // 50mb

const usecutfile = ({ uploadurl, checkfilefn, mergefilefn, callback }) => {
	const store = usestore();
	const { t } = usei18n()

	const file = ref(null)
	const filename = computed(() => file.value?.name || '')
	const filesize = computed(() => file.value?.size || 0)

	const filemd5 = ref('')
	const filechunklist = ref([])
	const filechunkhashlist = ref([])
	const chunktotal = ref(0)
	// 上传id
	const uploadid = ref('')
	// 上传进度
	let uploadprogress = 0
	// 每一个分片已上传的大小
	let chunkuploadedsize = {}
	// 断点续传时已上传的大小
	let uploadedsize = 0

	// 监听上传文件弹框的文件改变
	async function handleuploadchange(fileobj) {
		file.value = fileobj
	}

	// 开始处理文件分块
	async function handlecutfile() {
		const worker = new worker(new url('@/workers/cutfile.js', import.meta.url), {
			type: 'module',
		})

		// 文件切块的过程不可点击
		worker.postmessage({ file: file.value.raw })
		worker.onmessage = (e) => {
			handlecutsuccess(e.data)
			worker.terminate()
		}
	}

	// 切片文件成功
	async function handlecutsuccess(data) {
		filemd5.value = data.filemd5
		filechunklist.value = data.filechunklist
		filechunkhashlist.value = data.filechunkhashlist
		chunktotal.value = filechunklist.value.length

		uploadfile()
	}

	// 上传文件
	async function uploadfile() {
		const data = await checkfile()
		if (!data) return

		const { chunk_upload, upload_id } = data
		uploadid.value = upload_id

		if (chunk_upload.length === 0) {
			// 上传整个文件
			return await handleuploadrequest()
		}
		// 上传未上传的分片，过滤已上传的分片 - 断点续传
		if (chunk_upload.length !== chunktotal.value) {
			uploadedsize = chunk_upload.length * defaultchunksize

			return await handleuploadrequest(chunk_upload)
		}
		// 上传完成 - 秒传
		store.dispatch('upload/setsampleuploading', false)
		modal.msgsuccess(t('upload.uploadedtip'))
		resetdata()
		return true
	}

	// 检查文件是否已经上传过
	async function checkfile() {
		// 这个接口要配置防响应拦截
		const params = {
			filename: filename.value,
			file_hash: filemd5.value,
			total_chunks: chunktotal.value,
		}
		const { code, msg, data } = await checkfilefn(params)
		// 已经上传过返回对应hash值
		if (code === 0) {
			return data
		}

		modal.msgerror(msg)
		store.dispatch('upload/setsampleuploading', false)
		return false
	}

	// 处理分片文件的上传
	function uploadchunk(chunk, index, filemd5) {
		const params = {
			chunk_id: index + 1,
			file_hash: filemd5,
			upload_id: uploadid.value,
			chunk_hash: filechunkhashlist.value[index],
		}

		const formdata = new formdata()
		formdata.append('file_chunk', chunk)

		return {
			url: uploadurl,
			method: 'post',
			timeout: 5 * 60 * 1000,
			data: formdata,
			params,
			skipinterceptor: true,
			headers: {
				// 以前的人写了避免重复提交
				repeatsubmit: false,
			},
			onuploadprogress: (progressevent) => {
				chunkuploadedsize[index] = progressevent.loaded

				// 计算uploadedsize的总和
				const size = object.values(chunkuploadedsize).reduce((total, item) => {
					return total + item
				}, 0)

				// 计算总的上传进度
				uploadprogress = ((size + uploadedsize) / filesize.value) * 100

				store.dispatch('upload/setprogress', uploadprogress)
			}
		}
	}

	// 通过请求池的方式上传文件
	async function handleuploadrequest(uploadedchunks = []) {
		const requestlist = []
		for (let i = 0; i < filechunklist.value.length; i++) {
			if (uploadedchunks.indexof(i + 1) === -1) {
				requestlist.push(uploadchunk(filechunklist.value[i], i, filemd5.value))
			}
		}

		// 方法一：使用请求池的方式发送请求
		await processrequests(requestlist)

		// 方法二：使用promise.all一次性发送全部请求，uploadchunk需要返回request({})
		// await promise.all(requestlist)
		// // 上传完成后，合并文件
		// await mergefilefn({ upload_id: uploadid.value })
		// return true
	}

	async function processrequests(requestlist) {
		maplimit(requestlist, 3, async (reqitem) => {
			await request(reqitem)
		}, async (err) => {
			if (err) {
				console.log('err: ', err)
				modal.msgerror(t('upload.error'))
				store.dispatch('upload/setsampleuploading', false)
				resetdata()
				return false
			}
			await mergefilefn({ upload_id: uploadid.value })
			modal.msgsuccess(t('upload.success'))
			callback && callback()
			store.dispatch('upload/setsampleuploading', false)
			resetdata()
			return true
		})
	}

	// 上传成功，还原数据
	function resetdata() {
		filemd5.value = ''
		filechunklist.value = []
		filechunkhashlist.value = []
		chunktotal.value = 0
		uploadid.value = ''
		uploadprogress = 0
		chunkuploadedsize = {}
		uploadedsize = 0
	}

	return {
		file,
		handleuploadchange,
		handlecutfile,
		handlecutsuccess,
		uploadfile,
		resetdata,
	}
}

export default usecutfile

额外–上传组件封装

<template>
	<el-dialog
		v-model="visible"
		:title="title"
		:width="width"
		append-to-body
		class="common-center-dialog"
		@close="emit('update:visible', false)"
	>
		<el-upload
			ref="uploadref"
			:headers="getheaders"
			:limit="1"
			:accept="accept"
			:action="actionurl"
			:show-file-list="showfilelist"
			:before-upload="handlebeforeupload"
			:on-change="handlechange"
			:on-success="handlesuccess"
			:on-error="handleerror"
			:on-exceed="handleexceed"
			:on-remove="handleremove"
			:auto-upload="autoupload"
			:disabled="loading"
			drag
		>
			<el-icon class="el-icon--upload">
				<upload-filled />
			</el-icon>
			<div class="el-upload__text">
				{{ t('upload.drag') }}
				<em>{{ t('upload.upload') }}</em>
			</div>
			<template #tip>
				<div class="el-upload__tip text-center">
					<span>{{ tiptext || t('upload.onlycsv') }}</span>
					<span v-if="templateurl || customdownload">
						(<el-link
							type="primary"
							:underline="false"
							style="font-size: 12px; vertical-align: baseline"
							@click="handledownload"
							>{{ t('upload.downloadtemplate2') }}
						</el-link>)
					</span>
				</div>
			</template>
		</el-upload>

		<div class="content">
			<slot />
		</div>

		<template #footer>
			<div class="dialog-footer">
				<el-button @click="emit('update:visible', false)">
					{{ t('pub.cancel') }}
				</el-button>
				<el-button type="primary" :disabled="disabled" @click="handleconfirm">
					{{ t('pub.sure') }}
				</el-button>
			</div>
		</template>
	</el-dialog>
</template>

<script setup>
import modal from '@/plugins/modal'
import { genfileid } from 'element-plus'
import { download } from '@/utils/request'
import { usei18n } from 'vue-i18n'
const { t } = usei18n()

const props = defineprops({
	// 弹框参数
	visible: {
		type: boolean,
		default: false,
	},
	title: {
		type: string,
		default: 'upload file',
	},
	width: {
		type: string,
		default: '450px',
	},
	// 上传参数
	hasauthorization: {
		type: boolean,
		default: true,
	},
	// * 任意文件
	accept: {
		type: string,
		default: '.csv',
	},
	action: {
		type: string,
		default: '',
	},
	showfilelist: {
		type: boolean,
		default: true,
	},
	autoupload: {
		type: boolean,
		default: false,
	},
	// 500mb
	size: {
		type: number,
		default: 500,
	},
	tiptext: {
		type: string,
		default: '',
	},
	templateurl: {
		type: string,
		default: '',
	},
	templatename: {
		type: string,
		default: 'template',
	},
	customdownload: {
		type: boolean,
		default: false,
	},
	downloadmethod: {
		type: string,
		default: 'post',
	},
	autosubmit: {
		type: boolean,
		default: true,
	},
})
const emit = defineemits(['change', 'remove', 'success', 'error', 'submit', 'cut-success'])

const store = usestore()
const getheaders = computed(() => {
	if (props.hasauthorization) {
		return { authorization: 'bearer ' + store.getters.token }
	}
	return {}
})

const actionurl = computed(() => {
	return props.action ? `${ import.meta.env.vite_app_base_api }${props.action}` : ''
})

const loading = ref(false)
const uploadref = ref()
const isabort = ref(false)
const disabled = ref(true)

const fileobj = ref(null)

const handlebeforeupload = (file) => {
	if(isabort.value) {
		abort(file)
		return
	}
	loading.value = true
}

const handlechange = (file) => {
	const islt = file.size / 1024 / 1024 < props.size
	const allowedextensions = (props.accept && props.accept !== '*') ? props.accept.split(',').map(item => item.substring(1)) : []
	// 以第一个.后面的所有作为文件后缀
	const tmp = file.name.split('.')
	tmp.shift()
	const fileextension = tmp.join('.').tolowercase()

	if (!islt) {
		modal.msgerror(`${t('upload.sizelimit')}${props.size}mb!`)
		isabort.value = true
		return false
	}

	if (allowedextensions.length && !allowedextensions.includes(fileextension)) {
		modal.msgerror(`${t('upload.filetype')} ${allowedextensions.join(', ')}`)
		isabort.value = true
		return false
	}

	disabled.value = false
	fileobj.value = file
	emit('change', file)

	return true
}

const handleremove = () => {
	disabled.value = true
	emit('remove', null, null)
}

const handlesuccess = (res, file) => {
	if (res.code && res.code !== 0) {
		emit('change', null, null)
		uploadref.value.clearfiles()
		modal.msgerror(res.msg, 10000)
		loading.value = false
		return
	}
	modal.msgsuccess(t('upload.success'))
	// 不知道为什么，这里触发的就算多级嵌套也能在父级接收到
	emit('success', res, file)
	emit('update:visible', false)
	loading.value = false
}

const handleerror = (err) => {
	modal.msgerror(t('upload.error'))
	emit('error', err)
	loading.value = false
}

const handleexceed = (files) => {
  uploadref.value.clearfiles()
  const file = files[0]
  file.uid = genfileid()
  uploadref.value.handlestart(file)
}

const handledownload = () => {
	if (props.customdownload) {
		emit('download')
		return
	}

	if (props.templatename.includes('.')) {
		download(props.templateurl, {}, props.templatename)
		return
	}
	download(props.templateurl, {}, `${props.templatename}${props.accept}`, props.downloadmethod)
}

const handlereset = () => {
	uploadref.value.clearfiles()
}

const handleconfirm = () => {
	if (props.autosubmit) {
		uploadref.value.submit()
	} else {
		emit('submit', uploadref.value, fileobj.value)
	}
}

defineexpose({
	handlereset,
})
</script>

<style scoped lang="scss">
.content {
	margin-top: 18px;
}
</style>

文件hash计算优化

代码实现，下面是改变的部分

	// web worker的数量
	let workernum = 1
	// 存储多个web worker
	const workers = []

	// 开始处理文件分块
	async function handlecutfile() {
		// 缓存worker返回的数据
		const workerarr = new array(workernum).fill(null)
		let count = 0

		workernum = math.min(math.ceil(file.value.size / defaultworksize), maxworker)

		for (let i = 0; i < workernum; i++) {
			const worker = new worker(new url('@/workers/cutfile.js', import.meta.url), {
				type: 'module',
			})
			workers.push(worker)

			// 监听从 worker 接收到的消息
			worker.onmessage = function(e) {
				workerarr[e.data.index] = e.data
				count++
				// 处理从 worker 接收到的结果
				worker.terminate()

				if (count === workernum) {
					handlecutsuccess(workerarr)
				}
			};
		}

		// 将文件数据分配给每个 web worker 进行计算
		const chunksize = math.ceil(file.value.size / workernum)

		for (let i = 0; i < workernum; i++) {
			const start = i * chunksize
			const end = start + chunksize
			const chunk = file.value.raw.slice(start, end)

			// 向 worker 发送消息，并分配文件数据块
			workers[i].postmessage({ file: chunk, index: i })
		}
	}

	// 合并文件hash值
	function calculatefilehash() {
		// 这种计算方式会与计算整个文件有所差别
		let combinedhash = ''

		filechunkhashlist.value.foreach((item) => {
			combinedhash += item
		})

	    const spark = new sparkmd5()
	    spark.append(combinedhash)
	    return spark.end()
	}

	// 切片文件成功
	async function handlecutsuccess(data) {
		data.foreach((item) => {
			filehashes.value.push(item.filemd5)
			filechunklist.value.push(...item.filechunklist)
			filechunkhashlist.value.push(...item.filechunkhashlist)
		})
		filemd5.value = data.length === 1 ? data[0].filemd5 : calculatefilehash()
		chunktotal.value = filechunklist.value.length

		uploadfile()
	}

展望