From 16eb3c7989452d2aa571a32eb09db754fd46bfef Mon Sep 17 00:00:00 2001 From: davi-thu Date: Thu, 24 Apr 2025 15:13:03 +0800 Subject: [PATCH 1/3] Upload video files and audio files on the browser. --- src/components/FileProcessor.vue | 153 ++++++ src/router/index.js | 5 + src/views/FileProcessing.vue | 245 ++++++++++ src/views/FileUpload/Index.vue | 632 +++++++++++++++++++++++++ src/views/FileUpload/index.less | 83 ++++ src/views/FileUpload/videoProcessor.js | 88 ++++ 6 files changed, 1206 insertions(+) create mode 100644 src/components/FileProcessor.vue create mode 100644 src/views/FileProcessing.vue create mode 100644 src/views/FileUpload/Index.vue create mode 100644 src/views/FileUpload/index.less create mode 100644 src/views/FileUpload/videoProcessor.js diff --git a/src/components/FileProcessor.vue b/src/components/FileProcessor.vue new file mode 100644 index 0000000..efe250e --- /dev/null +++ b/src/components/FileProcessor.vue @@ -0,0 +1,153 @@ + + + \ No newline at end of file diff --git a/src/router/index.js b/src/router/index.js index bc75041..0e1372b 100644 --- a/src/router/index.js +++ b/src/router/index.js @@ -1,5 +1,6 @@ import { createRouter, createWebHistory } from 'vue-router'; import AudioVideoCall from '../views/AudioVideoCall/Index.vue'; +import FileUpload from '@/views/FileUpload/Index.vue'; // 2025-04-21 // 定义路由 const routes = [ @@ -7,6 +8,10 @@ const routes = [ path: '/', name: 'Home', component: AudioVideoCall + }, + { + path: '/upload', + component: FileUpload } ]; diff --git a/src/views/FileProcessing.vue b/src/views/FileProcessing.vue new file mode 100644 index 0000000..bdd8aa2 --- /dev/null +++ b/src/views/FileProcessing.vue @@ -0,0 +1,245 @@ + + + + + \ No newline at end of file diff --git a/src/views/FileUpload/Index.vue b/src/views/FileUpload/Index.vue new file mode 100644 index 0000000..dbe77ad --- /dev/null +++ b/src/views/FileUpload/Index.vue @@ -0,0 +1,632 @@ + + + + + \ No newline at end of file diff --git a/src/views/FileUpload/index.less b/src/views/FileUpload/index.less new file mode 100644 index 0000000..2137a59 --- /dev/null +++ b/src/views/FileUpload/index.less @@ -0,0 +1,83 @@ +.file-upload { + // 定义变量，便于维护和复用 + @header-height: 59px; + @standard-padding: 24px; + @border-color: rgba(224, 224, 224, 0.6); + @primary-color: #409eff; + @border-radius: 8px; + + background-color: #fff; + height: 100%; + display: flex; + flex-direction: column; + + &__header { + height: @header-height; + border-bottom: 1px solid @border-color; + padding: 0 @standard-padding; + + h3 { + color: #131212; + font-size: 20px; + font-weight: 500; + line-height: @header-height; + margin: 0; + } + } + + &__content { + padding: @standard-padding; + flex: 1; + overflow-y: auto; + + .upload-section { + display: flex; + gap: @standard-padding; + margin-bottom: @standard-padding; + + // 使用CSS Grid作为备选布局方案 + @media (max-width: 768px) { + display: grid; + grid-template-columns: 1fr; + gap: 16px; + } + } + + .upload-box { + flex: 1; + padding: 16px; + border: 1px dashed #dcdfe6; + border-radius: @border-radius; + transition: border-color 0.3s ease; + + &:hover { + border-color: @primary-color; + } + + h4 { + margin: 0 0 12px; + font-weight: 500; + } + + .file-info { + margin-top: 8px; + color: @primary-color; + word-break: break-all; + font-size: 14px; + } + } + + .process-progress { + margin-top: 16px; + padding: 12px; + background-color: #f8f9fa; + border-radius: 4px; + + // 添加进度条动画 + &.processing { + animation: progress-pulse 2s infinite; + } + } + } + } + \ No newline at end of file diff --git a/src/views/FileUpload/videoProcessor.js b/src/views/FileUpload/videoProcessor.js new file mode 100644 index 0000000..ab55446 --- /dev/null +++ b/src/views/FileUpload/videoProcessor.js @@ -0,0 +1,88 @@ +// 处理视频文件,提取帧 +export async function processVideoFrames(videoFile, onProgress) { + return new Promise((resolve, reject) => { + const video = document.createElement('video') + const canvas = document.createElement('canvas') + const ctx = canvas.getContext('2d') + const frames = [] + + video.src = URL.createObjectURL(videoFile) + + video.onloadedmetadata = () => { + canvas.width = video.videoWidth + canvas.height = video.videoHeight + + // 设置提取帧的时间间隔(ms) + const frameInterval = 200 // 每200ms提取一帧 + let currentTime = 0 + const duration = video.duration + const totalFrames = Math.ceil(duration * 1000 / frameInterval) + let processedFrames = 0 + + function extractFrame() { + if (currentTime <= duration) { + video.currentTime = currentTime + + video.onseeked = () => { + // 在Canvas上绘制当前视频帧 + ctx.drawImage(video, 0, 0) + + // 将帧转换为base64 + canvas.toBlob((blob) => { + const reader = new FileReader() + reader.onloadend = () => { + const base64data = reader.result.split(',')[1] + frames.push(base64data) + + processedFrames++ + onProgress(Math.min(Math.floor(processedFrames / totalFrames * 100), 100)) + + currentTime += frameInterval / 1000 + if (processedFrames < totalFrames) { + extractFrame() + } else { + URL.revokeObjectURL(video.src) + resolve(frames) + } + } + reader.readAsDataURL(blob) + }, 'image/jpeg', 0.8) + } + } + } + + extractFrame() + } + + video.onerror = (error) => { + URL.revokeObjectURL(video.src) + reject(new Error('视频文件加载失败: ' + error.message)) + } + }) + } + + // 将文件转换为base64 + export function readFileAsBase64(file, onProgress) { + return new Promise((resolve, reject) => { + const reader = new FileReader() + + reader.onload = () => { + const base64 = reader.result.split(',')[1] + onProgress(100) + resolve(base64) + } + + reader.onprogress = (event) => { + if (event.lengthComputable) { + const progress = Math.floor((event.loaded / event.total) * 100) + onProgress(progress) + } + } + + reader.onerror = () => reject(new Error('文件读取失败')) + + reader.readAsDataURL(file) + }) + } + + \ No newline at end of file From e478f01b67985e04d8a8c90fba176b523636d412 Mon Sep 17 00:00:00 2001 From: davi-thu Date: Thu, 24 Apr 2025 18:25:03 +0800 Subject: [PATCH 2/3] Debugging Http API Upload. --- src/server/index.js | 20 + .../server - \345\211\257\346\234\254 (2).js" | 396 ++++++++++++++++ .../server - \345\211\257\346\234\254 (3).js" | 418 +++++++++++++++++ .../server - \345\211\257\346\234\254.js" | 198 ++++++++ ...06\344\270\200\351\203\250\345\210\206.js" | 384 ++++++++++++++++ ...06\344\270\200\351\203\250\345\210\206.js" | 383 ++++++++++++++++ ...56\344\270\200\347\202\271\344\272\206.js" | 407 +++++++++++++++++ src/server/server.js | 408 +++++++++++++++++ src/server/server.js.err | 424 ++++++++++++++++++ src/server/server.js.err2 | 404 +++++++++++++++++ src/views/FileUpload/videoProcessor.js | 224 +++++---- 11 files changed, 3583 insertions(+), 83 deletions(-) create mode 100644 src/server/index.js create mode 100644 "src/server/server - \345\211\257\346\234\254 (2).js" create mode 100644 "src/server/server - \345\211\257\346\234\254 (3).js" create mode 100644 "src/server/server - \345\211\257\346\234\254.js" create mode 100644 "src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" create mode 100644 "src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" create mode 100644 "src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" create mode 100644 src/server/server.js create mode 100644 src/server/server.js.err create mode 100644 src/server/server.js.err2 diff --git a/src/server/index.js b/src/server/index.js new file mode 100644 index 0000000..4372109 --- /dev/null +++ b/src/server/index.js @@ -0,0 +1,20 @@ +import 'dotenv/config'; +import express from 'express'; +import cors from 'cors'; +import uploadRouter from './server.js'; + +const app = express(); +const port = process.env.PORT || 3000; + +app.use(cors()); +app.use(express.json()); +app.use('/api', uploadRouter); + +app.listen(port, () => { + console.log(`Server is running on port ${port}`); + console.log('Environment:', { + VITE_APP_DOMAIN: process.env.VITE_APP_DOMAIN, + VITE_APP_PROXY_PATH: process.env.VITE_APP_PROXY_PATH, + API_KEY: process.env.API_KEY ? '已设置' : '未设置' + }); +}); \ No newline at end of file diff --git "a/src/server/server - \345\211\257\346\234\254 (2).js" "b/src/server/server - \345\211\257\346\234\254 (2).js" new file mode 100644 index 0000000..0c9b251 --- /dev/null +++ "b/src/server/server - \345\211\257\346\234\254 (2).js" @@ -0,0 +1,396 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 先测试导入 +try { + console.log('正在导入视频处理模块...'); + const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); + console.log('视频处理模块导入成功:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 + }); +} catch (error) { + console.error('导入视频处理模块失败:', error); +} + +// 导入视频处理相关函数 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + + + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + if (err) { + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 进行文件处理前的日志 + console.log('开始调用 AI 处理...'); + + try { + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + } catch (processError) { + console.error('AI 处理错误:', processError); + throw processError; + } + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理文件 + if (req.files) { + for (const field of ['video', 'audio']) { + if (req.files[field]) { + try { + await fs.unlink(req.files[field][0].path); + } catch (e) { + console.error(`清理${field}文件失败:`, e); + } + } + } + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + console.log('进入 processFilesAndCallAI 函数'); + try { + console.log('开始 AI 处理流程...'); + const env = checkEnvironmentVariables(); + + // 检查视频处理模块 + console.log('检查视频处理模块导入状态:', + typeof processVideoFrames === 'function' ? '成功' : '失败', + typeof readFileAsBase64 === 'function' ? '成功' : '失败' + ); + + // 创建文件对象 + console.log('创建文件对象...'); + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + console.log('文件对象创建完成'); + console.log('视频文件:', videoFile.name, videoFile.size, '字节'); + console.log('音频文件:', audioFile.name, audioFile.size, '字节'); + + // WebSocket URL + const wsUrl = `${env.domain}${env.proxyPath}/v4/realtime?Authorization=${env.apiKey}`; + console.log('准备连接 WebSocket:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + return new Promise((resolve, reject) => { + try { + console.log('创建 WebSocket 连接...'); + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + })); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 + console.log('开始发送视频帧...'); + for (const frame of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: Date.now(), + video_frame: frame + })); + } + console.log('视频帧发送完成'); + + // 4. 处理音频 + console.log('开始处理音频...'); + const audioBase64 = await readFileAsBase64(audioFile, (progress) => { + console.log(`音频处理进度: ${progress}%`); + }); + console.log('音频数据准备完成，长度:', audioBase64.length); + + // 5. 发送音频 + console.log('发送音频数据...'); + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64 + })); + console.log('音频数据发送完成'); + + // 6. 发送提交指令 + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 连接错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + + } catch (error) { + console.error('创建 WebSocket 连接时出错:', error); + reject(error); + } + }); + } catch (error) { + console.error('处理文件时出错:', error); + throw error; + } +} + + +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + // ... 前面的错误处理代码保持不变 ... + + try { + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('开始处理文件并调用 AI 接口...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // 清理临时文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { + messages + } + }); + + } catch (error) { + console.error('AI 处理过程中出错:', error); + // 清理临时文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'AI_PROCESS_ERROR' + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git "a/src/server/server - \345\211\257\346\234\254 (3).js" "b/src/server/server - \345\211\257\346\234\254 (3).js" new file mode 100644 index 0000000..3e7f2e9 --- /dev/null +++ "b/src/server/server - \345\211\257\346\234\254 (3).js" @@ -0,0 +1,418 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 先测试导入 +try { + console.log('正在导入视频处理模块...'); + const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); + console.log('视频处理模块导入成功:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 + }); +} catch (error) { + console.error('导入视频处理模块失败:', error); +} + +// 导入视频处理相关函数 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + + + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + if (err) { + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 进行文件处理前的日志 + console.log('开始调用 AI 处理...'); + + try { + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + } catch (processError) { + console.error('AI 处理错误:', processError); + throw processError; + } + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理文件 + if (req.files) { + for (const field of ['video', 'audio']) { + if (req.files[field]) { + try { + await fs.unlink(req.files[field][0].path); + } catch (e) { + console.error(`清理${field}文件失败:`, e); + } + } + } + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + let sessionUpdated = false; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + const sessionConfig = { + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + }; + ws.send(JSON.stringify(sessionConfig)); + + // 2. 等待会话配置确认 + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error('等待会话配置超时')); + }, 5000); + + const handler = (data) => { + const message = JSON.parse(data.toString()); + if (message.type === 'session.updated') { + clearTimeout(timeout); + ws.removeListener('message', handler); + resolve(); + } + }; + ws.on('message', handler); + }); + + // 3. 处理和发送视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + + console.log('开始发送视频帧...'); + for (const { timestamp, frame } of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: timestamp, + video_frame: frame + })); + } + console.log('视频帧发送完成'); + + // 4. 等待确保视频帧发送完成 + await new Promise(resolve => setTimeout(resolve, 500)); + + // 5. 发送音频开始标记 + console.log('发送音频开始标记...'); + ws.send(JSON.stringify({ + type: 'audio.start', + client_timestamp: Date.now() + })); + + // 6. 处理并发送音频数据 + console.log('开始处理音频...'); + const audioBase64 = await readFileAsBase64(audioFile); + console.log('音频数据准备完成, 长度:', audioBase64.length); + + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64, + role: 'user', + receive_voice: true + })); + console.log('音频数据发送完成'); + + // 7. 发送音频结束标记 + console.log('发送音频结束标记...'); + ws.send(JSON.stringify({ + type: 'audio.end', + client_timestamp: Date.now() + })); + + // 8. 等待后发送提交指令 + await new Promise(resolve => setTimeout(resolve, 500)); + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} + +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + // ... 前面的错误处理代码保持不变 ... + + try { + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('开始处理文件并调用 AI 接口...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // 清理临时文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { + messages + } + }); + + } catch (error) { + console.error('AI 处理过程中出错:', error); + // 清理临时文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'AI_PROCESS_ERROR' + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git "a/src/server/server - \345\211\257\346\234\254.js" "b/src/server/server - \345\211\257\346\234\254.js" new file mode 100644 index 0000000..f607f62 --- /dev/null +++ "b/src/server/server - \345\211\257\346\234\254.js" @@ -0,0 +1,198 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + if (err instanceof multer.MulterError) { + console.error('Multer 错误:', err); + return res.status(400).json({ + success: false, + message: '上传失败', + error: err.message, + code: 'MULTER_ERROR' + }); + } else if (err) { + console.error('其他错误:', err); + return res.status(400).json({ + success: false, + message: '上传失败', + error: err.message, + code: 'UPLOAD_ERROR' + }); + } + + try { + if (!req.files || !req.files.video || !req.files.audio) { + console.error('缺少必要的文件'); + return res.status(400).json({ + success: false, + message: '缺少必要的文件', + receivedFiles: req.files, + code: 'MISSING_FILES' + }); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('成功接收文件:'); + console.log('视频文件:', { + path: videoFile.path, + size: videoFile.size, + mimetype: videoFile.mimetype + }); + console.log('音频文件:', { + path: audioFile.path, + size: audioFile.size, + mimetype: audioFile.mimetype + }); + + // 检查文件大小 + const videoStats = await fs.stat(videoFile.path); + const audioStats = await fs.stat(audioFile.path); + + console.log('文件系统大小检查:'); + console.log('视频文件大小:', videoStats.size); + console.log('音频文件大小:', audioStats.size); + + // 验证文件大小是否合理 + if (videoStats.size < 1024) { // 小于 1KB + throw new Error('视频文件大小异常'); + } + if (audioStats.size < 1024) { // 小于 1KB + throw new Error('音频文件大小异常'); + } + + // 读取文件头部来验证文件类型 + const videoBuffer = await fs.readFile(videoFile.path, { length: 4096 }); + const audioBuffer = await fs.readFile(audioFile.path, { length: 4096 }); + + // 简单的文件类型验证 + if (!isValidVideoFile(videoBuffer)) { + throw new Error('无效的视频文件格式'); + } + if (!isValidAudioFile(audioBuffer)) { + throw new Error('无效的音频文件格式'); + } + + res.json({ + success: true, + message: '文件上传成功', + data: { + video: { + filename: videoFile.filename, + size: videoStats.size, + mimetype: videoFile.mimetype + }, + audio: { + filename: audioFile.filename, + size: audioStats.size, + mimetype: audioFile.mimetype + } + } + }); + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理可能已上传的文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'PROCESS_ERROR' + }); + } + }); +}); + +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +export default router; \ No newline at end of file diff --git "a/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" "b/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" new file mode 100644 index 0000000..df0a2a8 --- /dev/null +++ "b/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" @@ -0,0 +1,384 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 先测试导入 +try { + console.log('正在导入视频处理模块...'); + const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); + console.log('视频处理模块导入成功:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 + }); +} catch (error) { + console.error('导入视频处理模块失败:', error); +} + +// 导入视频处理相关函数 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + + + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + if (err) { + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 进行文件处理前的日志 + console.log('开始调用 AI 处理...'); + + try { + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + } catch (processError) { + console.error('AI 处理错误:', processError); + throw processError; + } + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理文件 + if (req.files) { + for (const field of ['video', 'audio']) { + if (req.files[field]) { + try { + await fs.unlink(req.files[field][0].path); + } catch (e) { + console.error(`清理${field}文件失败:`, e); + } + } + } + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + const sessionConfig = { + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + }; + ws.send(JSON.stringify(sessionConfig)); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 + console.log('开始发送视频帧...'); + for (const frame of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: Date.now(), // 恢复使用当前时间戳 + video_frame: frame.frame // 使用 frame 对象中的 frame 属性 + })); + } + console.log('视频帧发送完成'); + + // 添加等待 + await new Promise(resolve => setTimeout(resolve, 1000)); + + // 4. 处理并发送音频数据 + console.log('开始处理音频...'); + const audioBase64 = await readFileAsBase64(audioFile); + console.log('音频数据准备完成'); + + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64 + })); + console.log('音频数据发送完成'); + + // 5. 发送提交指令 + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + // ... 前面的错误处理代码保持不变 ... + + try { + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('开始处理文件并调用 AI 接口...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // 清理临时文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { + messages + } + }); + + } catch (error) { + console.error('AI 处理过程中出错:', error); + // 清理临时文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'AI_PROCESS_ERROR' + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git "a/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" "b/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" new file mode 100644 index 0000000..3da960c --- /dev/null +++ "b/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" @@ -0,0 +1,383 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 先测试导入 +try { + console.log('正在导入视频处理模块...'); + const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); + console.log('视频处理模块导入成功:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 + }); +} catch (error) { + console.error('导入视频处理模块失败:', error); +} + +// 导入视频处理相关函数 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + + + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + if (err) { + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 进行文件处理前的日志 + console.log('开始调用 AI 处理...'); + + try { + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + } catch (processError) { + console.error('AI 处理错误:', processError); + throw processError; + } + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理文件 + if (req.files) { + for (const field of ['video', 'audio']) { + if (req.files[field]) { + try { + await fs.unlink(req.files[field][0].path); + } catch (e) { + console.error(`清理${field}文件失败:`, e); + } + } + } + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + const sessionConfig = { + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + }; + ws.send(JSON.stringify(sessionConfig)); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 + console.log('开始发送视频帧...'); + for (const frame of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: Date.now(), + video_frame: frame + })); + } + console.log('视频帧发送完成'); + + // 4. 处理并发送音频数据 + console.log('开始处理音频...'); + const audioBase64 = await readFileAsBase64(audioFile, (progress) => { + console.log(`音频处理进度: ${progress}%`); + }); + console.log('音频数据准备完成'); + + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64 + })); + console.log('音频数据发送完成'); + + // 5. 发送提交指令 + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + // ... 前面的错误处理代码保持不变 ... + + try { + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('开始处理文件并调用 AI 接口...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // 清理临时文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { + messages + } + }); + + } catch (error) { + console.error('AI 处理过程中出错:', error); + // 清理临时文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'AI_PROCESS_ERROR' + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git "a/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" "b/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" new file mode 100644 index 0000000..e662b60 --- /dev/null +++ "b/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" @@ -0,0 +1,407 @@ +import path from 'path'; +import multer from 'multer'; +import express from 'express'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 只保留一种导入方式，推荐使用 ES 模块的静态导入 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +// 在代码开始处验证函数是否正确导入 +console.log('正在检查视频处理模块...'); +console.log('视频处理模块检查结果:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 +}); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +//const uploadDir = join(__dirname, '../../uploads'); +//await fs.mkdir(uploadDir, { recursive: true }); + +// 使用异步方式配置 storage +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + const uploadDir = path.join(process.cwd(), 'uploads'); + try { + await fs.access(uploadDir); + } catch (error) { + if (error.code === 'ENOENT') { + await fs.mkdir(uploadDir, { recursive: true }); + } + } + cb(null, uploadDir); + }, + filename: function (req, file, cb) { + const timestamp = Date.now(); + const randomString = Math.random().toString(36).substring(7); + cb(null, `${file.fieldname}-${timestamp}-${randomString}${path.extname(file.originalname)}`); + } +}); + +const fileFilter = (req, file, cb) => { + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +router.post('/upload', (req, res) => { + console.log('接收到上传请求'); + + upload(req, res, async function (err) { + console.log('multer 处理完成'); + + try { + if (err) { + console.error('文件上传错误:', err); + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + console.error('文件缺失:', req.files); + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('接收到的文件信息:'); + console.log('视频文件:', { + path: videoFile.path, + size: videoFile.size, + mimetype: videoFile.mimetype + }); + console.log('音频文件:', { + path: audioFile.path, + size: audioFile.size, + mimetype: audioFile.mimetype + }); + + // 检查文件是否真实存在 + console.log('检查文件是否存在:'); + //console.log('视频文件存在:', fs.existsSync(videoFile.path)); + //console.log('音频文件存在:', fs.existsSync(audioFile.path)); + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // ... 其余代码保持不变 + } catch (error) { + console.error('处理过程中出错:', error); + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + + +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // 检查 MP4 文件头 + const header = buffer.slice(4, 8); + return header.toString() === 'ftyp'; +} + +function isValidAudioFile(buffer) { + // 检查 WAV 文件头 + const header = buffer.slice(0, 4); + return header.toString() === 'RIFF'; +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + })); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 - + console.log('开始发送视频帧...'); + for (const frameData of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: 0, // 使用固定时间戳 + video_frame: frameData // 直接发送 base64 字符串 + })); + } + console.log('视频帧发送完成'); + + // 等待一下确保视频帧处理完成 + await new Promise(resolve => setTimeout(resolve, 1000)); + + // 4. 处理并发送音频数据 + // 在 server.js 中修改音频文件处理 + // 4. 处理并发送音频数据 + console.log('开始处理音频...'); + console.log('音频文件路径:', audioPath); + + try { + await fs.access(audioPath); + const audioBase64 = await readFileAsBase64(audioFile); + console.log('音频数据准备完成'); + + // 发送音频开始标记 + ws.send(JSON.stringify({ + type: 'audio.start', + client_timestamp: Date.now() + })); + + // 发送音频数据 + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64, + role: 'user', + receive_voice: true + })); + console.log('音频数据发送完成'); + + // 发送音频结束标记 + ws.send(JSON.stringify({ + type: 'audio.end', + client_timestamp: Date.now() + })); + + } catch (error) { + console.error('音频文件处理失败:', error); + throw error; + } + + // 5. 发送提交指令 + await new Promise(resolve => setTimeout(resolve, 500)); + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + // 处理 multer 错误 + if (err instanceof multer.MulterError) { + throw new Error(`文件上传错误: ${err.code}`); + } else if (err) { + throw new Error(`文件上传错误: ${err.message}`); + } + + // 验证文件是否存在 + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 记录文件信息 + console.log('接收到的文件:'); + console.log('视频文件:', videoFile.path); + console.log('音频文件:', audioPath); + + // 确认文件存在 + if (!fs.existsSync(videoFile.path) || !fs.existsSync(audioFile.path)) { + throw new Error('文件保存失败'); + } + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path).catch(e => console.error('清理视频文件失败:', e)), + fs.unlink(audioFile.path).catch(e => console.error('清理音频文件失败:', e)) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + + } catch (error) { + console.error('处理过程中出错:', error); + + // 清理文件（如果存在） + if (req.files) { + await Promise.all(Object.values(req.files).flat().map(file => + fs.unlink(file.path).catch(e => + console.error(`清理文件 ${file.path} 失败:`, e) + ) + )); + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git a/src/server/server.js b/src/server/server.js new file mode 100644 index 0000000..fb6d942 --- /dev/null +++ b/src/server/server.js @@ -0,0 +1,408 @@ +//视频发送出错，需要继续分析 +import path from 'path'; +import multer from 'multer'; +import express from 'express'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 只保留一种导入方式，推荐使用 ES 模块的静态导入 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +// 在代码开始处验证函数是否正确导入 +console.log('正在检查视频处理模块...'); +console.log('视频处理模块检查结果:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 +}); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +//const uploadDir = join(__dirname, '../../uploads'); +//await fs.mkdir(uploadDir, { recursive: true }); + +// 使用异步方式配置 storage +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + const uploadDir = path.join(process.cwd(), 'uploads'); + try { + await fs.access(uploadDir); + } catch (error) { + if (error.code === 'ENOENT') { + await fs.mkdir(uploadDir, { recursive: true }); + } + } + cb(null, uploadDir); + }, + filename: function (req, file, cb) { + const timestamp = Date.now(); + const randomString = Math.random().toString(36).substring(7); + cb(null, `${file.fieldname}-${timestamp}-${randomString}${path.extname(file.originalname)}`); + } +}); + +const fileFilter = (req, file, cb) => { + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +router.post('/upload', (req, res) => { + console.log('接收到上传请求'); + + upload(req, res, async function (err) { + console.log('multer 处理完成'); + + try { + if (err) { + console.error('文件上传错误:', err); + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + console.error('文件缺失:', req.files); + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('接收到的文件信息:'); + console.log('视频文件:', { + path: videoFile.path, + size: videoFile.size, + mimetype: videoFile.mimetype + }); + console.log('音频文件:', { + path: audioFile.path, + size: audioFile.size, + mimetype: audioFile.mimetype + }); + + // 检查文件是否真实存在 + console.log('检查文件是否存在:'); + //console.log('视频文件存在:', fs.existsSync(videoFile.path)); + //console.log('音频文件存在:', fs.existsSync(audioFile.path)); + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // ... 其余代码保持不变 + } catch (error) { + console.error('处理过程中出错:', error); + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + + +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // 检查 MP4 文件头 + const header = buffer.slice(4, 8); + return header.toString() === 'ftyp'; +} + +function isValidAudioFile(buffer) { + // 检查 WAV 文件头 + const header = buffer.slice(0, 4); + return header.toString() === 'RIFF'; +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + })); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 - + console.log('开始发送视频帧...'); + for (const frameData of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: 0, // 使用固定时间戳 + video_frame: frameData // 直接发送 base64 字符串 + })); + } + console.log('视频帧发送完成'); + + // 等待一下确保视频帧处理完成 + await new Promise(resolve => setTimeout(resolve, 1000)); + + // 4. 处理并发送音频数据 + // 在 server.js 中修改音频文件处理 + // 4. 处理并发送音频数据 + console.log('开始处理音频...'); + console.log('音频文件路径:', audioPath); + + try { + await fs.access(audioPath); + const audioBase64 = await readFileAsBase64(audioFile); + console.log('音频数据准备完成'); + + // 发送音频开始标记 + ws.send(JSON.stringify({ + type: 'audio.start', + client_timestamp: Date.now() + })); + + // 发送音频数据 + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64, + role: 'user', + receive_voice: true + })); + console.log('音频数据发送完成'); + + // 发送音频结束标记 + ws.send(JSON.stringify({ + type: 'audio.end', + client_timestamp: Date.now() + })); + + } catch (error) { + console.error('音频文件处理失败:', error); + throw error; + } + + // 5. 发送提交指令 + await new Promise(resolve => setTimeout(resolve, 500)); + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + // 处理 multer 错误 + if (err instanceof multer.MulterError) { + throw new Error(`文件上传错误: ${err.code}`); + } else if (err) { + throw new Error(`文件上传错误: ${err.message}`); + } + + // 验证文件是否存在 + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 记录文件信息 + console.log('接收到的文件:'); + console.log('视频文件:', videoFile.path); + console.log('音频文件:', audioPath); + + // 确认文件存在 + if (!fs.existsSync(videoFile.path) || !fs.existsSync(audioFile.path)) { + throw new Error('文件保存失败'); + } + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path).catch(e => console.error('清理视频文件失败:', e)), + fs.unlink(audioFile.path).catch(e => console.error('清理音频文件失败:', e)) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + + } catch (error) { + console.error('处理过程中出错:', error); + + // 清理文件（如果存在） + if (req.files) { + await Promise.all(Object.values(req.files).flat().map(file => + fs.unlink(file.path).catch(e => + console.error(`清理文件 ${file.path} 失败:`, e) + ) + )); + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git a/src/server/server.js.err b/src/server/server.js.err new file mode 100644 index 0000000..3c80208 --- /dev/null +++ b/src/server/server.js.err @@ -0,0 +1,424 @@ +import express from 'express'; +import multer from 'multer'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 先测试导入 +try { + console.log('正在导入视频处理模块...'); + const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); + console.log('视频处理模块导入成功:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 + }); +} catch (error) { + console.error('导入视频处理模块失败:', error); +} + +// 导入视频处理相关函数 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +const uploadDir = join(__dirname, '../../uploads'); +await fs.mkdir(uploadDir, { recursive: true }); + + + +// 配置 multer +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + try { + await fs.access(uploadDir); + cb(null, uploadDir); + } catch (error) { + console.error('上传目录不存在或无访问权限:', error); + cb(new Error('上传目录配置错误')); + } + }, + filename: function (req, file, cb) { + const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); + const ext = file.originalname.split('.').pop(); + cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); + } +}); + +// 文件过滤器 +const fileFilter = (req, file, cb) => { + console.log('正在处理文件:', file.originalname); + console.log('文件类型:', file.mimetype); + + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +// 在处理上传请求的路由中添加文件大小验证 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + if (err) { + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 进行文件处理前的日志 + console.log('开始调用 AI 处理...'); + + try { + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + } catch (processError) { + console.error('AI 处理错误:', processError); + throw processError; + } + + } catch (error) { + console.error('处理过程中出错:', error); + // 清理文件 + if (req.files) { + for (const field of ['video', 'audio']) { + if (req.files[field]) { + try { + await fs.unlink(req.files[field][0].path); + } catch (e) { + console.error(`清理${field}文件失败:`, e); + } + } + } + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // MP4 文件通常以 'ftyp' 标记开始 + return buffer.includes('ftyp'); +} + +function isValidAudioFile(buffer) { + // WAV 文件通常以 'RIFF' 标记开始 + return buffer.includes('RIFF'); +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + path: videoPath, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + path: audioPath, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.VITE_APP_API_KEY; + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + +return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let sessionUpdated = false; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 等待会话配置确认 + await new Promise((resolveSession, rejectSession) => { + const timeout = setTimeout(() => { + rejectSession(new Error('等待会话配置超时')); + }, 10000); + + // 发送会话配置 + console.log('发送会话配置...'); + ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + })); + + // 监听会话更新确认 + const sessionHandler = (data) => { + try { + const message = JSON.parse(data); + if (message.type === 'session.updated') { + clearTimeout(timeout); + ws.removeListener('message', sessionHandler); + sessionUpdated = true; + resolveSession(); + } + } catch (error) { + console.error('处理会话消息时出错:', error); + } + }; + ws.on('message', sessionHandler); + }); + + // 确保会话已更新 + if (!sessionUpdated) { + throw new Error('会话配置未确认'); + } + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const frameBuffer = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + + // 3. 发送视频帧并等待确认 + console.log('开始发送视频帧...'); + const videoFrame = { + type: 'video.append', + client_timestamp: 0, + video_frame: frameBuffer.toString('base64') + }; + + await new Promise((resolveVideo, rejectVideo) => { + const timeout = setTimeout(() => { + rejectVideo(new Error('视频帧发送确认超时')); + }, 10000); + + const videoHandler = (data) => { + try { + const message = JSON.parse(data); + if (message.type === 'video.appended') { + clearTimeout(timeout); + ws.removeListener('message', videoHandler); + resolveVideo(); + } + // 处理错误消息 + if (message.type === 'error') { + clearTimeout(timeout); + ws.removeListener('message', videoHandler); + rejectVideo(new Error(message.error?.message || '视频帧处理失败')); + } + } catch (error) { + console.error('处理视频确认消息时出错:', error); + } + }; + + ws.on('message', videoHandler); + ws.send(JSON.stringify(videoFrame)); + }); + + console.log('视频帧已确认接收'); + + // 4. 处理音频数据 + console.log('开始处理音频...'); + const audioBuffer = await fs.readFile(audioPath); + const audioBase64 = audioBuffer.toString('base64'); + + // 5. 发送音频数据 + ws.send(JSON.stringify({ + type: 'audio.start', + client_timestamp: Date.now() + })); + + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64, + role: 'user', + receive_voice: true + })); + + ws.send(JSON.stringify({ + type: 'audio.end', + client_timestamp: Date.now() + })); + + // 6. 发送提交指令 + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + // 全局消息处理 + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + if (message.type === 'error') { + console.error('收到错误消息:', message); + reject(new Error(message.error?.message || '处理失败')); + } + + if (message.type === 'response.audio_txt' || message.type === 'response.text') { + console.log('收到响应内容:', message.delta); + // 处理响应内容... + } + } catch (err) { + console.error('处理消息时出错:', err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (!sessionUpdated) { + reject(new Error('会话未完成配置就关闭了')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} + +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + // ... 前面的错误处理代码保持不变 ... + + try { + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('开始处理文件并调用 AI 接口...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // 清理临时文件 + await Promise.all([ + fs.unlink(videoFile.path), + fs.unlink(audioFile.path) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { + messages + } + }); + + } catch (error) { + console.error('AI 处理过程中出错:', error); + // 清理临时文件 + if (req.files) { + if (req.files.video) { + try { + await fs.unlink(req.files.video[0].path); + } catch (e) { + console.error('清理视频文件失败:', e); + } + } + if (req.files.audio) { + try { + await fs.unlink(req.files.audio[0].path); + } catch (e) { + console.error('清理音频文件失败:', e); + } + } + } + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message, + code: 'AI_PROCESS_ERROR' + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git a/src/server/server.js.err2 b/src/server/server.js.err2 new file mode 100644 index 0000000..f7c5eeb --- /dev/null +++ b/src/server/server.js.err2 @@ -0,0 +1,404 @@ +import path from 'path'; +import multer from 'multer'; +import express from 'express'; +import { WebSocket } from 'ws'; +import { promises as fs } from 'fs'; +import { fileURLToPath } from 'url'; +import { dirname, join } from 'path'; + +// 只保留一种导入方式，推荐使用 ES 模块的静态导入 +import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; + +// 在代码开始处验证函数是否正确导入 +console.log('正在检查视频处理模块...'); +console.log('视频处理模块检查结果:', { + processVideoFrames: typeof processVideoFrames, + readFileAsBase64: typeof readFileAsBase64 +}); + +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); + + +const router = express.Router(); + +// 确保上传目录存在 +//const uploadDir = join(__dirname, '../../uploads'); +//await fs.mkdir(uploadDir, { recursive: true }); + +// 使用异步方式配置 storage +const storage = multer.diskStorage({ + destination: async function (req, file, cb) { + const uploadDir = path.join(process.cwd(), 'uploads'); + try { + await fs.access(uploadDir); + } catch (error) { + if (error.code === 'ENOENT') { + await fs.mkdir(uploadDir, { recursive: true }); + } + } + cb(null, uploadDir); + }, + filename: function (req, file, cb) { + const timestamp = Date.now(); + const randomString = Math.random().toString(36).substring(7); + cb(null, `${file.fieldname}-${timestamp}-${randomString}${path.extname(file.originalname)}`); + } +}); + +const fileFilter = (req, file, cb) => { + if (file.fieldname === 'video') { + if (!file.mimetype.startsWith('video/')) { + return cb(new Error('只接受视频文件')); + } + } else if (file.fieldname === 'audio') { + if (!file.mimetype.startsWith('audio/')) { + return cb(new Error('只接受音频文件')); + } + } + cb(null, true); +}; + +const upload = multer({ + storage: storage, + fileFilter: fileFilter, + limits: { + fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB + } +}).fields([ + { name: 'video', maxCount: 1 }, + { name: 'audio', maxCount: 1 } +]); + +// 处理文件上传的路由 +router.post('/upload', (req, res) => { + console.log('接收到上传请求'); + + upload(req, res, async function (err) { + console.log('multer 处理完成'); + + try { + if (err) { + console.error('文件上传错误:', err); + throw new Error('文件上传错误: ' + err.message); + } + + if (!req.files || !req.files.video || !req.files.audio) { + console.error('文件缺失:', req.files); + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + console.log('接收到的文件信息:'); + console.log('视频文件:', { + path: videoFile.path, + size: videoFile.size, + mimetype: videoFile.mimetype + }); + console.log('音频文件:', { + path: audioFile.path, + size: audioFile.size, + mimetype: audioFile.mimetype + }); + + // 检查文件是否真实存在 + console.log('检查文件是否存在:'); + //console.log('视频文件存在:', fs.existsSync(videoFile.path)); + //console.log('音频文件存在:', fs.existsSync(audioFile.path)); + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + + // ... 其余代码保持不变 + } catch (error) { + console.error('处理过程中出错:', error); + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + + +// 简单的文件类型验证函数 +function isValidVideoFile(buffer) { + // 检查 MP4 文件头 + const header = buffer.slice(4, 8); + return header.toString() === 'ftyp'; +} + +function isValidAudioFile(buffer) { + // 检查 WAV 文件头 + const header = buffer.slice(0, 4); + return header.toString() === 'RIFF'; +} + +// 添加环境变量检查 +const checkEnvironmentVariables = () => { + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; + const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; + + console.log('环境变量检查:'); + console.log('VITE_APP_DOMAIN:', domain); + console.log('VITE_APP_PROXY_PATH:', proxyPath); + console.log('API_KEY:', apiKey ? '已设置' : '未设置'); + + return { domain, proxyPath, apiKey }; +}; + +async function processFilesAndCallAI(videoPath, audioPath) { + try { + console.log('开始 AI 处理流程...'); + + // 修正环境变量的使用 + const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; + const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 + const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 + + // 检查环境变量 + console.log('环境变量检查:'); + console.log('DOMAIN:', domain); + console.log('PROXY_PATH:', proxyPath); + console.log('API_KEY 是否存在:', !!apiKey); + + const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; + console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); + + // 创建文件对象 + const videoFile = { + name: videoPath.split('/').pop(), + type: 'video/mp4', + size: (await fs.stat(videoPath)).size, + async arrayBuffer() { + return await fs.readFile(videoPath); + } + }; + + const audioFile = { + name: audioPath.split('/').pop(), + type: 'audio/wav', + size: (await fs.stat(audioPath)).size, + async arrayBuffer() { + return await fs.readFile(audioPath); + } + }; + + return new Promise((resolve, reject) => { + const ws = new WebSocket(wsUrl); + let messages = []; + + ws.on('open', async () => { + try { + console.log('WebSocket 连接已建立'); + + // 1. 发送会话配置 + console.log('发送会话配置...'); + ws.send(JSON.stringify({ + type: 'session.update', + session: { + turn_detection: { + type: 'client_vad', + }, + beta_fields: { + chat_mode: 'video_passive', + }, + output_audio_format: "mp3", + input_audio_format: "wav", + } + })); + + // 2. 处理视频帧 + console.log('开始处理视频帧...'); + const videoFrames = await processVideoFrames(videoFile, (progress) => { + console.log(`视频处理进度: ${progress}%`); + }); + console.log(`提取到 ${videoFrames.length} 个视频帧`); + + // 3. 发送视频帧 - 只修改这部分 + console.log('开始发送视频帧...'); + for (const frameData of videoFrames) { + ws.send(JSON.stringify({ + type: 'video.append', + client_timestamp: Date.now(), + video_frame: { + data: frameData, // base64 字符串 + format: 'jpg', // 指定格式 + width: 640, // 添加宽度 + height: 360 // 添加高度 + } + })); + } + console.log('视频帧发送完成'); + + // 4. 处理并发送音频数据 + console.log('开始处理音频...'); + try { + await fs.access(audioPath); + const audioBase64 = await readFileAsBase64(audioFile); + console.log('音频数据准备完成'); + + // 先发送音频开始标记 + ws.send(JSON.stringify({ + type: 'audio.start', + client_timestamp: Date.now() + })); + + // 音频数据 + ws.send(JSON.stringify({ + type: 'audio.append', + client_timestamp: Date.now(), + audio: audioBase64, + role: 'user', + receive_voice: true + })); + + // 音频结束标记 + ws.send(JSON.stringify({ + type: 'audio.end', + client_timestamp: Date.now() + })); + + } catch (error) { + console.error('音频文件处理失败:', error); + throw error; + } + + // 5. 发送提交指令 + await new Promise(resolve => setTimeout(resolve, 500)); + console.log('发送提交指令...'); + ws.send(JSON.stringify({ + type: 'commit', + client_timestamp: Date.now() + })); + console.log('提交指令发送完成'); + + } catch (error) { + console.error('处理过程中出错:', error); + ws.close(); + reject(error); + } + }); + + ws.on('message', (data) => { + try { + const message = JSON.parse(data); + console.log('收到 WebSocket 消息:', message.type); + + switch (message.type) { + case 'response.audio_txt': + case 'response.text': + console.log('收到响应内容:', message.delta); + messages.push(message.delta); + break; + + case 'response.audio_done': + console.log('AI 处理完成'); + ws.close(); + break; + + case 'error': + console.error('收到错误消息:', message.error); + ws.close(); + reject(new Error(message.error?.message || '处理失败')); + break; + } + } catch (err) { + console.error('处理消息时出错:', err); + ws.close(); + reject(err); + } + }); + + ws.on('error', (error) => { + console.error('WebSocket 错误:', error); + reject(error); + }); + + ws.on('close', () => { + console.log('WebSocket 连接已关闭'); + if (messages.length > 0) { + resolve(messages); + } else { + reject(new Error('处理完成但没有收到任何消息')); + } + }); + }); + } catch (error) { + console.error('AI 处理过程中出错:', error); + throw error; + } +} +// 在上传路由中调用 AI 处理 +router.post('/upload', (req, res) => { + upload(req, res, async function (err) { + console.log('开始处理上传请求'); + + try { + // 处理 multer 错误 + if (err instanceof multer.MulterError) { + throw new Error(`文件上传错误: ${err.code}`); + } else if (err) { + throw new Error(`文件上传错误: ${err.message}`); + } + + // 验证文件是否存在 + if (!req.files || !req.files.video || !req.files.audio) { + throw new Error('缺少必要的文件'); + } + + const videoFile = req.files.video[0]; + const audioFile = req.files.audio[0]; + + // 记录文件信息 + console.log('接收到的文件:'); + console.log('视频文件:', videoFile.path); + console.log('音频文件:', audioPath); + + // 确认文件存在 + if (!fs.existsSync(videoFile.path) || !fs.existsSync(audioFile.path)) { + throw new Error('文件保存失败'); + } + + console.log('开始调用 AI 处理...'); + const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); + console.log('AI 处理返回的消息:', messages); + + // 清理文件 + await Promise.all([ + fs.unlink(videoFile.path).catch(e => console.error('清理视频文件失败:', e)), + fs.unlink(audioFile.path).catch(e => console.error('清理音频文件失败:', e)) + ]); + + res.json({ + success: true, + message: '处理成功', + data: { messages } + }); + + } catch (error) { + console.error('处理过程中出错:', error); + + // 清理文件（如果存在） + if (req.files) { + await Promise.all(Object.values(req.files).flat().map(file => + fs.unlink(file.path).catch(e => + console.error(`清理文件 ${file.path} 失败:`, e) + ) + )); + } + + res.status(500).json({ + success: false, + message: '处理失败', + error: error.message + }); + } + }); +}); + +export default router; \ No newline at end of file diff --git a/src/views/FileUpload/videoProcessor.js b/src/views/FileUpload/videoProcessor.js index ab55446..3c6c39c 100644 --- a/src/views/FileUpload/videoProcessor.js +++ b/src/views/FileUpload/videoProcessor.js @@ -1,88 +1,146 @@ -// 处理视频文件,提取帧 -export async function processVideoFrames(videoFile, onProgress) { - return new Promise((resolve, reject) => { - const video = document.createElement('video') - const canvas = document.createElement('canvas') - const ctx = canvas.getContext('2d') - const frames = [] - - video.src = URL.createObjectURL(videoFile) - - video.onloadedmetadata = () => { - canvas.width = video.videoWidth - canvas.height = video.videoHeight - - // 设置提取帧的时间间隔(ms) - const frameInterval = 200 // 每200ms提取一帧 - let currentTime = 0 - const duration = video.duration - const totalFrames = Math.ceil(duration * 1000 / frameInterval) - let processedFrames = 0 - - function extractFrame() { - if (currentTime <= duration) { - video.currentTime = currentTime - - video.onseeked = () => { - // 在Canvas上绘制当前视频帧 - ctx.drawImage(video, 0, 0) - - // 将帧转换为base64 - canvas.toBlob((blob) => { - const reader = new FileReader() - reader.onloadend = () => { - const base64data = reader.result.split(',')[1] - frames.push(base64data) - - processedFrames++ - onProgress(Math.min(Math.floor(processedFrames / totalFrames * 100), 100)) - - currentTime += frameInterval / 1000 - if (processedFrames < totalFrames) { - extractFrame() - } else { - URL.revokeObjectURL(video.src) - resolve(frames) - } - } - reader.readAsDataURL(blob) - }, 'image/jpeg', 0.8) - } - } - } - - extractFrame() - } - - video.onerror = (error) => { - URL.revokeObjectURL(video.src) - reject(new Error('视频文件加载失败: ' + error.message)) +//视频发送出错，需要继续分析 +// src/views/FileUpload/videoProcessor.js +// 判断运行环境 +const isNode = typeof window === 'undefined' || typeof document === 'undefined'; + +import ffmpeg from 'fluent-ffmpeg'; +import { promises as fs } from 'fs'; +import { file as tmpFile, dir as tmpDir } from 'tmp-promise'; +import { join } from 'path'; +import { createCanvas, loadImage } from 'canvas'; + +// 添加递归删除目录的辅助函数 +async function removeDir(dir) { + try { + const items = await fs.readdir(dir); + for (const item of items) { + const path = join(dir, item); + const stat = await fs.stat(path); + if (stat.isDirectory()) { + await removeDir(path); + } else { + await fs.unlink(path); } - }) + } + await fs.rmdir(dir); + } catch (error) { + console.error('删除目录失败:', error); + throw error; } - - // 将文件转换为base64 - export function readFileAsBase64(file, onProgress) { - return new Promise((resolve, reject) => { - const reader = new FileReader() - - reader.onload = () => { - const base64 = reader.result.split(',')[1] - onProgress(100) - resolve(base64) - } - - reader.onprogress = (event) => { - if (event.lengthComputable) { - const progress = Math.floor((event.loaded / event.total) * 100) - onProgress(progress) - } +} + +export const processVideoFrames = async (file, onProgress) => { + if (isNode) { + let videoPath = null; + let framesDir = null; + + try { + // 创建临时文件和目录 + const tmpVideoFile = await tmpFile({ postfix: '.mp4' }); + const tmpFramesDir = await tmpDir(); + videoPath = tmpVideoFile.path; + framesDir = tmpFramesDir.path; + + // 写入视频文件 + const videoBuffer = Buffer.from(await file.arrayBuffer()); + await fs.writeFile(videoPath, videoBuffer); + + // 提取视频帧 + return new Promise((resolve, reject) => { + ffmpeg(videoPath) + .screenshots({ + count: 1, + folder: framesDir, + filename: 'frame-%i.jpg', + size: '640x?' + }) + .on('end', async () => { + try { + // 读取生成的帧 + const frames = await fs.readdir(framesDir); + // 在处理视频帧时，直接返回 base64 字符串 + const frameResults = []; + for (let i = 0; i < frames.length; i++) { + const framePath = join(framesDir, frames[i]); + const frameBuffer = await fs.readFile(framePath); + frameResults.push(frameBuffer.toString('base64')); // 直接返回 base64 字符串 + } + + // 注释掉清理临时文件的代码 + /* + try { + await fs.unlink(videoPath); + console.log('临时视频文件已删除'); + await removeDir(framesDir); + console.log('临时帧目录已删除'); + } catch (cleanupError) { + console.error('清理临时文件失败:', cleanupError); + } + */ + + resolve(frameResults); + } catch (error) { + console.error('处理视频帧时出错:', error); + reject(error); + } + }) + .on('error', async (err) => { + console.error('FFmpeg 错误:', err); + // 注释掉错误处理中的清理代码 + /* + try { + if (videoPath) await fs.unlink(videoPath).catch(() => {}); + if (framesDir) await removeDir(framesDir).catch(() => {}); + } catch (cleanupError) { + console.error('清理临时文件失败:', cleanupError); + } + */ + reject(err); + }); + }); + } catch (error) { + // 注释掉最外层错误处理中的清理代码 + /* + try { + if (videoPath) await fs.unlink(videoPath).catch(() => {}); + if (framesDir) await removeDir(framesDir).catch(() => {}); + } catch (cleanupError) { + console.error('清理临时文件失败:', cleanupError); } - - reader.onerror = () => reject(new Error('文件读取失败')) - - reader.readAsDataURL(file) - }) + */ + console.error('视频处理初始化错误:', error); + throw error; + } + } else { + // 浏览器环境代码保持不变... } +}; + + +// 将文件转换为base64 +// 音频处理函数保持不变 +export const readFileAsBase64 = async (file, onProgress) => { + try { + const arrayBuffer = await file.arrayBuffer(); + if (isNode) { + // Node.js 环境 + return Buffer.from(arrayBuffer).toString('base64'); + } else { + // 浏览器环境 + return new Promise((resolve) => { + const reader = new FileReader(); + reader.onload = () => { + const base64 = reader.result.split(',')[1]; + if (onProgress) onProgress(100); + resolve(base64); + }; + reader.readAsDataURL(new Blob([arrayBuffer])); + }); + } + } catch (error) { + console.error('处理音频文件时出错:', error); + throw error; + } +}; + - \ No newline at end of file From e96e81f3cbc0618a483ce9cf90fe2a97836f7eaf Mon Sep 17 00:00:00 2001 From: davi-thu Date: Thu, 24 Apr 2025 18:27:42 +0800 Subject: [PATCH 3/3] Debugging Http API Upload 2. --- .../server - \345\211\257\346\234\254 (2).js" | 396 ---------------- .../server - \345\211\257\346\234\254 (3).js" | 418 ----------------- .../server - \345\211\257\346\234\254.js" | 198 -------- ...06\344\270\200\351\203\250\345\210\206.js" | 384 ---------------- ...06\344\270\200\351\203\250\345\210\206.js" | 383 ---------------- ...56\344\270\200\347\202\271\344\272\206.js" | 407 ----------------- src/server/server.js.err | 424 ------------------ src/server/server.js.err2 | 404 ----------------- 8 files changed, 3014 deletions(-) delete mode 100644 "src/server/server - \345\211\257\346\234\254 (2).js" delete mode 100644 "src/server/server - \345\211\257\346\234\254 (3).js" delete mode 100644 "src/server/server - \345\211\257\346\234\254.js" delete mode 100644 "src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" delete mode 100644 "src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" delete mode 100644 "src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" delete mode 100644 src/server/server.js.err delete mode 100644 src/server/server.js.err2 diff --git "a/src/server/server - \345\211\257\346\234\254 (2).js" "b/src/server/server - \345\211\257\346\234\254 (2).js" deleted file mode 100644 index 0c9b251..0000000 --- "a/src/server/server - \345\211\257\346\234\254 (2).js" +++ /dev/null @@ -1,396 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 先测试导入 -try { - console.log('正在导入视频处理模块...'); - const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); - console.log('视频处理模块导入成功:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 - }); -} catch (error) { - console.error('导入视频处理模块失败:', error); -} - -// 导入视频处理相关函数 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - - - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - if (err) { - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 进行文件处理前的日志 - console.log('开始调用 AI 处理...'); - - try { - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - } catch (processError) { - console.error('AI 处理错误:', processError); - throw processError; - } - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理文件 - if (req.files) { - for (const field of ['video', 'audio']) { - if (req.files[field]) { - try { - await fs.unlink(req.files[field][0].path); - } catch (e) { - console.error(`清理${field}文件失败:`, e); - } - } - } - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - console.log('进入 processFilesAndCallAI 函数'); - try { - console.log('开始 AI 处理流程...'); - const env = checkEnvironmentVariables(); - - // 检查视频处理模块 - console.log('检查视频处理模块导入状态:', - typeof processVideoFrames === 'function' ? '成功' : '失败', - typeof readFileAsBase64 === 'function' ? '成功' : '失败' - ); - - // 创建文件对象 - console.log('创建文件对象...'); - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - console.log('文件对象创建完成'); - console.log('视频文件:', videoFile.name, videoFile.size, '字节'); - console.log('音频文件:', audioFile.name, audioFile.size, '字节'); - - // WebSocket URL - const wsUrl = `${env.domain}${env.proxyPath}/v4/realtime?Authorization=${env.apiKey}`; - console.log('准备连接 WebSocket:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - return new Promise((resolve, reject) => { - try { - console.log('创建 WebSocket 连接...'); - const ws = new WebSocket(wsUrl); - let messages = []; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - ws.send(JSON.stringify({ - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - })); - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - console.log(`提取到 ${videoFrames.length} 个视频帧`); - - // 3. 发送视频帧 - console.log('开始发送视频帧...'); - for (const frame of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: Date.now(), - video_frame: frame - })); - } - console.log('视频帧发送完成'); - - // 4. 处理音频 - console.log('开始处理音频...'); - const audioBase64 = await readFileAsBase64(audioFile, (progress) => { - console.log(`音频处理进度: ${progress}%`); - }); - console.log('音频数据准备完成，长度:', audioBase64.length); - - // 5. 发送音频 - console.log('发送音频数据...'); - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64 - })); - console.log('音频数据发送完成'); - - // 6. 发送提交指令 - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 连接错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - - } catch (error) { - console.error('创建 WebSocket 连接时出错:', error); - reject(error); - } - }); - } catch (error) { - console.error('处理文件时出错:', error); - throw error; - } -} - - -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - // ... 前面的错误处理代码保持不变 ... - - try { - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('开始处理文件并调用 AI 接口...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // 清理临时文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { - messages - } - }); - - } catch (error) { - console.error('AI 处理过程中出错:', error); - // 清理临时文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'AI_PROCESS_ERROR' - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git "a/src/server/server - \345\211\257\346\234\254 (3).js" "b/src/server/server - \345\211\257\346\234\254 (3).js" deleted file mode 100644 index 3e7f2e9..0000000 --- "a/src/server/server - \345\211\257\346\234\254 (3).js" +++ /dev/null @@ -1,418 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 先测试导入 -try { - console.log('正在导入视频处理模块...'); - const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); - console.log('视频处理模块导入成功:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 - }); -} catch (error) { - console.error('导入视频处理模块失败:', error); -} - -// 导入视频处理相关函数 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - - - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - if (err) { - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 进行文件处理前的日志 - console.log('开始调用 AI 处理...'); - - try { - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - } catch (processError) { - console.error('AI 处理错误:', processError); - throw processError; - } - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理文件 - if (req.files) { - for (const field of ['video', 'audio']) { - if (req.files[field]) { - try { - await fs.unlink(req.files[field][0].path); - } catch (e) { - console.error(`清理${field}文件失败:`, e); - } - } - } - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - console.log('开始 AI 处理流程...'); - - // 修正环境变量的使用 - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 - const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 - - // 检查环境变量 - console.log('环境变量检查:'); - console.log('DOMAIN:', domain); - console.log('PROXY_PATH:', proxyPath); - console.log('API_KEY 是否存在:', !!apiKey); - - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - // 创建文件对象 - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let messages = []; - let sessionUpdated = false; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - const sessionConfig = { - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - }; - ws.send(JSON.stringify(sessionConfig)); - - // 2. 等待会话配置确认 - await new Promise((resolve, reject) => { - const timeout = setTimeout(() => { - reject(new Error('等待会话配置超时')); - }, 5000); - - const handler = (data) => { - const message = JSON.parse(data.toString()); - if (message.type === 'session.updated') { - clearTimeout(timeout); - ws.removeListener('message', handler); - resolve(); - } - }; - ws.on('message', handler); - }); - - // 3. 处理和发送视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - - console.log('开始发送视频帧...'); - for (const { timestamp, frame } of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: timestamp, - video_frame: frame - })); - } - console.log('视频帧发送完成'); - - // 4. 等待确保视频帧发送完成 - await new Promise(resolve => setTimeout(resolve, 500)); - - // 5. 发送音频开始标记 - console.log('发送音频开始标记...'); - ws.send(JSON.stringify({ - type: 'audio.start', - client_timestamp: Date.now() - })); - - // 6. 处理并发送音频数据 - console.log('开始处理音频...'); - const audioBase64 = await readFileAsBase64(audioFile); - console.log('音频数据准备完成, 长度:', audioBase64.length); - - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64, - role: 'user', - receive_voice: true - })); - console.log('音频数据发送完成'); - - // 7. 发送音频结束标记 - console.log('发送音频结束标记...'); - ws.send(JSON.stringify({ - type: 'audio.end', - client_timestamp: Date.now() - })); - - // 8. 等待后发送提交指令 - await new Promise(resolve => setTimeout(resolve, 500)); - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} - -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - // ... 前面的错误处理代码保持不变 ... - - try { - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('开始处理文件并调用 AI 接口...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // 清理临时文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { - messages - } - }); - - } catch (error) { - console.error('AI 处理过程中出错:', error); - // 清理临时文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'AI_PROCESS_ERROR' - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git "a/src/server/server - \345\211\257\346\234\254.js" "b/src/server/server - \345\211\257\346\234\254.js" deleted file mode 100644 index f607f62..0000000 --- "a/src/server/server - \345\211\257\346\234\254.js" +++ /dev/null @@ -1,198 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - if (err instanceof multer.MulterError) { - console.error('Multer 错误:', err); - return res.status(400).json({ - success: false, - message: '上传失败', - error: err.message, - code: 'MULTER_ERROR' - }); - } else if (err) { - console.error('其他错误:', err); - return res.status(400).json({ - success: false, - message: '上传失败', - error: err.message, - code: 'UPLOAD_ERROR' - }); - } - - try { - if (!req.files || !req.files.video || !req.files.audio) { - console.error('缺少必要的文件'); - return res.status(400).json({ - success: false, - message: '缺少必要的文件', - receivedFiles: req.files, - code: 'MISSING_FILES' - }); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('成功接收文件:'); - console.log('视频文件:', { - path: videoFile.path, - size: videoFile.size, - mimetype: videoFile.mimetype - }); - console.log('音频文件:', { - path: audioFile.path, - size: audioFile.size, - mimetype: audioFile.mimetype - }); - - // 检查文件大小 - const videoStats = await fs.stat(videoFile.path); - const audioStats = await fs.stat(audioFile.path); - - console.log('文件系统大小检查:'); - console.log('视频文件大小:', videoStats.size); - console.log('音频文件大小:', audioStats.size); - - // 验证文件大小是否合理 - if (videoStats.size < 1024) { // 小于 1KB - throw new Error('视频文件大小异常'); - } - if (audioStats.size < 1024) { // 小于 1KB - throw new Error('音频文件大小异常'); - } - - // 读取文件头部来验证文件类型 - const videoBuffer = await fs.readFile(videoFile.path, { length: 4096 }); - const audioBuffer = await fs.readFile(audioFile.path, { length: 4096 }); - - // 简单的文件类型验证 - if (!isValidVideoFile(videoBuffer)) { - throw new Error('无效的视频文件格式'); - } - if (!isValidAudioFile(audioBuffer)) { - throw new Error('无效的音频文件格式'); - } - - res.json({ - success: true, - message: '文件上传成功', - data: { - video: { - filename: videoFile.filename, - size: videoStats.size, - mimetype: videoFile.mimetype - }, - audio: { - filename: audioFile.filename, - size: audioStats.size, - mimetype: audioFile.mimetype - } - } - }); - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理可能已上传的文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'PROCESS_ERROR' - }); - } - }); -}); - -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -export default router; \ No newline at end of file diff --git "a/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" "b/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" deleted file mode 100644 index df0a2a8..0000000 --- "a/src/server/server - \345\217\210\345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" +++ /dev/null @@ -1,384 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 先测试导入 -try { - console.log('正在导入视频处理模块...'); - const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); - console.log('视频处理模块导入成功:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 - }); -} catch (error) { - console.error('导入视频处理模块失败:', error); -} - -// 导入视频处理相关函数 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - - - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - if (err) { - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 进行文件处理前的日志 - console.log('开始调用 AI 处理...'); - - try { - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - } catch (processError) { - console.error('AI 处理错误:', processError); - throw processError; - } - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理文件 - if (req.files) { - for (const field of ['video', 'audio']) { - if (req.files[field]) { - try { - await fs.unlink(req.files[field][0].path); - } catch (e) { - console.error(`清理${field}文件失败:`, e); - } - } - } - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - console.log('开始 AI 处理流程...'); - - // 修正环境变量的使用 - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 - const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 - - // 检查环境变量 - console.log('环境变量检查:'); - console.log('DOMAIN:', domain); - console.log('PROXY_PATH:', proxyPath); - console.log('API_KEY 是否存在:', !!apiKey); - - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - // 创建文件对象 - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let messages = []; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - const sessionConfig = { - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - }; - ws.send(JSON.stringify(sessionConfig)); - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - console.log(`提取到 ${videoFrames.length} 个视频帧`); - - // 3. 发送视频帧 - console.log('开始发送视频帧...'); - for (const frame of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: Date.now(), // 恢复使用当前时间戳 - video_frame: frame.frame // 使用 frame 对象中的 frame 属性 - })); - } - console.log('视频帧发送完成'); - - // 添加等待 - await new Promise(resolve => setTimeout(resolve, 1000)); - - // 4. 处理并发送音频数据 - console.log('开始处理音频...'); - const audioBase64 = await readFileAsBase64(audioFile); - console.log('音频数据准备完成'); - - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64 - })); - console.log('音频数据发送完成'); - - // 5. 发送提交指令 - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - // ... 前面的错误处理代码保持不变 ... - - try { - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('开始处理文件并调用 AI 接口...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // 清理临时文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { - messages - } - }); - - } catch (error) { - console.error('AI 处理过程中出错:', error); - // 清理临时文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'AI_PROCESS_ERROR' - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git "a/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" "b/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" deleted file mode 100644 index 3da960c..0000000 --- "a/src/server/server - \345\257\271\344\272\206\344\270\200\351\203\250\345\210\206.js" +++ /dev/null @@ -1,383 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 先测试导入 -try { - console.log('正在导入视频处理模块...'); - const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); - console.log('视频处理模块导入成功:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 - }); -} catch (error) { - console.error('导入视频处理模块失败:', error); -} - -// 导入视频处理相关函数 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - - - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - if (err) { - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 进行文件处理前的日志 - console.log('开始调用 AI 处理...'); - - try { - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - } catch (processError) { - console.error('AI 处理错误:', processError); - throw processError; - } - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理文件 - if (req.files) { - for (const field of ['video', 'audio']) { - if (req.files[field]) { - try { - await fs.unlink(req.files[field][0].path); - } catch (e) { - console.error(`清理${field}文件失败:`, e); - } - } - } - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - console.log('开始 AI 处理流程...'); - - // 修正环境变量的使用 - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 - const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 - - // 检查环境变量 - console.log('环境变量检查:'); - console.log('DOMAIN:', domain); - console.log('PROXY_PATH:', proxyPath); - console.log('API_KEY 是否存在:', !!apiKey); - - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - // 创建文件对象 - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let messages = []; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - const sessionConfig = { - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - }; - ws.send(JSON.stringify(sessionConfig)); - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - console.log(`提取到 ${videoFrames.length} 个视频帧`); - - // 3. 发送视频帧 - console.log('开始发送视频帧...'); - for (const frame of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: Date.now(), - video_frame: frame - })); - } - console.log('视频帧发送完成'); - - // 4. 处理并发送音频数据 - console.log('开始处理音频...'); - const audioBase64 = await readFileAsBase64(audioFile, (progress) => { - console.log(`音频处理进度: ${progress}%`); - }); - console.log('音频数据准备完成'); - - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64 - })); - console.log('音频数据发送完成'); - - // 5. 发送提交指令 - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - // ... 前面的错误处理代码保持不变 ... - - try { - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('开始处理文件并调用 AI 接口...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // 清理临时文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { - messages - } - }); - - } catch (error) { - console.error('AI 处理过程中出错:', error); - // 清理临时文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'AI_PROCESS_ERROR' - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git "a/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" "b/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" deleted file mode 100644 index e662b60..0000000 --- "a/src/server/server - \345\260\261\345\267\256\344\270\200\347\202\271\344\272\206.js" +++ /dev/null @@ -1,407 +0,0 @@ -import path from 'path'; -import multer from 'multer'; -import express from 'express'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 只保留一种导入方式，推荐使用 ES 模块的静态导入 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -// 在代码开始处验证函数是否正确导入 -console.log('正在检查视频处理模块...'); -console.log('视频处理模块检查结果:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 -}); - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -//const uploadDir = join(__dirname, '../../uploads'); -//await fs.mkdir(uploadDir, { recursive: true }); - -// 使用异步方式配置 storage -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - const uploadDir = path.join(process.cwd(), 'uploads'); - try { - await fs.access(uploadDir); - } catch (error) { - if (error.code === 'ENOENT') { - await fs.mkdir(uploadDir, { recursive: true }); - } - } - cb(null, uploadDir); - }, - filename: function (req, file, cb) { - const timestamp = Date.now(); - const randomString = Math.random().toString(36).substring(7); - cb(null, `${file.fieldname}-${timestamp}-${randomString}${path.extname(file.originalname)}`); - } -}); - -const fileFilter = (req, file, cb) => { - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -router.post('/upload', (req, res) => { - console.log('接收到上传请求'); - - upload(req, res, async function (err) { - console.log('multer 处理完成'); - - try { - if (err) { - console.error('文件上传错误:', err); - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - console.error('文件缺失:', req.files); - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('接收到的文件信息:'); - console.log('视频文件:', { - path: videoFile.path, - size: videoFile.size, - mimetype: videoFile.mimetype - }); - console.log('音频文件:', { - path: audioFile.path, - size: audioFile.size, - mimetype: audioFile.mimetype - }); - - // 检查文件是否真实存在 - console.log('检查文件是否存在:'); - //console.log('视频文件存在:', fs.existsSync(videoFile.path)); - //console.log('音频文件存在:', fs.existsSync(audioFile.path)); - - console.log('开始调用 AI 处理...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // ... 其余代码保持不变 - } catch (error) { - console.error('处理过程中出错:', error); - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); - - -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // 检查 MP4 文件头 - const header = buffer.slice(4, 8); - return header.toString() === 'ftyp'; -} - -function isValidAudioFile(buffer) { - // 检查 WAV 文件头 - const header = buffer.slice(0, 4); - return header.toString() === 'RIFF'; -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - console.log('开始 AI 处理流程...'); - - // 修正环境变量的使用 - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 - const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 - - // 检查环境变量 - console.log('环境变量检查:'); - console.log('DOMAIN:', domain); - console.log('PROXY_PATH:', proxyPath); - console.log('API_KEY 是否存在:', !!apiKey); - - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - // 创建文件对象 - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let messages = []; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - ws.send(JSON.stringify({ - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - })); - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - console.log(`提取到 ${videoFrames.length} 个视频帧`); - - // 3. 发送视频帧 - - console.log('开始发送视频帧...'); - for (const frameData of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: 0, // 使用固定时间戳 - video_frame: frameData // 直接发送 base64 字符串 - })); - } - console.log('视频帧发送完成'); - - // 等待一下确保视频帧处理完成 - await new Promise(resolve => setTimeout(resolve, 1000)); - - // 4. 处理并发送音频数据 - // 在 server.js 中修改音频文件处理 - // 4. 处理并发送音频数据 - console.log('开始处理音频...'); - console.log('音频文件路径:', audioPath); - - try { - await fs.access(audioPath); - const audioBase64 = await readFileAsBase64(audioFile); - console.log('音频数据准备完成'); - - // 发送音频开始标记 - ws.send(JSON.stringify({ - type: 'audio.start', - client_timestamp: Date.now() - })); - - // 发送音频数据 - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64, - role: 'user', - receive_voice: true - })); - console.log('音频数据发送完成'); - - // 发送音频结束标记 - ws.send(JSON.stringify({ - type: 'audio.end', - client_timestamp: Date.now() - })); - - } catch (error) { - console.error('音频文件处理失败:', error); - throw error; - } - - // 5. 发送提交指令 - await new Promise(resolve => setTimeout(resolve, 500)); - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - // 处理 multer 错误 - if (err instanceof multer.MulterError) { - throw new Error(`文件上传错误: ${err.code}`); - } else if (err) { - throw new Error(`文件上传错误: ${err.message}`); - } - - // 验证文件是否存在 - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 记录文件信息 - console.log('接收到的文件:'); - console.log('视频文件:', videoFile.path); - console.log('音频文件:', audioPath); - - // 确认文件存在 - if (!fs.existsSync(videoFile.path) || !fs.existsSync(audioFile.path)) { - throw new Error('文件保存失败'); - } - - console.log('开始调用 AI 处理...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path).catch(e => console.error('清理视频文件失败:', e)), - fs.unlink(audioFile.path).catch(e => console.error('清理音频文件失败:', e)) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - - } catch (error) { - console.error('处理过程中出错:', error); - - // 清理文件（如果存在） - if (req.files) { - await Promise.all(Object.values(req.files).flat().map(file => - fs.unlink(file.path).catch(e => - console.error(`清理文件 ${file.path} 失败:`, e) - ) - )); - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git a/src/server/server.js.err b/src/server/server.js.err deleted file mode 100644 index 3c80208..0000000 --- a/src/server/server.js.err +++ /dev/null @@ -1,424 +0,0 @@ -import express from 'express'; -import multer from 'multer'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 先测试导入 -try { - console.log('正在导入视频处理模块...'); - const { processVideoFrames, readFileAsBase64 } = await import('../views/FileUpload/videoProcessor.js'); - console.log('视频处理模块导入成功:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 - }); -} catch (error) { - console.error('导入视频处理模块失败:', error); -} - -// 导入视频处理相关函数 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -const uploadDir = join(__dirname, '../../uploads'); -await fs.mkdir(uploadDir, { recursive: true }); - - - -// 配置 multer -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - try { - await fs.access(uploadDir); - cb(null, uploadDir); - } catch (error) { - console.error('上传目录不存在或无访问权限:', error); - cb(new Error('上传目录配置错误')); - } - }, - filename: function (req, file, cb) { - const uniqueSuffix = Date.now() + '-' + Math.round(Math.random() * 1E9); - const ext = file.originalname.split('.').pop(); - cb(null, `${file.fieldname}-${uniqueSuffix}.${ext}`); - } -}); - -// 文件过滤器 -const fileFilter = (req, file, cb) => { - console.log('正在处理文件:', file.originalname); - console.log('文件类型:', file.mimetype); - - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -// 在处理上传请求的路由中添加文件大小验证 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - if (err) { - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 进行文件处理前的日志 - console.log('开始调用 AI 处理...'); - - try { - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - } catch (processError) { - console.error('AI 处理错误:', processError); - throw processError; - } - - } catch (error) { - console.error('处理过程中出错:', error); - // 清理文件 - if (req.files) { - for (const field of ['video', 'audio']) { - if (req.files[field]) { - try { - await fs.unlink(req.files[field][0].path); - } catch (e) { - console.error(`清理${field}文件失败:`, e); - } - } - } - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // MP4 文件通常以 'ftyp' 标记开始 - return buffer.includes('ftyp'); -} - -function isValidAudioFile(buffer) { - // WAV 文件通常以 'RIFF' 标记开始 - return buffer.includes('RIFF'); -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - path: videoPath, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - path: audioPath, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.VITE_APP_API_KEY; - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - -return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let sessionUpdated = false; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 等待会话配置确认 - await new Promise((resolveSession, rejectSession) => { - const timeout = setTimeout(() => { - rejectSession(new Error('等待会话配置超时')); - }, 10000); - - // 发送会话配置 - console.log('发送会话配置...'); - ws.send(JSON.stringify({ - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - })); - - // 监听会话更新确认 - const sessionHandler = (data) => { - try { - const message = JSON.parse(data); - if (message.type === 'session.updated') { - clearTimeout(timeout); - ws.removeListener('message', sessionHandler); - sessionUpdated = true; - resolveSession(); - } - } catch (error) { - console.error('处理会话消息时出错:', error); - } - }; - ws.on('message', sessionHandler); - }); - - // 确保会话已更新 - if (!sessionUpdated) { - throw new Error('会话配置未确认'); - } - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const frameBuffer = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - - // 3. 发送视频帧并等待确认 - console.log('开始发送视频帧...'); - const videoFrame = { - type: 'video.append', - client_timestamp: 0, - video_frame: frameBuffer.toString('base64') - }; - - await new Promise((resolveVideo, rejectVideo) => { - const timeout = setTimeout(() => { - rejectVideo(new Error('视频帧发送确认超时')); - }, 10000); - - const videoHandler = (data) => { - try { - const message = JSON.parse(data); - if (message.type === 'video.appended') { - clearTimeout(timeout); - ws.removeListener('message', videoHandler); - resolveVideo(); - } - // 处理错误消息 - if (message.type === 'error') { - clearTimeout(timeout); - ws.removeListener('message', videoHandler); - rejectVideo(new Error(message.error?.message || '视频帧处理失败')); - } - } catch (error) { - console.error('处理视频确认消息时出错:', error); - } - }; - - ws.on('message', videoHandler); - ws.send(JSON.stringify(videoFrame)); - }); - - console.log('视频帧已确认接收'); - - // 4. 处理音频数据 - console.log('开始处理音频...'); - const audioBuffer = await fs.readFile(audioPath); - const audioBase64 = audioBuffer.toString('base64'); - - // 5. 发送音频数据 - ws.send(JSON.stringify({ - type: 'audio.start', - client_timestamp: Date.now() - })); - - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64, - role: 'user', - receive_voice: true - })); - - ws.send(JSON.stringify({ - type: 'audio.end', - client_timestamp: Date.now() - })); - - // 6. 发送提交指令 - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - // 全局消息处理 - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - if (message.type === 'error') { - console.error('收到错误消息:', message); - reject(new Error(message.error?.message || '处理失败')); - } - - if (message.type === 'response.audio_txt' || message.type === 'response.text') { - console.log('收到响应内容:', message.delta); - // 处理响应内容... - } - } catch (err) { - console.error('处理消息时出错:', err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (!sessionUpdated) { - reject(new Error('会话未完成配置就关闭了')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} - -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - // ... 前面的错误处理代码保持不变 ... - - try { - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('开始处理文件并调用 AI 接口...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // 清理临时文件 - await Promise.all([ - fs.unlink(videoFile.path), - fs.unlink(audioFile.path) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { - messages - } - }); - - } catch (error) { - console.error('AI 处理过程中出错:', error); - // 清理临时文件 - if (req.files) { - if (req.files.video) { - try { - await fs.unlink(req.files.video[0].path); - } catch (e) { - console.error('清理视频文件失败:', e); - } - } - if (req.files.audio) { - try { - await fs.unlink(req.files.audio[0].path); - } catch (e) { - console.error('清理音频文件失败:', e); - } - } - } - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message, - code: 'AI_PROCESS_ERROR' - }); - } - }); -}); - -export default router; \ No newline at end of file diff --git a/src/server/server.js.err2 b/src/server/server.js.err2 deleted file mode 100644 index f7c5eeb..0000000 --- a/src/server/server.js.err2 +++ /dev/null @@ -1,404 +0,0 @@ -import path from 'path'; -import multer from 'multer'; -import express from 'express'; -import { WebSocket } from 'ws'; -import { promises as fs } from 'fs'; -import { fileURLToPath } from 'url'; -import { dirname, join } from 'path'; - -// 只保留一种导入方式，推荐使用 ES 模块的静态导入 -import { processVideoFrames, readFileAsBase64 } from '../views/FileUpload/videoProcessor.js'; - -// 在代码开始处验证函数是否正确导入 -console.log('正在检查视频处理模块...'); -console.log('视频处理模块检查结果:', { - processVideoFrames: typeof processVideoFrames, - readFileAsBase64: typeof readFileAsBase64 -}); - -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); - - -const router = express.Router(); - -// 确保上传目录存在 -//const uploadDir = join(__dirname, '../../uploads'); -//await fs.mkdir(uploadDir, { recursive: true }); - -// 使用异步方式配置 storage -const storage = multer.diskStorage({ - destination: async function (req, file, cb) { - const uploadDir = path.join(process.cwd(), 'uploads'); - try { - await fs.access(uploadDir); - } catch (error) { - if (error.code === 'ENOENT') { - await fs.mkdir(uploadDir, { recursive: true }); - } - } - cb(null, uploadDir); - }, - filename: function (req, file, cb) { - const timestamp = Date.now(); - const randomString = Math.random().toString(36).substring(7); - cb(null, `${file.fieldname}-${timestamp}-${randomString}${path.extname(file.originalname)}`); - } -}); - -const fileFilter = (req, file, cb) => { - if (file.fieldname === 'video') { - if (!file.mimetype.startsWith('video/')) { - return cb(new Error('只接受视频文件')); - } - } else if (file.fieldname === 'audio') { - if (!file.mimetype.startsWith('audio/')) { - return cb(new Error('只接受音频文件')); - } - } - cb(null, true); -}; - -const upload = multer({ - storage: storage, - fileFilter: fileFilter, - limits: { - fileSize: 50 * 1024 * 1024, // 限制文件大小为 50MB - } -}).fields([ - { name: 'video', maxCount: 1 }, - { name: 'audio', maxCount: 1 } -]); - -// 处理文件上传的路由 -router.post('/upload', (req, res) => { - console.log('接收到上传请求'); - - upload(req, res, async function (err) { - console.log('multer 处理完成'); - - try { - if (err) { - console.error('文件上传错误:', err); - throw new Error('文件上传错误: ' + err.message); - } - - if (!req.files || !req.files.video || !req.files.audio) { - console.error('文件缺失:', req.files); - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - console.log('接收到的文件信息:'); - console.log('视频文件:', { - path: videoFile.path, - size: videoFile.size, - mimetype: videoFile.mimetype - }); - console.log('音频文件:', { - path: audioFile.path, - size: audioFile.size, - mimetype: audioFile.mimetype - }); - - // 检查文件是否真实存在 - console.log('检查文件是否存在:'); - //console.log('视频文件存在:', fs.existsSync(videoFile.path)); - //console.log('音频文件存在:', fs.existsSync(audioFile.path)); - - console.log('开始调用 AI 处理...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - - // ... 其余代码保持不变 - } catch (error) { - console.error('处理过程中出错:', error); - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); - - -// 简单的文件类型验证函数 -function isValidVideoFile(buffer) { - // 检查 MP4 文件头 - const header = buffer.slice(4, 8); - return header.toString() === 'ftyp'; -} - -function isValidAudioFile(buffer) { - // 检查 WAV 文件头 - const header = buffer.slice(0, 4); - return header.toString() === 'RIFF'; -} - -// 添加环境变量检查 -const checkEnvironmentVariables = () => { - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; - const apiKey = process.env.API_KEY || "d20c08612ef746beb7038a326131d475.cv71sxB0l7w8yOAO"; - - console.log('环境变量检查:'); - console.log('VITE_APP_DOMAIN:', domain); - console.log('VITE_APP_PROXY_PATH:', proxyPath); - console.log('API_KEY:', apiKey ? '已设置' : '未设置'); - - return { domain, proxyPath, apiKey }; -}; - -async function processFilesAndCallAI(videoPath, audioPath) { - try { - console.log('开始 AI 处理流程...'); - - // 修正环境变量的使用 - const domain = process.env.VITE_APP_DOMAIN || 'https://api.zhipu.ai'; - const proxyPath = process.env.VITE_APP_PROXY_PATH || '/eastai'; // 修正为正确的默认值 - const apiKey = process.env.VITE_APP_API_KEY; // 使用相同的环境变量名 - - // 检查环境变量 - console.log('环境变量检查:'); - console.log('DOMAIN:', domain); - console.log('PROXY_PATH:', proxyPath); - console.log('API_KEY 是否存在:', !!apiKey); - - const wsUrl = `${domain}${proxyPath}/v4/realtime?Authorization=${apiKey}`; - console.log('WebSocket URL:', wsUrl.replace(/Authorization=.*$/, 'Authorization=***')); - - // 创建文件对象 - const videoFile = { - name: videoPath.split('/').pop(), - type: 'video/mp4', - size: (await fs.stat(videoPath)).size, - async arrayBuffer() { - return await fs.readFile(videoPath); - } - }; - - const audioFile = { - name: audioPath.split('/').pop(), - type: 'audio/wav', - size: (await fs.stat(audioPath)).size, - async arrayBuffer() { - return await fs.readFile(audioPath); - } - }; - - return new Promise((resolve, reject) => { - const ws = new WebSocket(wsUrl); - let messages = []; - - ws.on('open', async () => { - try { - console.log('WebSocket 连接已建立'); - - // 1. 发送会话配置 - console.log('发送会话配置...'); - ws.send(JSON.stringify({ - type: 'session.update', - session: { - turn_detection: { - type: 'client_vad', - }, - beta_fields: { - chat_mode: 'video_passive', - }, - output_audio_format: "mp3", - input_audio_format: "wav", - } - })); - - // 2. 处理视频帧 - console.log('开始处理视频帧...'); - const videoFrames = await processVideoFrames(videoFile, (progress) => { - console.log(`视频处理进度: ${progress}%`); - }); - console.log(`提取到 ${videoFrames.length} 个视频帧`); - - // 3. 发送视频帧 - 只修改这部分 - console.log('开始发送视频帧...'); - for (const frameData of videoFrames) { - ws.send(JSON.stringify({ - type: 'video.append', - client_timestamp: Date.now(), - video_frame: { - data: frameData, // base64 字符串 - format: 'jpg', // 指定格式 - width: 640, // 添加宽度 - height: 360 // 添加高度 - } - })); - } - console.log('视频帧发送完成'); - - // 4. 处理并发送音频数据 - console.log('开始处理音频...'); - try { - await fs.access(audioPath); - const audioBase64 = await readFileAsBase64(audioFile); - console.log('音频数据准备完成'); - - // 先发送音频开始标记 - ws.send(JSON.stringify({ - type: 'audio.start', - client_timestamp: Date.now() - })); - - // 音频数据 - ws.send(JSON.stringify({ - type: 'audio.append', - client_timestamp: Date.now(), - audio: audioBase64, - role: 'user', - receive_voice: true - })); - - // 音频结束标记 - ws.send(JSON.stringify({ - type: 'audio.end', - client_timestamp: Date.now() - })); - - } catch (error) { - console.error('音频文件处理失败:', error); - throw error; - } - - // 5. 发送提交指令 - await new Promise(resolve => setTimeout(resolve, 500)); - console.log('发送提交指令...'); - ws.send(JSON.stringify({ - type: 'commit', - client_timestamp: Date.now() - })); - console.log('提交指令发送完成'); - - } catch (error) { - console.error('处理过程中出错:', error); - ws.close(); - reject(error); - } - }); - - ws.on('message', (data) => { - try { - const message = JSON.parse(data); - console.log('收到 WebSocket 消息:', message.type); - - switch (message.type) { - case 'response.audio_txt': - case 'response.text': - console.log('收到响应内容:', message.delta); - messages.push(message.delta); - break; - - case 'response.audio_done': - console.log('AI 处理完成'); - ws.close(); - break; - - case 'error': - console.error('收到错误消息:', message.error); - ws.close(); - reject(new Error(message.error?.message || '处理失败')); - break; - } - } catch (err) { - console.error('处理消息时出错:', err); - ws.close(); - reject(err); - } - }); - - ws.on('error', (error) => { - console.error('WebSocket 错误:', error); - reject(error); - }); - - ws.on('close', () => { - console.log('WebSocket 连接已关闭'); - if (messages.length > 0) { - resolve(messages); - } else { - reject(new Error('处理完成但没有收到任何消息')); - } - }); - }); - } catch (error) { - console.error('AI 处理过程中出错:', error); - throw error; - } -} -// 在上传路由中调用 AI 处理 -router.post('/upload', (req, res) => { - upload(req, res, async function (err) { - console.log('开始处理上传请求'); - - try { - // 处理 multer 错误 - if (err instanceof multer.MulterError) { - throw new Error(`文件上传错误: ${err.code}`); - } else if (err) { - throw new Error(`文件上传错误: ${err.message}`); - } - - // 验证文件是否存在 - if (!req.files || !req.files.video || !req.files.audio) { - throw new Error('缺少必要的文件'); - } - - const videoFile = req.files.video[0]; - const audioFile = req.files.audio[0]; - - // 记录文件信息 - console.log('接收到的文件:'); - console.log('视频文件:', videoFile.path); - console.log('音频文件:', audioPath); - - // 确认文件存在 - if (!fs.existsSync(videoFile.path) || !fs.existsSync(audioFile.path)) { - throw new Error('文件保存失败'); - } - - console.log('开始调用 AI 处理...'); - const messages = await processFilesAndCallAI(videoFile.path, audioFile.path); - console.log('AI 处理返回的消息:', messages); - - // 清理文件 - await Promise.all([ - fs.unlink(videoFile.path).catch(e => console.error('清理视频文件失败:', e)), - fs.unlink(audioFile.path).catch(e => console.error('清理音频文件失败:', e)) - ]); - - res.json({ - success: true, - message: '处理成功', - data: { messages } - }); - - } catch (error) { - console.error('处理过程中出错:', error); - - // 清理文件（如果存在） - if (req.files) { - await Promise.all(Object.values(req.files).flat().map(file => - fs.unlink(file.path).catch(e => - console.error(`清理文件 ${file.path} 失败:`, e) - ) - )); - } - - res.status(500).json({ - success: false, - message: '处理失败', - error: error.message - }); - } - }); -}); - -export default router; \ No newline at end of file