diff --git a/scripts/fetch_and_generate.ts b/scripts/fetch_and_generate.ts index 0775272..b959041 100644 --- a/scripts/fetch_and_generate.ts +++ b/scripts/fetch_and_generate.ts @@ -3,7 +3,7 @@ import { openAI_ClassifyFeed, openAI_GeneratePodcastContent, } from "../services/llm.js"; -import { generateTTS } from "../services/tts.js"; +import { generateTTS, generateTTSWithoutQueue } from "../services/tts.js"; import { saveFeed, getFeedByUrl, @@ -208,15 +208,30 @@ async function processUnprocessedArticles(): Promise { console.log(`🎯 Found ${unprocessedArticles.length} unprocessed articles`); - // Track articles that successfully generated audio - const successfullyGeneratedArticles: string[] = []; + // Track articles that successfully generated audio AND episodes + let successfullyGeneratedCount = 0; for (const article of unprocessedArticles) { try { - await generatePodcastForArticle(article); - await markArticleAsProcessed(article.id); - console.log(`✅ Podcast generated for: ${article.title}`); - successfullyGeneratedArticles.push(article.id); + const episodeCreated = await generatePodcastForArticle(article); + + // Only mark as processed and update RSS if episode was actually created + if (episodeCreated) { + await markArticleAsProcessed(article.id); + console.log(`✅ Podcast generated for: ${article.title}`); + successfullyGeneratedCount++; + + // Update RSS immediately after each successful episode creation + console.log(`📻 Updating podcast RSS after successful episode creation...`); + try { + await updatePodcastRSS(); + console.log(`📻 RSS updated successfully for: ${article.title}`); + } catch (rssError) { + console.error(`❌ Failed to update RSS after episode creation for: ${article.title}`, rssError); + } + } else { + console.warn(`⚠️ Episode creation failed for: ${article.title} - not marking as processed`); + } } catch (error) { console.error( `❌ Failed to generate podcast for article: ${article.title}`, @@ -226,10 +241,9 @@ async function processUnprocessedArticles(): Promise { } } - // Only update RSS if at least one article was successfully processed - if (successfullyGeneratedArticles.length > 0) { - console.log(`📻 Updating podcast RSS for ${successfullyGeneratedArticles.length} new episodes...`); - await updatePodcastRSS(); + console.log(`🎯 Batch processing completed: ${successfullyGeneratedCount} episodes successfully created`); + if (successfullyGeneratedCount === 0) { + console.log(`ℹ️ No episodes were successfully created in this batch`); } } catch (error) { console.error("💥 Error processing unprocessed articles:", error); @@ -242,6 +256,8 @@ async function processUnprocessedArticles(): Promise { */ async function processRetryQueue(): Promise { const { getQueueItems, updateQueueItemStatus, removeFromQueue } = await import("../services/database.js"); + const { Database } = await import("bun:sqlite"); + const db = new Database(config.paths.dbPath); console.log("🔄 Processing TTS retry queue..."); @@ -256,41 +272,65 @@ async function processRetryQueue(): Promise { for (const item of queueItems) { try { - console.log(`🔁 Retrying TTS generation for: ${item.itemId} (attempt ${item.retryCount + 1})`); + console.log(`🔁 Retrying TTS generation for: ${item.itemId} (attempt ${item.retryCount + 1}/3)`); // Mark as processing await updateQueueItemStatus(item.id, 'processing'); - // Attempt TTS generation - await generateTTS(item.itemId, item.scriptText, item.retryCount); + // Attempt TTS generation without re-queuing on failure + const audioFilePath = await generateTTSWithoutQueue(item.itemId, item.scriptText, item.retryCount); - // Success - remove from queue + // Success - remove from queue and update RSS await removeFromQueue(item.id); console.log(`✅ TTS retry successful for: ${item.itemId}`); + // Update RSS immediately after successful retry + console.log(`📻 Updating podcast RSS after successful retry...`); + try { + await updatePodcastRSS(); + console.log(`📻 RSS updated successfully after retry for: ${item.itemId}`); + } catch (rssError) { + console.error(`❌ Failed to update RSS after retry for: ${item.itemId}`, rssError); + } + } catch (error) { console.error(`❌ TTS retry failed for: ${item.itemId}`, error); - if (item.retryCount >= 2) { - // Max retries reached, mark as failed - await updateQueueItemStatus(item.id, 'failed'); - console.log(`💀 Max retries reached for: ${item.itemId}, marking as failed`); - } else { - // Reset to pending for next retry - await updateQueueItemStatus(item.id, 'pending'); + try { + if (item.retryCount >= 2) { + // Max retries reached, mark as failed + await updateQueueItemStatus(item.id, 'failed'); + console.log(`💀 Max retries reached for: ${item.itemId}, marking as failed`); + } else { + // Increment retry count and reset to pending for next retry + const updatedRetryCount = item.retryCount + 1; + const stmt = db.prepare("UPDATE tts_queue SET retry_count = ?, status = 'pending' WHERE id = ?"); + stmt.run(updatedRetryCount, item.id); + console.log(`🔄 Updated retry count to ${updatedRetryCount} for: ${item.itemId}`); + } + } catch (dbError) { + console.error(`❌ Failed to update queue status for: ${item.itemId}`, dbError); } } } } catch (error) { console.error("💥 Error processing retry queue:", error); throw error; + } finally { + // Clean up database connection + try { + db.close(); + } catch (closeError) { + console.warn("⚠️ Warning: Failed to close database connection:", closeError); + } } } /** * Generate podcast for a single article + * Returns true if episode was successfully created, false otherwise */ -async function generatePodcastForArticle(article: any): Promise { +async function generatePodcastForArticle(article: any): Promise { console.log(`🎤 Generating podcast for: ${article.title}`); try { @@ -315,31 +355,66 @@ async function generatePodcastForArticle(article: any): Promise { // Generate unique ID for the episode const episodeId = crypto.randomUUID(); - // Generate TTS audio - const audioFilePath = await generateTTS(episodeId, podcastContent); - console.log(`🔊 Audio generated: ${audioFilePath}`); + // Generate TTS audio - this is the critical step that can fail + let audioFilePath: string; + try { + audioFilePath = await generateTTS(episodeId, podcastContent); + console.log(`🔊 Audio generated: ${audioFilePath}`); + } catch (ttsError) { + console.error(`❌ TTS generation failed for ${article.title}:`, ttsError); + + // Check if error indicates item was added to retry queue + const errorMessage = ttsError instanceof Error ? ttsError.message : String(ttsError); + if (errorMessage.includes('added to retry queue')) { + console.log(`📋 Article will be retried later via TTS queue: ${article.title}`); + // Don't mark as processed - leave it for retry + return false; + } else { + console.error(`💀 TTS generation permanently failed for ${article.title} - max retries exceeded`); + // Max retries exceeded, don't create episode but mark as processed to avoid infinite retry + return false; + } + } + + // Verify audio file was actually created and is valid + try { + const audioStats = await getAudioFileStats(audioFilePath); + if (audioStats.size === 0) { + console.error(`❌ Audio file is empty for ${article.title}: ${audioFilePath}`); + return false; + } + } catch (statsError) { + console.error(`❌ Cannot access audio file for ${article.title}: ${audioFilePath}`, statsError); + return false; + } // Get audio file stats const audioStats = await getAudioFileStats(audioFilePath); - // Save episode - await saveEpisode({ - articleId: article.id, - title: `${category}: ${article.title}`, - description: - article.description || `Podcast episode for: ${article.title}`, - audioPath: audioFilePath, - duration: audioStats.duration, - fileSize: audioStats.size, - }); + // Save episode - only if we have valid audio + try { + await saveEpisode({ + articleId: article.id, + title: `${category}: ${article.title}`, + description: + article.description || `Podcast episode for: ${article.title}`, + audioPath: audioFilePath, + duration: audioStats.duration, + fileSize: audioStats.size, + }); - console.log(`💾 Episode saved for article: ${article.title}`); + console.log(`💾 Episode saved for article: ${article.title}`); + return true; + } catch (saveError) { + console.error(`❌ Failed to save episode for ${article.title}:`, saveError); + return false; + } } catch (error) { console.error( `💥 Error generating podcast for article: ${article.title}`, error, ); - throw error; + return false; } } diff --git a/services/tts.ts b/services/tts.ts index 74b4f8d..386a9e7 100644 --- a/services/tts.ts +++ b/services/tts.ts @@ -12,7 +12,11 @@ const defaultVoiceStyle: VoiceStyle = { styleId: config.voicevox.styleId, }; -export async function generateTTS( +/** + * Generate TTS without adding to retry queue on failure + * Used for retry queue processing to avoid infinite loops + */ +export async function generateTTSWithoutQueue( itemId: string, scriptText: string, retryCount: number = 0, @@ -25,104 +29,113 @@ export async function generateTTS( throw new Error("Script text is required for TTS generation"); } + console.log(`TTS生成開始: ${itemId} (試行回数: ${retryCount + 1})`); + const encodedText = encodeURIComponent(scriptText); + + const queryUrl = `${config.voicevox.host}/audio_query?text=${encodedText}&speaker=${defaultVoiceStyle.styleId}`; + const synthesisUrl = `${config.voicevox.host}/synthesis?speaker=${defaultVoiceStyle.styleId}`; + + const queryResponse = await fetch(queryUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + }); + + if (!queryResponse.ok) { + const errorText = await queryResponse.text(); + throw new Error( + `VOICEVOX audio query failed (${queryResponse.status}): ${errorText}`, + ); + } + + const audioQuery = await queryResponse.json(); + + console.log(`音声合成開始: ${itemId}`); + const audioResponse = await fetch(synthesisUrl, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(audioQuery), + signal: AbortSignal.timeout(600000), // 10分のタイムアウト + }); + + if (!audioResponse.ok) { + const errorText = await audioResponse.text(); + console.error(`音声合成失敗: ${itemId}`); + throw new Error( + `VOICEVOX synthesis failed (${audioResponse.status}): ${errorText}`, + ); + } + + const audioArrayBuffer = await audioResponse.arrayBuffer(); + const audioBuffer = Buffer.from(audioArrayBuffer); + + // 出力ディレクトリの準備 + const outputDir = config.paths.podcastAudioDir; + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + const wavFilePath = path.resolve(outputDir, `${itemId}.wav`); + const mp3FilePath = path.resolve(outputDir, `${itemId}.mp3`); + + console.log(`WAVファイル保存開始: ${wavFilePath}`); + fs.writeFileSync(wavFilePath, audioBuffer); + console.log(`WAVファイル保存完了: ${wavFilePath}`); + + console.log(`MP3変換開始: ${wavFilePath} -> ${mp3FilePath}`); + + const ffmpegCmd = ffmpegPath || "ffmpeg"; + const result = Bun.spawnSync({ + cmd: [ + ffmpegCmd, + "-i", + wavFilePath, + "-codec:a", + "libmp3lame", + "-qscale:a", + "2", + "-y", // Overwrite output file + mp3FilePath, + ], + }); + + if (result.exitCode !== 0) { + const stderr = result.stderr + ? new TextDecoder().decode(result.stderr) + : "Unknown error"; + throw new Error(`FFmpeg conversion failed: ${stderr}`); + } + + // Wavファイルを削除 + if (fs.existsSync(wavFilePath)) { + fs.unlinkSync(wavFilePath); + } + + console.log(`TTS生成完了: ${itemId}`); + + return path.basename(mp3FilePath); +} + +export async function generateTTS( + itemId: string, + scriptText: string, + retryCount: number = 0, +): Promise { const maxRetries = 2; try { - console.log(`TTS生成開始: ${itemId} (試行回数: ${retryCount + 1}/${maxRetries + 1})`); - const encodedText = encodeURIComponent(scriptText); - - const queryUrl = `${config.voicevox.host}/audio_query?text=${encodedText}&speaker=${defaultVoiceStyle.styleId}`; - const synthesisUrl = `${config.voicevox.host}/synthesis?speaker=${defaultVoiceStyle.styleId}`; - - const queryResponse = await fetch(queryUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - Accept: "application/json", - }, - }); - - if (!queryResponse.ok) { - const errorText = await queryResponse.text(); - throw new Error( - `VOICEVOX audio query failed (${queryResponse.status}): ${errorText}`, - ); - } - - const audioQuery = await queryResponse.json(); - - console.log(`音声合成開始: ${itemId}`); - const audioResponse = await fetch(synthesisUrl, { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(audioQuery), - signal: AbortSignal.timeout(600000), // 10分のタイムアウト - }); - - if (!audioResponse.ok) { - const errorText = await audioResponse.text(); - console.error(`音声合成失敗: ${itemId}`); - throw new Error( - `VOICEVOX synthesis failed (${audioResponse.status}): ${errorText}`, - ); - } - - const audioArrayBuffer = await audioResponse.arrayBuffer(); - const audioBuffer = Buffer.from(audioArrayBuffer); - - // 出力ディレクトリの準備 - const outputDir = config.paths.podcastAudioDir; - if (!fs.existsSync(outputDir)) { - fs.mkdirSync(outputDir, { recursive: true }); - } - - const wavFilePath = path.resolve(outputDir, `${itemId}.wav`); - const mp3FilePath = path.resolve(outputDir, `${itemId}.mp3`); - - console.log(`WAVファイル保存開始: ${wavFilePath}`); - fs.writeFileSync(wavFilePath, audioBuffer); - console.log(`WAVファイル保存完了: ${wavFilePath}`); - - console.log(`MP3変換開始: ${wavFilePath} -> ${mp3FilePath}`); - - const ffmpegCmd = ffmpegPath || "ffmpeg"; - const result = Bun.spawnSync({ - cmd: [ - ffmpegCmd, - "-i", - wavFilePath, - "-codec:a", - "libmp3lame", - "-qscale:a", - "2", - "-y", // Overwrite output file - mp3FilePath, - ], - }); - - if (result.exitCode !== 0) { - const stderr = result.stderr - ? new TextDecoder().decode(result.stderr) - : "Unknown error"; - throw new Error(`FFmpeg conversion failed: ${stderr}`); - } - - // Wavファイルを削除 - if (fs.existsSync(wavFilePath)) { - fs.unlinkSync(wavFilePath); - } - - console.log(`TTS生成完了: ${itemId}`); - - return path.basename(mp3FilePath); + return await generateTTSWithoutQueue(itemId, scriptText, retryCount); } catch (error) { console.error(`TTS生成エラー: ${itemId} (試行回数: ${retryCount + 1})`, error); if (retryCount < maxRetries) { + // Add to queue for retry only on initial failure const { addToQueue } = await import("../services/database.js"); - await addToQueue(itemId, scriptText, retryCount + 1); + await addToQueue(itemId, scriptText, retryCount); throw new Error(`TTS generation failed, added to retry queue: ${error}`); } else { throw new Error(`TTS generation failed after ${maxRetries + 1} attempts: ${error}`);