From 3452d7c5417bc99ac610907afda78ffd27d722ce Mon Sep 17 00:00:00 2001 From: Satsuki Akiba Date: Sat, 7 Jun 2025 11:15:39 +0900 Subject: [PATCH] Update --- scripts/fetch_and_generate.ts | 130 ++++++++++++++++++++-------------- 1 file changed, 75 insertions(+), 55 deletions(-) diff --git a/scripts/fetch_and_generate.ts b/scripts/fetch_and_generate.ts index 085c6be..e0ce6f7 100644 --- a/scripts/fetch_and_generate.ts +++ b/scripts/fetch_and_generate.ts @@ -4,13 +4,13 @@ import { openAI_GeneratePodcastContent, } from "../services/llm.js"; import { generateTTS } from "../services/tts.js"; -import { - saveFeed, - getFeedByUrl, - saveArticle, - getUnprocessedArticles, +import { + saveFeed, + getFeedByUrl, + saveArticle, + getUnprocessedArticles, markArticleAsProcessed, - saveEpisode + saveEpisode, } from "../services/database.js"; import { updatePodcastRSS } from "../services/podcast.js"; import { config } from "../services/config.js"; @@ -34,14 +34,14 @@ interface FeedItem { export async function batchProcess(): Promise { try { console.log("🚀 Starting enhanced batch process..."); - + // Load feed URLs from file const feedUrls = await loadFeedUrls(); if (feedUrls.length === 0) { console.log("ℹī¸ No feed URLs found."); return; } - + console.log(`📡 Processing ${feedUrls.length} feeds...`); // Process each feed URL @@ -57,10 +57,10 @@ export async function batchProcess(): Promise { // Process unprocessed articles and generate podcasts await processUnprocessedArticles(); - // Update RSS feed - await updatePodcastRSS(); - - console.log("✅ Enhanced batch process completed:", new Date().toISOString()); + console.log( + "✅ Enhanced batch process completed:", + new Date().toISOString(), + ); } catch (error) { console.error("đŸ’Ĩ Batch process failed:", error); throw error; @@ -78,7 +78,9 @@ async function loadFeedUrls(): Promise { .map((url) => url.trim()) .filter((url) => url.length > 0 && !url.startsWith("#")); } catch (err) { - console.warn(`⚠ī¸ Failed to read feed URLs file: ${config.paths.feedUrlsFile}`); + console.warn( + `⚠ī¸ Failed to read feed URLs file: ${config.paths.feedUrlsFile}`, + ); console.warn("📝 Please create the file with one RSS URL per line."); return []; } @@ -88,12 +90,12 @@ async function loadFeedUrls(): Promise { * Process a single feed URL and discover new articles */ async function processFeedUrl(url: string): Promise { - if (!url || !url.startsWith('http')) { + if (!url || !url.startsWith("http")) { throw new Error(`Invalid feed URL: ${url}`); } - + console.log(`🔍 Processing feed: ${url}`); - + try { // Parse RSS feed const parser = new Parser(); @@ -108,7 +110,7 @@ async function processFeedUrl(url: string): Promise { title: feed.title, description: feed.description, lastUpdated: new Date().toISOString(), - active: true + active: true, }); feedRecord = await getFeedByUrl(url); } @@ -118,8 +120,11 @@ async function processFeedUrl(url: string): Promise { } // Process feed items and save new articles - const newArticlesCount = await discoverNewArticles(feedRecord, feed.items || []); - + const newArticlesCount = await discoverNewArticles( + feedRecord, + feed.items || [], + ); + // Update feed last updated timestamp if (newArticlesCount > 0) { await saveFeed({ @@ -127,12 +132,13 @@ async function processFeedUrl(url: string): Promise { title: feedRecord.title, description: feedRecord.description, lastUpdated: new Date().toISOString(), - active: feedRecord.active + active: feedRecord.active, }); } - console.log(`📊 Feed processed: ${feed.title || url} (${newArticlesCount} new articles)`); - + console.log( + `📊 Feed processed: ${feed.title || url} (${newArticlesCount} new articles)`, + ); } catch (error) { console.error(`đŸ’Ĩ Error processing feed ${url}:`, error); throw error; @@ -142,9 +148,12 @@ async function processFeedUrl(url: string): Promise { /** * Discover and save new articles from feed items */ -async function discoverNewArticles(feed: any, items: FeedItem[]): Promise { +async function discoverNewArticles( + feed: any, + items: FeedItem[], +): Promise { let newArticlesCount = 0; - + for (const item of items) { if (!item.title || !item.link) { console.warn("⚠ī¸ Skipping item without title or link"); @@ -160,7 +169,7 @@ async function discoverNewArticles(feed: any, items: FeedItem[]): Promise { console.log("🎧 Processing unprocessed articles..."); - + try { // Get unprocessed articles (limit to prevent overwhelming) const unprocessedArticles = await getUnprocessedArticles(20); - + if (unprocessedArticles.length === 0) { console.log("ℹī¸ No unprocessed articles found."); return; @@ -199,12 +207,15 @@ async function processUnprocessedArticles(): Promise { await generatePodcastForArticle(article); await markArticleAsProcessed(article.id); console.log(`✅ Podcast generated for: ${article.title}`); + await updatePodcastRSS(); // Update RSS after each article } catch (error) { - console.error(`❌ Failed to generate podcast for article: ${article.title}`, error); + console.error( + `❌ Failed to generate podcast for article: ${article.title}`, + error, + ); // Don't mark as processed if generation failed } } - } catch (error) { console.error("đŸ’Ĩ Error processing unprocessed articles:", error); throw error; @@ -216,28 +227,29 @@ async function processUnprocessedArticles(): Promise { */ async function generatePodcastForArticle(article: any): Promise { console.log(`🎤 Generating podcast for: ${article.title}`); - + try { // Get feed information for context const feed = await getFeedByUrl(article.feedId); const feedTitle = feed?.title || "Unknown Feed"; - + // Classify the article/feed - const category = await openAI_ClassifyFeed(`${feedTitle}: ${article.title}`); + const category = await openAI_ClassifyFeed( + `${feedTitle}: ${article.title}`, + ); console.log(`🏷ī¸ Article classified as: ${category}`); // Generate podcast content for this single article - const podcastContent = await openAI_GeneratePodcastContent( - article.title, - [{ + const podcastContent = await openAI_GeneratePodcastContent(article.title, [ + { title: article.title, - link: article.link - }] - ); + link: article.link, + }, + ]); // Generate unique ID for the episode const episodeId = crypto.randomUUID(); - + // Generate TTS audio const audioFilePath = await generateTTS(episodeId, podcastContent); console.log(`🔊 Audio generated: ${audioFilePath}`); @@ -249,16 +261,19 @@ async function generatePodcastForArticle(article: any): Promise { await saveEpisode({ articleId: article.id, title: `${category}: ${article.title}`, - description: article.description || `Podcast episode for: ${article.title}`, + description: + article.description || `Podcast episode for: ${article.title}`, audioPath: audioFilePath, duration: audioStats.duration, - fileSize: audioStats.size + fileSize: audioStats.size, }); console.log(`💾 Episode saved for article: ${article.title}`); - } catch (error) { - console.error(`đŸ’Ĩ Error generating podcast for article: ${article.title}`, error); + console.error( + `đŸ’Ĩ Error generating podcast for article: ${article.title}`, + error, + ); throw error; } } @@ -266,18 +281,23 @@ async function generatePodcastForArticle(article: any): Promise { /** * Get audio file statistics */ -async function getAudioFileStats(audioFileName: string): Promise<{ duration?: number, size: number }> { +async function getAudioFileStats( + audioFileName: string, +): Promise<{ duration?: number; size: number }> { try { const audioPath = `${config.paths.podcastAudioDir}/${audioFileName}`; const stats = await fs.stat(audioPath); - + return { size: stats.size, // TODO: Add duration calculation using ffprobe if needed - duration: undefined + duration: undefined, }; } catch (error) { - console.warn(`⚠ī¸ Could not get audio file stats for ${audioFileName}:`, error); + console.warn( + `⚠ī¸ Could not get audio file stats for ${audioFileName}:`, + error, + ); return { size: 0 }; } } @@ -381,17 +401,17 @@ async function getAudioFileStats(audioFileName: string): Promise<{ duration?: nu // Export function for use in server export async function addNewFeedUrl(feedUrl: string): Promise { - if (!feedUrl || !feedUrl.startsWith('http')) { - throw new Error('Invalid feed URL'); + if (!feedUrl || !feedUrl.startsWith("http")) { + throw new Error("Invalid feed URL"); } - + try { // Add to feeds table await saveFeed({ url: feedUrl, - active: true + active: true, }); - + console.log(`✅ Feed URL added: ${feedUrl}`); } catch (error) { console.error(`❌ Failed to add feed URL: ${feedUrl}`, error); @@ -405,4 +425,4 @@ if (import.meta.main) { console.error("đŸ’Ĩ Batch process failed:", err); process.exit(1); }); -} \ No newline at end of file +}