anosatsuk124/VoiceRSSSummary
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
Satsuki Akiba
2025-06-07 11:15:39 +09:00
parent 17e9580e23
commit 3452d7c541
1 changed files with 75 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
scripts/fetch_and_generate.ts
View File

@ -10,7 +10,7 @@ import {
saveArticle,
getUnprocessedArticles,
markArticleAsProcessed,
saveEpisode
saveEpisode,
} from "../services/database.js";
import { updatePodcastRSS } from "../services/podcast.js";
import { config } from "../services/config.js";
@ -57,10 +57,10 @@ export async function batchProcess(): Promise<void> {
// Process unprocessed articles and generate podcasts
await processUnprocessedArticles();
// Update RSS feed
await updatePodcastRSS();
console.log("✅ Enhanced batch process completed:", new Date().toISOString());
console.log(
"✅ Enhanced batch process completed:",
new Date().toISOString(),
);
} catch (error) {
console.error("💥 Batch process failed:", error);
throw error;
@ -78,7 +78,9 @@ async function loadFeedUrls(): Promise<string[]> {
.map((url) => url.trim())
.filter((url) => url.length > 0 && !url.startsWith("#"));
} catch (err) {
console.warn(`⚠️ Failed to read feed URLs file: ${config.paths.feedUrlsFile}`);
console.warn(
`⚠️ Failed to read feed URLs file: ${config.paths.feedUrlsFile}`,
);
console.warn("📝 Please create the file with one RSS URL per line.");
return [];
}
@ -88,7 +90,7 @@ async function loadFeedUrls(): Promise<string[]> {
* Process a single feed URL and discover new articles
*/
async function processFeedUrl(url: string): Promise<void> {
if (!url || !url.startsWith('http')) {
if (!url || !url.startsWith("http")) {
throw new Error(`Invalid feed URL: ${url}`);
}
@ -108,7 +110,7 @@ async function processFeedUrl(url: string): Promise<void> {
title: feed.title,
description: feed.description,
lastUpdated: new Date().toISOString(),
active: true
active: true,
});
feedRecord = await getFeedByUrl(url);
}
@ -118,7 +120,10 @@ async function processFeedUrl(url: string): Promise<void> {
}
// Process feed items and save new articles
const newArticlesCount = await discoverNewArticles(feedRecord, feed.items || []);
const newArticlesCount = await discoverNewArticles(
feedRecord,
feed.items || [],
);
// Update feed last updated timestamp
if (newArticlesCount > 0) {
@ -127,12 +132,13 @@ async function processFeedUrl(url: string): Promise<void> {
title: feedRecord.title,
description: feedRecord.description,
lastUpdated: new Date().toISOString(),
active: feedRecord.active
active: feedRecord.active,
});
}
console.log(`📊 Feed processed: ${feed.title || url} (${newArticlesCount} new articles)`);
console.log(
`📊 Feed processed: ${feed.title || url} (${newArticlesCount} new articles)`,
);
} catch (error) {
console.error(`💥 Error processing feed ${url}:`, error);
throw error;
@ -142,7 +148,10 @@ async function processFeedUrl(url: string): Promise<void> {
/**
* Discover and save new articles from feed items
*/
async function discoverNewArticles(feed: any, items: FeedItem[]): Promise<number> {
async function discoverNewArticles(
feed: any,
items: FeedItem[],
): Promise<number> {
let newArticlesCount = 0;
for (const item of items) {
@ -160,7 +169,7 @@ async function discoverNewArticles(feed: any, items: FeedItem[]): Promise<number
description: item.description || item.contentSnippet,
content: item.content,
pubDate: item.pubDate || new Date().toISOString(),
processed: false
processed: false,
});
// Check if this is truly a new article
@ -168,7 +177,6 @@ async function discoverNewArticles(feed: any, items: FeedItem[]): Promise<number
newArticlesCount++;
console.log(`📄 New article discovered: ${item.title}`);
}
} catch (error) {
console.error(`❌ Error saving article: ${item.title}`, error);
}
@ -199,12 +207,15 @@ async function processUnprocessedArticles(): Promise<void> {
await generatePodcastForArticle(article);
await markArticleAsProcessed(article.id);
console.log(`✅ Podcast generated for: ${article.title}`);
await updatePodcastRSS(); // Update RSS after each article
} catch (error) {
console.error(`❌ Failed to generate podcast for article: ${article.title}`, error);
console.error(
`❌ Failed to generate podcast for article: ${article.title}`,
error,
);
// Don't mark as processed if generation failed
}
}
} catch (error) {
console.error("💥 Error processing unprocessed articles:", error);
throw error;
@ -223,17 +234,18 @@ async function generatePodcastForArticle(article: any): Promise<void> {
const feedTitle = feed?.title || "Unknown Feed";
// Classify the article/feed
const category = await openAI_ClassifyFeed(`${feedTitle}: ${article.title}`);
const category = await openAI_ClassifyFeed(
`${feedTitle}: ${article.title}`,
);
console.log(`🏷️ Article classified as: ${category}`);
// Generate podcast content for this single article
const podcastContent = await openAI_GeneratePodcastContent(
article.title,
[{
const podcastContent = await openAI_GeneratePodcastContent(article.title, [
{
title: article.title,
link: article.link
}]
);
link: article.link,
},
]);
// Generate unique ID for the episode
const episodeId = crypto.randomUUID();
@ -249,16 +261,19 @@ async function generatePodcastForArticle(article: any): Promise<void> {
await saveEpisode({
articleId: article.id,
title: `${category}: ${article.title}`,
description: article.description || `Podcast episode for: ${article.title}`,
description:
article.description || `Podcast episode for: ${article.title}`,
audioPath: audioFilePath,
duration: audioStats.duration,
fileSize: audioStats.size
fileSize: audioStats.size,
});
console.log(`💾 Episode saved for article: ${article.title}`);
} catch (error) {
console.error(`💥 Error generating podcast for article: ${article.title}`, error);
console.error(
`💥 Error generating podcast for article: ${article.title}`,
error,
);
throw error;
}
}
@ -266,7 +281,9 @@ async function generatePodcastForArticle(article: any): Promise<void> {
/**
* Get audio file statistics
*/
async function getAudioFileStats(audioFileName: string): Promise<{ duration?: number, size: number }> {
async function getAudioFileStats(
audioFileName: string,
): Promise<{ duration?: number; size: number }> {
try {
const audioPath = `${config.paths.podcastAudioDir}/${audioFileName}`;
const stats = await fs.stat(audioPath);
@ -274,10 +291,13 @@ async function getAudioFileStats(audioFileName: string): Promise<{ duration?: nu
return {
size: stats.size,
// TODO: Add duration calculation using ffprobe if needed
duration: undefined
duration: undefined,
};
} catch (error) {
console.warn(`⚠️ Could not get audio file stats for ${audioFileName}:`, error);
console.warn(
`⚠️ Could not get audio file stats for ${audioFileName}:`,
error,
);
return { size: 0 };
}
}
@ -381,15 +401,15 @@ async function getAudioFileStats(audioFileName: string): Promise<{ duration?: nu
// Export function for use in server
export async function addNewFeedUrl(feedUrl: string): Promise<void> {
if (!feedUrl || !feedUrl.startsWith('http')) {
throw new Error('Invalid feed URL');
if (!feedUrl || !feedUrl.startsWith("http")) {
throw new Error("Invalid feed URL");
}
try {
// Add to feeds table
await saveFeed({
url: feedUrl,
active: true
active: true,
});
console.log(`✅ Feed URL added: ${feedUrl}`);