log stream improvement
This commit is contained in:
@@ -166,9 +166,21 @@ async function main(): Promise<void> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const bootstrapTime = Date.now() - bootstrapStart;
|
const bootstrapTime = Date.now() - bootstrapStart;
|
||||||
const stats = vfs.getStats();
|
let stats = vfs.getStats();
|
||||||
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
|
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
|
||||||
|
|
||||||
|
// Fallback: If stream was empty but Redis has files, bootstrap from Redis keys
|
||||||
|
// This handles the case where meta-core restarted and cleared the stream
|
||||||
|
if (stats.fileCount === 0) {
|
||||||
|
const indexCount = await redisClient.getIndexCount();
|
||||||
|
if (indexCount > 0) {
|
||||||
|
logger.info(`Stream was empty but Redis has ${indexCount} files - bootstrapping from Redis keys...`);
|
||||||
|
await vfs.refresh();
|
||||||
|
stats = vfs.getStats();
|
||||||
|
logger.info(`Redis key bootstrap complete: ${stats.fileCount} files`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Start live stream consumer from where replay left off
|
// Start live stream consumer from where replay left off
|
||||||
logger.info(`Starting live stream consumer from ${lastId}...`);
|
logger.info(`Starting live stream consumer from ${lastId}...`);
|
||||||
redisClient.startSimpleStreamConsumer(
|
redisClient.startSimpleStreamConsumer(
|
||||||
|
|||||||
@@ -445,6 +445,24 @@ export class RedisClient implements Partial<IKVClient> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get count of files in Redis index
|
||||||
|
* More efficient than getAllHashIds() when you only need the count
|
||||||
|
*/
|
||||||
|
async getIndexCount(): Promise<number> {
|
||||||
|
if (!this.client || !this.isConnected) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const indexKey = `${this.config.prefix}file:__index__`;
|
||||||
|
return await this.client.scard(indexKey);
|
||||||
|
} catch (error) {
|
||||||
|
logger.error('Failed to get index count:', error);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Scan Redis keys matching pattern
|
* Scan Redis keys matching pattern
|
||||||
*/
|
*/
|
||||||
|
|||||||
Reference in New Issue
Block a user