Skip to content
Snippets Groups Projects

feat: make redis chunk sizes configurable and side actions for action-picker

Merged Daniel Schmidt requested to merge instantnoodl/redis-config-chunk-size into master
1 file
+ 44
3
Compare changes
  • Side-by-side
  • Inline
+ 44
3
@@ -24,7 +24,7 @@ const redisConfig = new Config('redis-config', {
});
/** Batch size for processing embeddings */
const BATCH_SIZE = 10;
const BATCH_SIZE = 25;
/** Redis manager singleton class */
class RedisManager {
@@ -309,6 +309,28 @@ async function processBatch(chunks: any[], prefix: string, entry: EntityData<Wik
);
}
/** Chunk size for text-embedding-3-small */
let CHUNK_SIZE = 250;
/** Chunk overlap for text-embedding-3-small */
let CHUNK_OVERLAP = 30;
/**
* Set the chunk size
* @param size The chunk size
*/
export function setChunkSize(size: number) {
CHUNK_SIZE = size;
}
/**
* Set the chunk overlap
* @param size The chunk overlap
*/
export function setChunkOverlap(size: number) {
CHUNK_OVERLAP = size;
}
/**
* Ensures wiki entries are properly stored in Redis
* @async
@@ -351,7 +373,8 @@ export async function ensureWikiEntries(ids?: string[] | string) {
}
const chunks = await splitHtml(entry.content, {
chunkSize: 1000,
chunkSize: CHUNK_SIZE,
chunkOverlap: CHUNK_OVERLAP,
});
logger.info(
{
@@ -362,7 +385,7 @@ export async function ensureWikiEntries(ids?: string[] | string) {
// Process chunks in batches
for (let i = 0; i < chunks.length; i += BATCH_SIZE) {
logger.debug(
logger.info(
{
chunkIndex: i,
chunkCount: chunks.length,
@@ -442,3 +465,21 @@ export function extractDataFromID(id: string, dataType: 'entry-id' | 'prefix') {
return id.split(':')[1]?.split('-')[0] ?? '';
}
}
/**
* Forces a re-embedding of all wiki entries
*/
export async function forceReEmbedding() {
const rawWikiCollection = await getRawCollection<EntityData<WikiEntry>>(WikiEntry._collectionName);
await rawWikiCollection.updateMany(
{
_class: WikiEntry._class,
published: true,
},
{
$set: {
needEmbeddingSync: true,
},
},
);
}
Loading