feat: use pgvecto.rs (#3605)

This commit is contained in:
Jason Rasmussen
2023-12-08 11:15:46 -05:00
committed by GitHub
parent 429ad28810
commit 1e99ba8167
99 changed files with 1935 additions and 2583 deletions

View File

@@ -0,0 +1,107 @@
export type ModelInfo = {
dimSize: number;
};
export const CLIP_MODEL_INFO: Record<string, ModelInfo> = {
RN50__openai: {
dimSize: 1024,
},
RN50__yfcc15m: {
dimSize: 1024,
},
RN50__cc12m: {
dimSize: 1024,
},
RN101__openai: {
dimSize: 512,
},
RN101__yfcc15m: {
dimSize: 512,
},
RN50x4__openai: {
dimSize: 640,
},
RN50x16__openai: {
dimSize: 768,
},
RN50x64__openai: {
dimSize: 1024,
},
'ViT-B-32__openai': {
dimSize: 512,
},
'ViT-B-32__laion2b_e16': {
dimSize: 512,
},
'ViT-B-32__laion400m_e31': {
dimSize: 512,
},
'ViT-B-32__laion400m_e32': {
dimSize: 512,
},
'ViT-B-32__laion2b-s34b-b79k': {
dimSize: 512,
},
'ViT-B-16__openai': {
dimSize: 512,
},
'ViT-B-16__laion400m_e31': {
dimSize: 512,
},
'ViT-B-16__laion400m_e32': {
dimSize: 512,
},
'ViT-B-16-plus-240__laion400m_e31': {
dimSize: 640,
},
'ViT-B-16-plus-240__laion400m_e32': {
dimSize: 640,
},
'ViT-L-14__openai': {
dimSize: 768,
},
'ViT-L-14__laion400m_e31': {
dimSize: 768,
},
'ViT-L-14__laion400m_e32': {
dimSize: 768,
},
'ViT-L-14__laion2b-s32b-b82k': {
dimSize: 768,
},
'ViT-L-14-336__openai': {
dimSize: 768,
},
'ViT-H-14__laion2b-s32b-b79k': {
dimSize: 1024,
},
'ViT-g-14__laion2b-s12b-b42k': {
dimSize: 1024,
},
'LABSE-Vit-L-14': {
dimSize: 768,
},
'XLM-Roberta-Large-Vit-B-32': {
dimSize: 512,
},
'XLM-Roberta-Large-Vit-B-16Plus': {
dimSize: 640,
},
'XLM-Roberta-Large-Vit-L-14': {
dimSize: 768,
},
};
export function cleanModelName(modelName: string): string {
const tokens = modelName.split('/');
return tokens[tokens.length - 1].replace(/:/g, '_');
}
export function getCLIPModelInfo(modelName: string): ModelInfo {
const modelInfo = CLIP_MODEL_INFO[cleanModelName(modelName)];
if (!modelInfo) {
throw new Error(`Unknown CLIP model: ${modelName}`);
}
return modelInfo;
}