chatbot : step 1
This commit is contained in:
morteza-mortezai
@@ -3,38 +3,38 @@ import { StringOutputParser } from "@langchain/core/output_parsers";
|
||||
import { PromptTemplate } from "@langchain/core/prompts";
|
||||
import { RunnableSequence } from "@langchain/core/runnables";
|
||||
import { ChatGoogleGenerativeAI, GoogleGenerativeAIEmbeddings } from "@langchain/google-genai";
|
||||
import { EntityManager } from "@mikro-orm/core";
|
||||
import { Injectable, Logger, OnModuleInit } from "@nestjs/common";
|
||||
import { ConfigService } from "@nestjs/config";
|
||||
import { injectable } from "inversify";
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
|
||||
import { MemoryVectorStore } from "langchain/vectorstores/memory";
|
||||
|
||||
import { Company } from "../../companies/entities/company.entity";
|
||||
import { Industry } from "../../industries/entities/industry.entity";
|
||||
import { CHATBOT_CONSTANTS } from "../constants/chatbot.constants";
|
||||
import { IChatContext, IChatbotResponse } from "../interfaces/chatbot.interface";
|
||||
import { Logger } from "../../../core/logging/logger";
|
||||
|
||||
@Injectable()
|
||||
export class LangChainService implements OnModuleInit {
|
||||
private readonly logger = new Logger(LangChainService.name);
|
||||
@injectable()
|
||||
export class LangChainService {
|
||||
private readonly logger: Logger;
|
||||
private embeddings: GoogleGenerativeAIEmbeddings;
|
||||
private llm: ChatGoogleGenerativeAI;
|
||||
private vectorStore: MemoryVectorStore;
|
||||
private textSplitter: RecursiveCharacterTextSplitter;
|
||||
private isInitialized = false;
|
||||
|
||||
constructor(
|
||||
private em: EntityManager,
|
||||
private configService: ConfigService,
|
||||
) {
|
||||
constructor() {
|
||||
this.logger = new Logger("LangChainService");
|
||||
const apiKey = process.env.GEMINI_API_KEY;
|
||||
if (!apiKey) {
|
||||
throw new Error("GEMINI_API_KEY is required");
|
||||
}
|
||||
|
||||
// Initialize Google Gemini components
|
||||
this.embeddings = new GoogleGenerativeAIEmbeddings({
|
||||
apiKey: this.configService.getOrThrow("GEMINI_API_KEY"),
|
||||
apiKey,
|
||||
modelName: "embedding-001", // Google's embedding model
|
||||
});
|
||||
|
||||
this.llm = new ChatGoogleGenerativeAI({
|
||||
apiKey: this.configService.getOrThrow("GEMINI_API_KEY"),
|
||||
apiKey,
|
||||
model: CHATBOT_CONSTANTS.DEFAULT_MODEL,
|
||||
temperature: CHATBOT_CONSTANTS.DEFAULT_TEMPERATURE,
|
||||
maxOutputTokens: CHATBOT_CONSTANTS.DEFAULT_MAX_TOKENS,
|
||||
@@ -45,18 +45,20 @@ export class LangChainService implements OnModuleInit {
|
||||
chunkOverlap: 200,
|
||||
separators: ["\n\n", "\n", ".", "!", "?", "؟", "!", ".", " ", ""],
|
||||
});
|
||||
}
|
||||
|
||||
async onModuleInit() {
|
||||
await this.initializeVectorStore();
|
||||
// Initialize vector store asynchronously
|
||||
this.initializeVectorStore().catch((error) => {
|
||||
this.logger.error("Failed to initialize vector store", error);
|
||||
});
|
||||
}
|
||||
|
||||
private async initializeVectorStore() {
|
||||
try {
|
||||
this.logger.log(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.LOADING_DATA);
|
||||
this.logger.info(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.LOADING_DATA);
|
||||
|
||||
// Load company and industry data for training
|
||||
const documents = await this.loadCompanyAndIndustryDocuments();
|
||||
// Load documents for training
|
||||
// TODO: Load actual data from your database models (Product, Category, etc.)
|
||||
const documents = await this.loadDocuments();
|
||||
|
||||
if (documents.length === 0) {
|
||||
this.logger.warn(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.NO_DATA_FOUND);
|
||||
@@ -68,7 +70,7 @@ export class LangChainService implements OnModuleInit {
|
||||
// Split documents into chunks
|
||||
const splitDocs = await this.textSplitter.splitDocuments(documents);
|
||||
|
||||
this.logger.log(
|
||||
this.logger.info(
|
||||
CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.DOCUMENTS_SPLIT.replace("{totalDocs}", documents.length.toString()).replace(
|
||||
"{chunks}",
|
||||
splitDocs.length.toString(),
|
||||
@@ -79,7 +81,7 @@ export class LangChainService implements OnModuleInit {
|
||||
this.vectorStore = await MemoryVectorStore.fromDocuments(splitDocs, this.embeddings);
|
||||
|
||||
this.isInitialized = true;
|
||||
this.logger.log(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.VECTOR_STORE_READY);
|
||||
this.logger.info(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.VECTOR_STORE_READY);
|
||||
} catch (error) {
|
||||
this.logger.error(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.VECTOR_STORE_ERROR, error);
|
||||
// Fallback to empty vector store
|
||||
@@ -88,131 +90,21 @@ export class LangChainService implements OnModuleInit {
|
||||
}
|
||||
}
|
||||
|
||||
private async loadCompanyAndIndustryDocuments(): Promise<Document[]> {
|
||||
private async loadDocuments(): Promise<Document[]> {
|
||||
const documents: Document[] = [];
|
||||
const em = this.em.fork();
|
||||
|
||||
try {
|
||||
// Load companies data with their products and services
|
||||
const companies = await em.find(Company, { isActive: true, deletedAt: null }, { populate: ["industry", "business", "products", "services"] });
|
||||
|
||||
companies.forEach((company) => {
|
||||
// Company basic information using template
|
||||
const companyContent = this.replaceTemplate(CHATBOT_CONSTANTS.COMPANY_DATA_TEMPLATES.COMPANY_INFO, {
|
||||
name: company.name,
|
||||
ceo: company.chiefExecutiveOfficer,
|
||||
email: company.email,
|
||||
phone: company.phone,
|
||||
registrationNumber: company.identificationNumber,
|
||||
establishmentDate: new Date(company.dateOfEstablishment).toLocaleDateString("fa-IR"),
|
||||
address: company.address,
|
||||
website: company.websiteUrl || CHATBOT_CONSTANTS.DEFAULT_VALUES.NO_WEBSITE,
|
||||
description: company.description,
|
||||
industry: company.industry?.title || CHATBOT_CONSTANTS.DEFAULT_VALUES.UNSPECIFIED,
|
||||
status: company.status,
|
||||
business: company.business?.name || CHATBOT_CONSTANTS.DEFAULT_VALUES.UNSPECIFIED,
|
||||
});
|
||||
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: companyContent,
|
||||
metadata: {
|
||||
type: "company",
|
||||
id: company.id,
|
||||
name: company.name,
|
||||
industry: company.industry?.title,
|
||||
businessId: company.business?.id,
|
||||
status: company.status,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
// Company products using template
|
||||
if (company.products && company.products.getItems().length > 0) {
|
||||
company.products.getItems().forEach((product) => {
|
||||
const productContent = this.replaceTemplate(CHATBOT_CONSTANTS.COMPANY_DATA_TEMPLATES.COMPANY_PRODUCT, {
|
||||
companyName: company.name,
|
||||
productTitle: product.title,
|
||||
industry: company.industry?.title || CHATBOT_CONSTANTS.DEFAULT_VALUES.UNSPECIFIED,
|
||||
companyDescription: company.description,
|
||||
companyAddress: company.address,
|
||||
companyPhone: company.phone,
|
||||
companyEmail: company.email,
|
||||
});
|
||||
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: productContent,
|
||||
metadata: {
|
||||
type: "product",
|
||||
id: product.id,
|
||||
title: product.title,
|
||||
companyId: company.id,
|
||||
companyName: company.name,
|
||||
industry: company.industry?.title,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
// Company services using template
|
||||
if (company.services && company.services.getItems().length > 0) {
|
||||
company.services.getItems().forEach((service) => {
|
||||
const serviceContent = this.replaceTemplate(CHATBOT_CONSTANTS.COMPANY_DATA_TEMPLATES.COMPANY_SERVICE, {
|
||||
companyName: company.name,
|
||||
serviceTitle: service.title,
|
||||
industry: company.industry?.title || CHATBOT_CONSTANTS.DEFAULT_VALUES.UNSPECIFIED,
|
||||
companyDescription: company.description,
|
||||
companyAddress: company.address,
|
||||
companyPhone: company.phone,
|
||||
companyEmail: company.email,
|
||||
});
|
||||
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: serviceContent,
|
||||
metadata: {
|
||||
type: "service",
|
||||
id: service.id,
|
||||
title: service.title,
|
||||
companyId: company.id,
|
||||
companyName: company.name,
|
||||
industry: company.industry?.title,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
// Load industries data using template
|
||||
const industries = await em.find(Industry, { isActive: true, deletedAt: null }, { populate: ["companies", "business"] });
|
||||
industries.forEach((industry) => {
|
||||
const companiesInIndustry = industry.companies?.getItems().filter((c) => c.isActive && !c.deletedAt) || [];
|
||||
|
||||
const industryContent = this.replaceTemplate(CHATBOT_CONSTANTS.COMPANY_DATA_TEMPLATES.INDUSTRY_INFO, {
|
||||
title: industry.title,
|
||||
status: industry.isActive ? CHATBOT_CONSTANTS.DEFAULT_VALUES.ACTIVE : CHATBOT_CONSTANTS.DEFAULT_VALUES.INACTIVE,
|
||||
companiesCount: companiesInIndustry.length.toString(),
|
||||
business: industry.business?.name || CHATBOT_CONSTANTS.DEFAULT_VALUES.UNSPECIFIED,
|
||||
companiesList: companiesInIndustry.map((company) => `- ${company.name}`).join("\n"),
|
||||
});
|
||||
|
||||
documents.push(
|
||||
new Document({
|
||||
pageContent: industryContent,
|
||||
metadata: {
|
||||
type: "industry",
|
||||
id: industry.id,
|
||||
title: industry.title,
|
||||
isActive: industry.isActive,
|
||||
companiesCount: companiesInIndustry.length,
|
||||
businessId: industry.business?.id,
|
||||
},
|
||||
}),
|
||||
);
|
||||
});
|
||||
// TODO: Load actual data from your database
|
||||
// Example:
|
||||
// const products = await ProductModel.find({ ... }).limit(100);
|
||||
// products.forEach((product) => {
|
||||
// documents.push(
|
||||
// new Document({
|
||||
// pageContent: `Product: ${product.title_fa} - Description: ${product.description}`,
|
||||
// metadata: { type: "product", id: product._id.toString() },
|
||||
// }),
|
||||
// );
|
||||
// });
|
||||
|
||||
// Add comprehensive Farsi guidance documents from constants
|
||||
const companyGuidanceDocuments = [
|
||||
@@ -252,7 +144,7 @@ export class LangChainService implements OnModuleInit {
|
||||
|
||||
documents.push(...companyGuidanceDocuments);
|
||||
|
||||
this.logger.log(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.DOCUMENTS_LOADED.replace("{count}", documents.length.toString()));
|
||||
this.logger.info(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.DOCUMENTS_LOADED.replace("{count}", documents.length.toString()));
|
||||
return documents;
|
||||
} catch (error) {
|
||||
this.logger.error(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.LOADING_ERROR, error);
|
||||
@@ -363,7 +255,8 @@ export class LangChainService implements OnModuleInit {
|
||||
}
|
||||
|
||||
async refreshVectorStore(): Promise<void> {
|
||||
this.logger.log(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.REFRESHING_VECTOR_STORE);
|
||||
this.logger.info(CHATBOT_CONSTANTS.LANGCHAIN_MESSAGES.REFRESHING_VECTOR_STORE);
|
||||
this.isInitialized = false;
|
||||
await this.initializeVectorStore();
|
||||
}
|
||||
|
||||
@@ -372,17 +265,6 @@ export class LangChainService implements OnModuleInit {
|
||||
return Math.ceil(text.length / 4);
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace placeholders in template with actual values
|
||||
*/
|
||||
private replaceTemplate(template: string, values: Record<string, string>): string {
|
||||
let result = template;
|
||||
Object.entries(values).forEach(([key, value]) => {
|
||||
result = result.replace(new RegExp(`{${key}}`, "g"), value);
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
get initialized(): boolean {
|
||||
return this.isInitialized;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user