From e838f48b89152990e5d86026d37e4c3f27fab48a Mon Sep 17 00:00:00 2001 From: geekgeekrun Date: Wed, 16 Apr 2025 03:00:32 +0800 Subject: [PATCH] add LlmModelUsageRecord table --- .../src/entity/LlmModelUsageRecord.ts | 61 +++++++++++++++++++ packages/sqlite-plugin/src/handlers.ts | 19 ++++++ packages/sqlite-plugin/src/index.ts | 2 + packages/ui/package.json | 1 + .../ui/src/main/features/llm-request-log.ts | 29 +++++++++ .../boss-operation.ts | 38 +++++++++++- packages/utils/gpt-request.mjs | 2 +- pnpm-lock.yaml | 7 +++ 8 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 packages/sqlite-plugin/src/entity/LlmModelUsageRecord.ts create mode 100644 packages/ui/src/main/features/llm-request-log.ts diff --git a/packages/sqlite-plugin/src/entity/LlmModelUsageRecord.ts b/packages/sqlite-plugin/src/entity/LlmModelUsageRecord.ts new file mode 100644 index 0000000..a796c32 --- /dev/null +++ b/packages/sqlite-plugin/src/entity/LlmModelUsageRecord.ts @@ -0,0 +1,61 @@ +import { requireTypeorm } from "../utils/module-loader"; +const { Entity, PrimaryGeneratedColumn, Column } = requireTypeorm() + +@Entity() +export class LlmModelUsageRecord { + @PrimaryGeneratedColumn() + id: number; + + @Column() + providerCompleteApiUrl: string + + @Column() + model: string + + @Column() + providerApiSecretMd5: string + + @Column({ + nullable: true + }) + completionTokens?: number; + + @Column({ + nullable: true + }) + promptTokens?: number; + + @Column({ + nullable: true + }) + promptCacheHitTokens?: number + + @Column({ + nullable: true + }) + promptCacheMissTokens?: number + + @Column({ + nullable: true + }) + totalTokens?: number; + + @Column() + requestStartTime: Date + + @Column({ + nullable: true + }) + requestEndTime?: Date + + @Column() + hasError: boolean + + @Column() + errorMessage: string + + @Column({ + nullable: true + }) + requestScene?: number +} diff --git a/packages/sqlite-plugin/src/handlers.ts b/packages/sqlite-plugin/src/handlers.ts index ec000c3..89bca04 100644 --- a/packages/sqlite-plugin/src/handlers.ts +++ b/packages/sqlite-plugin/src/handlers.ts @@ -10,6 +10,7 @@ import { CompanyInfoChangeLog } from "./entity/CompanyInfoChangeLog"; import { JobInfoChangeLog } from "./entity/JobInfoChangeLog"; import { MarkAsNotSuitLog } from "./entity/MarkAsNotSuitLog"; import { ChatMessageRecord } from "./entity/ChatMessageRecord"; +import { LlmModelUsageRecord } from "./entity/LlmModelUsageRecord"; function getBossInfoIfIsEqual (savedOne, currentOne) { if (savedOne === currentOne) { @@ -307,3 +308,21 @@ export async function saveChatMessageRecord( //#endregion return } + +export async function saveGptCompletionRequestRecord( + ds: DataSource, + records: LlmModelUsageRecord[] +) { + //#region mark-as-not-suit-log + const list = records.map(it => { + const o = new LlmModelUsageRecord() + for (const k of Object.keys(it)) { + o[k] = it[k] + } + return o + }) + const chatMessageRecordRepository = ds.getRepository(LlmModelUsageRecord); + await chatMessageRecordRepository.save(list); + //#endregion + return +} diff --git a/packages/sqlite-plugin/src/index.ts b/packages/sqlite-plugin/src/index.ts index 02c2a17..555b859 100644 --- a/packages/sqlite-plugin/src/index.ts +++ b/packages/sqlite-plugin/src/index.ts @@ -19,6 +19,7 @@ import { VJobLibrary } from "./entity/VJobLibrary"; import { VCompanyLibrary } from "./entity/VCompanyLibrary" import { VMarkAsNotSuitLog } from "./entity/VMarkAsNotSuitLog" import { ChatMessageRecord } from './entity/ChatMessageRecord' +import { LlmModelUsageRecord } from './entity/LlmModelUsageRecord' import sqlite3 from 'sqlite3'; import { saveChatStartupRecord, saveJobInfoFromRecommendPage, saveMarkAsNotSuitRecord } from "./handlers"; @@ -53,6 +54,7 @@ export function initDb(dbFilePath) { MarkAsNotSuitLog, VMarkAsNotSuitLog, ChatMessageRecord, + LlmModelUsageRecord, ], migrations: [ UpdateChatStartupLogTable1729182577167, diff --git a/packages/ui/package.json b/packages/ui/package.json index 93d3979..fa23227 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -73,6 +73,7 @@ "normalize.css": "^8.0.1", "prettier": "^3.2.4", "sass": "^1.70.0", + "spark-md5": "^3.0.2", "terser": "^5.37.0", "typescript": "^5.3.3", "unocss": "^0.58.5", diff --git a/packages/ui/src/main/features/llm-request-log.ts b/packages/ui/src/main/features/llm-request-log.ts new file mode 100644 index 0000000..741a74c --- /dev/null +++ b/packages/ui/src/main/features/llm-request-log.ts @@ -0,0 +1,29 @@ +import { saveGptCompletionRequestRecord } from '@geekgeekrun/sqlite-plugin/dist/handlers' + +export const RequestSceneEnum = { + testing: 1, + readNoReplyAutoReminder: 2, + geekAutoStartChatWithBoss: 3 +} +export const providerApiSecretToMd5Map = {} + +let dbInitPromise +export const recordGptCompletionRequest = async (payload) => { + const { getPublicDbFilePath } = await import( + '@geekgeekrun/geek-auto-start-chat-with-boss/runtime-file-utils.mjs' + ) + const { initDb } = await import('@geekgeekrun/sqlite-plugin') + const SparkMD5 = await import('spark-md5') + + if (!dbInitPromise) { + dbInitPromise = initDb(getPublicDbFilePath()) + } + const ds = await dbInitPromise + const o = { ...payload } + if (!providerApiSecretToMd5Map[o.providerApiSecret]) { + providerApiSecretToMd5Map[o.providerApiSecret] = SparkMD5.hash(o.providerApiSecret) + } + o.providerApiSecretMd5 = providerApiSecretToMd5Map[o.providerApiSecret] + delete o.providerApiSecret + await saveGptCompletionRequestRecord(ds, [o]) +} diff --git a/packages/ui/src/main/flow/READ_NO_REPLY_AUTO_REMINDER/boss-operation.ts b/packages/ui/src/main/flow/READ_NO_REPLY_AUTO_REMINDER/boss-operation.ts index cc7cd3e..3781895 100644 --- a/packages/ui/src/main/flow/READ_NO_REPLY_AUTO_REMINDER/boss-operation.ts +++ b/packages/ui/src/main/flow/READ_NO_REPLY_AUTO_REMINDER/boss-operation.ts @@ -1,6 +1,7 @@ import { Page } from 'puppeteer' import { sleepWithRandomDelay, sleep } from '@geekgeekrun/utils/sleep.mjs' import { completes } from '@geekgeekrun/utils/gpt-request.mjs' +import { recordGptCompletionRequest, RequestSceneEnum } from '../../features/llm-request-log' import { readConfigFile, readStorageFile, @@ -8,6 +9,7 @@ import { } from '@geekgeekrun/geek-auto-start-chat-with-boss/runtime-file-utils.mjs' import { formatResumeJsonToMarkdown } from '../../../common/utils/resume' import { SINGLE_ITEM_DEFAULT_SERVE_WEIGHT } from '../../../common/constant' +import { LlmModelUsageRecord } from '@geekgeekrun/sqlite-plugin/dist/entity/LlmModelUsageRecord' export const sendLookForwardReplyEmotion = async (page: Page) => { const emotionEntryButtonProxy = await page.$('.chat-conversation .message-controls .btn-emotion') @@ -52,7 +54,7 @@ const pickLlmConfigFromList = (llmConfigList) => { return null } const index = Math.floor(pool.length * Math.random()) - return llmConfigList.find(it => it.id === pool[index]) ?? null + return llmConfigList.find((it) => it.id === pool[index]) ?? null } // let _index = 0 @@ -149,11 +151,22 @@ export const sendGptContent = async (page: Page, chatRecords) => { const llmConfigList = await readConfigFile('llm.json') const llmConfig = pickLlmConfigFromList(llmConfigList) if (!llmConfig) { - throw new Error(`CANNOT_FIND_A_USABLE_MODEL`); + throw new Error(`CANNOT_FIND_A_USABLE_MODEL`) } console.log(llmConfig.providerCompleteApiUrl) + const llmRequestRecord: Omit & { + providerApiSecret: string + } = { + providerCompleteApiUrl: llmConfig.providerCompleteApiUrl, + model: llmConfig.model, + providerApiSecret: llmConfig.providerApiSecret, + requestStartTime: new Date(), + hasError: false, + errorMessage: '', + requestScene: RequestSceneEnum.readNoReplyAutoReminder + } try { - res = await completes( + const completion = await completes( { baseURL: llmConfig.providerCompleteApiUrl, apiKey: llmConfig.providerApiSecret, @@ -161,9 +174,28 @@ export const sendGptContent = async (page: Page, chatRecords) => { }, chatList ) + res = completion?.choices?.[0] ?? null + Object.assign(llmRequestRecord, { + completionTokens: completion.usage?.completion_token ?? null, + promptCacheHitTokens: completion.usage?.prompt_cache_hit_tokens ?? null, + promptCacheMissTokens: completion.usage?.prompt_cache_miss_tokens ?? null, + promptTokens: completion.usage?.prompt_tokens ?? null, + totalTokens: completion.usage?.total_tokens ?? null + } as LlmModelUsageRecord) } catch (err) { console.log('request failed', err) blockModelSet.add(llmConfig.id) + Object.assign(llmRequestRecord, { + hasError: true, + errorMessage: err?.message ?? '' + }) + } finally { + llmRequestRecord.requestEndTime = new Date() + try { + await recordGptCompletionRequest(llmRequestRecord) + } catch (err) { + console.log('CANNOT_SAVE_LLM_COMPLETION_LOG', err) + } } } console.log(res) diff --git a/packages/utils/gpt-request.mjs b/packages/utils/gpt-request.mjs index ff5a48a..deec331 100644 --- a/packages/utils/gpt-request.mjs +++ b/packages/utils/gpt-request.mjs @@ -22,5 +22,5 @@ export async function completes( }); console.log(completion.choices[0].message.content); - return completion.choices?.[0] ?? null; + return completion; } \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8114f88..d370d0c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -229,6 +229,9 @@ importers: sass: specifier: ^1.70.0 version: 1.70.0 + spark-md5: + specifier: ^3.0.2 + version: 3.0.2 terser: specifier: ^5.37.0 version: 5.37.0 @@ -5779,6 +5782,10 @@ packages: requiresBuild: true dev: true + /spark-md5@3.0.2: + resolution: {integrity: sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==} + dev: true + /sprintf-js@1.1.3: resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==} requiresBuild: true