add LlmModelUsageRecord table

packages/sqlite-plugin/src/entity/LlmModelUsageRecord.ts

@@ -0,0 +1,61 @@
+import { requireTypeorm } from "../utils/module-loader";
+const { Entity, PrimaryGeneratedColumn, Column } = requireTypeorm()
+
+@Entity()
+export class LlmModelUsageRecord {
+  @PrimaryGeneratedColumn()
+  id: number;
+
+  @Column()
+  providerCompleteApiUrl: string
+
+  @Column()
+  model: string
+
+  @Column()
+  providerApiSecretMd5: string
+
+  @Column({
+    nullable: true
+  })
+  completionTokens?: number;
+
+  @Column({
+    nullable: true
+  })
+  promptTokens?: number;
+
+  @Column({
+    nullable: true
+  })
+  promptCacheHitTokens?: number
+
+  @Column({
+    nullable: true
+  })
+  promptCacheMissTokens?: number
+
+  @Column({
+    nullable: true
+  })
+  totalTokens?: number;
+
+  @Column()
+  requestStartTime: Date
+
+  @Column({
+    nullable: true
+  })
+  requestEndTime?: Date
+
+  @Column()
+  hasError: boolean
+
+  @Column()
+  errorMessage: string
+
+  @Column({
+    nullable: true
+  })
+  requestScene?: number
+}

packages/sqlite-plugin/src/handlers.ts

@@ -10,6 +10,7 @@ import { CompanyInfoChangeLog } from "./entity/CompanyInfoChangeLog";
 import { JobInfoChangeLog } from "./entity/JobInfoChangeLog";
 import { MarkAsNotSuitLog } from "./entity/MarkAsNotSuitLog";
 import { ChatMessageRecord } from "./entity/ChatMessageRecord";
+import { LlmModelUsageRecord } from "./entity/LlmModelUsageRecord";
 
 function getBossInfoIfIsEqual (savedOne, currentOne) {
   if (savedOne === currentOne) {
@@ -307,3 +308,21 @@ export async function saveChatMessageRecord(
   //#endregion
   return
 }
+
+export async function saveGptCompletionRequestRecord(
+  ds: DataSource,
+  records: LlmModelUsageRecord[]
+) {
+  //#region mark-as-not-suit-log
+  const list = records.map(it => {
+    const o = new LlmModelUsageRecord()
+    for (const k of Object.keys(it)) {
+      o[k] = it[k]
+    }
+    return o
+  })
+  const chatMessageRecordRepository = ds.getRepository(LlmModelUsageRecord);
+  await chatMessageRecordRepository.save(list);
+  //#endregion
+  return
+}

packages/sqlite-plugin/src/index.ts

@@ -19,6 +19,7 @@ import { VJobLibrary } from "./entity/VJobLibrary";
 import { VCompanyLibrary } from "./entity/VCompanyLibrary"
 import { VMarkAsNotSuitLog } from "./entity/VMarkAsNotSuitLog"
 import { ChatMessageRecord } from './entity/ChatMessageRecord'
+import { LlmModelUsageRecord } from './entity/LlmModelUsageRecord'
 
 import sqlite3 from 'sqlite3';
 import { saveChatStartupRecord, saveJobInfoFromRecommendPage, saveMarkAsNotSuitRecord } from "./handlers";
@@ -53,6 +54,7 @@ export function initDb(dbFilePath) {
       MarkAsNotSuitLog,
       VMarkAsNotSuitLog,
       ChatMessageRecord,
+      LlmModelUsageRecord,
     ],
     migrations: [
       UpdateChatStartupLogTable1729182577167,

packages/ui/package.json

@@ -73,6 +73,7 @@
     "normalize.css": "^8.0.1",
     "prettier": "^3.2.4",
     "sass": "^1.70.0",
+    "spark-md5": "^3.0.2",
     "terser": "^5.37.0",
     "typescript": "^5.3.3",
     "unocss": "^0.58.5",

packages/ui/src/main/features/llm-request-log.ts

@@ -0,0 +1,29 @@
+import { saveGptCompletionRequestRecord } from '@geekgeekrun/sqlite-plugin/dist/handlers'
+
+export const RequestSceneEnum = {
+  testing: 1,
+  readNoReplyAutoReminder: 2,
+  geekAutoStartChatWithBoss: 3
+}
+export const providerApiSecretToMd5Map = {}
+
+let dbInitPromise
+export const recordGptCompletionRequest = async (payload) => {
+  const { getPublicDbFilePath } = await import(
+    '@geekgeekrun/geek-auto-start-chat-with-boss/runtime-file-utils.mjs'
+  )
+  const { initDb } = await import('@geekgeekrun/sqlite-plugin')
+  const SparkMD5 = await import('spark-md5')
+
+  if (!dbInitPromise) {
+    dbInitPromise = initDb(getPublicDbFilePath())
+  }
+  const ds = await dbInitPromise
+  const o = { ...payload }
+  if (!providerApiSecretToMd5Map[o.providerApiSecret]) {
+    providerApiSecretToMd5Map[o.providerApiSecret] = SparkMD5.hash(o.providerApiSecret)
+  }
+  o.providerApiSecretMd5 = providerApiSecretToMd5Map[o.providerApiSecret]
+  delete o.providerApiSecret
+  await saveGptCompletionRequestRecord(ds, [o])
+}

packages/ui/src/main/flow/READ_NO_REPLY_AUTO_REMINDER/boss-operation.ts

@@ -1,6 +1,7 @@
 import { Page } from 'puppeteer'
 import { sleepWithRandomDelay, sleep } from '@geekgeekrun/utils/sleep.mjs'
 import { completes } from '@geekgeekrun/utils/gpt-request.mjs'
+import { recordGptCompletionRequest, RequestSceneEnum } from '../../features/llm-request-log'
 import {
   readConfigFile,
   readStorageFile,
@@ -8,6 +9,7 @@ import {
 } from '@geekgeekrun/geek-auto-start-chat-with-boss/runtime-file-utils.mjs'
 import { formatResumeJsonToMarkdown } from '../../../common/utils/resume'
 import { SINGLE_ITEM_DEFAULT_SERVE_WEIGHT } from '../../../common/constant'
+import { LlmModelUsageRecord } from '@geekgeekrun/sqlite-plugin/dist/entity/LlmModelUsageRecord'
 
 export const sendLookForwardReplyEmotion = async (page: Page) => {
   const emotionEntryButtonProxy = await page.$('.chat-conversation .message-controls .btn-emotion')
@@ -52,7 +54,7 @@ const pickLlmConfigFromList = (llmConfigList) => {
     return null
   }
   const index = Math.floor(pool.length * Math.random())
-  return llmConfigList.find(it => it.id === pool[index]) ?? null
+  return llmConfigList.find((it) => it.id === pool[index]) ?? null
 }
 
 // let _index = 0
@@ -149,11 +151,22 @@ export const sendGptContent = async (page: Page, chatRecords) => {
     const llmConfigList = await readConfigFile('llm.json')
     const llmConfig = pickLlmConfigFromList(llmConfigList)
     if (!llmConfig) {
-      throw new Error(`CANNOT_FIND_A_USABLE_MODEL`);
+      throw new Error(`CANNOT_FIND_A_USABLE_MODEL`)
     }
     console.log(llmConfig.providerCompleteApiUrl)
+    const llmRequestRecord: Omit<LlmModelUsageRecord, 'id' | 'providerApiSecretMd5'> & {
+      providerApiSecret: string
+    } = {
+      providerCompleteApiUrl: llmConfig.providerCompleteApiUrl,
+      model: llmConfig.model,
+      providerApiSecret: llmConfig.providerApiSecret,
+      requestStartTime: new Date(),
+      hasError: false,
+      errorMessage: '',
+      requestScene: RequestSceneEnum.readNoReplyAutoReminder
+    }
     try {
-      res = await completes(
+      const completion = await completes(
         {
           baseURL: llmConfig.providerCompleteApiUrl,
           apiKey: llmConfig.providerApiSecret,
@@ -161,9 +174,28 @@ export const sendGptContent = async (page: Page, chatRecords) => {
         },
         chatList
       )
+      res = completion?.choices?.[0] ?? null
+      Object.assign(llmRequestRecord, {
+        completionTokens: completion.usage?.completion_token ?? null,
+        promptCacheHitTokens: completion.usage?.prompt_cache_hit_tokens ?? null,
+        promptCacheMissTokens: completion.usage?.prompt_cache_miss_tokens ?? null,
+        promptTokens: completion.usage?.prompt_tokens ?? null,
+        totalTokens: completion.usage?.total_tokens ?? null
+      } as LlmModelUsageRecord)
     } catch (err) {
       console.log('request failed', err)
       blockModelSet.add(llmConfig.id)
+      Object.assign(llmRequestRecord, {
+        hasError: true,
+        errorMessage: err?.message ?? ''
+      })
+    } finally {
+      llmRequestRecord.requestEndTime = new Date()
+      try {
+        await recordGptCompletionRequest(llmRequestRecord)
+      } catch (err) {
+        console.log('CANNOT_SAVE_LLM_COMPLETION_LOG', err)
+      }
     }
   }
   console.log(res)

packages/utils/gpt-request.mjs

@@ -22,5 +22,5 @@ export async function completes(
   });
 
   console.log(completion.choices[0].message.content);
-  return completion.choices?.[0] ?? null;
+  return completion;
 }

pnpm-lock.yaml

@@ -229,6 +229,9 @@ importers:
       sass:
         specifier: ^1.70.0
         version: 1.70.0
+      spark-md5:
+        specifier: ^3.0.2
+        version: 3.0.2
       terser:
         specifier: ^5.37.0
         version: 5.37.0
@@ -5779,6 +5782,10 @@ packages:
     requiresBuild: true
     dev: true
 
+  /spark-md5@3.0.2:
+    resolution: {integrity: sha512-wcFzz9cDfbuqe0FZzfi2or1sgyIrsDwmPwfZC4hiNidPdPINjeUwNfv5kldczoEAcjl9Y1L3SM7Uz2PUEQzxQw==}
+    dev: true
+
   /sprintf-js@1.1.3:
     resolution: {integrity: sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==}
     requiresBuild: true