samreshu_backend/tests/services/llm.service.test.ts

import { describe, it, expect, vi, beforeEach } from 'vitest';

vi.mock('../../src/config/env.js', () => ({
  env: {
    LLM_BASE_URL: 'http://test',
    LLM_MODEL: 'test-model',
    LLM_FALLBACK_MODEL: 'fallback-model',
    LLM_API_KEY: 'key',
    LLM_TIMEOUT_MS: 5000,
    LLM_TEMPERATURE: 0.7,
    LLM_MAX_TOKENS: 2048,
    LLM_MAX_RETRIES: 1,
    LLM_RETRY_DELAY_MS: 10,
  },
}));

import { LlmService } from '../../src/services/llm/llm.service.js';

const mockConfig = {
  baseUrl: 'http://llm.test/v1',
  model: 'test-model',
  fallbackModel: 'fallback-model',
  apiKey: 'test-key',
  timeoutMs: 5000,
  temperature: 0.7,
  maxTokens: 2048,
  maxRetries: 1,
  retryDelayMs: 10,
};

const validQuestionsJson = JSON.stringify({
  questions: [
    {
      questionText: 'What is 2+2?',
      type: 'single_choice',
      options: [{ key: 'a', text: '4' }, { key: 'b', text: '3' }],
      correctAnswer: 'a',
      explanation: 'Basic math',
    },
  ],
});

describe('LlmService', () => {
  let mockFetch: ReturnType<typeof vi.fn>;

  beforeEach(() => {
    mockFetch = vi.fn();
    vi.stubGlobal('fetch', mockFetch);
  });

  describe('chat', () => {
    it('returns content from LLM response', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: 'Hello!' } }],
        }),
      });

      const service = new LlmService(mockConfig);
      const result = await service.chat([
        { role: 'user', content: 'Hi' },
      ]);

      expect(result).toBe('Hello!');
      expect(mockFetch).toHaveBeenCalledWith(
        'http://llm.test/v1/chat/completions',
        expect.objectContaining({
          method: 'POST',
          headers: expect.objectContaining({
            'Content-Type': 'application/json',
            'Authorization': 'Bearer test-key',
          }),
        })
      );
    });
  });

  describe('chatWithMeta', () => {
    it('returns content and model name', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: 'Response' } }],
        }),
      });

      const service = new LlmService(mockConfig);
      const result = await service.chatWithMeta([{ role: 'user', content: 'Q' }]);

      expect(result.content).toBe('Response');
      expect(result.model).toBe('test-model');
    });

    it('retries on failure then succeeds', async () => {
      mockFetch
        .mockRejectedValueOnce(new Error('Network error'))
        .mockResolvedValueOnce({
          ok: true,
          json: () => Promise.resolve({
            choices: [{ message: { content: 'Retry OK' } }],
          }),
        });

      const service = new LlmService({ ...mockConfig, maxRetries: 1 });
      const result = await service.chatWithMeta([{ role: 'user', content: 'Q' }]);

      expect(result.content).toBe('Retry OK');
      expect(mockFetch).toHaveBeenCalledTimes(2);
    });

    it('falls back to fallbackModel when primary fails', async () => {
      // Primary model: 2 attempts (initial + 1 retry), both fail
      mockFetch
        .mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Error', text: () => Promise.resolve('') })
        .mockResolvedValueOnce({ ok: false, status: 500, statusText: 'Error', text: () => Promise.resolve('') })
        .mockResolvedValueOnce({
          ok: true,
          json: () => Promise.resolve({
            choices: [{ message: { content: 'Fallback OK' } }],
          }),
        });

      const service = new LlmService(mockConfig);
      const result = await service.chatWithMeta([{ role: 'user', content: 'Q' }]);

      expect(result.content).toBe('Fallback OK');
      expect(result.model).toBe('fallback-model');
    });

    it('throws when all attempts fail', async () => {
      mockFetch.mockRejectedValue(new Error('Network error'));

      const service = new LlmService({ ...mockConfig, maxRetries: 0 });
      await expect(
        service.chatWithMeta([{ role: 'user', content: 'Q' }])
      ).rejects.toThrow();
    });
  });

  describe('generateQuestions', () => {
    it('returns validated questions with meta', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: validQuestionsJson } }],
        }),
      });

      const service = new LlmService(mockConfig);
      const result = await service.generateQuestions({
        stack: 'js',
        level: 'beginner',
        count: 1,
      });

      expect(result.questions).toHaveLength(1);
      expect(result.questions[0].questionText).toBe('What is 2+2?');
      expect(result.questions[0].stack).toBe('js');
      expect(result.questions[0].level).toBe('beginner');
      expect(result.meta.llmModel).toBe('test-model');
      expect(result.meta.promptHash).toBeDefined();
      expect(result.meta.generationTimeMs).toBeGreaterThanOrEqual(0);
    });

    it('extracts JSON from markdown code block', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: '```json\n' + validQuestionsJson + '\n```' } }],
        }),
      });

      const service = new LlmService(mockConfig);
      const result = await service.generateQuestions({
        stack: 'ts',
        level: 'intermediate',
        count: 1,
      });

      expect(result.questions).toHaveLength(1);
      expect(result.questions[0].type).toBe('single_choice');
    });

    it('throws when response validation fails', async () => {
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: '{"invalid": "response"}' } }],
        }),
      });

      const service = new LlmService(mockConfig);
      await expect(
        service.generateQuestions({ stack: 'js', level: 'beginner', count: 1 })
      ).rejects.toThrow('LLM response validation failed');
    });

    it('throws when single_choice has no options', async () => {
      const invalidJson = JSON.stringify({
        questions: [
          {
            questionText: 'Q?',
            type: 'single_choice',
            options: [],
            correctAnswer: 'a',
            explanation: 'e',
          },
        ],
      });
      mockFetch.mockResolvedValueOnce({
        ok: true,
        json: () => Promise.resolve({
          choices: [{ message: { content: invalidJson } }],
        }),
      });

      const service = new LlmService(mockConfig);
      await expect(
        service.generateQuestions({ stack: 'js', level: 'beginner', count: 1 })
      ).rejects.toThrow('Question validation failed');
    });
  });
});