10장: 실전 프로젝트 -- AI 테스트 자동화 파이프라인

ecommerce-app/
  src/
    services/        # 비즈니스 로직 (결제, 주문, 사용자)
    api/             # REST API 엔드포인트
    components/      # React UI 컴포넌트
    pages/           # 페이지 컴포넌트
  tests/
    unit/            # AI 생성 단위 테스트
    integration/     # AI 생성 통합 테스트
    e2e/             # AI 기반 E2E 테스트
    visual/          # 시각적 회귀 테스트
    mutation/        # 변이 테스트 설정
  .github/
    workflows/       # AI QA 파이프라인
  scripts/
    ai-test/         # AI 테스트 도구 스크립트

{
  "language": "typescript",
  "testFramework": "jest",
  "testDirectory": "tests/unit",
  "coverageTarget": 85,
  "generateOnSave": false,
  "generateOnPR": true,
  "patterns": {
    "include": ["src/services/**/*.ts", "src/utils/**/*.ts"],
    "exclude": ["**/*.d.ts", "**/*.test.ts"]
  },
  "style": {
    "naming": "describe-it",
    "assertion": "expect",
    "groupBy": "method"
  }
}

name: AI Unit Test Generation
 
on:
  pull_request:
    paths:
      - "src/services/**"
      - "src/utils/**"
 
jobs:
  generate-tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
 
      - name: Get changed source files
        id: changes
        run: |
          FILES=$(git diff --name-only origin/main...HEAD -- 'src/services/*.ts' 'src/utils/*.ts' | tr '\n' ' ')
          echo "files=$FILES" >> $GITHUB_OUTPUT
 
      - name: Generate unit tests
        if: steps.changes.outputs.files != ''
        run: |
          npx qodo generate \
            --files="${{ steps.changes.outputs.files }}" \
            --output=tests/unit/ \
            --coverage-target=85
 
      - name: Run generated tests
        run: npx jest tests/unit/ --ci --coverage
 
      - name: Check coverage threshold
        run: |
          node scripts/check-coverage.js \
            --threshold=85 \
            --report=coverage/coverage-summary.json

import { readFileSync } from "fs";
import { parse } from "yaml";
 
interface ApiTestConfig {
  schemaPath: string;
  outputDir: string;
  baseUrl: string;
  authStrategy: "bearer" | "session" | "none";
}
 
async function generateApiTests(config: ApiTestConfig): Promise<void> {
  const schema = parse(readFileSync(config.schemaPath, "utf-8"));
 
  for (const [path, methods] of Object.entries(schema.paths)) {
    for (const [method, spec] of Object.entries(methods as Record<string, any>)) {
      const testCases = deriveTestCases(path, method, spec);
 
      const testCode = generateTestCode({
        path,
        method,
        testCases,
        baseUrl: config.baseUrl,
        authStrategy: config.authStrategy,
      });
 
      writeTestFile(config.outputDir, path, method, testCode);
    }
  }
}
 
function deriveTestCases(
  path: string,
  method: string,
  spec: any
): TestCase[] {
  const cases: TestCase[] = [];
 
  // 정상 요청 테스트
  cases.push({
    name: `${method.toUpperCase()} ${path} - 정상 요청`,
    type: "happy",
    expectedStatus: parseInt(Object.keys(spec.responses)[0]),
  });
 
  // 유효성 검증 실패 테스트
  if (spec.requestBody?.required) {
    cases.push({
      name: `${method.toUpperCase()} ${path} - 필수 필드 누락`,
      type: "validation",
      expectedStatus: 400,
    });
  }
 
  // 인증 실패 테스트
  if (spec.security) {
    cases.push({
      name: `${method.toUpperCase()} ${path} - 인증 없음`,
      type: "auth",
      expectedStatus: 401,
    });
  }
 
  return cases;
}

import { PostgreSqlContainer } from "@testcontainers/postgresql";
import { RedisContainer } from "@testcontainers/redis";
 
let pgContainer: any;
let redisContainer: any;
 
export async function setupIntegrationEnv() {
  // 병렬로 컨테이너 시작
  const [pg, redis] = await Promise.all([
    new PostgreSqlContainer("postgres:16").start(),
    new RedisContainer("redis:7").start(),
  ]);
 
  pgContainer = pg;
  redisContainer = redis;
 
  process.env.DATABASE_URL = pg.getConnectionUri();
  process.env.REDIS_URL = `redis://${redis.getHost()}:${redis.getPort()}`;
 
  // 마이그레이션 실행
  await runMigrations(process.env.DATABASE_URL);
 
  return { pg, redis };
}
 
export async function teardownIntegrationEnv() {
  await pgContainer?.stop();
  await redisContainer?.stop();
}

import { test as base, expect } from "@playwright/test";
 
interface AiFixtures {
  aiPage: AiPage;
}
 
class AiPage {
  constructor(private page: any) {}
 
  /**
   * 의미 기반 요소 탐색 - 셀렉터가 실패하면 대체 전략 시도
   */
  async findAndClick(description: string): Promise<void> {
    const strategies = [
      () => this.page.getByRole("button", { name: description }),
      () => this.page.getByRole("link", { name: description }),
      () => this.page.getByText(description, { exact: false }),
      () => this.page.getByLabel(description),
      () => this.page.getByTestId(description.toLowerCase().replace(/\s/g, "-")),
    ];
 
    for (const strategy of strategies) {
      const locator = strategy();
      if ((await locator.count()) > 0) {
        await locator.first().click();
        return;
      }
    }
 
    throw new Error(`클릭할 요소를 찾을 수 없습니다: ${description}`);
  }
 
  async findAndFill(description: string, value: string): Promise<void> {
    const strategies = [
      () => this.page.getByLabel(description),
      () => this.page.getByPlaceholder(description),
      () => this.page.getByRole("textbox", { name: description }),
    ];
 
    for (const strategy of strategies) {
      const locator = strategy();
      if ((await locator.count()) > 0) {
        await locator.first().fill(value);
        return;
      }
    }
 
    throw new Error(`입력할 요소를 찾을 수 없습니다: ${description}`);
  }
}
 
export const test = base.extend<AiFixtures>({
  aiPage: async ({ page }, use) => {
    await use(new AiPage(page));
  },
});

import { test, expect } from "./fixtures";
 
test.describe("구매 플로우", () => {
  test("상품 검색부터 결제 완료까지", async ({ page, aiPage }) => {
    await page.goto("/");
 
    // 1. 상품 검색
    await aiPage.findAndFill("검색", "무선 키보드");
    await page.keyboard.press("Enter");
    await expect(page.getByText("검색 결과")).toBeVisible();
 
    // 2. 상품 선택
    await page.getByText("무선 키보드 프로").first().click();
    await expect(page).toHaveURL(/products/);
 
    // 3. 장바구니 추가
    await aiPage.findAndClick("장바구니에 추가");
    await expect(page.getByText("추가되었습니다")).toBeVisible();
 
    // 4. 장바구니로 이동
    await aiPage.findAndClick("장바구니");
    await expect(page.getByText("무선 키보드 프로")).toBeVisible();
 
    // 5. 결제 진행
    await aiPage.findAndClick("결제하기");
 
    // 6. 배송 정보 입력
    await aiPage.findAndFill("수령인", "홍길동");
    await aiPage.findAndFill("연락처", "010-1234-5678");
    await aiPage.findAndFill("주소", "서울특별시 강남구 테헤란로 123");
 
    // 7. 결제 수단 선택
    await aiPage.findAndClick("카드 결제");
 
    // 8. 결제 완료 확인
    await aiPage.findAndClick("결제 완료");
    await expect(page.getByText("주문이 완료되었습니다")).toBeVisible({
      timeout: 10000,
    });
  });
});

import { test } from "@playwright/test";
import {
  Eyes,
  Target,
  Configuration,
  BatchInfo,
  BrowserType,
  DeviceName,
  ScreenOrientation,
} from "@applitools/eyes-playwright";
 
test.describe("페이지별 시각적 검증", () => {
  let eyes: Eyes;
 
  test.beforeEach(async () => {
    eyes = new Eyes();
    const config = new Configuration();
    config.setBatch(new BatchInfo("Release Visual Regression"));
    config.setApiKey(process.env.APPLITOOLS_API_KEY!);
 
    // 다중 브라우저 + 디바이스
    config.addBrowser(1920, 1080, BrowserType.CHROME);
    config.addBrowser(1366, 768, BrowserType.FIREFOX);
    config.addDeviceEmulation(DeviceName.iPhone_14, ScreenOrientation.PORTRAIT);
    config.addDeviceEmulation(DeviceName.Pixel_7, ScreenOrientation.PORTRAIT);
 
    eyes.setConfiguration(config);
  });
 
  test.afterEach(async () => {
    await eyes.close();
  });
 
  const pages = [
    { name: "메인 페이지", path: "/" },
    { name: "상품 목록", path: "/products" },
    { name: "로그인", path: "/login" },
    { name: "회원가입", path: "/signup" },
  ];
 
  for (const pageInfo of pages) {
    test(`${pageInfo.name} 시각적 검증`, async ({ page }) => {
      await eyes.open(page, "E-Commerce", pageInfo.name);
      await page.goto(pageInfo.path);
 
      await eyes.check(
        pageInfo.name,
        Target.window()
          .fully()
          .ignoreRegions(".ad-banner", "[data-dynamic]")
          .strict()
      );
    });
  }
});

{
  "$schema": "./node_modules/@stryker-mutator/core/schema/stryker-schema.json",
  "mutate": ["src/services/**/*.ts", "!src/**/*.test.ts"],
  "testRunner": "jest",
  "jest": {
    "configFile": "jest.config.ts"
  },
  "checkers": ["typescript"],
  "reporters": ["html", "json", "clear-text"],
  "thresholds": {
    "high": 80,
    "low": 60,
    "break": 50
  },
  "concurrency": 4,
  "timeoutMS": 15000,
  "incremental": true,
  "incrementalFile": ".stryker-cache/incremental.json"
}

name: AI QA Complete Pipeline
 
on:
  pull_request:
    branches: [main]
 
concurrency:
  group: ai-qa-${{ github.head_ref }}
  cancel-in-progress: true
 
env:
  NODE_VERSION: "22"
 
jobs:
  # 0단계: 변경 분석
  analyze:
    runs-on: ubuntu-latest
    outputs:
      risk: ${{ steps.analyze.outputs.risk }}
      affected-tests: ${{ steps.analyze.outputs.tests }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Analyze changes
        id: analyze
        run: node scripts/ai-test/analyze-changes.js
 
  # 1단계: 단위 테스트 (항상 실행, 병렬 4 샤드)
  unit-test:
    needs: analyze
    runs-on: ubuntu-latest
    strategy:
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - name: Run unit tests
        run: pnpm test:unit -- --shard=${{ matrix.shard }}/4 --ci --coverage
      - uses: actions/upload-artifact@v4
        with:
          name: coverage-unit-${{ matrix.shard }}
          path: coverage/
 
  # 2단계: 통합 테스트 (중간 이상 위험도)
  integration-test:
    needs: [analyze, unit-test]
    if: needs.analyze.outputs.risk != 'low'
    runs-on: ubuntu-latest
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_DB: testdb
          POSTGRES_PASSWORD: testpass
        ports:
          - 5432:5432
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - name: Run integration tests
        run: pnpm test:integration --ci
        env:
          DATABASE_URL: postgresql://postgres:testpass@localhost:5432/testdb
 
  # 3단계: E2E 테스트 (높은 위험도)
  e2e-test:
    needs: [analyze, integration-test]
    if: needs.analyze.outputs.risk == 'high'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: npx playwright install --with-deps
      - name: Run E2E tests
        run: pnpm test:e2e
      - uses: actions/upload-artifact@v4
        if: failure()
        with:
          name: e2e-traces
          path: test-results/
 
  # 4단계: 시각적 테스트 (UI 변경 시)
  visual-test:
    needs: [analyze, e2e-test]
    if: contains(needs.analyze.outputs.affected-tests, 'visual')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - run: npx playwright install --with-deps
      - name: Run visual tests
        run: pnpm test:visual
        env:
          APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
 
  # 5단계: 변이 테스트 (증분, 주 1회 전체)
  mutation-test:
    needs: unit-test
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: "pnpm"
      - run: pnpm install --frozen-lockfile
      - name: Run incremental mutation testing
        run: |
          CHANGED=$(git diff --name-only origin/main...HEAD -- 'src/services/*.ts' | tr '\n' ',')
          if [ -n "$CHANGED" ]; then
            npx stryker run --mutate="$CHANGED"
          fi
 
  # 6단계: 품질 게이트
  quality-gate:
    needs: [unit-test, integration-test, e2e-test, visual-test, mutation-test]
    if: always()
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          pattern: coverage-*
          merge-multiple: true
          path: coverage/
      - name: Merge coverage reports
        run: node scripts/ai-test/merge-coverage.js
      - name: Evaluate quality gate
        run: |
          node scripts/ai-test/quality-gate.js \
            --coverage-threshold=80 \
            --mutation-threshold=60
      - name: Post PR comment
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v7
        with:
          script: |
            const report = require('./quality-report.json');
            const body = formatQualityReport(report);
            github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: body,
            });

interface DashboardData {
  timestamp: string;
  coverage: {
    line: number;
    branch: number;
    function: number;
    mutation: number;
  };
  stability: {
    ciPassRate: number;
    flakyRate: number;
    mttrMinutes: number;
    skipRate: number;
  };
  efficiency: {
    totalDurationMinutes: number;
    testsPerMinute: number;
    autoHealedCount: number;
    aiGeneratedTests: number;
  };
  value: {
    defectEscapeRate: number;
    aiFoundDefects: number;
    blockedDeployments: number;
  };
}

연간 ROI = (연간 절감액 - 연간 비용) / 연간 비용 x 100%
 
예시:
  도구 비용: 월 200만 원 = 연 2,400만 원
  인프라 추가 비용: 연 600만 원
  학습 비용: 연 300만 원 (1회성)
  총 비용: 연 3,300만 원
 
  테스트 작성 절감: 연 4,800만 원 (개발자 4명 x 월 100만 원 절감)
  유지보수 절감: 연 1,200만 원
  결함 비용 절감: 연 2,000만 원 (프로덕션 결함 10건 감소 x 건당 200만 원)
  CI 비용 절감: 연 360만 원
  총 절감: 연 8,360만 원
 
  ROI = (8,360 - 3,300) / 3,300 x 100% = 153%