ef_front/parsingServer/server.js

import http from 'http';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { parse } from 'csv-parse/sync';
import playwright from 'playwright-extra';
const { chromium } = playwright;
import AdmZip from 'adm-zip';

// 전역 헤드리스 설정(환경변수 HEADLESS=true/false로 제어 가능) - 기본값: true
const HEADLESS = process.env.HEADLESS ? process.env.HEADLESS !== 'false' : true;
// ESM에서 __dirname 대체
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

// 최신 다운로드된 unzipped 폴더 경로 찾기
function findLatestUnzippedDir(baseDir) {
  if (!fs.existsSync(baseDir)) return null;
  const entries = fs.readdirSync(baseDir)
    .map(name => ({ name, full: path.join(baseDir, name) }))
    .filter(e => fs.statSync(e.full).isDirectory() && /unzipped$/i.test(e.name))
    .sort((a, b) => fs.statSync(b.full).mtimeMs - fs.statSync(a.full).mtimeMs);
  return entries[0]?.full || null;
}

// CSV 파일 선택(기본: '표 데이터.csv')
function chooseCsvFile(unzipDir, preferredPattern = /표 데이터\.csv$/) {
  const files = fs.readdirSync(unzipDir).filter(f => f.toLowerCase().endsWith('.csv'));
  if (files.length === 0) return null;
  const picked = preferredPattern ? (files.find(f => preferredPattern.test(f)) || files[0]) : files[0];
  return path.join(unzipDir, picked);
}

// CSV → JSON 배열 파싱
function parseCsvToJson(csvPath) {
  const csvContent = fs.readFileSync(csvPath, 'utf-8');
  return parse(csvContent, {
    columns: true,
    skip_empty_lines: true,
    bom: true,
    relax_column_count: true,
    trim: true,
  });
}

// 공통 JSON 응답 유틸
function sendJson(res, status, obj, extraHeaders = {}) {
  res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Access-Control-Allow-Origin': '*', ...extraHeaders });
  res.end(JSON.stringify(obj));
}

// 요청 본문(JSON) 파서
function parseRequestBody(req) {
  return new Promise((resolve, reject) => {
    let body = '';
    req.on('data', (chunk) => { body += chunk; });
    req.on('end', () => {
      try {
        resolve(body ? JSON.parse(body) : {});
      } catch (e) {
        reject(e);
      }
    });
    req.on('error', reject);
  });
}

// ====== 아래부터는 새로 내보내기를 수행하기 위한 자동화 유틸 ======
function ensureDirExists(dirPath) {
  if (!fs.existsSync(dirPath)) {
    fs.mkdirSync(dirPath, { recursive: true });
  }
}

const loadSession = async (filePath) => {
  const jsonData = fs.readFileSync(filePath, 'utf-8');
  return JSON.parse(jsonData);
};

async function ensureChecked(dlg, page, label) {
  const cb = dlg.getByRole('checkbox', { name: label, exact: true });
  await cb.scrollIntoViewIfNeeded();
  let state = await cb.getAttribute('aria-checked');
  if (state !== 'true') {
    await cb.click({ force: true });
    for (let i = 0; i < 5; i++) {
      await page.waitForTimeout(100);
      state = await cb.getAttribute('aria-checked');
      if (state === 'true') break;
    }
    if (state !== 'true') throw new Error(`체크 실패: ${label}`);
  }
}

async function createBrowser() {
  const optionsBrowser = {
    headless: HEADLESS,
    args: [
      '--disable-blink-features=AutomationControlled',
      '--no-sandbox',
      '--disable-web-security',
      '--disable-infobars',
      '--disable-extensions',
      '--start-maximized',
      '--window-size=1280,720',
    ],
  };
  return chromium.launch(optionsBrowser);
}

async function createContext(browser, sessionState) {
  const optionsContext = {
    userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36',
    locale: 'ko-KR',
    viewport: { width: 1280, height: 720 },
    deviceScaleFactor: 1,
    acceptDownloads: true,
    storageState: sessionState,
    extraHTTPHeaders: {
      'sec-ch-ua': '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
      'sec-ch-ua-arch': '"arm"',
      'sec-ch-ua-bitness': '"64"',
      'sec-ch-ua-form-factors': '"Desktop"',
      'sec-ch-ua-full-version': '"139.0.7258.154"',
      'sec-ch-ua-full-version-list': '"Not;A=Brand";v="99.0.0.0", "Google Chrome";v="139.0.7258.154", "Chromium";v="139.0.7258.154"',
      'sec-ch-ua-mobile': '?0',
      'sec-ch-ua-model': '""',
      'sec-ch-ua-platform': '"macOS"',
      'sec-ch-ua-platform-version': '"15.6.1"',
      'sec-ch-ua-wow64': '?0',
    }
  };
  return browser.newContext(optionsContext);
}

async function openAnalyticsAdvanced(page) {
  await page.goto('https://studio.youtube.com/');
  await page.locator('ytcp-navigation-drawer').getByRole('button', { name: '분석', exact: true }).click();
  await page.getByRole('link', { name: '고급 모드', exact: true }).click();
}

function formatKoreanDateFromYMD(ymd) {
  // 입력: '20250901' → 출력: '2025. 9. 1.'
  if (!/^\d{8}$/.test(ymd)) return null;
  const yyyy = ymd.slice(0, 4);
  const mm = String(parseInt(ymd.slice(4, 6), 10));
  const dd = String(parseInt(ymd.slice(6, 8), 10));
  return `${yyyy}. ${mm}. ${dd}.`;
}

// '2025. 9. 1.' → '20250901' 로 변환
function dotKoreanDateToYMD(dot) {
  const m = dot?.match(/^(\d{4})\.\s*(\d{1,2})\.\s*(\d{1,2})\.?$/);
  if (!m) return null;
  const yyyy = m[1];
  const mm = String(parseInt(m[2], 10)).padStart(2, '0');
  const dd = String(parseInt(m[3], 10)).padStart(2, '0');
  return `${yyyy}${mm}${dd}`;
}

// 입력칸의 기존 값을 지우고 새 값 입력
async function clearAndType(inputLocator, page, value) {
  await inputLocator.click();
  await page.keyboard.press(process.platform === 'darwin' ? 'Meta+A' : 'Control+A');
  await page.keyboard.press('Backspace');
  await inputLocator.type(value);
}

async function configureDateRangeSingleDay(page, ymdTarget) {
  await page.locator('yta-time-picker #picker-trigger ytcp-dropdown-trigger[role="button"]').click();
  await page.locator('tp-yt-paper-item[test-id="week"]').click();
  await page.locator('yta-time-picker #picker-trigger ytcp-dropdown-trigger[role="button"]').click();
  await page.locator('tp-yt-paper-item[test-id="fixed"]').click();

  const caldlg = page.locator('tp-yt-paper-dialog:has(ytcp-date-period-picker)');
  await caldlg.waitFor({ state: 'visible' });
  const endInput = caldlg.locator('#end-date input');
  await endInput.waitFor({ state: 'visible' });


  

  let startVal;
  let endVal;
  if (ymdTarget === 'latest') {
    // 가장 최신 날짜(endInput 값)를 시작일에 복사하여 최신 하루 데이터 요청
    endVal = await endInput.inputValue();
    const startInput = caldlg.locator('#start-date input');
    await clearAndType(startInput, page, endVal);
    startVal = endVal;
  } else if (ymdTarget) {
    // 최신 종료일(=현재 UI가 가진 가장 최근 일자)과 비교하여 이후면 에러
    const latestEndVal = await endInput.inputValue();
    const latestYmd = dotKoreanDateToYMD(latestEndVal);
    if (!latestYmd) throw new Error('최신 종료일 파싱에 실패했습니다.');
    if (!/^\d{8}$/.test(ymdTarget)) throw new Error('잘못된 날짜 형식입니다. (예: 20250901)');
    if (ymdTarget > latestYmd) throw new Error(`요청한 날짜(${ymdTarget})가 최신 데이터(${latestYmd})보다 이후입니다.`);

    const formatted = formatKoreanDateFromYMD(ymdTarget);
    if (!formatted) throw new Error(`잘못된 날짜 형식입니다. (예: 20250901)`);
    // 시작/종료 모두 동일 날짜로 설정
    const startInput = caldlg.locator('#start-date input');
    await clearAndType(startInput, page, formatted);
    await clearAndType(endInput, page, formatted);
    startVal = formatted;
    endVal = formatted;
  } else {
    // 지정 날짜가 없으면 기존 종료일 값을 읽어 시작일에 복사
    endVal = await endInput.inputValue();
    const startInput = caldlg.locator('#start-date input');
    await clearAndType(startInput, page, endVal);
    startVal = endVal;
  }
  await caldlg.locator('#apply-button[aria-disabled="false"] button').click();
  return { startDate: startVal, endDate: endVal };
}

async function configureMetrics(page) {
  await page.locator('yta-explore-column-picker-dropdown[title="측정항목"] ytcp-dropdown-trigger').click();
  const dlg = page.getByRole('dialog', { name: '측정항목' });
  await page.locator('h2.picker-text', { hasText: 'Premium' }).click();
  await dlg.getByRole('button', { name: '전체 선택 해제' }).click();
  await ensureChecked(dlg, page, '조회수');
  await ensureChecked(dlg, page, '유효 조회수');
  await ensureChecked(dlg, page, '시청 시간(단위: 시간)');
  await ensureChecked(dlg, page, 'YouTube Premium 조회수');
  await dlg.getByRole('button', { name: '적용' }).click();
}

async function exportCsvAndExtract(page, downloadDir) {
  ensureDirExists(downloadDir);
  const [download] = await Promise.all([
    page.waitForEvent('download'),
    (async () => {
      await page.locator('ytcp-icon-button#export-button, ytcp-icon-button[aria-label="현재 화면 내보내기"]').click();
      await page.locator('tp-yt-paper-item[test-id="CSV"]').click();
    })()
  ]);
  const suggested = download.suggestedFilename();
  const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
  const zipPath = path.join(downloadDir, `${timestamp}-${suggested || 'export.zip'}`);
  await download.saveAs(zipPath);
  const unzipDir = path.join(downloadDir, `${timestamp}-unzipped`);
  ensureDirExists(unzipDir);
  const zip = new AdmZip(zipPath);
  zip.extractAllTo(unzipDir, true);
  return { zipPath, unzipDir };
}

async function runFreshExport(ymdTarget) {
  let browser;
  try {
    browser = await createBrowser();
    const sessionData = await loadSession(path.join(__dirname, 'everlogin.json'));
    const context = await createContext(browser, sessionData);
    const page = await context.newPage();
    await openAnalyticsAdvanced(page);
    const { startDate, endDate } = await configureDateRangeSingleDay(page, ymdTarget);
    await configureMetrics(page);
    const downloadDir = path.resolve(process.cwd(), 'downloads');
    const { unzipDir } = await exportCsvAndExtract(page, downloadDir);
    const csvPath = chooseCsvFile(unzipDir, /표 데이터\.csv$/);
    if (!csvPath) throw new Error('CSV 파일을 찾지 못했습니다.');
    const data = parseCsvToJson(csvPath);
    return { file: path.basename(csvPath), count: data.length, data, startDate, endDate };
  } finally {
    try { await browser?.close(); } catch {}
  }
}

let isRunning = false;
const server = http.createServer((req, res) => {
  // CORS preflight 처리
  if (req.method === 'OPTIONS') {
    res.writeHead(204, {
      'Access-Control-Allow-Origin': '*',
      'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
      'Access-Control-Allow-Headers': 'Content-Type',
    });
    res.end();
    return;
  }

  if (req.url?.startsWith('/data')) {
    const urlObj = new URL(req.url, 'http://localhost');
    const ymd = urlObj.searchParams.get('date') || undefined;
    if (isRunning) {
      sendJson(res, 429, { error: '이미 요청 처리 중입니다. 잠시 후 다시 시도하세요.' });
      return;
    }
    isRunning = true;
    (async () => {
      try {
        const result = await runFreshExport(ymd);
        const payload = {
          ...result,
          // 요청된 ymd 대신 실제 적용된 기간의 종료일(=단일일자)로 반환
          date: result.endDate,
        };
        sendJson(res, 200, payload);
      } catch (err) {
        console.error('[data] export error:', err);
        sendJson(res, 500, { error: String(err) });
      } finally {
        isRunning = false;
      }
    })();
    return;
  }

  if (req.url?.startsWith('/isCodeMatch')) {
    if (req.method !== 'POST') {
      sendJson(res, 405, { error: 'Method Not Allowed' }, { 'Allow': 'POST, OPTIONS' });
      return;
    }
    (async () => {
      try {
        const body = await parseRequestBody(req);
        if (!body || typeof body.handle !== 'string') {
          sendJson(res, 400, { error: 'Missing handle' });
          return;
        }
        if (!body || typeof body.code !== 'string') {
          sendJson(res, 400, { error: 'Missing code' });
          return;
        }

        const handle = body.handle;
        const registerCode = body.code;
        console.log('[isCodeMatch] handle:', handle, 'code:', registerCode);
        // const registerCode = body.code; // 필요시 사용

        // Playwright로 채널 페이지 접속 및 정보 추출
        let browser;
        try {
          browser = await chromium.launch({ headless: HEADLESS, args: ['--no-sandbox'] });
          const context = await browser.newContext();
          const page = await context.newPage();
          await page.goto(`https://www.youtube.com/${handle}`, { waitUntil: 'domcontentloaded', timeout: 20000 });

          const header = page.locator('#page-header');
          const descSpan = header.locator('yt-description-preview-view-model .truncated-text-wiz__truncated-text-content span.yt-core-attributed-string').first();

          let foundtext = null;
          try {
            await descSpan.waitFor({ state: 'visible', timeout: 3000 });
            foundtext = await descSpan.innerText();
          } catch {
            const descBtn = header.getByRole('button', { name: /^설명:/ });
            try {
              await descBtn.waitFor({ state: 'attached', timeout: 5000 });
              const aria = await descBtn.getAttribute('aria-label');
              if (aria) {
                const parts = aria.split('설명:');
                if (parts.length > 1) {
                  const rest = parts[1];
                  foundtext = rest.split('...', 1)[0].trim();
                } else {
                  foundtext = aria;
                }
              }
            } catch {}
          }

          // 아바타 이미지 src 추출
          let handleText = null;
          // 채널 핸들(@...) 추출: 헤더 텍스트 → canonical 폴백
          try {
            const handleNode = header.locator('.yt-core-attributed-string:has-text("@")').first();
            await handleNode.waitFor({ state: 'visible', timeout: 4000 });
            const txt = (await handleNode.innerText())?.trim();
            const m = txt?.match(/@[^\s•|]+/);
            if (m && m[0]) handleText = m[0];
          } catch {}
          if (!handleText) {
            try {
              const href = await page.locator('link[rel="canonical"]').getAttribute('href');
              if (href) {
                const mm = href.match(/https?:\/\/www\.youtube\.com\/(%2F)?(@[^/?#]+)/);
                if (mm && mm[2]) handleText = decodeURIComponent(mm[2]);
              }
            } catch {}
          }
          let avatar = null;
          try {
            const img = header.locator('img[src^="https://yt3.googleusercontent.com/"]').first();
            await img.waitFor({ state: 'attached', timeout: 5000 });
            avatar = await img.getAttribute('src');
          } catch {}

          await context.close();
          await browser.close();
          sendJson(res, 200, { success: true, foundtext, avatar, handle: handleText });
        } catch (e) {
          try { await browser?.close(); } catch {}
          console.error('[isCodeMatch] playwright error:', e);
          sendJson(res, 400, { success: false, message: `${handle} connection failed` });
        }
      } catch (e) {
        console.error('[isCodeMatch] handler error:', e);
        sendJson(res, 400, { success: false, message: `unknown error ${String(e)}` });
      }
    })();
    return;
  }

  if (req.url?.startsWith('/gethandle')) {
    if (req.method !== 'POST') {
      sendJson(res, 405, { error: 'Method Not Allowed' }, { 'Allow': 'POST, OPTIONS' });
      return;
    }
    (async () => {
      try {
        const body = await parseRequestBody(req);
        const ids = Array.isArray(body?.ids) ? body.ids : [];
        console.log('[gethandle] received ids:', ids);

        let browser;
        let context;
        let page;
        const results = [];
        try {
          browser = await chromium.launch({ headless: HEADLESS, args: ['--no-sandbox'] });
          context = await browser.newContext();
          page = await context.newPage();

          for (const id of ids) {
            const url = `https://www.youtube.com/shorts/${id}`;
            let handleText = null;
            let avatar = null;
            try {
              console.log(`[gethandle] (${id}) goto shorts URL:`, url);
              await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });

              // 우선 채널 핸들 앵커( '/@' 로 시작) 탐색
              const handleAnchor = page.locator('ytd-reel-player-overlay-renderer a[href^="/@"]').first();
              try {
                console.log(`[gethandle] (${id}) try overlay handle selector: ytd-reel-player-overlay-renderer a[href^="/@"]`);
                await handleAnchor.waitFor({ state: 'visible', timeout: 5000 });
                handleText = (await handleAnchor.innerText())?.trim() || null;
                console.log(`[gethandle] (${id}) overlay handle found:`, handleText);
              } catch {
                // 대안: 텍스트가 '@'로 시작하는 앵커
                const alt = page.locator('a:has-text("@")').first();
                try {
                  console.log(`[gethandle] (${id}) overlay handle not found, try alt anchor with '@' text`);
                  await alt.waitFor({ state: 'attached', timeout: 4000 });
                  const t = (await alt.innerText())?.trim();
                  handleText = t && t.startsWith('@') ? t : handleText;
                  if (handleText) console.log(`[gethandle] (${id}) alt handle found:`, handleText);
                } catch {}
              }

              // 추가 대안: 메타패널 내부의 링크들 중 '@' 포함
              if (!handleText) {
                console.log(`[gethandle] (${id}) try metapanel handle extraction`);
                const metaLink = page.locator('#metapanel a').first();
                try {
                  await metaLink.waitFor({ state: 'attached', timeout: 3000 });
                  const t = (await metaLink.innerText())?.trim();
                  handleText = t && t.startsWith('@') ? t : handleText;
                  if (handleText) console.log(`[gethandle] (${id}) metapanel handle found:`, handleText);
                } catch {}
              }

              // 최종 대안: 채널 이름 영역의 채널 링크로 이동 후 채널 페이지에서 @handle/아바타 추출
              if (!handleText) {
                try {
                  console.log(`[gethandle] (${id}) overlay handle not found, try channel link route`);
                  const channelNameAnchor = page.locator('ytd-channel-name a[href^="/channel/"]').first();
                  await channelNameAnchor.waitFor({ state: 'attached', timeout: 4000 });
                  const href = await channelNameAnchor.getAttribute('href');
                  if (href) {
                    const channelUrl = `https://www.youtube.com${href}`;
                    console.log(`[gethandle] (${id}) navigate to channel URL:`, channelUrl);
                    await page.goto(channelUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
                    // 채널 핸들(@...)
                    try {
                      console.log(`[gethandle] (${id}) try channel page handle selector: a[href^="/@"]`);
                      const chHandle = page.locator('a[href^="/@"]').first();
                      await chHandle.waitFor({ state: 'visible', timeout: 5000 });
                      const t = (await chHandle.innerText())?.trim();
                      if (t && t.startsWith('@')) handleText = t;
                      if (handleText) console.log(`[gethandle] (${id}) channel page handle found:`, handleText);
                    } catch {}
                    // 대체1: 채널 헤더 내 텍스트 기반(@...) 추출
                    if (!handleText) {
                      try {
                        console.log(`[gethandle] (${id}) try channel header text selector: #page-header .yt-core-attributed-string:has-text("@")`);
                        const textNode = page.locator('#page-header .yt-core-attributed-string:has-text("@")').first();
                        await textNode.waitFor({ state: 'visible', timeout: 4000 });
                        const txt = (await textNode.innerText())?.trim();
                        const m = txt?.match(/@[^\s•|]+/);
                        if (m && m[0]) handleText = m[0];
                        if (handleText) console.log(`[gethandle] (${id}) channel header text handle found:`, handleText);
                      } catch {}
                    }
                    // 대체2: canonical 링크에서 @handle 파싱
                    if (!handleText) {
                      try {
                        console.log(`[gethandle] (${id}) try canonical link for handle`);
                        const href = await page.locator('link[rel="canonical"]').getAttribute('href');
                        if (href) {
                          const m = href.match(/https?:\/\/www\.youtube\.com\/(%2F)?(@[^/?#]+)/);
                          if (m && m[2]) handleText = decodeURIComponent(m[2]);
                        }
                        if (handleText) console.log(`[gethandle] (${id}) canonical handle found:`, handleText);
                      } catch {}
                    }
                    // 채널 아바타
                    try {
                      console.log(`[gethandle] (${id}) try channel page avatar selector: img[src*="yt3"]`);
                      const chAvatar = page.locator('img[src*="yt3"], img[src*="yt3.ggpht.com"]').first();
                      await chAvatar.waitFor({ state: 'attached', timeout: 4000 });
                      const src = await chAvatar.getAttribute('src');
                      if (src) avatar = src;
                      if (avatar) console.log(`[gethandle] (${id}) channel page avatar found:`, avatar);
                    } catch {}
                  }
                } catch {}
              }

              // 아바타 이미지 추출(overlay 내 yt3 도메인 이미지)
              try {
                if (!avatar) console.log(`[gethandle] (${id}) try overlay avatar selector: ytd-reel-player-overlay-renderer img[src*="yt3"]`);
                const avatarImg = page.locator('ytd-reel-player-overlay-renderer img[src*="yt3"]').first();
                await avatarImg.waitFor({ state: 'attached', timeout: 4000 });
                avatar = await avatarImg.getAttribute('src');
                if (avatar) console.log(`[gethandle] (${id}) overlay avatar found:`, avatar);
              } catch {}

              results.push({ id, url, handle: handleText, avatar });
            } catch (e) {
              console.error('[gethandle] navigate/parsing error for id', id, e);
              results.push({ id, url, error: 'navigation_failed' });
            }
          }
        } finally {
          try { await page?.close(); } catch {}
          try { await context?.close(); } catch {}
          try { await browser?.close(); } catch {}
        }

        sendJson(res, 200, { success: true, count: results.length, items: results });
      } catch (e) {
        console.error('[gethandle] handler error:', e);
        sendJson(res, 400, { success: false, message: `invalid body ${String(e)}` });
      }
    })();
    return;
  }

  // 기본 루트
  res.writeHead(200, { 'Content-Type': 'text/plain; charset=utf-8' });
  res.end('OK. GET /data 로 CSV JSON을 가져오세요.');
});

const PORT = process.env.PORT ? Number(process.env.PORT) : 9556;
server.listen(PORT, () => {
  console.log(`HTTP server listening on http://localhost:${PORT}`);
});