motaju/ef_front
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

middle 2

Browse Source
This commit is contained in:
motaju
2025-09-09 00:15:08 +00:00
parent 0cce0322a1
commit fd34eb370f
30 changed files with 1953 additions and 35881 deletions
Download Patch File Download Diff File Expand all files Collapse all files

503
parsingServer/server.js Normal file
View File

@@ -0,0 +1,503 @@
import http from 'http';
import fs from 'fs';
import path from 'path';
import { fileURLToPath } from 'url';
import { parse } from 'csv-parse/sync';
import playwright from 'playwright-extra';
const { chromium } = playwright;
import AdmZip from 'adm-zip';
// 전역 헤드리스 설정(환경변수 HEADLESS=true/false로 제어 가능) - 기본값: true
const HEADLESS = process.env.HEADLESS ? process.env.HEADLESS !== 'false' : true;
// ESM에서 __dirname 대체
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// 최신 다운로드된 unzipped 폴더 경로 찾기
function findLatestUnzippedDir(baseDir) {
if (!fs.existsSync(baseDir)) return null;
const entries = fs.readdirSync(baseDir)
.map(name => ({ name, full: path.join(baseDir, name) }))
.filter(e => fs.statSync(e.full).isDirectory() && /unzipped$/i.test(e.name))
.sort((a, b) => fs.statSync(b.full).mtimeMs - fs.statSync(a.full).mtimeMs);
return entries[0]?.full || null;
}
// CSV 파일 선택(기본: '표 데이터.csv')
function chooseCsvFile(unzipDir, preferredPattern = /표 데이터\.csv$/) {
const files = fs.readdirSync(unzipDir).filter(f => f.toLowerCase().endsWith('.csv'));
if (files.length === 0) return null;
const picked = preferredPattern ? (files.find(f => preferredPattern.test(f)) || files[0]) : files[0];
return path.join(unzipDir, picked);
}
// CSV → JSON 배열 파싱
function parseCsvToJson(csvPath) {
const csvContent = fs.readFileSync(csvPath, 'utf-8');
return parse(csvContent, {
columns: true,
skip_empty_lines: true,
bom: true,
relax_column_count: true,
trim: true,
});
}
// 공통 JSON 응답 유틸
function sendJson(res, status, obj, extraHeaders = {}) {
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8', 'Access-Control-Allow-Origin': '*', ...extraHeaders });
res.end(JSON.stringify(obj));
}
// 요청 본문(JSON) 파서
function parseRequestBody(req) {
return new Promise((resolve, reject) => {
let body = '';
req.on('data', (chunk) => { body += chunk; });
req.on('end', () => {
try {
resolve(body ? JSON.parse(body) : {});
} catch (e) {
reject(e);
}
});
req.on('error', reject);
});
}
// ====== 아래부터는 새로 내보내기를 수행하기 위한 자동화 유틸 ======
function ensureDirExists(dirPath) {
if (!fs.existsSync(dirPath)) {
fs.mkdirSync(dirPath, { recursive: true });
}
}
const loadSession = async (filePath) => {
const jsonData = fs.readFileSync(filePath, 'utf-8');
return JSON.parse(jsonData);
};
async function ensureChecked(dlg, page, label) {
const cb = dlg.getByRole('checkbox', { name: label, exact: true });
await cb.scrollIntoViewIfNeeded();
let state = await cb.getAttribute('aria-checked');
if (state !== 'true') {
await cb.click({ force: true });
for (let i = 0; i < 5; i++) {
await page.waitForTimeout(100);
state = await cb.getAttribute('aria-checked');
if (state === 'true') break;
}
if (state !== 'true') throw new Error(`체크 실패: ${label}`);
}
}
async function createBrowser() {
const optionsBrowser = {
headless: HEADLESS,
args: [
'--disable-blink-features=AutomationControlled',
'--no-sandbox',
'--disable-web-security',
'--disable-infobars',
'--disable-extensions',
'--start-maximized',
'--window-size=1280,720',
],
};
return chromium.launch(optionsBrowser);
}
async function createContext(browser, sessionState) {
const optionsContext = {
userAgent: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36',
locale: 'ko-KR',
viewport: { width: 1280, height: 720 },
deviceScaleFactor: 1,
acceptDownloads: true,
storageState: sessionState,
extraHTTPHeaders: {
'sec-ch-ua': '"Not;A=Brand";v="99", "Google Chrome";v="139", "Chromium";v="139"',
'sec-ch-ua-arch': '"arm"',
'sec-ch-ua-bitness': '"64"',
'sec-ch-ua-form-factors': '"Desktop"',
'sec-ch-ua-full-version': '"139.0.7258.154"',
'sec-ch-ua-full-version-list': '"Not;A=Brand";v="99.0.0.0", "Google Chrome";v="139.0.7258.154", "Chromium";v="139.0.7258.154"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-model': '""',
'sec-ch-ua-platform': '"macOS"',
'sec-ch-ua-platform-version': '"15.6.1"',
'sec-ch-ua-wow64': '?0',
}
};
return browser.newContext(optionsContext);
}
async function openAnalyticsAdvanced(page) {
await page.goto('https://studio.youtube.com/');
await page.locator('ytcp-navigation-drawer').getByRole('button', { name: '분석', exact: true }).click();
await page.getByRole('link', { name: '고급 모드', exact: true }).click();
}
function formatKoreanDateFromYMD(ymd) {
// 입력: '20250901' → 출력: '2025. 9. 1.'
if (!/^\d{8}$/.test(ymd)) return null;
const yyyy = ymd.slice(0, 4);
const mm = String(parseInt(ymd.slice(4, 6), 10));
const dd = String(parseInt(ymd.slice(6, 8), 10));
return `${yyyy}. ${mm}. ${dd}.`;
}
// 입력칸의 기존 값을 지우고 새 값 입력
async function clearAndType(inputLocator, page, value) {
await inputLocator.click();
await page.keyboard.press(process.platform === 'darwin' ? 'Meta+A' : 'Control+A');
await page.keyboard.press('Backspace');
await inputLocator.type(value);
}
async function configureDateRangeSingleDay(page, ymdTarget) {
await page.locator('yta-time-picker #picker-trigger ytcp-dropdown-trigger[role="button"]').click();
await page.locator('tp-yt-paper-item[test-id="week"]').click();
await page.locator('yta-time-picker #picker-trigger ytcp-dropdown-trigger[role="button"]').click();
await page.locator('tp-yt-paper-item[test-id="fixed"]').click();
const caldlg = page.locator('tp-yt-paper-dialog:has(ytcp-date-period-picker)');
await caldlg.waitFor({ state: 'visible' });
const endInput = caldlg.locator('#end-date input');
await endInput.waitFor({ state: 'visible' });
let startVal;
let endVal;
if (ymdTarget === 'latest') {
// 가장 최신 날짜(endInput 값)를 시작일에 복사하여 최신 하루 데이터 요청
endVal = await endInput.inputValue();
const startInput = caldlg.locator('#start-date input');
await clearAndType(startInput, page, endVal);
startVal = endVal;
} else if (ymdTarget) {
const formatted = formatKoreanDateFromYMD(ymdTarget);
if (!formatted) throw new Error(`잘못된 날짜 형식입니다. (예: 20250901)`);
// 시작/종료 모두 동일 날짜로 설정
const startInput = caldlg.locator('#start-date input');
await clearAndType(startInput, page, formatted);
await clearAndType(endInput, page, formatted);
startVal = formatted;
endVal = formatted;
} else {
// 지정 날짜가 없으면 기존 종료일 값을 읽어 시작일에 복사
endVal = await endInput.inputValue();
const startInput = caldlg.locator('#start-date input');
await clearAndType(startInput, page, endVal);
startVal = endVal;
}
await caldlg.locator('#apply-button[aria-disabled="false"] button').click();
return { startDate: startVal, endDate: endVal };
}
async function configureMetrics(page) {
await page.locator('yta-explore-column-picker-dropdown[title="측정항목"] ytcp-dropdown-trigger').click();
const dlg = page.getByRole('dialog', { name: '측정항목' });
await page.locator('h2.picker-text', { hasText: 'Premium' }).click();
await dlg.getByRole('button', { name: '전체 선택 해제' }).click();
await ensureChecked(dlg, page, '조회수');
await ensureChecked(dlg, page, '유효 조회수');
await ensureChecked(dlg, page, '시청 시간(단위: 시간)');
await ensureChecked(dlg, page, 'YouTube Premium 조회수');
await dlg.getByRole('button', { name: '적용' }).click();
}
async function exportCsvAndExtract(page, downloadDir) {
ensureDirExists(downloadDir);
const [download] = await Promise.all([
page.waitForEvent('download'),
(async () => {
await page.locator('ytcp-icon-button#export-button, ytcp-icon-button[aria-label="현재 화면 내보내기"]').click();
await page.locator('tp-yt-paper-item[test-id="CSV"]').click();
})()
]);
const suggested = download.suggestedFilename();
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
const zipPath = path.join(downloadDir, `${timestamp}-${suggested || 'export.zip'}`);
await download.saveAs(zipPath);
const unzipDir = path.join(downloadDir, `${timestamp}-unzipped`);
ensureDirExists(unzipDir);
const zip = new AdmZip(zipPath);
zip.extractAllTo(unzipDir, true);
return { zipPath, unzipDir };
}
async function runFreshExport(ymdTarget) {
let browser;
try {
browser = await createBrowser();
const sessionData = await loadSession(path.join(__dirname, 'everlogin.json'));
const context = await createContext(browser, sessionData);
const page = await context.newPage();
await openAnalyticsAdvanced(page);
const { startDate, endDate } = await configureDateRangeSingleDay(page, ymdTarget);
await configureMetrics(page);
const downloadDir = path.resolve(process.cwd(), 'downloads');
const { unzipDir } = await exportCsvAndExtract(page, downloadDir);
const csvPath = chooseCsvFile(unzipDir, /표 데이터\.csv$/);
if (!csvPath) throw new Error('CSV 파일을 찾지 못했습니다.');
const data = parseCsvToJson(csvPath);
return { file: path.basename(csvPath), count: data.length, data, startDate, endDate };
} finally {
try { await browser?.close(); } catch {}
}
}
let isRunning = false;
const server = http.createServer((req, res) => {
// CORS preflight 처리
if (req.method === 'OPTIONS') {
res.writeHead(204, {
'Access-Control-Allow-Origin': '*',
'Access-Control-Allow-Methods': 'GET,POST,OPTIONS',
'Access-Control-Allow-Headers': 'Content-Type',
});
res.end();
return;
}
if (req.url?.startsWith('/data')) {
const urlObj = new URL(req.url, 'http://localhost');
const ymd = urlObj.searchParams.get('date') || undefined;
if (isRunning) {
sendJson(res, 429, { error: '이미 요청 처리 중입니다. 잠시 후 다시 시도하세요.' });
return;
}
isRunning = true;
(async () => {
try {
const result = await runFreshExport(ymd);
const payload = {
...result,
date: ymd === 'latest' ? result.endDate : (ymd || result.endDate),
};
sendJson(res, 200, payload);
} catch (err) {
console.error('[data] export error:', err);
sendJson(res, 500, { error: String(err) });
} finally {
isRunning = false;
}
})();
return;
}
if (req.url?.startsWith('/isCodeMatch')) {
if (req.method !== 'POST') {
sendJson(res, 405, { error: 'Method Not Allowed' }, { 'Allow': 'POST, OPTIONS' });
return;
}
(async () => {
try {
const body = await parseRequestBody(req);
if (!body || typeof body.handle !== 'string') {
sendJson(res, 400, { error: 'Missing handle' });
return;
}
if (!body || typeof body.code !== 'string') {
sendJson(res, 400, { error: 'Missing code' });
return;
}
const handle = body.handle;
const registerCode = body.code;
console.log('[isCodeMatch] handle:', handle, 'code:', registerCode);
// const registerCode = body.code; // 필요시 사용
// Playwright로 채널 페이지 접속 및 정보 추출
let browser;
try {
browser = await chromium.launch({ headless: HEADLESS, args: ['--no-sandbox'] });
const context = await browser.newContext();
const page = await context.newPage();
await page.goto(`https://www.youtube.com/${handle}`, { waitUntil: 'domcontentloaded', timeout: 20000 });
const header = page.locator('#page-header');
const descSpan = header.locator('yt-description-preview-view-model .truncated-text-wiz__truncated-text-content span.yt-core-attributed-string').first();
let foundtext = null;
try {
await descSpan.waitFor({ state: 'visible', timeout: 3000 });
foundtext = await descSpan.innerText();
} catch {
const descBtn = header.getByRole('button', { name: /^설명:/ });
try {
await descBtn.waitFor({ state: 'attached', timeout: 5000 });
const aria = await descBtn.getAttribute('aria-label');
if (aria) {
const parts = aria.split('설명:');
if (parts.length > 1) {
const rest = parts[1];
foundtext = rest.split('...', 1)[0].trim();
} else {
foundtext = aria;
}
}
} catch {}
}
// 아바타 이미지 src 추출
let avatar = null;
try {
const img = header.locator('img[src^="https://yt3.googleusercontent.com/"]').first();
await img.waitFor({ state: 'attached', timeout: 5000 });
avatar = await img.getAttribute('src');
} catch {}
await context.close();
await browser.close();
sendJson(res, 200, { success: true, foundtext, avatar });
} catch (e) {
try { await browser?.close(); } catch {}
console.error('[isCodeMatch] playwright error:', e);
sendJson(res, 400, { success: false, message: `${handle} connection failed` });
}
} catch (e) {
console.error('[isCodeMatch] handler error:', e);
sendJson(res, 400, { success: false, message: `unknown error ${String(e)}` });
}
})();
return;
}
if (req.url?.startsWith('/gethandle')) {
if (req.method !== 'POST') {
sendJson(res, 405, { error: 'Method Not Allowed' }, { 'Allow': 'POST, OPTIONS' });
return;
}
(async () => {
try {
const body = await parseRequestBody(req);
const ids = Array.isArray(body?.ids) ? body.ids : [];
console.log('[gethandle] received ids:', ids);
let browser;
let context;
let page;
const results = [];
try {
browser = await chromium.launch({ headless: HEADLESS, args: ['--no-sandbox'] });
context = await browser.newContext();
page = await context.newPage();
for (const id of ids) {
const url = `https://www.youtube.com/shorts/${id}`;
let handleText = null;
let avatar = null;
try {
console.log(`[gethandle] (${id}) goto shorts URL:`, url);
await page.goto(url, { waitUntil: 'domcontentloaded', timeout: 20000 });
// 우선 채널 핸들 앵커( '/@' 로 시작) 탐색
const handleAnchor = page.locator('ytd-reel-player-overlay-renderer a[href^="/@"]').first();
try {
console.log(`[gethandle] (${id}) try overlay handle selector: ytd-reel-player-overlay-renderer a[href^="/@"]`);
await handleAnchor.waitFor({ state: 'visible', timeout: 5000 });
handleText = (await handleAnchor.innerText())?.trim() || null;
console.log(`[gethandle] (${id}) overlay handle found:`, handleText);
} catch {
// 대안: 텍스트가 '@'로 시작하는 앵커
const alt = page.locator('a:has-text("@")').first();
try {
console.log(`[gethandle] (${id}) overlay handle not found, try alt anchor with '@' text`);
await alt.waitFor({ state: 'attached', timeout: 4000 });
const t = (await alt.innerText())?.trim();
handleText = t && t.startsWith('@') ? t : handleText;
if (handleText) console.log(`[gethandle] (${id}) alt handle found:`, handleText);
} catch {}
}
// 추가 대안: 메타패널 내부의 링크들 중 '@' 포함
if (!handleText) {
console.log(`[gethandle] (${id}) try metapanel handle extraction`);
const metaLink = page.locator('#metapanel a').first();
try {
await metaLink.waitFor({ state: 'attached', timeout: 3000 });
const t = (await metaLink.innerText())?.trim();
handleText = t && t.startsWith('@') ? t : handleText;
if (handleText) console.log(`[gethandle] (${id}) metapanel handle found:`, handleText);
} catch {}
}
// 최종 대안: 채널 이름 영역의 채널 링크로 이동 후 채널 페이지에서 @handle/아바타 추출
if (!handleText) {
try {
console.log(`[gethandle] (${id}) overlay handle not found, try channel link route`);
const channelNameAnchor = page.locator('ytd-channel-name a[href^="/channel/"]').first();
await channelNameAnchor.waitFor({ state: 'attached', timeout: 4000 });
const href = await channelNameAnchor.getAttribute('href');
if (href) {
const channelUrl = `https://www.youtube.com${href}`;
console.log(`[gethandle] (${id}) navigate to channel URL:`, channelUrl);
await page.goto(channelUrl, { waitUntil: 'domcontentloaded', timeout: 20000 });
// 채널 핸들(@...)
try {
console.log(`[gethandle] (${id}) try channel page handle selector: a[href^="/@"]`);
const chHandle = page.locator('a[href^="/@"]').first();
await chHandle.waitFor({ state: 'visible', timeout: 5000 });
const t = (await chHandle.innerText())?.trim();
if (t && t.startsWith('@')) handleText = t;
if (handleText) console.log(`[gethandle] (${id}) channel page handle found:`, handleText);
} catch {}
// 채널 아바타
try {
console.log(`[gethandle] (${id}) try channel page avatar selector: img[src*="yt3"]`);
const chAvatar = page.locator('img[src*="yt3"], img[src*="yt3.ggpht.com"]').first();
await chAvatar.waitFor({ state: 'attached', timeout: 4000 });
const src = await chAvatar.getAttribute('src');
if (src) avatar = src;
if (avatar) console.log(`[gethandle] (${id}) channel page avatar found:`, avatar);
} catch {}
}
} catch {}
}
// 아바타 이미지 추출(overlay 내 yt3 도메인 이미지)
try {
if (!avatar) console.log(`[gethandle] (${id}) try overlay avatar selector: ytd-reel-player-overlay-renderer img[src*="yt3"]`);
const avatarImg = page.locator('ytd-reel-player-overlay-renderer img[src*="yt3"]').first();
await avatarImg.waitFor({ state: 'attached', timeout: 4000 });
avatar = await avatarImg.getAttribute('src');
if (avatar) console.log(`[gethandle] (${id}) overlay avatar found:`, avatar);
} catch {}
results.push({ id, url, handle: handleText, avatar });
} catch (e) {
console.error('[gethandle] navigate/parsing error for id', id, e);
results.push({ id, url, error: 'navigation_failed' });
}
}
} finally {
try { await page?.close(); } catch {}
try { await context?.close(); } catch {}
try { await browser?.close(); } catch {}
}
sendJson(res, 200, { success: true, count: results.length, items: results });
} catch (e) {
console.error('[gethandle] handler error:', e);
sendJson(res, 400, { success: false, message: `invalid body ${String(e)}` });
}
})();
return;
}
// 기본 루트
res.writeHead(200, { 'Content-Type': 'text/plain; charset=utf-8' });
res.end('OK. GET /data 로 CSV JSON을 가져오세요.');
});
const PORT = process.env.PORT ? Number(process.env.PORT) : 9556;
server.listen(PORT, () => {
console.log(`HTTP server listening on http://localhost:${PORT}`);
});