/** * Copyright (c) 2019 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); you * may not use this file except in compliance with the License. You may * obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or * implied. See the License for the specific language governing * permissions and limitations under the License. */ 'use strict'; // If you add or remove job types, do not forget to fix the colspans below. const JOB_TYPES = [ { id: 'linux-gcc7-x86_64-release', label: 'rel' }, { id: 'linux-clang-x86_64-debug', label: 'dbg' }, { id: 'linux-clang-x86_64-tsan', label: 'tsan' }, { id: 'linux-clang-x86_64-msan', label: 'msan' }, { id: 'linux-clang-x86_64-asan_lsan', label: '{a,l}san' }, { id: 'linux-clang-x86-asan_lsan', label: 'x86 {a,l}san' }, { id: 'linux-clang-x86_64-libfuzzer', label: 'fuzzer' }, { id: 'linux-clang-x86_64-bazel', label: 'bazel' }, { id: 'ui-clang-x86_64-release', label: 'rel' }, { id: 'android-clang-arm-release', label: 'rel' }, { id: 'android-clang-arm-asan', label: 'asan' }, ]; const STATS_LINK = 'https://app.google.stackdriver.com/dashboards/5008687313278081798?project=perfetto-ci'; const state = { // An array of recent CL objects retrieved from Gerrit. gerritCls: [], // A map of sha1 -> Gerrit commit object. // See https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-commit gerritCommits: {}, // A map of git-log ranges to commit objects: // 'dead..beef' -> [commit1, 2] gerritLogs: {}, // Maps 'cls/1234-1' or 'branches/xxxx' -> array of job ids. dbJobSets: {}, // Maps 'jobId' -> DB job object, as perf /ci/jobs/jobID. // A jobId looks like 20190702143507-1008614-9-android-clang-arm. dbJobs: {}, // Maps 'worker id' -> DB wokrker object, as per /ci/workers. dbWorker: {}, // Maps 'master-YYMMDD' -> DB branch object, as perf /ci/branches/xxx. dbBranches: {}, getBranchKeys: () => Object.keys(state.dbBranches).sort().reverse(), // Maps 'CL number' -> true|false. Retains the collapsed/expanded information // for each row in the CLs table. expandCl: {}, postsubmitShown: 3, // Lines that will be appended to the terminal on the next redraw() cycle. termLines: [ 'Hover a CL icon to see the log tail.', 'Click on it to load the full log.' ], termJobId: undefined, // The job id currently being shown by the terminal. termClear: false, // If true the next redraw will clear the terminal. redrawPending: false, // State for the Jobs page. These are arrays of job ids. jobsQueued: [], jobsRunning: [], jobsRecent: [], // Firebase DB listeners (the objects returned by the .ref() operator). realTimeLogRef: undefined, // Ref for the real-time log streaming. workersRef: undefined, jobsRunningRef: undefined, jobsQueuedRef: undefined, jobsRecentRef: undefined, clRefs: {}, // '1234-1' -> Ref subscribed to updates on the given cl. jobRefs: {}, // '....-arm-asan' -> Ref subscribed updates on the given job. branchRefs: {} // 'master' -> Ref subscribed updates on the given branch. }; let term = undefined; let fitAddon = undefined; let searchAddon = undefined; function main() { firebase.initializeApp({ databaseURL: cfg.DB_ROOT }); m.route(document.body, '/cls', { '/cls': CLsPageRenderer, '/cls/:cl': CLsPageRenderer, '/logs/:jobId': LogsPageRenderer, '/jobs': JobsPageRenderer, '/jobs/:jobId': JobsPageRenderer, }); setInterval(fetchGerritCLs, 15000); fetchGerritCLs(); fetchCIStatusForBranch('master'); } // ----------------------------------------------------------------------------- // Rendering functions // ----------------------------------------------------------------------------- function renderHeader() { const active = id => m.route.get().startsWith(`/${id}`) ? '.active' : ''; const logUrl = 'https://goto.google.com/perfetto-ci-logs-'; const docsUrl = 'https://perfetto.dev/docs/design-docs/continuous-integration'; return m( 'header', m('a[href=/#!/cls]', m('h1', 'Perfetto ', m('span', 'CI'))), m( 'nav', m(`div${active('cls')}`, m('a[href=/#!/cls]', 'CLs')), m(`div${active('jobs')}`, m('a[href=/#!/jobs]', 'Jobs')), m(`div${active('stats')}`, m(`a[href=${STATS_LINK}][target=_blank]`, 'Stats')), m(`div`, m(`a[href=${docsUrl}][target=_blank]`, 'Docs')), m( `div.logs`, 'Logs', m('div', m(`a[href=${logUrl}controller][target=_blank]`, 'Controller')), m('div', m(`a[href=${logUrl}workers][target=_blank]`, 'Workers')), m('div', m(`a[href=${logUrl}frontend][target=_blank]`, 'Frontend')), ), )); } var CLsPageRenderer = { view: function (vnode) { const allCols = 4 + JOB_TYPES.length; const postsubmitHeader = m('tr', m(`td.header[colspan=${allCols}]`, 'Post-submit') ); const postsubmitLoadMore = m('tr', m(`td[colspan=${allCols}]`, m('a[href=#]', { onclick: () => state.postsubmitShown += 10 }, 'Load more' ) ) ); const presubmitHeader = m('tr', m(`td.header[colspan=${allCols}]`, 'Pre-submit') ); let branchRows = []; const branchKeys = state.getBranchKeys(); for (let i = 0; i < branchKeys.length && i < state.postsubmitShown; i++) { const rowsForBranch = renderPostsubmitRow(branchKeys[i]); branchRows = branchRows.concat(rowsForBranch); } let clRows = []; for (const gerritCl of state.gerritCls) { if (vnode.attrs.cl && gerritCl.num != vnode.attrs.cl) continue; clRows = clRows.concat(renderCLRow(gerritCl)); } let footer = []; if (vnode.attrs.cl) { footer = m('footer', `Showing only CL ${vnode.attrs.cl} - `, m(`a[href=#!/cls]`, 'Click here to see all CLs') ); } return [ renderHeader(), m('main#cls', m('div.table-scrolling-container', m('table.main-table', m('thead', m('tr', m('td[rowspan=4]', 'Subject'), m('td[rowspan=4]', 'Status'), m('td[rowspan=4]', 'Owner'), m('td[rowspan=4]', 'Updated'), m('td[colspan=11]', 'Bots'), ), m('tr', m('td[colspan=9]', 'linux'), m('td[colspan=2]', 'android'), ), m('tr', m('td', 'gcc7'), m('td[colspan=7]', 'clang'), m('td[colspan=1]', 'ui'), m('td[colspan=2]', 'clang-arm'), ), m('tr#cls_header', JOB_TYPES.map(job => m(`td#${job.id}`, job.label)) ), ), m('tbody', postsubmitHeader, branchRows, postsubmitLoadMore, presubmitHeader, clRows, ) ), footer, ), m(TermRenderer), ), ]; } }; function getLastUpdate(lastUpdate) { const lastUpdateMins = Math.ceil((Date.now() - lastUpdate) / 60000); if (lastUpdateMins < 60) return lastUpdateMins + ' mins ago'; if (lastUpdateMins < 60 * 24) return Math.ceil(lastUpdateMins / 60) + ' hours ago'; return lastUpdate.toLocaleDateString(); } function renderCLRow(cl) { const expanded = !!state.expandCl[cl.num]; const toggleExpand = () => { state.expandCl[cl.num] ^= 1; fetchCIJobsForAllPatchsetOfCL(cl.num); } const rows = []; // Create the row for the latest patchset (as fetched by Gerrit). rows.push(m(`tr.${cl.status}`, m('td', m(`i.material-icons.expand${expanded ? '.expanded' : ''}`, { onclick: toggleExpand }, 'arrow_right' ), m(`a[href=${cfg.GERRIT_REVIEW_URL}/+/${cl.num}/${cl.psNum}]`, `${cl.subject}`, m('span.ps', `#${cl.psNum}`)) ), m('td', cl.status), m('td', stripEmail(cl.owner)), m('td', getLastUpdate(cl.lastUpdate)), JOB_TYPES.map(x => renderClJobCell(`cls/${cl.num}-${cl.psNum}`, x.id)) )); // If the usere clicked on the expand button, show also the other patchsets // present in the CI DB. for (let psNum = cl.psNum; expanded && psNum > 0; psNum--) { const src = `cls/${cl.num}-${psNum}`; const jobs = state.dbJobSets[src]; if (!jobs) continue; rows.push(m(`tr.nested`, m('td', m(`a[href=${cfg.GERRIT_REVIEW_URL}/+/${cl.num}/${psNum}]`, ' Patchset', m('span.ps', `#${psNum}`)) ), m('td', ''), m('td', ''), m('td', ''), JOB_TYPES.map(x => renderClJobCell(src, x.id)) )); } return rows; } function renderPostsubmitRow(key) { const branch = state.dbBranches[key]; console.assert(branch !== undefined); const subject = branch.subject; let rows = []; rows.push(m(`tr`, m('td', m(`a[href=${cfg.REPO_URL}/+/${branch.rev}]`, subject, m('span.ps', `#${branch.rev.substr(0, 8)}`) ) ), m('td', ''), m('td', stripEmail(branch.author)), m('td', getLastUpdate(new Date(branch.time_committed))), JOB_TYPES.map(x => renderClJobCell(`branches/${key}`, x.id)) )); const allKeys = state.getBranchKeys(); const curIdx = allKeys.indexOf(key); if (curIdx >= 0 && curIdx < allKeys.length - 1) { const nextKey = allKeys[curIdx + 1]; const range = `${state.dbBranches[nextKey].rev}..${branch.rev}`; const logs = (state.gerritLogs[range] || []).slice(1); for (const log of logs) { if (log.parents.length < 2) continue; // Show only merge commits. rows.push( m('tr.nested', m('td', m(`a[href=${cfg.REPO_URL}/+/${log.commit}]`, log.message.split('\n')[0], m('span.ps', `#${log.commit.substr(0, 8)}`) ) ), m('td', ''), m('td', stripEmail(log.author.email)), m('td', getLastUpdate(parseGerritTime(log.committer.time))), m(`td[colspan=${JOB_TYPES.length}]`, 'No post-submit was run for this revision' ), ) ); } } return rows; } function renderJobLink(jobId, jobStatus) { const ICON_MAP = { 'COMPLETED': 'check_circle', 'STARTED': 'hourglass_full', 'QUEUED': 'schedule', 'FAILED': 'bug_report', 'CANCELLED': 'cancel', 'INTERRUPTED': 'cancel', 'TIMED_OUT': 'notification_important', }; const icon = ICON_MAP[jobStatus] || 'clear'; const eventHandlers = jobId ? { onmouseover: () => showLogTail(jobId) } : {}; const logUrl = jobId ? `#!/logs/${jobId}` : '#'; return m(`a.${jobStatus}[href=${logUrl}][title=${jobStatus}]`, eventHandlers, m(`i.material-icons`, icon) ); } function renderClJobCell(src, jobType) { let jobStatus = 'UNKNOWN'; let jobId = undefined; // To begin with check that the given CL/PS is present in the DB (the // AppEngine cron job might have not seen that at all yet). // If it is, find the global job id for the given jobType for the passed CL. for (const id of (state.dbJobSets[src] || [])) { const job = state.dbJobs[id]; if (job !== undefined && job.type == jobType) { // We found the job object that corresponds to jobType for the given CL. jobStatus = job.status; jobId = id; } } return m('td.job', renderJobLink(jobId, jobStatus)); } const TermRenderer = { oncreate: function(vnode) { console.log('Creating terminal object'); fitAddon = new FitAddon.FitAddon(); searchAddon = new SearchAddon.SearchAddon(); term = new Terminal({ rows: 6, fontFamily: 'monospace', fontSize: 12, scrollback: 100000, disableStdin: true, }); term.loadAddon(fitAddon); term.loadAddon(searchAddon); term.open(vnode.dom); fitAddon.fit(); if (vnode.attrs.focused) term.focus(); }, onremove: function(vnode) { term.dispose(); fitAddon.dispose(); searchAddon.dispose(); }, onupdate: function(vnode) { fitAddon.fit(); if (state.termClear) { term.clear(); state.termClear = false; } for (const line of state.termLines) { term.write(line + '\r\n'); } state.termLines = []; }, view: function() { return m('.term-container', { onkeydown: (e) => { if (e.key === 'f' && (e.ctrlKey || e.metaKey)) { document.querySelector('.term-search').select(); e.preventDefault(); } } }, m('input[type=text][placeholder=search and press Enter].term-search', { onkeydown: (e) => { if (e.key !== 'Enter') return; if (e.shiftKey) { searchAddon.findNext(e.target.value); } else { searchAddon.findPrevious(e.target.value); } e.stopPropagation(); e.preventDefault(); } }) ); } }; const LogsPageRenderer = { oncreate: function (vnode) { showFullLog(vnode.attrs.jobId); }, view: function () { return [ renderHeader(), m(TermRenderer, { focused: true }) ]; } } const JobsPageRenderer = { oncreate: function (vnode) { fetchRecentJobsStatus(); fetchWorkers(); }, createWorkerTable: function () { const makeWokerRow = workerId => { const worker = state.dbWorker[workerId]; if (worker.status === 'TERMINATED') return []; return m('tr', m('td', worker.host), m('td', workerId), m('td', worker.status), m('td', getLastUpdate(new Date(worker.last_update))), m('td', m(`a[href=#!/jobs/${worker.job_id}]`, worker.job_id)), ); }; return m('table.main-table', m('thead', m('tr', m('td[colspan=5]', 'Workers')), m('tr', m('td', 'Host'), m('td', 'Worker'), m('td', 'Status'), m('td', 'Last ping'), m('td', 'Job'), ) ), m('tbody', Object.keys(state.dbWorker).map(makeWokerRow)) ); }, createJobsTable: function (vnode, title, jobIds) { const tStr = function (tStart, tEnd) { return new Date(tEnd - tStart).toUTCString().substr(17, 9); }; const makeJobRow = function (jobId) { const job = state.dbJobs[jobId] || {}; let cols = [ m('td.job.align-left', renderJobLink(jobId, job ? job.status : undefined), m(`span.status.${job.status}`, job.status) ) ]; if (job) { const tQ = Date.parse(job.time_queued); const tS = Date.parse(job.time_started); const tE = Date.parse(job.time_ended) || Date.now(); let cell = m(''); if (job.src === undefined) { cell = '?'; } else if (job.src.startsWith('cls/')) { const cl_and_ps = job.src.substr(4).replace('-', '/'); const href = `${cfg.GERRIT_REVIEW_URL}/+/${cl_and_ps}`; cell = m(`a[href=${href}][target=_blank]`, cl_and_ps); } else if (job.src.startsWith('branches/')) { cell = job.src.substr(9).split('-')[0] } cols.push(m('td', cell)); cols.push(m('td', `${job.type}`)); cols.push(m('td', `${job.worker || ''}`)); cols.push(m('td', `${job.time_queued}`)); cols.push(m(`td[title=Start ${job.time_started}]`, `${tStr(tQ, tS)}`)); cols.push(m(`td[title=End ${job.time_ended}]`, `${tStr(tS, tE)}`)); } else { cols.push(m('td[colspan=6]', jobId)); } return m(`tr${vnode.attrs.jobId === jobId ? '.selected' : ''}`, cols) }; return m('table.main-table', m('thead', m('tr', m('td[colspan=7]', title)), m('tr', m('td', 'Status'), m('td', 'CL'), m('td', 'Type'), m('td', 'Worker'), m('td', 'T queued'), m('td', 'Queue time'), m('td', 'Run time'), ) ), m('tbody', jobIds.map(makeJobRow)) ); }, view: function (vnode) { return [ renderHeader(), m('main', m('.jobs-list', this.createWorkerTable(), this.createJobsTable(vnode, 'Queued + Running jobs', state.jobsRunning.concat(state.jobsQueued)), this.createJobsTable(vnode, 'Last 100 jobs', state.jobsRecent), ), ) ]; } }; // ----------------------------------------------------------------------------- // Business logic (handles fetching from Gerrit and Firebase DB). // ----------------------------------------------------------------------------- function parseGerritTime(str) { // Gerrit timestamps are UTC (as per public docs) but obviously they are not // encoded in ISO format. return new Date(`${str} UTC`); } function stripEmail(email) { return email.replace('@google.com', '@'); } // Fetches the list of CLs from gerrit and updates the state. async function fetchGerritCLs() { console.log('Fetching CL list from Gerrit'); let uri = '/gerrit/changes/?-age:7days'; uri += '+-is:abandoned&o=DETAILED_ACCOUNTS&o=CURRENT_REVISION'; const response = await fetch(uri); state.gerritCls = []; if (response.status !== 200) { setTimeout(fetchGerritCLs, 3000); // Retry. return; } const json = (await response.text()); const cls = []; for (const e of JSON.parse(json)) { const revHash = Object.keys(e.revisions)[0]; const cl = { subject: e.subject, status: e.status, num: e._number, revHash: revHash, psNum: e.revisions[revHash]._number, lastUpdate: parseGerritTime(e.updated), owner: e.owner.email, }; cls.push(cl); fetchCIJobsForCLOrBranch(`cls/${cl.num}-${cl.psNum}`); } state.gerritCls = cls; scheduleRedraw(); } async function fetchGerritCommit(sha1) { const response = await fetch(`/gerrit/commits/${sha1}`); console.assert(response.status === 200); const json = (await response.text()); state.gerritCommits[sha1] = JSON.parse(json); scheduleRedraw(); } async function fetchGerritLog(first, second) { const range = `${first}..${second}`; const response = await fetch(`/gerrit/log/${range}`); if (response.status !== 200) return; const json = await response.text(); state.gerritLogs[range] = JSON.parse(json).log; scheduleRedraw(); } // Retrieves the status of a given (CL, PS) in the DB. function fetchCIJobsForCLOrBranch(src) { if (src in state.clRefs) return; // Aslready have a listener for this key. const ref = firebase.database().ref(`/ci/${src}`); state.clRefs[src] = ref; ref.on('value', (e) => { const obj = e.val(); if (!obj) return; state.dbJobSets[src] = Object.keys(obj.jobs); for (var jobId of state.dbJobSets[src]) { fetchCIStatusForJob(jobId); } scheduleRedraw(); }); } function fetchCIJobsForAllPatchsetOfCL(cl) { let ref = firebase.database().ref('/ci/cls').orderByKey(); ref = ref.startAt(`${cl}-0`).endAt(`${cl}-~`); ref.once('value', (e) => { const patchsets = e.val() || {}; for (const clAndPs in patchsets) { const jobs = Object.keys(patchsets[clAndPs].jobs); state.dbJobSets[`cls/${clAndPs}`] = jobs; for (var jobId of jobs) { fetchCIStatusForJob(jobId); } } scheduleRedraw(); }); } function fetchCIStatusForJob(jobId) { if (jobId in state.jobRefs) return; // Already have a listener for this key. const ref = firebase.database().ref(`/ci/jobs/${jobId}`); state.jobRefs[jobId] = ref; ref.on('value', (e) => { if (e.val()) state.dbJobs[jobId] = e.val(); scheduleRedraw(); }); } function fetchCIStatusForBranch(branch) { if (branch in state.branchRefs) return; // Already have a listener. const db = firebase.database(); const ref = db.ref('/ci/branches').orderByKey().limitToLast(20); state.branchRefs[branch] = ref; ref.on('value', (e) => { const resp = e.val(); if (!resp) return; // key looks like 'master-YYYYMMDDHHMMSS', where YMD is the commit datetime. // Iterate in most-recent-first order. const keys = Object.keys(resp).sort().reverse(); for (let i = 0; i < keys.length; i++) { const key = keys[i]; const branchInfo = resp[key]; state.dbBranches[key] = branchInfo; fetchCIJobsForCLOrBranch(`branches/${key}`); if (i < keys.length - 1) { fetchGerritLog(resp[keys[i + 1]].rev, branchInfo.rev); } } scheduleRedraw(); }); } function fetchWorkers() { if (state.workersRef !== undefined) return; // Aslready have a listener. const ref = firebase.database().ref('/ci/workers'); state.workersRef = ref; ref.on('value', (e) => { state.dbWorker = e.val() || {}; scheduleRedraw(); }); } async function showLogTail(jobId) { if (state.termJobId === jobId) return; // Already on it. const TAIL = 20; state.termClear = true; state.termLines = [ `Fetching last ${TAIL} lines for ${jobId}.`, `Click on the CI icon to see the full log.` ]; state.termJobId = jobId; scheduleRedraw(); const ref = firebase.database().ref(`/ci/logs/${jobId}`); const lines = (await ref.orderByKey().limitToLast(TAIL).once('value')).val(); if (state.termJobId !== jobId || !lines) return; const lastKey = appendLogLinesAndRedraw(lines); startRealTimeLogs(jobId, lastKey); } async function showFullLog(jobId) { state.termClear = true; state.termLines = [`Fetching full for ${jobId} ...`]; state.termJobId = jobId; scheduleRedraw(); // Suspend any other real-time logging in progress. stopRealTimeLogs(); // Starts a chain of async tasks that fetch the current log lines in batches. state.termJobId = jobId; const ref = firebase.database().ref(`/ci/logs/${jobId}`).orderByKey(); let lastKey = ''; const BATCH = 1000; for (; ;) { const batchRef = ref.startAt(`${lastKey}!`).limitToFirst(BATCH); const logs = (await batchRef.once('value')).val(); if (!logs) break; lastKey = appendLogLinesAndRedraw(logs); } startRealTimeLogs(jobId, lastKey) } function startRealTimeLogs(jobId, lastLineKey) { stopRealTimeLogs(); console.log('Starting real-time logs for ', jobId); state.termJobId = jobId; let ref = firebase.database().ref(`/ci/logs/${jobId}`); ref = ref.orderByKey().startAt(`${lastLineKey}!`); state.realTimeLogRef = ref; state.realTimeLogRef.on('child_added', res => { const line = res.val(); if (state.termJobId !== jobId || !line) return; const lines = {}; lines[res.key] = line; appendLogLinesAndRedraw(lines); }); } function stopRealTimeLogs() { if (state.realTimeLogRef !== undefined) { state.realTimeLogRef.off(); state.realTimeLogRef = undefined; } } function appendLogLinesAndRedraw(lines) { const keys = Object.keys(lines).sort(); for (var key of keys) { const date = new Date(null); date.setSeconds(parseInt(key.substr(0, 6), 16) / 1000); const timeString = date.toISOString().substr(11, 8); const isErr = lines[key].indexOf('FAILED:') >= 0; let line = `[${timeString}] ${lines[key]}`; if (isErr) line = `\u001b[33m${line}\u001b[0m`; state.termLines.push(line); } scheduleRedraw(); return keys[keys.length - 1]; } async function fetchRecentJobsStatus() { const db = firebase.database(); if (state.jobsQueuedRef === undefined) { state.jobsQueuedRef = db.ref(`/ci/jobs_queued`).on('value', e => { state.jobsQueued = Object.keys(e.val() || {}).sort().reverse(); for (const jobId of state.jobsQueued) fetchCIStatusForJob(jobId); scheduleRedraw(); }); } if (state.jobsRunningRef === undefined) { state.jobsRunningRef = db.ref(`/ci/jobs_running`).on('value', e => { state.jobsRunning = Object.keys(e.val() || {}).sort().reverse(); for (const jobId of state.jobsRunning) fetchCIStatusForJob(jobId); scheduleRedraw(); }); } if (state.jobsRecentRef === undefined) { state.jobsRecentRef = db.ref(`/ci/jobs`).orderByKey().limitToLast(100); state.jobsRecentRef.on('value', e => { state.jobsRecent = Object.keys(e.val() || {}).sort().reverse(); for (const jobId of state.jobsRecent) fetchCIStatusForJob(jobId); scheduleRedraw(); }); } } function scheduleRedraw() { if (state.redrawPending) return; state.redrawPending = true; window.requestAnimationFrame(() => { state.redrawPending = false; m.redraw(); }); } main();