1 /* 2 * Copyright (c) 2018 Google Inc. All Rights Reserved. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you 5 * may not use this file except in compliance with the License. You may 6 * obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 13 * implied. See the License for the specific language governing 14 * permissions and limitations under the License. 15 */ 16 17 package com.android.vts.servlet; 18 19 import com.android.vts.util.GcsHelper; 20 import com.google.appengine.api.memcache.ErrorHandlers; 21 import com.google.appengine.api.memcache.MemcacheService; 22 import com.google.appengine.api.memcache.MemcacheServiceFactory; 23 import com.google.auth.oauth2.ServiceAccountCredentials; 24 import com.google.cloud.storage.Blob; 25 import com.google.cloud.storage.Bucket; 26 import com.google.cloud.storage.Storage; 27 import com.google.cloud.storage.Storage.BlobListOption; 28 import com.google.cloud.storage.StorageOptions; 29 import com.google.gson.Gson; 30 import org.apache.commons.io.IOUtils; 31 32 import javax.servlet.RequestDispatcher; 33 import javax.servlet.ServletConfig; 34 import javax.servlet.ServletException; 35 import javax.servlet.http.HttpServletRequest; 36 import javax.servlet.http.HttpServletResponse; 37 38 import java.io.ByteArrayInputStream; 39 import java.io.File; 40 import java.io.IOException; 41 import java.io.InputStream; 42 import java.nio.charset.StandardCharsets; 43 import java.nio.file.Path; 44 import java.nio.file.Paths; 45 46 import java.util.ArrayList; 47 import java.util.HashMap; 48 import java.util.Iterator; 49 import java.util.List; 50 import java.util.Map; 51 import java.util.Objects; 52 import java.util.Optional; 53 import java.util.logging.Level; 54 import java.util.zip.ZipEntry; 55 import java.util.zip.ZipInputStream; 56 57 /** 58 * A GCS log servlet read log zip file from Google Cloud Storage bucket and show the content in it 59 * from the zip file by unarchiving it 60 */ 61 @SuppressWarnings("serial") 62 public class ShowGcsLogServlet extends BaseServlet { 63 64 private static final String GCS_LOG_JSP = "WEB-INF/jsp/show_gcs_log.jsp"; 65 66 /** Google Cloud Storage project's key file to access the storage */ 67 private static String GCS_KEY_FILE; 68 /** Google Cloud Storage project's default bucket name for vtslab test result files */ 69 private static String GCS_BUCKET_NAME; 70 /** Google Cloud Storage project's default bucket name for vtslab infra log files */ 71 private static String GCS_INFRA_LOG_BUCKET_NAME; 72 73 /** 74 * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full 75 * control of the bucket. 76 */ 77 private InputStream keyFileInputStream; 78 79 /** This is the instance of java google storage library */ 80 private Storage storage; 81 82 /** This is the instance of App Engine memcache service java library */ 83 private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService(); 84 85 /** GCS Test Report Bucket instance */ 86 private Bucket vtsReportBucket; 87 88 /** GCS Infra Log Bucket instance */ 89 private Bucket vtsInfraLogBucket; 90 91 @Override init(ServletConfig cfg)92 public void init(ServletConfig cfg) throws ServletException { 93 super.init(cfg); 94 95 GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile"); 96 GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName"); 97 GCS_INFRA_LOG_BUCKET_NAME = systemConfigProp.getProperty("gcs.infraLogBucketName"); 98 99 String keyFilePath = "keys/" + GCS_KEY_FILE; 100 101 byte[] keyFileByteArray = new byte[0]; 102 try { 103 keyFileByteArray = 104 IOUtils.toByteArray( 105 this.getClass().getClassLoader().getResourceAsStream(keyFilePath)); 106 } catch (IOException e) { 107 e.printStackTrace(); 108 } 109 this.keyFileInputStream = new ByteArrayInputStream(keyFileByteArray); 110 InputStream vtsReportInputStream = new ByteArrayInputStream(keyFileByteArray); 111 InputStream vtsInfraInputStream = new ByteArrayInputStream(keyFileByteArray); 112 113 Optional<Storage> optionalVtsReportStorage = GcsHelper.getStorage(vtsReportInputStream); 114 if (optionalVtsReportStorage.isPresent()) { 115 this.storage = optionalVtsReportStorage.get(); 116 this.vtsReportBucket = storage.get(GCS_BUCKET_NAME); 117 } else { 118 logger.log(Level.SEVERE, "Error on getting storage instance!"); 119 throw new ServletException("Creating storage instance exception!"); 120 } 121 122 Optional<Storage> optionalVtsInfraStorage = GcsHelper.getStorage(vtsInfraInputStream); 123 if (optionalVtsInfraStorage.isPresent()) { 124 this.storage = optionalVtsInfraStorage.get(); 125 this.vtsInfraLogBucket = storage.get(GCS_INFRA_LOG_BUCKET_NAME); 126 } else { 127 logger.log(Level.SEVERE, "Error on getting storage instance!"); 128 throw new ServletException("Creating storage instance exception!"); 129 } 130 syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO)); 131 } 132 133 @Override getNavParentType()134 public PageType getNavParentType() { 135 return PageType.TOT; 136 } 137 138 @Override getBreadcrumbLinks(HttpServletRequest request)139 public List<Page> getBreadcrumbLinks(HttpServletRequest request) { 140 return null; 141 } 142 143 @Override doGetHandler(HttpServletRequest request, HttpServletResponse response)144 public void doGetHandler(HttpServletRequest request, HttpServletResponse response) 145 throws IOException { 146 if (keyFileInputStream == null) { 147 request.setAttribute("error_title", "GCS Key file Error"); 148 request.setAttribute("error_message", "The GCS Key file is not existed!"); 149 RequestDispatcher dispatcher = request.getRequestDispatcher(ERROR_MESSAGE_JSP); 150 try { 151 dispatcher.forward(request, response); 152 } catch (ServletException e) { 153 logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); 154 } 155 } else { 156 String pathInfo = request.getPathInfo(); 157 if (Objects.nonNull(pathInfo)) { 158 if (pathInfo.equalsIgnoreCase("/download")) { 159 downloadHandler(request, response); 160 } else { 161 logger.log(Level.INFO, "Path Info => " + pathInfo); 162 logger.log(Level.WARNING, "Unknown path access!"); 163 } 164 } else { 165 defaultHandler(request, response); 166 } 167 } 168 } 169 downloadHandler(HttpServletRequest request, HttpServletResponse response)170 private void downloadHandler(HttpServletRequest request, HttpServletResponse response) 171 throws IOException { 172 String file = request.getParameter("file") == null ? "/" : request.getParameter("file"); 173 Path filePathInfo = Paths.get(file); 174 175 Blob blobFile = vtsInfraLogBucket.get(filePathInfo.toString()); 176 177 if (blobFile.exists()) { 178 response.setContentType("application/octet-stream"); 179 response.setContentLength(blobFile.getSize().intValue()); 180 response.setHeader( 181 "Content-Disposition", 182 "attachment; filename=\"" + filePathInfo.getFileName() + "\""); 183 184 response.getOutputStream().write(blobFile.getContent()); 185 } else { 186 request.setAttribute("error_title", "Infra Log File Not Found"); 187 request.setAttribute("error_message", "Please contact the administrator!"); 188 RequestDispatcher dispatcher = request.getRequestDispatcher(ERROR_MESSAGE_JSP); 189 try { 190 dispatcher.forward(request, response); 191 } catch (ServletException e) { 192 logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); 193 } 194 } 195 } 196 defaultHandler(HttpServletRequest request, HttpServletResponse response)197 private void defaultHandler(HttpServletRequest request, HttpServletResponse response) 198 throws IOException { 199 200 String action = 201 request.getParameter("action") == null ? "read" : request.getParameter("action"); 202 String path = request.getParameter("path") == null ? "/" : request.getParameter("path"); 203 String entry = request.getParameter("entry") == null ? "" : request.getParameter("entry"); 204 Path pathInfo = Paths.get(path); 205 206 List<String> dirList = new ArrayList<>(); 207 List<String> fileList = new ArrayList<>(); 208 List<String> entryList = new ArrayList<>(); 209 Map<String, Object> resultMap = new HashMap<>(); 210 String entryContent = ""; 211 212 if (pathInfo.toString().endsWith(".zip")) { 213 214 Blob blobFile = (Blob) this.syncCache.get(path.toString()); 215 if (blobFile == null) { 216 blobFile = vtsReportBucket.get(path); 217 this.syncCache.put(path.toString(), blobFile); 218 } 219 220 if (action.equalsIgnoreCase("read")) { 221 InputStream blobInputStream = new ByteArrayInputStream(blobFile.getContent()); 222 ZipInputStream zipInputStream = new ZipInputStream(blobInputStream); 223 224 ZipEntry zipEntry; 225 while ((zipEntry = zipInputStream.getNextEntry()) != null) { 226 if (zipEntry.isDirectory()) { 227 228 } else { 229 if (entry.length() > 0) { 230 logger.log(Level.INFO, "param entry => " + entry); 231 if (zipEntry.getName().equals(entry)) { 232 logger.log(Level.INFO, "matched !!!! " + zipEntry.getName()); 233 entryContent = 234 IOUtils.toString( 235 zipInputStream, StandardCharsets.UTF_8.name()); 236 } 237 } else { 238 entryList.add(zipEntry.getName()); 239 } 240 } 241 } 242 resultMap.put("entryList", entryList); 243 resultMap.put("entryContent", entryContent); 244 245 String json = new Gson().toJson(resultMap); 246 response.setContentType("application/json"); 247 response.setCharacterEncoding("UTF-8"); 248 response.getWriter().write(json); 249 } else { 250 response.setContentType("application/octet-stream"); 251 response.setContentLength(blobFile.getSize().intValue()); 252 response.setHeader( 253 "Content-Disposition", 254 "attachment; filename=\"" + pathInfo.getFileName() + "\""); 255 256 response.getOutputStream().write(blobFile.getContent()); 257 } 258 259 } else { 260 261 logger.log(Level.INFO, "path info => " + pathInfo); 262 logger.log(Level.INFO, "path name count => " + pathInfo.getNameCount()); 263 264 BlobListOption[] listOptions; 265 if (pathInfo.getNameCount() == 0) { 266 listOptions = new BlobListOption[] {BlobListOption.currentDirectory()}; 267 } else { 268 String prefixPathString = path.endsWith("/") ? path : path.concat("/"); 269 if (pathInfo.getNameCount() <= 1) { 270 dirList.add("/"); 271 } else { 272 dirList.add(getParentDirPath(prefixPathString)); 273 } 274 275 listOptions = 276 new BlobListOption[] { 277 BlobListOption.currentDirectory(), 278 BlobListOption.prefix(prefixPathString) 279 }; 280 } 281 282 Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll(); 283 Iterator<Blob> blobIterator = blobIterable.iterator(); 284 while (blobIterator.hasNext()) { 285 Blob blob = blobIterator.next(); 286 logger.log(Level.INFO, "blob name => " + blob); 287 if (blob.isDirectory()) { 288 logger.log(Level.INFO, "directory name => " + blob.getName()); 289 dirList.add(blob.getName()); 290 } else { 291 logger.log(Level.INFO, "file name => " + blob.getName()); 292 fileList.add(Paths.get(blob.getName()).getFileName().toString()); 293 } 294 } 295 296 response.setStatus(HttpServletResponse.SC_OK); 297 request.setAttribute("entryList", entryList); 298 request.setAttribute("dirList", dirList); 299 request.setAttribute("fileList", fileList); 300 request.setAttribute("path", path); 301 RequestDispatcher dispatcher = request.getRequestDispatcher(GCS_LOG_JSP); 302 try { 303 dispatcher.forward(request, response); 304 } catch (ServletException e) { 305 logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); 306 } 307 } 308 } 309 getParentDirPath(String fileOrDirPath)310 private String getParentDirPath(String fileOrDirPath) { 311 boolean endsWithSlashCheck = fileOrDirPath.endsWith(File.separator); 312 return fileOrDirPath.substring( 313 0, 314 fileOrDirPath.lastIndexOf( 315 File.separatorChar, 316 endsWithSlashCheck 317 ? fileOrDirPath.length() - 2 318 : fileOrDirPath.length() - 1)); 319 } 320 } 321