1 /*
2 * Copyright (C) 2021 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "host/libs/vm_manager/gem5_manager.h"
18
19 #include <string.h>
20 #include <sys/socket.h>
21 #include <sys/stat.h>
22 #include <sys/types.h>
23 #include <sys/un.h>
24 #include <sys/wait.h>
25 #include <unistd.h>
26
27 #include <cstdlib>
28 #include <fstream>
29 #include <sstream>
30 #include <string>
31 #include <thread>
32 #include <unordered_map>
33 #include <utility>
34 #include <vector>
35
36 #include <android-base/strings.h>
37 #include <android-base/logging.h>
38 #include <vulkan/vulkan.h>
39
40 #include "common/libs/fs/shared_select.h"
41 #include "common/libs/utils/files.h"
42 #include "common/libs/utils/result.h"
43 #include "common/libs/utils/subprocess.h"
44 #include "common/libs/utils/users.h"
45 #include "host/libs/config/command_source.h"
46 #include "host/libs/config/cuttlefish_config.h"
47 #include "host/libs/config/known_paths.h"
48
49 using cuttlefish::StringFromEnv;
50
51 namespace cuttlefish {
52 namespace vm_manager {
53 namespace {
54
LogAndSetEnv(const char * key,const std::string & value)55 void LogAndSetEnv(const char* key, const std::string& value) {
56 setenv(key, value.c_str(), 1);
57 LOG(INFO) << key << "=" << value;
58 }
59
GenerateGem5File(const CuttlefishConfig & config,const CuttlefishConfig::InstanceSpecific & instance)60 void GenerateGem5File(const CuttlefishConfig& config,
61 const CuttlefishConfig::InstanceSpecific& instance) {
62 // Gem5 specific config, currently users have to change these config locally (without throug launch_cvd input flag) to meet their design
63 // TODO: Add these config into launch_cvd input flag or parse from one json file
64 std::string cpu_class = "AtomicSimpleCPU";
65 std::string l1_icache_class = "None";
66 std::string l1_dcache_class = "None";
67 std::string walk_cache_class = "None";
68 std::string l2_Cache_class = "None";
69 std::string cpu_freq = "4GHz";
70 int num_cores = 1;
71 std::string mem_type = "DDR3_1600_8x8";
72 int mem_channels = 1;
73 std::string mem_ranks = "None";
74
75 // start generating starter_fs.py
76 std::string fs_path = instance.gem5_binary_dir() +
77 "/configs/example/arm/starter_fs.py";
78 std::ofstream starter_fs_ofstream(fs_path.c_str());
79 starter_fs_ofstream << fs_header << "\n";
80
81 // global vars in python
82 starter_fs_ofstream << "default_disk = 'linaro-minimal-aarch64.img'\n";
83
84 // main function
85 starter_fs_ofstream << "def main():\n";
86
87 // args
88 starter_fs_ofstream << " parser = argparse.ArgumentParser(epilog=__doc__)\n";
89 starter_fs_ofstream << " parser.add_argument(\"--disk-image\", action=\"append\", type=str, default=[])\n";
90 starter_fs_ofstream << " parser.add_argument(\"--mem-type\", default=\"" << mem_type << "\", choices=ObjectList.mem_list.get_names())\n";
91 starter_fs_ofstream << " parser.add_argument(\"--mem-channels\", type=int, default=" << mem_channels << ")\n";
92 starter_fs_ofstream << " parser.add_argument(\"--mem-ranks\", type=int, default=" << mem_ranks << ")\n";
93 starter_fs_ofstream << " parser.add_argument(\"--mem-size\", action=\"store\", type=str, default=\"" << instance.memory_mb() << "MB\")\n";
94 starter_fs_ofstream << " parser.add_argument(\"--restore\", type=str, default=None)\n";
95 starter_fs_ofstream << " args = parser.parse_args()\n";
96
97 // instantiate system
98 starter_fs_ofstream << " root = Root(full_system=True)\n";
99 starter_fs_ofstream << " mem_mode = " << cpu_class << ".memory_mode()\n";
100 starter_fs_ofstream << " has_caches = True if mem_mode == \"timing\" else False\n";
101 starter_fs_ofstream << " root.system = devices.SimpleSystem(has_caches, args.mem_size, mem_mode=mem_mode, workload=ArmFsLinux(object_file=SysPaths.binary(\"" << config.assembly_dir() << "/kernel\")))\n";
102
103 // mem config and pci instantiate
104 starter_fs_ofstream << fs_mem_pci;
105
106 // system settings
107 starter_fs_ofstream << " root.system.cpu_cluster = [devices.CpuCluster(root.system, " << num_cores << ", \"" << cpu_freq << "\", \"1.0V\", " << cpu_class << ", " << l1_icache_class << ", " << l1_dcache_class << ", " << walk_cache_class << ", " << l2_Cache_class << ")]\n";
108 starter_fs_ofstream << " root.system.addCaches(has_caches, last_cache_level=2)\n";
109 starter_fs_ofstream << " root.system.realview.setupBootLoader(root.system, SysPaths.binary)\n";
110 starter_fs_ofstream << " root.system.workload.dtb_filename = os.path.join(m5.options.outdir, 'system.dtb')\n";
111 starter_fs_ofstream << " root.system.generateDtb(root.system.workload.dtb_filename)\n";
112 starter_fs_ofstream << " root.system.workload.initrd_filename = \"" << instance.PerInstancePath("initrd.img") << "\"\n";
113 starter_fs_ofstream << " root_dir = \"" << StringFromEnv("HOME", ".") << "\"\n";
114
115 //kernel cmd
116 starter_fs_ofstream << fs_kernel_cmd << "\n";
117
118 // execute main
119 starter_fs_ofstream << fs_exe_main << "\n";
120 }
121
122 } // namespace
123
Gem5Manager(Arch arch)124 Gem5Manager::Gem5Manager(Arch arch) : arch_(arch) {}
125
IsSupported()126 bool Gem5Manager::IsSupported() {
127 return HostSupportsQemuCli();
128 }
129
130 Result<std::unordered_map<std::string, std::string>>
ConfigureGraphics(const CuttlefishConfig::InstanceSpecific & instance)131 Gem5Manager::ConfigureGraphics(
132 const CuttlefishConfig::InstanceSpecific& instance) {
133 // TODO: Add support for the gem5 gpu models
134
135 // Override the default HAL search paths in all cases. We do this because
136 // the HAL search path allows for fallbacks, and fallbacks in conjunction
137 // with properities lead to non-deterministic behavior while loading the
138 // HALs.
139
140 std::unordered_map<std::string, std::string> bootconfig_args;
141
142 if (instance.gpu_mode() == kGpuModeGuestSwiftshader) {
143 LOG(INFO) << "We are in SwiftShader mode";
144 bootconfig_args = {
145 {"androidboot.cpuvulkan.version", std::to_string(VK_API_VERSION_1_1)},
146 {"androidboot.hardware.gralloc", "minigbm"},
147 {"androidboot.hardware.hwcomposer", "ranchu"},
148 {"androidboot.hardware.hwcomposer.mode", "noop"},
149 {"androidboot.hardware.hwcomposer.display_finder_mode", "gem5"},
150 {"androidboot.hardware.egl", "angle"},
151 {"androidboot.hardware.vulkan", "pastel"},
152 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
153 };
154 } else if (instance.gpu_mode() == kGpuModeGfxstream) {
155 LOG(INFO) << "We are in Gfxstream mode";
156 bootconfig_args = {
157 {"androidboot.cpuvulkan.version", "0"},
158 {"androidboot.hardware.gralloc", "minigbm"},
159 {"androidboot.hardware.hwcomposer", "ranchu"},
160 {"androidboot.hardware.hwcomposer.display_finder_mode", "gem5"},
161 {"androidboot.hardware.egl", "emulation"},
162 {"androidboot.hardware.vulkan", "ranchu"},
163 {"androidboot.hardware.gltransport", "virtio-gpu-pipe"},
164 {"androidboot.opengles.version", "196609"}, // OpenGL ES 3.1
165 };
166 } else if (instance.gpu_mode() == kGpuModeNone) {
167 return {};
168 } else {
169 return CF_ERR("Unknown GPU mode " << instance.gpu_mode());
170 }
171
172 if (!instance.gpu_angle_feature_overrides_enabled().empty()) {
173 bootconfig_args["androidboot.hardware.angle_feature_overrides_enabled"] =
174 instance.gpu_angle_feature_overrides_enabled();
175 }
176 if (!instance.gpu_angle_feature_overrides_disabled().empty()) {
177 bootconfig_args["androidboot.hardware.angle_feature_overrides_disabled"] =
178 instance.gpu_angle_feature_overrides_disabled();
179 }
180
181 return bootconfig_args;
182 }
183
184 Result<std::unordered_map<std::string, std::string>>
ConfigureBootDevices(int,bool)185 Gem5Manager::ConfigureBootDevices(int /*num_disks*/, bool /*have_gpu*/) {
186 switch (arch_) {
187 case Arch::Arm:
188 case Arch::Arm64:
189 return {{{"androidboot.boot_devices", "30000000.pci"}}};
190 // TODO: Add x86 support
191 default:
192 return CF_ERR("Unhandled arch");
193 }
194 }
195
StartCommands(const CuttlefishConfig & config)196 Result<std::vector<MonitorCommand>> Gem5Manager::StartCommands(
197 const CuttlefishConfig& config) {
198 auto instance = config.ForDefaultInstance();
199
200 auto stop = [](Subprocess* proc) {
201 return KillSubprocess(proc) == StopperResult::kStopSuccess
202 ? StopperResult::kStopCrash
203 : StopperResult::kStopFailure;
204 };
205 std::string gem5_binary = instance.gem5_binary_dir();
206 switch (arch_) {
207 case Arch::Arm:
208 case Arch::Arm64:
209 gem5_binary += "/build/ARM/gem5.opt";
210 break;
211 case Arch::RiscV64:
212 gem5_binary += "/build/RISCV/gem5.opt";
213 break;
214 case Arch::X86:
215 case Arch::X86_64:
216 gem5_binary += "/build/X86/gem5.opt";
217 break;
218 }
219 // generate Gem5 starter_fs.py before we execute it
220 GenerateGem5File(config, instance);
221
222 Command gem5_cmd(gem5_binary, stop);
223
224 // Always enable listeners, because auto mode will disable once it detects
225 // gem5 is not run interactively
226 gem5_cmd.AddParameter("--listener-mode=on");
227
228 // Add debug-flags and debug-file before the script (i.e. starter_fs.py).
229 // We check the flags are not empty first since they are optional
230 if(!config.gem5_debug_flags().empty()) {
231 gem5_cmd.AddParameter("--debug-flags=", config.gem5_debug_flags());
232 if(!instance.gem5_debug_file().empty()) {
233 gem5_cmd.AddParameter("--debug-file=", instance.gem5_debug_file());
234 }
235 }
236
237 gem5_cmd.AddParameter(instance.gem5_binary_dir(),
238 "/configs/example/arm/starter_fs.py");
239
240 // restore checkpoint case
241 if (instance.gem5_checkpoint_dir() != "") {
242 gem5_cmd.AddParameter("--restore=",
243 instance.gem5_checkpoint_dir());
244 }
245
246 gem5_cmd.AddParameter("--mem-size=", instance.memory_mb() * 1024ULL * 1024ULL);
247 for (const auto& disk : instance.virtual_disk_paths()) {
248 gem5_cmd.AddParameter("--disk-image=", disk);
249 }
250
251 LogAndSetEnv("M5_PATH", config.assembly_dir());
252
253 std::vector<MonitorCommand> commands;
254 commands.emplace_back(std::move(gem5_cmd), true);
255 return commands;
256 }
257
258 } // namespace vm_manager
259 } // namespace cuttlefish
260