• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright (c) 2011 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
5 #include "chrome/browser/importer/firefox3_importer.h"
6 
7 #include <set>
8 
9 #include "app/sql/connection.h"
10 #include "app/sql/statement.h"
11 #include "base/file_util.h"
12 #include "base/memory/scoped_ptr.h"
13 #include "base/message_loop.h"
14 #include "base/stl_util-inl.h"
15 #include "base/string_util.h"
16 #include "base/utf_string_conversions.h"
17 #include "chrome/browser/browser_process.h"
18 #include "chrome/browser/history/history_types.h"
19 #include "chrome/browser/importer/firefox2_importer.h"
20 #include "chrome/browser/importer/firefox_importer_utils.h"
21 #include "chrome/browser/importer/importer_bridge.h"
22 #include "chrome/browser/importer/nss_decryptor.h"
23 #include "chrome/browser/search_engines/template_url.h"
24 #include "chrome/common/time_format.h"
25 #include "content/browser/browser_thread.h"
26 #include "googleurl/src/gurl.h"
27 #include "grit/generated_resources.h"
28 #include "webkit/glue/password_form.h"
29 
30 namespace {
31 
32 // Original definition is in http://mxr.mozilla.org/firefox/source/toolkit/
33 //  components/places/public/nsINavBookmarksService.idl
34 enum BookmarkItemType {
35   TYPE_BOOKMARK = 1,
36   TYPE_FOLDER = 2,
37   TYPE_SEPARATOR = 3,
38   TYPE_DYNAMIC_CONTAINER = 4
39 };
40 
41 }  // namespace
42 
43 struct Firefox3Importer::BookmarkItem {
44   int parent;
45   int id;
46   GURL url;
47   string16 title;
48   BookmarkItemType type;
49   std::string keyword;
50   base::Time date_added;
51   int64 favicon;
52   bool empty_folder;
53 };
54 
Firefox3Importer()55 Firefox3Importer::Firefox3Importer() {
56 #if defined(OS_LINUX)
57   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::UI));
58   locale_ = g_browser_process->GetApplicationLocale();
59 #endif
60 }
61 
~Firefox3Importer()62 Firefox3Importer::~Firefox3Importer() {
63 }
64 
StartImport(const importer::SourceProfile & source_profile,uint16 items,ImporterBridge * bridge)65 void Firefox3Importer::StartImport(
66     const importer::SourceProfile& source_profile,
67     uint16 items,
68     ImporterBridge* bridge) {
69 #if defined(OS_LINUX)
70   DCHECK(BrowserThread::CurrentlyOn(BrowserThread::FILE));
71 #endif
72   bridge_ = bridge;
73   source_path_ = source_profile.source_path;
74   app_path_ = source_profile.app_path;
75 
76   // The order here is important!
77   bridge_->NotifyStarted();
78   if ((items & importer::HOME_PAGE) && !cancelled())
79     ImportHomepage();  // Doesn't have a UI item.
80 
81   // Note history should be imported before bookmarks because bookmark import
82   // will also import favicons and we store favicon for a URL only if the URL
83   // exist in history or bookmarks.
84   if ((items & importer::HISTORY) && !cancelled()) {
85     bridge_->NotifyItemStarted(importer::HISTORY);
86     ImportHistory();
87     bridge_->NotifyItemEnded(importer::HISTORY);
88   }
89 
90   if ((items & importer::FAVORITES) && !cancelled()) {
91     bridge_->NotifyItemStarted(importer::FAVORITES);
92     ImportBookmarks();
93     bridge_->NotifyItemEnded(importer::FAVORITES);
94   }
95   if ((items & importer::SEARCH_ENGINES) && !cancelled()) {
96     bridge_->NotifyItemStarted(importer::SEARCH_ENGINES);
97     ImportSearchEngines();
98     bridge_->NotifyItemEnded(importer::SEARCH_ENGINES);
99   }
100   if ((items & importer::PASSWORDS) && !cancelled()) {
101     bridge_->NotifyItemStarted(importer::PASSWORDS);
102     ImportPasswords();
103     bridge_->NotifyItemEnded(importer::PASSWORDS);
104   }
105   bridge_->NotifyEnded();
106 }
107 
ImportHistory()108 void Firefox3Importer::ImportHistory() {
109   FilePath file = source_path_.AppendASCII("places.sqlite");
110   if (!file_util::PathExists(file))
111     return;
112 
113   sql::Connection db;
114   if (!db.Open(file))
115     return;
116 
117   // |visit_type| represent the transition type of URLs (typed, click,
118   // redirect, bookmark, etc.) We eliminate some URLs like sub-frames and
119   // redirects, since we don't want them to appear in history.
120   // Firefox transition types are defined in:
121   //   toolkit/components/places/public/nsINavHistoryService.idl
122   const char* query = "SELECT h.url, h.title, h.visit_count, "
123                       "h.hidden, h.typed, v.visit_date "
124                       "FROM moz_places h JOIN moz_historyvisits v "
125                       "ON h.id = v.place_id "
126                       "WHERE v.visit_type <= 3";
127 
128   sql::Statement s(db.GetUniqueStatement(query));
129   if (!s)
130     return;
131 
132   std::vector<history::URLRow> rows;
133   while (s.Step() && !cancelled()) {
134     GURL url(s.ColumnString(0));
135 
136     // Filter out unwanted URLs.
137     if (!CanImportURL(url))
138       continue;
139 
140     history::URLRow row(url);
141     row.set_title(s.ColumnString16(1));
142     row.set_visit_count(s.ColumnInt(2));
143     row.set_hidden(s.ColumnInt(3) == 1);
144     row.set_typed_count(s.ColumnInt(4));
145     row.set_last_visit(base::Time::FromTimeT(s.ColumnInt64(5)/1000000));
146 
147     rows.push_back(row);
148   }
149 
150   if (!rows.empty() && !cancelled())
151     bridge_->SetHistoryItems(rows, history::SOURCE_FIREFOX_IMPORTED);
152 }
153 
ImportBookmarks()154 void Firefox3Importer::ImportBookmarks() {
155   FilePath file = source_path_.AppendASCII("places.sqlite");
156   if (!file_util::PathExists(file))
157     return;
158 
159   sql::Connection db;
160   if (!db.Open(file))
161     return;
162 
163   // Get the bookmark folders that we are interested in.
164   int toolbar_folder_id = -1;
165   int menu_folder_id = -1;
166   int unsorted_folder_id = -1;
167   LoadRootNodeID(&db, &toolbar_folder_id, &menu_folder_id, &unsorted_folder_id);
168 
169   // Load livemark IDs.
170   std::set<int> livemark_id;
171   LoadLivemarkIDs(&db, &livemark_id);
172 
173   // Load the default bookmarks. Its storage is the same as Firefox 2.
174   std::set<GURL> default_urls;
175   Firefox2Importer::LoadDefaultBookmarks(app_path_, &default_urls);
176 
177   BookmarkList list;
178   GetTopBookmarkFolder(&db, toolbar_folder_id, &list);
179   GetTopBookmarkFolder(&db, menu_folder_id, &list);
180   GetTopBookmarkFolder(&db, unsorted_folder_id, &list);
181   size_t count = list.size();
182   for (size_t i = 0; i < count; ++i)
183     GetWholeBookmarkFolder(&db, &list, i, NULL);
184 
185   std::vector<ProfileWriter::BookmarkEntry> bookmarks;
186   std::vector<TemplateURL*> template_urls;
187   FaviconMap favicon_map;
188 
189   // TODO(jcampan): http://b/issue?id=1196285 we do not support POST based
190   //                keywords yet.  We won't include them in the list.
191   std::set<int> post_keyword_ids;
192   const char* query = "SELECT b.id FROM moz_bookmarks b "
193       "INNER JOIN moz_items_annos ia ON ia.item_id = b.id "
194       "INNER JOIN moz_anno_attributes aa ON ia.anno_attribute_id = aa.id "
195       "WHERE aa.name = 'bookmarkProperties/POSTData'";
196   sql::Statement s(db.GetUniqueStatement(query));
197   if (s) {
198     while (s.Step() && !cancelled())
199       post_keyword_ids.insert(s.ColumnInt(0));
200   } else {
201     NOTREACHED();
202     return;
203   }
204 
205   string16 firefox_folder =
206       bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
207   for (size_t i = 0; i < list.size(); ++i) {
208     BookmarkItem* item = list[i];
209 
210     if (item->type == TYPE_FOLDER) {
211       // Folders are added implicitly on adding children,
212       // so now we pass only empty folders to add them explicitly.
213       if (!item->empty_folder)
214         continue;
215     } else if (item->type == TYPE_BOOKMARK) {
216       // Import only valid bookmarks
217       if (!CanImportURL(item->url))
218         continue;
219     } else {
220       continue;
221     }
222 
223     // Skip the default bookmarks and unwanted URLs.
224     if (default_urls.find(item->url) != default_urls.end() ||
225         post_keyword_ids.find(item->id) != post_keyword_ids.end())
226       continue;
227 
228     // Find the bookmark path by tracing their links to parent folders.
229     std::vector<string16> path;
230     BookmarkItem* child = item;
231     bool found_path = false;
232     bool is_in_toolbar = false;
233     while (child->parent >= 0) {
234       BookmarkItem* parent = list[child->parent];
235       if (parent->id == toolbar_folder_id) {
236         // This bookmark entry should be put in the bookmark bar.
237         // But we put it in the Firefox group after first run, so
238         // that do not mess up the bookmark bar.
239         if (import_to_bookmark_bar()) {
240           is_in_toolbar = true;
241         } else {
242           path.insert(path.begin(), parent->title);
243           path.insert(path.begin(), firefox_folder);
244         }
245         found_path = true;
246         break;
247       } else if (parent->id == menu_folder_id ||
248                  parent->id == unsorted_folder_id) {
249         // After the first run, the item will be placed in a folder in
250         // the "Other bookmarks".
251         if (!import_to_bookmark_bar())
252           path.insert(path.begin(), firefox_folder);
253         found_path = true;
254         break;
255       } else if (livemark_id.find(parent->id) != livemark_id.end()) {
256         // If the entry is under a livemark folder, we don't import it.
257         break;
258       }
259       path.insert(path.begin(), parent->title);
260       child = parent;
261     }
262 
263     if (!found_path)
264       continue;
265 
266     ProfileWriter::BookmarkEntry entry;
267     entry.creation_time = item->date_added;
268     entry.title = item->title;
269     entry.url = item->url;
270     entry.path = path;
271     entry.in_toolbar = is_in_toolbar;
272     entry.is_folder = item->type == TYPE_FOLDER;
273 
274     bookmarks.push_back(entry);
275 
276     if (item->type == TYPE_BOOKMARK) {
277       if (item->favicon)
278         favicon_map[item->favicon].insert(item->url);
279 
280       // This bookmark has a keyword, we import it to our TemplateURL model.
281       TemplateURL* t_url = Firefox2Importer::CreateTemplateURL(
282           item->title, UTF8ToUTF16(item->keyword), item->url);
283       if (t_url)
284         template_urls.push_back(t_url);
285     }
286   }
287 
288   STLDeleteContainerPointers(list.begin(), list.end());
289 
290   // Write into profile.
291   if (!bookmarks.empty() && !cancelled()) {
292     const string16& first_folder_name =
293         bridge_->GetLocalizedString(IDS_BOOKMARK_GROUP_FROM_FIREFOX);
294     int options = 0;
295     if (import_to_bookmark_bar())
296       options = ProfileWriter::IMPORT_TO_BOOKMARK_BAR;
297     bridge_->AddBookmarkEntries(bookmarks, first_folder_name, options);
298   }
299   if (!template_urls.empty() && !cancelled()) {
300     bridge_->SetKeywords(template_urls, -1, false);
301   } else {
302     STLDeleteContainerPointers(template_urls.begin(), template_urls.end());
303   }
304   if (!favicon_map.empty() && !cancelled()) {
305     std::vector<history::ImportedFaviconUsage> favicons;
306     LoadFavicons(&db, favicon_map, &favicons);
307     bridge_->SetFavicons(favicons);
308   }
309 }
310 
ImportPasswords()311 void Firefox3Importer::ImportPasswords() {
312   // Initializes NSS3.
313   NSSDecryptor decryptor;
314   if (!decryptor.Init(source_path_, source_path_) &&
315       !decryptor.Init(app_path_, source_path_)) {
316     return;
317   }
318 
319   std::vector<webkit_glue::PasswordForm> forms;
320   FilePath source_path = source_path_;
321   FilePath file = source_path.AppendASCII("signons.sqlite");
322   if (file_util::PathExists(file)) {
323     // Since Firefox 3.1, passwords are in signons.sqlite db.
324     decryptor.ReadAndParseSignons(file, &forms);
325   } else {
326     // Firefox 3.0 uses signons3.txt to store the passwords.
327     file = source_path.AppendASCII("signons3.txt");
328     if (!file_util::PathExists(file))
329       file = source_path.AppendASCII("signons2.txt");
330 
331     std::string content;
332     file_util::ReadFileToString(file, &content);
333     decryptor.ParseSignons(content, &forms);
334   }
335 
336   if (!cancelled()) {
337     for (size_t i = 0; i < forms.size(); ++i) {
338       bridge_->SetPasswordForm(forms[i]);
339     }
340   }
341 }
342 
ImportSearchEngines()343 void Firefox3Importer::ImportSearchEngines() {
344   std::vector<FilePath> files;
345   GetSearchEnginesXMLFiles(&files);
346 
347   std::vector<TemplateURL*> search_engines;
348   ParseSearchEnginesFromXMLFiles(files, &search_engines);
349   int default_index =
350       GetFirefoxDefaultSearchEngineIndex(search_engines, source_path_);
351   bridge_->SetKeywords(search_engines, default_index, true);
352 }
353 
ImportHomepage()354 void Firefox3Importer::ImportHomepage() {
355   GURL home_page = GetHomepage(source_path_);
356   if (home_page.is_valid() && !IsDefaultHomepage(home_page, app_path_)) {
357     bridge_->AddHomePage(home_page);
358   }
359 }
360 
GetSearchEnginesXMLFiles(std::vector<FilePath> * files)361 void Firefox3Importer::GetSearchEnginesXMLFiles(
362     std::vector<FilePath>* files) {
363   FilePath file = source_path_.AppendASCII("search.sqlite");
364   if (!file_util::PathExists(file))
365     return;
366 
367   sql::Connection db;
368   if (!db.Open(file))
369     return;
370 
371   const char* query = "SELECT engineid FROM engine_data "
372                       "WHERE engineid NOT IN "
373                       "(SELECT engineid FROM engine_data "
374                       "WHERE name='hidden') "
375                       "ORDER BY value ASC";
376 
377   sql::Statement s(db.GetUniqueStatement(query));
378   if (!s)
379     return;
380 
381   FilePath app_path = app_path_.AppendASCII("searchplugins");
382   FilePath profile_path = source_path_.AppendASCII("searchplugins");
383 
384   // Firefox doesn't store a search engine in its sqlite database unless the
385   // user has added a engine. So we get search engines from sqlite db as well
386   // as from the file system.
387   if (s.Step()) {
388     const std::wstring kAppPrefix = L"[app]/";
389     const std::wstring kProfilePrefix = L"[profile]/";
390     do {
391       FilePath file;
392       std::wstring engine = UTF8ToWide(s.ColumnString(0));
393 
394       // The string contains [app]/<name>.xml or [profile]/<name>.xml where
395       // the [app] and [profile] need to be replaced with the actual app or
396       // profile path.
397       size_t index = engine.find(kAppPrefix);
398       if (index != std::wstring::npos) {
399         // Remove '[app]/'.
400         file = app_path.Append(FilePath::FromWStringHack(
401                                    engine.substr(index + kAppPrefix.length())));
402       } else if ((index = engine.find(kProfilePrefix)) != std::wstring::npos) {
403         // Remove '[profile]/'.
404           file = profile_path.Append(
405               FilePath::FromWStringHack(
406                   engine.substr(index + kProfilePrefix.length())));
407       } else {
408         // Looks like absolute path to the file.
409         file = FilePath::FromWStringHack(engine);
410       }
411       files->push_back(file);
412     } while (s.Step() && !cancelled());
413   }
414 
415 #if defined(OS_LINUX)
416   // Ubuntu-flavored Firefox3 supports locale-specific search engines via
417   // locale-named subdirectories. They fall back to en-US.
418   // See http://crbug.com/53899
419   // TODO(jshin): we need to make sure our locale code matches that of
420   // Firefox.
421   FilePath locale_app_path = app_path.AppendASCII(locale_);
422   FilePath default_locale_app_path = app_path.AppendASCII("en-US");
423   if (file_util::DirectoryExists(locale_app_path))
424     app_path = locale_app_path;
425   else if (file_util::DirectoryExists(default_locale_app_path))
426     app_path = default_locale_app_path;
427 #endif
428 
429   // Get search engine definition from file system.
430   file_util::FileEnumerator engines(app_path, false,
431                                     file_util::FileEnumerator::FILES);
432   for (FilePath engine_path = engines.Next(); !engine_path.value().empty();
433        engine_path = engines.Next()) {
434     files->push_back(engine_path);
435   }
436 }
437 
LoadRootNodeID(sql::Connection * db,int * toolbar_folder_id,int * menu_folder_id,int * unsorted_folder_id)438 void Firefox3Importer::LoadRootNodeID(sql::Connection* db,
439                                       int* toolbar_folder_id,
440                                       int* menu_folder_id,
441                                       int* unsorted_folder_id) {
442   static const char* kToolbarFolderName = "toolbar";
443   static const char* kMenuFolderName = "menu";
444   static const char* kUnsortedFolderName = "unfiled";
445 
446   const char* query = "SELECT root_name, folder_id FROM moz_bookmarks_roots";
447   sql::Statement s(db->GetUniqueStatement(query));
448   if (!s)
449     return;
450 
451   while (s.Step()) {
452     std::string folder = s.ColumnString(0);
453     int id = s.ColumnInt(1);
454     if (folder == kToolbarFolderName)
455       *toolbar_folder_id = id;
456     else if (folder == kMenuFolderName)
457       *menu_folder_id = id;
458     else if (folder == kUnsortedFolderName)
459       *unsorted_folder_id = id;
460   }
461 }
462 
LoadLivemarkIDs(sql::Connection * db,std::set<int> * livemark)463 void Firefox3Importer::LoadLivemarkIDs(sql::Connection* db,
464                                        std::set<int>* livemark) {
465   static const char* kFeedAnnotation = "livemark/feedURI";
466   livemark->clear();
467 
468   const char* query = "SELECT b.item_id "
469                       "FROM moz_anno_attributes a "
470                       "JOIN moz_items_annos b ON a.id = b.anno_attribute_id "
471                       "WHERE a.name = ? ";
472   sql::Statement s(db->GetUniqueStatement(query));
473   if (!s)
474     return;
475 
476   s.BindString(0, kFeedAnnotation);
477   while (s.Step() && !cancelled())
478     livemark->insert(s.ColumnInt(0));
479 }
480 
GetTopBookmarkFolder(sql::Connection * db,int folder_id,BookmarkList * list)481 void Firefox3Importer::GetTopBookmarkFolder(sql::Connection* db,
482                                             int folder_id,
483                                             BookmarkList* list) {
484   const char* query = "SELECT b.title "
485                      "FROM moz_bookmarks b "
486                      "WHERE b.type = 2 AND b.id = ? "
487                      "ORDER BY b.position";
488   sql::Statement s(db->GetUniqueStatement(query));
489   if (!s)
490     return;
491 
492   s.BindInt(0, folder_id);
493   if (s.Step()) {
494     BookmarkItem* item = new BookmarkItem;
495     item->parent = -1;  // The top level folder has no parent.
496     item->id = folder_id;
497     item->title = s.ColumnString16(0);
498     item->type = TYPE_FOLDER;
499     item->favicon = 0;
500     item->empty_folder = true;
501     list->push_back(item);
502   }
503 }
504 
GetWholeBookmarkFolder(sql::Connection * db,BookmarkList * list,size_t position,bool * empty_folder)505 void Firefox3Importer::GetWholeBookmarkFolder(sql::Connection* db,
506                                               BookmarkList* list,
507                                               size_t position,
508                                               bool* empty_folder) {
509   if (position >= list->size()) {
510     NOTREACHED();
511     return;
512   }
513 
514   const char* query = "SELECT b.id, h.url, COALESCE(b.title, h.title), "
515          "b.type, k.keyword, b.dateAdded, h.favicon_id "
516          "FROM moz_bookmarks b "
517          "LEFT JOIN moz_places h ON b.fk = h.id "
518          "LEFT JOIN moz_keywords k ON k.id = b.keyword_id "
519          "WHERE b.type IN (1,2) AND b.parent = ? "
520          "ORDER BY b.position";
521   sql::Statement s(db->GetUniqueStatement(query));
522   if (!s)
523     return;
524 
525   s.BindInt(0, (*list)[position]->id);
526   BookmarkList temp_list;
527   while (s.Step()) {
528     BookmarkItem* item = new BookmarkItem;
529     item->parent = static_cast<int>(position);
530     item->id = s.ColumnInt(0);
531     item->url = GURL(s.ColumnString(1));
532     item->title = s.ColumnString16(2);
533     item->type = static_cast<BookmarkItemType>(s.ColumnInt(3));
534     item->keyword = s.ColumnString(4);
535     item->date_added = base::Time::FromTimeT(s.ColumnInt64(5)/1000000);
536     item->favicon = s.ColumnInt64(6);
537     item->empty_folder = true;
538 
539     temp_list.push_back(item);
540     if (empty_folder != NULL)
541       *empty_folder = false;
542   }
543 
544   // Appends all items to the list.
545   for (BookmarkList::iterator i = temp_list.begin();
546        i != temp_list.end(); ++i) {
547     list->push_back(*i);
548     // Recursive add bookmarks in sub-folders.
549     if ((*i)->type == TYPE_FOLDER)
550       GetWholeBookmarkFolder(db, list, list->size() - 1, &(*i)->empty_folder);
551   }
552 }
553 
LoadFavicons(sql::Connection * db,const FaviconMap & favicon_map,std::vector<history::ImportedFaviconUsage> * favicons)554 void Firefox3Importer::LoadFavicons(
555     sql::Connection* db,
556     const FaviconMap& favicon_map,
557     std::vector<history::ImportedFaviconUsage>* favicons) {
558   const char* query = "SELECT url, data FROM moz_favicons WHERE id=?";
559   sql::Statement s(db->GetUniqueStatement(query));
560   if (!s)
561     return;
562 
563   for (FaviconMap::const_iterator i = favicon_map.begin();
564        i != favicon_map.end(); ++i) {
565     s.BindInt64(0, i->first);
566     if (s.Step()) {
567       history::ImportedFaviconUsage usage;
568 
569       usage.favicon_url = GURL(s.ColumnString(0));
570       if (!usage.favicon_url.is_valid())
571         continue;  // Don't bother importing favicons with invalid URLs.
572 
573       std::vector<unsigned char> data;
574       s.ColumnBlobAsVector(1, &data);
575       if (data.empty())
576         continue;  // Data definitely invalid.
577 
578       if (!ReencodeFavicon(&data[0], data.size(), &usage.png_data))
579         continue;  // Unable to decode.
580 
581       usage.urls = i->second;
582       favicons->push_back(usage);
583     }
584     s.Reset();
585   }
586 }
587