Autopsy  4.19.3
Graphical digital forensics platform for The Sleuth Kit and other tools.
ExtractIE.java
Go to the documentation of this file.
1 /*
2  *
3  * Autopsy Forensic Browser
4  *
5  * Copyright 2012-2021 Basis Technology Corp.
6  *
7  * Copyright 2012 42six Solutions.
8  * Contact: aebadirad <at> 42six <dot> com
9  * Project Contact/Architect: carrier <at> sleuthkit <dot> org
10  *
11  * Licensed under the Apache License, Version 2.0 (the "License");
12  * you may not use this file except in compliance with the License.
13  * You may obtain a copy of the License at
14  *
15  * http://www.apache.org/licenses/LICENSE-2.0
16  *
17  * Unless required by applicable law or agreed to in writing, software
18  * distributed under the License is distributed on an "AS IS" BASIS,
19  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20  * See the License for the specific language governing permissions and
21  * limitations under the License.
22  */
23 package org.sleuthkit.autopsy.recentactivity;
24 
25 import java.io.BufferedReader;
26 import org.openide.util.NbBundle;
29 import java.io.File;
30 import java.io.FileInputStream;
31 import java.io.FileNotFoundException;
32 import java.io.IOException;
33 import java.io.InputStreamReader;
34 import java.nio.file.Paths;
35 import java.text.ParseException;
36 import java.text.SimpleDateFormat;
37 import java.util.ArrayList;
38 import java.util.List;
39 import java.util.logging.Level;
41 import java.util.Collection;
42 import java.util.Scanner;
43 import java.util.stream.Collectors;
44 import org.openide.modules.InstalledFileLocator;
45 import org.openide.util.NbBundle.Messages;
48 import org.sleuthkit.datamodel.BlackboardArtifact;
49 import org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE;
50 import org.sleuthkit.datamodel.BlackboardAttribute;
51 import org.sleuthkit.datamodel.BlackboardAttribute.ATTRIBUTE_TYPE;
52 import org.sleuthkit.datamodel.Content;
57 import org.sleuthkit.datamodel.AbstractFile;
58 import static org.sleuthkit.datamodel.BlackboardArtifact.ARTIFACT_TYPE.TSK_WEB_HISTORY;
59 import org.sleuthkit.datamodel.ReadContentInputStream;
60 import org.sleuthkit.datamodel.TskCoreException;
61 
66 class ExtractIE extends Extract {
67 
68  private static final Logger logger = Logger.getLogger(ExtractIE.class.getName());
69  private String PASCO_LIB_PATH;
70  private final String JAVA_PATH;
71  private static final String RESOURCE_URL_PREFIX = "res://";
72  private static final SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
73  private Content dataSource;
74  private final IngestJobContext context;
75 
76  @Messages({
77  "Progress_Message_IE_History=IE History",
78  "Progress_Message_IE_Bookmarks=IE Bookmarks",
79  "Progress_Message_IE_Cookies=IE Cookies",
80  "Progress_Message_IE_Downloads=IE Downloads",
81  "Progress_Message_IE_FormHistory=IE Form History",
82  "Progress_Message_IE_AutoFill=IE Auto Fill",
83  "Progress_Message_IE_Logins=IE Logins",})
84 
85  ExtractIE(IngestJobContext context) {
86  super(NbBundle.getMessage(ExtractIE.class, "ExtractIE.moduleName.text"), context);
87  JAVA_PATH = PlatformUtil.getJavaPath();
88  this.context = context;
89  }
90 
91  @Override
92  public void process(Content dataSource, DataSourceIngestModuleProgress progressBar) {
93  String moduleTempDir = RAImageIngestModule.getRATempPath(getCurrentCase(), "IE", context.getJobId());
94  String moduleTempResultsDir = Paths.get(moduleTempDir, "results").toString();
95 
96  this.dataSource = dataSource;
97  dataFound = false;
98 
99  progressBar.progress(Bundle.Progress_Message_IE_Bookmarks());
100  this.getBookmark();
101 
102  if (context.dataSourceIngestIsCancelled()) {
103  return;
104  }
105 
106  progressBar.progress(Bundle.Progress_Message_IE_Cookies());
107  this.getCookie();
108 
109  if (context.dataSourceIngestIsCancelled()) {
110  return;
111  }
112 
113  progressBar.progress(Bundle.Progress_Message_IE_History());
114  this.getHistory(moduleTempDir, moduleTempResultsDir);
115  }
116 
120  private void getBookmark() {
121  org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager();
122  List<AbstractFile> favoritesFiles;
123  try {
124  favoritesFiles = fileManager.findFiles(dataSource, "%.url", "Favorites"); //NON-NLS
125  } catch (TskCoreException ex) {
126  logger.log(Level.WARNING, "Error fetching 'url' files for Internet Explorer bookmarks.", ex); //NON-NLS
127  this.addErrorMessage(
128  NbBundle.getMessage(this.getClass(), "ExtractIE.getBookmark.errMsg.errGettingBookmarks",
129  this.getDisplayName()));
130  return;
131  }
132 
133  if (favoritesFiles.isEmpty()) {
134  logger.log(Level.INFO, "Didn't find any IE bookmark files."); //NON-NLS
135  return;
136  }
137 
138  dataFound = true;
139  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
140  for (AbstractFile fav : favoritesFiles) {
141  if (fav.getSize() == 0) {
142  continue;
143  }
144 
145  if (context.dataSourceIngestIsCancelled()) {
146  break;
147  }
148 
149  String url = getURLFromIEBookmarkFile(fav);
150 
151  String name = fav.getName();
152  Long datetime = fav.getCrtime();
153  String Tempdate = datetime.toString();
154  datetime = Long.valueOf(Tempdate);
155  String domain = extractDomain(url);
156 
157  try {
158  Collection<BlackboardAttribute> bbattributes = createBookmarkAttributes(
159  url,
160  name,
161  datetime,
162  NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text"),
163  domain);
164 
165  bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_BOOKMARK, fav, bbattributes));
166  } catch (TskCoreException ex) {
167  logger.log(Level.SEVERE, String.format("Failed to create %s for file %d", ARTIFACT_TYPE.TSK_WEB_BOOKMARK.getDisplayName(), fav.getId()), ex);
168  }
169  }
170 
171  if (!context.dataSourceIngestIsCancelled()) {
172  postArtifacts(bbartifacts);
173  }
174  }
175 
176  private String getURLFromIEBookmarkFile(AbstractFile fav) {
177  BufferedReader reader = new BufferedReader(new InputStreamReader(new ReadContentInputStream(fav)));
178  String line, url = "";
179  try {
180  line = reader.readLine();
181  while (null != line) {
182  // The actual shortcut line we are interested in is of the
183  // form URL=http://path/to/website
184  if (line.startsWith("URL")) { //NON-NLS
185  url = line.substring(line.indexOf("=") + 1);
186  break;
187  }
188  line = reader.readLine();
189  }
190  } catch (IOException ex) {
191  logger.log(Level.WARNING, "Failed to read from content: " + fav.getName(), ex); //NON-NLS
192  this.addErrorMessage(
193  NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg", this.getDisplayName(),
194  fav.getName()));
195  } catch (IndexOutOfBoundsException ex) {
196  logger.log(Level.WARNING, "Failed while getting URL of IE bookmark. Unexpected format of the bookmark file: " + fav.getName(), ex); //NON-NLS
197  this.addErrorMessage(
198  NbBundle.getMessage(this.getClass(), "ExtractIE.getURLFromIEBmkFile.errMsg2", this.getDisplayName(),
199  fav.getName()));
200  } finally {
201  try {
202  reader.close();
203  } catch (IOException ex) {
204  logger.log(Level.WARNING, "Failed to close reader.", ex); //NON-NLS
205  }
206  }
207 
208  return url;
209  }
210 
214  private void getCookie() {
215  org.sleuthkit.autopsy.casemodule.services.FileManager fileManager = currentCase.getServices().getFileManager();
216  List<AbstractFile> cookiesFiles;
217  try {
218  cookiesFiles = fileManager.findFiles(dataSource, "%.txt", "Cookies"); //NON-NLS
219  } catch (TskCoreException ex) {
220  logger.log(Level.WARNING, "Error getting cookie files for IE"); //NON-NLS
221  this.addErrorMessage(
222  NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errGettingFile", this.getDisplayName()));
223  return;
224  }
225 
226  if (cookiesFiles.isEmpty()) {
227  logger.log(Level.INFO, "Didn't find any IE cookies files."); //NON-NLS
228  return;
229  }
230 
231  dataFound = true;
232  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
233  for (AbstractFile cookiesFile : cookiesFiles) {
234  if (context.dataSourceIngestIsCancelled()) {
235  break;
236  }
237  if (cookiesFile.getSize() == 0) {
238  continue;
239  }
240 
241  byte[] t = new byte[(int) cookiesFile.getSize()];
242  try {
243  final int bytesRead = cookiesFile.read(t, 0, cookiesFile.getSize());
244  } catch (TskCoreException ex) {
245  logger.log(Level.WARNING, "Error reading bytes of Internet Explorer cookie.", ex); //NON-NLS
246  this.addErrorMessage(
247  NbBundle.getMessage(this.getClass(), "ExtractIE.getCookie.errMsg.errReadingIECookie",
248  this.getDisplayName(), cookiesFile.getName()));
249  continue;
250  }
251  String cookieString = new String(t);
252  String[] values = cookieString.split("\n");
253  String url = values.length > 2 ? values[2] : "";
254  String value = values.length > 1 ? values[1] : "";
255  String name = values.length > 0 ? values[0] : "";
256  Long datetime = cookiesFile.getCrtime();
257  String tempDate = datetime.toString();
258  datetime = Long.valueOf(tempDate);
259  String domain = extractDomain(url);
260 
261  Collection<BlackboardAttribute> bbattributes = new ArrayList<>();
262  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_URL,
263  RecentActivityExtracterModuleFactory.getModuleName(), url));
264  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DATETIME_CREATED,
265  RecentActivityExtracterModuleFactory.getModuleName(), datetime));
266  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_NAME,
267  RecentActivityExtracterModuleFactory.getModuleName(), (name != null) ? name : ""));
268  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_VALUE,
269  RecentActivityExtracterModuleFactory.getModuleName(), value));
270  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_PROG_NAME,
271  RecentActivityExtracterModuleFactory.getModuleName(),
272  NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text")));
273  if (domain != null && domain.isEmpty() == false) {
274  bbattributes.add(new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_DOMAIN,
275  RecentActivityExtracterModuleFactory.getModuleName(), domain));
276  }
277 
278  try {
279  bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_COOKIE, cookiesFile, bbattributes));
280  } catch (TskCoreException ex) {
281  logger.log(Level.SEVERE, String.format("Failed to create %s for file %d", BlackboardArtifact.Type.TSK_WEB_COOKIE.getDisplayName(), cookiesFile.getId()), ex);
282  }
283  }
284 
285  if (!context.dataSourceIngestIsCancelled()) {
286  postArtifacts(bbartifacts);
287  }
288  }
289 
297  private void getHistory(String moduleTempDir, String moduleTempResultsDir) {
298  logger.log(Level.INFO, "Pasco results path: {0}", moduleTempResultsDir); //NON-NLS
299  boolean foundHistory = false;
300 
301  final File pascoRoot = InstalledFileLocator.getDefault().locate("pasco2", ExtractIE.class.getPackage().getName(), false); //NON-NLS
302  if (pascoRoot == null) {
303  this.addErrorMessage(
304  NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.unableToGetHist", this.getDisplayName()));
305  logger.log(Level.SEVERE, "Error finding pasco program "); //NON-NLS
306  return;
307  }
308 
309  final String pascoHome = pascoRoot.getAbsolutePath();
310  logger.log(Level.INFO, "Pasco2 home: {0}", pascoHome); //NON-NLS
311 
312  PASCO_LIB_PATH = pascoHome + File.separator + "pasco2.jar" + File.pathSeparator //NON-NLS
313  + pascoHome + File.separator + "*";
314 
315  File resultsDir = new File(moduleTempResultsDir);
316  resultsDir.mkdirs();
317 
318  // get index.dat files
319  FileManager fileManager = currentCase.getServices().getFileManager();
320  List<AbstractFile> indexFiles;
321  try {
322  indexFiles = fileManager.findFiles(dataSource, "index.dat"); //NON-NLS
323  } catch (TskCoreException ex) {
324  this.addErrorMessage(NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errGettingHistFiles",
325  this.getDisplayName()));
326  logger.log(Level.WARNING, "Error fetching 'index.data' files for Internet Explorer history."); //NON-NLS
327  return;
328  }
329 
330  if (indexFiles.isEmpty()) {
331  String msg = NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.noHistFiles");
332  logger.log(Level.INFO, msg);
333  return;
334  }
335 
336  dataFound = true;
337  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
338  String temps;
339  String indexFileName;
340  for (AbstractFile indexFile : indexFiles) {
341  // Since each result represent an index.dat file,
342  // just create these files with the following notation:
343  // index<Number>.dat (i.e. index0.dat, index1.dat,..., indexN.dat)
344  // where <Number> is the obj_id of the file.
345  // Write each index.dat file to a temp directory.
346  //BlackboardArtifact bbart = fsc.newArtifact(ARTIFACT_TYPE.TSK_WEB_HISTORY);
347  indexFileName = "index" + Integer.toString((int) indexFile.getId()) + ".dat"; //NON-NLS
348  //indexFileName = "index" + Long.toString(bbart.getArtifactID()) + ".dat";
349  temps = moduleTempDir + File.separator + indexFileName; //NON-NLS
350  File datFile = new File(temps);
351  if (context.dataSourceIngestIsCancelled()) {
352  break;
353  }
354  try {
355  ContentUtils.writeToFile(indexFile, datFile, context::dataSourceIngestIsCancelled);
356  } catch (IOException e) {
357  logger.log(Level.WARNING, "Error while trying to write index.dat file " + datFile.getAbsolutePath(), e); //NON-NLS
358  this.addErrorMessage(
359  NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errWriteFile", this.getDisplayName(),
360  datFile.getAbsolutePath()));
361  continue;
362  }
363 
364  String filename = "pasco2Result." + indexFile.getId() + ".txt"; //NON-NLS
365  boolean bPascProcSuccess = executePasco(temps, filename, moduleTempResultsDir);
366  if (context.dataSourceIngestIsCancelled()) {
367  return;
368  }
369 
370  //At this point pasco2 proccessed the index files.
371  //Now fetch the results, parse them and the delete the files.
372  if (bPascProcSuccess) {
373  // Don't add TSK_OS_ACCOUNT artifacts to the ModuleDataEvent
374  bbartifacts.addAll(parsePascoOutput(indexFile, filename, moduleTempResultsDir).stream()
375  .filter(bbart -> bbart.getArtifactTypeID() == ARTIFACT_TYPE.TSK_WEB_HISTORY.getTypeID())
376  .collect(Collectors.toList()));
377  if (context.dataSourceIngestIsCancelled()) {
378  return;
379  }
380  foundHistory = true;
381 
382  //Delete index<n>.dat file since it was succcessfully by Pasco
383  datFile.delete();
384  } else {
385  logger.log(Level.WARNING, "pasco execution failed on: {0}", filename); //NON-NLS
386  this.addErrorMessage(
387  NbBundle.getMessage(this.getClass(), "ExtractIE.getHistory.errMsg.errProcHist", this.getDisplayName()));
388  }
389  }
390 
391  if (!context.dataSourceIngestIsCancelled()) {
392  postArtifacts(bbartifacts);
393  }
394  }
395 
405  @Messages({
406  "# {0} - sub module name",
407  "ExtractIE_executePasco_errMsg_errorRunningPasco={0}: Error analyzing Internet Explorer web history",})
408  private boolean executePasco(String indexFilePath, String outputFileName, String moduleTempResultsDir) {
409  boolean success = true;
410  try {
411  final String outputFileFullPath = moduleTempResultsDir + File.separator + outputFileName;
412  final String errFileFullPath = moduleTempResultsDir + File.separator + outputFileName + ".err"; //NON-NLS
413  logger.log(Level.INFO, "Writing pasco results to: {0}", outputFileFullPath); //NON-NLS
414  List<String> commandLine = new ArrayList<>();
415  commandLine.add(JAVA_PATH);
416  commandLine.add("-cp"); //NON-NLS
417  commandLine.add(PASCO_LIB_PATH);
418  commandLine.add("isi.pasco2.Main"); //NON-NLS
419  commandLine.add("-T"); //NON-NLS
420  commandLine.add("history"); //NON-NLS
421  commandLine.add(indexFilePath);
422  ProcessBuilder processBuilder = new ProcessBuilder(commandLine);
423  processBuilder.redirectOutput(new File(outputFileFullPath));
424  processBuilder.redirectError(new File(errFileFullPath));
425  /*
426  * NOTE on Pasco return codes: There is no documentation for Pasco.
427  * Looking at the Pasco source code I see that when something goes
428  * wrong Pasco returns a negative number as a return code. However,
429  * we should still attempt to parse the Pasco output even if that
430  * happens. I have seen many situations where Pasco output file
431  * contains a lot of useful data and only the last entry is
432  * corrupted.
433  */
434  ExecUtil.execute(processBuilder, new DataSourceIngestModuleProcessTerminator(context, true));
435  // @@@ Investigate use of history versus cache as type.
436  } catch (IOException ex) {
437  logger.log(Level.SEVERE, "Error executing Pasco to process Internet Explorer web history", ex); //NON-NLS
438  addErrorMessage(Bundle.ExtractIE_executePasco_errMsg_errorRunningPasco(getDisplayName()));
439  success = false;
440  }
441  return success;
442  }
443 
454  private Collection<BlackboardArtifact> parsePascoOutput(AbstractFile origFile, String pascoOutputFileName, String moduleTempResultsDir) {
455 
456  Collection<BlackboardArtifact> bbartifacts = new ArrayList<>();
457  String fnAbs = moduleTempResultsDir + File.separator + pascoOutputFileName;
458 
459  File file = new File(fnAbs);
460  if (file.exists() == false) {
461  this.addErrorMessage(
462  NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.notFound", this.getDisplayName(),
463  file.getName()));
464  logger.log(Level.WARNING, "Pasco Output not found: {0}", file.getPath()); //NON-NLS
465  return bbartifacts;
466  }
467 
468  // Make sure the file the is not empty or the Scanner will
469  // throw a "No Line found" Exception
470  if (file.length() == 0) {
471  return bbartifacts;
472  }
473 
474  Scanner fileScanner;
475  try {
476  fileScanner = new Scanner(new FileInputStream(file.toString()));
477  } catch (FileNotFoundException ex) {
478  this.addErrorMessage(
479  NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsing", this.getDisplayName(),
480  file.getName()));
481  logger.log(Level.WARNING, "Unable to find the Pasco file at " + file.getPath(), ex); //NON-NLS
482  return bbartifacts;
483  }
484  while (fileScanner.hasNext()) {
485 
486  if (context.dataSourceIngestIsCancelled()) {
487  return bbartifacts;
488  }
489 
490  String line = fileScanner.nextLine();
491  if (!line.startsWith("URL")) { //NON-NLS
492  continue;
493  }
494 
495  String[] lineBuff = line.split("\\t"); //NON-NLS
496 
497  if (lineBuff.length < 4) {
498  logger.log(Level.INFO, "Found unrecognized IE history format."); //NON-NLS
499  continue;
500  }
501 
502  String actime = lineBuff[3];
503  Long ftime = (long) 0;
504  String user = "";
505  String realurl = null;
506  String domain;
507 
508  /*
509  * We've seen two types of lines: URL http://XYZ.com .... URL
510  * Visited: Joe@http://XYZ.com ....
511  */
512  if (lineBuff[1].contains("@")) {
513  String url[] = lineBuff[1].split("@", 2);
514 
515  /*
516  * Verify the left portion of the URL is valid.
517  */
518  domain = extractDomain(url[0]);
519 
520  if (domain != null && domain.isEmpty() == false) {
521  /*
522  * Use the entire input for the URL.
523  */
524  realurl = lineBuff[1].trim();
525  } else {
526  /*
527  * Use the left portion of the input for the user, and the
528  * right portion for the host.
529  */
530  user = url[0];
531  user = user.replace("Visited:", ""); //NON-NLS
532  user = user.replace(":Host:", ""); //NON-NLS
533  user = user.replaceAll("(:)(.*?)(:)", "");
534  user = user.trim();
535  realurl = url[1];
536  realurl = realurl.replace("Visited:", ""); //NON-NLS
537  realurl = realurl.replaceAll(":(.*?):", "");
538  realurl = realurl.replace(":Host:", ""); //NON-NLS
539  realurl = realurl.trim();
540  domain = extractDomain(realurl);
541  }
542  } else {
543  /*
544  * Use the entire input for the URL.
545  */
546  realurl = lineBuff[1].trim();
547  domain = extractDomain(realurl);
548  }
549 
550  if (!actime.isEmpty()) {
551  try {
552  Long epochtime = dateFormatter.parse(actime).getTime();
553  ftime = epochtime / 1000;
554  } catch (ParseException e) {
555  this.addErrorMessage(
556  NbBundle.getMessage(this.getClass(), "ExtractIE.parsePascoOutput.errMsg.errParsingEntry",
557  this.getDisplayName()));
558  logger.log(Level.WARNING, String.format("Error parsing Pasco results, may have partial processing of corrupt file (id=%d)", origFile.getId()), e); //NON-NLS
559  }
560  }
561 
562  try {
563  Collection<BlackboardAttribute> bbattributes = createHistoryAttributes(
564  realurl,
565  ftime,
566  null,
567  null,
568  NbBundle.getMessage(this.getClass(), "ExtractIE.moduleName.text"),
569  domain,
570  user);
571 
572  bbartifacts.add(createArtifactWithAttributes(BlackboardArtifact.Type.TSK_WEB_HISTORY, origFile, bbattributes));
573  } catch (TskCoreException ex) {
574  logger.log(Level.SEVERE, String.format("Failed to create %s for file %d", BlackboardArtifact.Type.TSK_WEB_HISTORY.getDisplayName(), origFile.getId()), ex);
575  }
576  }
577  fileScanner.close();
578  return bbartifacts;
579  }
580 
589  private String extractDomain(String url) {
590  if (url == null || url.isEmpty()) {
591  return url;
592  }
593 
594  if (url.toLowerCase().startsWith(RESOURCE_URL_PREFIX)) {
595  /*
596  * Ignore URLs that begin with the matched text.
597  */
598  return null;
599  }
600 
601  return NetworkUtils.extractDomain(url);
602  }
603 }
List< AbstractFile > findFiles(String fileName)

Copyright © 2012-2022 Basis Technology. Generated on: Mon Apr 17 2023
This work is licensed under a Creative Commons Attribution-Share Alike 3.0 United States License.