"""Microsoft Internet Explorer cookie loading on Windows. Copyright 2002-2003 Johnny Lee (MSIE Perl code) Copyright 2002-2006 John J Lee (The Python port) This code is free software; you can redistribute it and/or modify it under the terms of the BSD or ZPL 2.1 licenses (see the file COPYING.txt included with the distribution). """ # XXX names and comments are not great here import os, re, time, struct, logging if os.name == "nt": import _winreg from _clientcookie import FileCookieJar, CookieJar, Cookie, \ MISSING_FILENAME_TEXT, LoadError debug = logging.getLogger("mechanize").debug def regload(path, leaf): key = _winreg.OpenKey(_winreg.HKEY_CURRENT_USER, path, 0, _winreg.KEY_ALL_ACCESS) try: value = _winreg.QueryValueEx(key, leaf)[0] except WindowsError: value = None return value WIN32_EPOCH = 0x019db1ded53e8000L # 1970 Jan 01 00:00:00 in Win32 FILETIME def epoch_time_offset_from_win32_filetime(filetime): """Convert from win32 filetime to seconds-since-epoch value. MSIE stores create and expire times as Win32 FILETIME, which is 64 bits of 100 nanosecond intervals since Jan 01 1601. mechanize expects time in 32-bit value expressed in seconds since the epoch (Jan 01 1970). """ if filetime < WIN32_EPOCH: raise ValueError("filetime (%d) is before epoch (%d)" % (filetime, WIN32_EPOCH)) return divmod((filetime - WIN32_EPOCH), 10000000L)[0] def binary_to_char(c): return "%02X" % ord(c) def binary_to_str(d): return "".join(map(binary_to_char, list(d))) class MSIEBase: magic_re = re.compile(r"Client UrlCache MMF Ver \d\.\d.*") padding = "\x0d\xf0\xad\x0b" msie_domain_re = re.compile(r"^([^/]+)(/.*)$") cookie_re = re.compile("Cookie\:.+\@([\x21-\xFF]+).*?" "(.+\@[\x21-\xFF]+\.txt)") # path under HKEY_CURRENT_USER from which to get location of index.dat reg_path = r"software\microsoft\windows" \ r"\currentversion\explorer\shell folders" reg_key = "Cookies" def __init__(self): self._delayload_domains = {} def _delayload_domain(self, domain): # if necessary, lazily load cookies for this domain delayload_info = self._delayload_domains.get(domain) if delayload_info is not None: cookie_file, ignore_discard, ignore_expires = delayload_info try: self.load_cookie_data(cookie_file, ignore_discard, ignore_expires) except (LoadError, IOError): debug("error reading cookie file, skipping: %s", cookie_file) else: del self._delayload_domains[domain] def _load_cookies_from_file(self, filename): debug("Loading MSIE cookies file: %s", filename) cookies = [] cookies_fh = open(filename) try: while 1: key = cookies_fh.readline() if key == "": break rl = cookies_fh.readline def getlong(rl=rl): return long(rl().rstrip()) def getstr(rl=rl): return rl().rstrip() key = key.rstrip() value = getstr() domain_path = getstr() flags = getlong() # 0x2000 bit is for secure I think lo_expire = getlong() hi_expire = getlong() lo_create = getlong() hi_create = getlong() sep = getstr() if "" in (key, value, domain_path, flags, hi_expire, lo_expire, hi_create, lo_create, sep) or (sep != "*"): break m = self.msie_domain_re.search(domain_path) if m: domain = m.group(1) path = m.group(2) cookies.append({"KEY": key, "VALUE": value, "DOMAIN": domain, "PATH": path, "FLAGS": flags, "HIXP": hi_expire, "LOXP": lo_expire, "HICREATE": hi_create, "LOCREATE": lo_create}) finally: cookies_fh.close() return cookies def load_cookie_data(self, filename, ignore_discard=False, ignore_expires=False): """Load cookies from file containing actual cookie data. Old cookies are kept unless overwritten by newly loaded ones. You should not call this method if the delayload attribute is set. I think each of these files contain all cookies for one user, domain, and path. filename: file containing cookies -- usually found in a file like C:\WINNT\Profiles\joe\Cookies\joe@blah[1].txt """ now = int(time.time()) cookie_data = self._load_cookies_from_file(filename) for cookie in cookie_data: flags = cookie["FLAGS"] secure = ((flags & 0x2000) != 0) filetime = (cookie["HIXP"] << 32) + cookie["LOXP"] expires = epoch_time_offset_from_win32_filetime(filetime) if expires < now: discard = True else: discard = False domain = cookie["DOMAIN"] initial_dot = domain.startswith(".") if initial_dot: domain_specified = True else: # MSIE 5 does not record whether the domain cookie-attribute # was specified. # Assuming it wasn't is conservative, because with strict # domain matching this will match less frequently; with regular # Netscape tail-matching, this will match at exactly the same # times that domain_specified = True would. It also means we # don't have to prepend a dot to achieve consistency with our # own & Mozilla's domain-munging scheme. domain_specified = False # assume path_specified is false # XXX is there other stuff in here? -- eg. comment, commentURL? c = Cookie(0, cookie["KEY"], cookie["VALUE"], None, False, domain, domain_specified, initial_dot, cookie["PATH"], False, secure, expires, discard, None, None, {"flags": flags}) if not ignore_discard and c.discard: continue if not ignore_expires and c.is_expired(now): continue CookieJar.set_cookie(self, c) def load_from_registry(self, ignore_discard=False, ignore_expires=False, username=None): """ username: only required on win9x """ cookies_dir = regload(self.reg_path, self.reg_key) filename = os.path.normpath(os.path.join(cookies_dir, "INDEX.DAT")) self.load(filename, ignore_discard, ignore_expires, username) def _really_load(self, index, filename, ignore_discard, ignore_expires, username): now = int(time.time()) if username is None: username = os.environ['USERNAME'].lower() cookie_dir = os.path.dirname(filename) data = index.read(256) if len(data) != 256: raise LoadError("%s file is too short" % filename) # Cookies' index.dat file starts with 32 bytes of signature # followed by an offset to the first record, stored as a little- # endian DWORD. sig, size, data = data[:32], data[32:36], data[36:] size = struct.unpack("= 0 if size != 2: assert to_read != 0 index.seek(to_read, 1) continue # read in rest of record if necessary if size > 2: more_data = index.read(to_read) if len(more_data) != to_read: break data = data + more_data cookie_re = ("Cookie\:%s\@([\x21-\xFF]+).*?" % username + "(%s\@[\x21-\xFF]+\.txt)" % username) m = re.search(cookie_re, data, re.I) if m: cookie_file = os.path.join(cookie_dir, m.group(2)) if not self.delayload: try: self.load_cookie_data(cookie_file, ignore_discard, ignore_expires) except (LoadError, IOError): debug("error reading cookie file, skipping: %s", cookie_file) else: domain = m.group(1) i = domain.find("/") if i != -1: domain = domain[:i] self._delayload_domains[domain] = ( cookie_file, ignore_discard, ignore_expires) class MSIECookieJar(MSIEBase, FileCookieJar): """FileCookieJar that reads from the Windows MSIE cookies database. MSIECookieJar can read the cookie files of Microsoft Internet Explorer (MSIE) for Windows version 5 on Windows NT and version 6 on Windows XP and Windows 98. Other configurations may also work, but are untested. Saving cookies in MSIE format is NOT supported. If you save cookies, they'll be in the usual Set-Cookie3 format, which you can read back in using an instance of the plain old CookieJar class. Don't save using the same filename that you loaded cookies from, because you may succeed in clobbering your MSIE cookies index file! You should be able to have LWP share Internet Explorer's cookies like this (note you need to supply a username to load_from_registry if you're on Windows 9x or Windows ME): cj = MSIECookieJar(delayload=1) # find cookies index file in registry and load cookies from it cj.load_from_registry() opener = mechanize.build_opener(mechanize.HTTPCookieProcessor(cj)) response = opener.open("http://example.com/") Iterating over a delayloaded MSIECookieJar instance will not cause any cookies to be read from disk. To force reading of all cookies from disk, call read_all_cookies. Note that the following methods iterate over self: clear_temporary_cookies, clear_expired_cookies, __len__, __repr__, __str__ and as_string. Additional methods: load_from_registry(ignore_discard=False, ignore_expires=False, username=None) load_cookie_data(filename, ignore_discard=False, ignore_expires=False) read_all_cookies() """ def __init__(self, filename=None, delayload=False, policy=None): MSIEBase.__init__(self) FileCookieJar.__init__(self, filename, delayload, policy) def set_cookie(self, cookie): if self.delayload: self._delayload_domain(cookie.domain) CookieJar.set_cookie(self, cookie) def _cookies_for_request(self, request): """Return a list of cookies to be returned to server.""" domains = self._cookies.copy() domains.update(self._delayload_domains) domains = domains.keys() cookies = [] for domain in domains: cookies.extend(self._cookies_for_domain(domain, request)) return cookies def _cookies_for_domain(self, domain, request): if not self._policy.domain_return_ok(domain, request): return [] debug("Checking %s for cookies to return", domain) if self.delayload: self._delayload_domain(domain) return CookieJar._cookies_for_domain(self, domain, request) def read_all_cookies(self): """Eagerly read in all cookies.""" if self.delayload: for domain in self._delayload_domains.keys(): self._delayload_domain(domain) def load(self, filename, ignore_discard=False, ignore_expires=False, username=None): """Load cookies from an MSIE 'index.dat' cookies index file. filename: full path to cookie index file username: only required on win9x """ if filename is None: if self.filename is not None: filename = self.filename else: raise ValueError(MISSING_FILENAME_TEXT) index = open(filename, "rb") try: self._really_load(index, filename, ignore_discard, ignore_expires, username) finally: index.close()