From 820cc38d4360b7d44bfaa47731175579f6276151 Mon Sep 17 00:00:00 2001 From: doronz Date: Mon, 11 Sep 2023 16:46:45 +0300 Subject: [PATCH] add macos support --- README.md | 35 +++++++++++-- harlogger/__main__.py | 44 ++++++++++++---- harlogger/http_transaction.py | 7 ++- harlogger/sniffers.py | 96 ++++++++++++++++++++++++----------- pyproject.toml | 2 +- requirements.txt | 3 +- 6 files changed, 140 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index c5c5edd..4074d2b 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ +- [Description](#description) +- [Installation](#installation) +- [Profile method for macOS host](#profile-method-for-macos-host) + * [Howto](#howto) +- [Profile method for non-jailbroken devices](#profile-method-for-non-jailbroken-devices) + * [Howto](#howto-1) +- [Secret preference method for jailbroken devices](#secret-preference-method-for-jailbroken-devices) + * [Howto](#howto-2) +- [Enable HTTP instrumentation method](#enable-http-instrumentation-method) + # Description Simple pure python utility for sniffing HTTP/HTTPS decrypted traffic recorded by one of Apple's not-so-well documented @@ -6,9 +16,26 @@ APIs. # Installation ```shell -python3 -m pip install --user -U harlogger +python3 -m pip install -U harlogger ``` +# Profile method for macOS host + +This method applies to Apple's CFNetwork profile. This profile is meant for debugging processes using the CFNetwork +framework. **This method doesn't include the request/response body.** + +## Howto + +- Download Apple's CFNetwork profile which can be found here: + https://developer.apple.com/services-account/download?path=/iOS/iOS_Logs/NetworkDiagnostic.mobileconfig + +- Install it using double-click + +- That's it! :) You can now just start sniffing out everything using: + ```shell + python3 -m harlogger profile + ``` + # Profile method for non-jailbroken devices This method applies to Apple's CFNetwork profile. This profile is meant for debugging processes using the CFNetwork @@ -23,7 +50,7 @@ framework. **This method doesn't include the request/response body.** ```shell # if you don't already have it - python3 -m pip install -U --user pymobiledevice3 + python3 -m pip install -U pymobiledevice3 # install the profile pymobiledevice3 profile install CFNetworkDiagnostics.mobileconfig @@ -31,7 +58,7 @@ framework. **This method doesn't include the request/response body.** - That's it! :) You can now just start sniffing out everything using: ```shell - python3 -m harlogger profile + python3 -m harlogger mobile profile ``` Output should look like: @@ -78,7 +105,7 @@ for `com.apple.CFNetwork` and trigger the `com.apple.CFNetwork.har-capture-updat Output should look like: ``` -➜ harlogger git:(master) ✗ python3 -m harlogger preference +➜ harlogger git:(master) ✗ python3 -m harlogger mobile preference ➡️ CFNetwork(1140) POST https://www.bing.com/fd/ls/lsp.aspx POST /fd/ls/lsp.aspx HTTP/2.0 Accept: */* diff --git a/harlogger/__main__.py b/harlogger/__main__.py index 46f8944..0da4266 100644 --- a/harlogger/__main__.py +++ b/harlogger/__main__.py @@ -2,7 +2,7 @@ from pymobiledevice3.cli.cli_common import Command from pymobiledevice3.lockdown_service_provider import LockdownServiceProvider -from harlogger.sniffers import Filters, SnifferPreference, SnifferProfile +from harlogger.sniffers import Filters, HostSnifferProfile, MobileSnifferProfile, SnifferPreference @click.group() @@ -10,7 +10,13 @@ def cli(): pass -@cli.command('profile', cls=Command) +@cli.group() +def mobile(): + """ Mobile sniffing options """ + pass + + +@mobile.command('profile', cls=Command) @click.option('pids', '-p', '--pid', type=click.INT, multiple=True, help='filter pid list') @click.option('--color/--no-color', default=True) @click.option('process_names', '-pn', '--process-name', multiple=True, help='filter process name list') @@ -19,19 +25,19 @@ def cli(): @click.option('--response/--no-response', is_flag=True, default=True, help='show responses') @click.option('-u', '--unique', is_flag=True, help='show only unique requests per image/pid/method/uri combination') @click.option('--black-list/--white-list', default=True, is_flag=True) -def cli_profile(service_provider: LockdownServiceProvider, pids, process_names, color, request, response, images, - unique, black_list): +def mobile_profile(service_provider: LockdownServiceProvider, pids, process_names, color, request, response, images, + unique, black_list): """ Sniff using CFNetworkDiagnostics.mobileconfig profile. This requires the specific Apple profile to be installed for the sniff to work. """ filters = Filters(pids, process_names, images, black_list) - SnifferProfile(service_provider, filters=filters, request=request, response=response, color=color, - unique=unique).sniff() + MobileSnifferProfile(service_provider, filters=filters, request=request, response=response, color=color, + unique=unique).sniff() -@cli.command('preference', cls=Command) +@mobile.command('preference', cls=Command) @click.option('-o', '--out', type=click.File('w'), help='file to store the har entries into upon exit (ctrl+c)') @click.option('pids', '-p', '--pid', type=click.INT, multiple=True, help='filter pid list') @click.option('--color/--no-color', default=True) @@ -41,8 +47,8 @@ def cli_profile(service_provider: LockdownServiceProvider, pids, process_names, @click.option('--response/--no-response', is_flag=True, default=True, help='show responses') @click.option('-u', '--unique', is_flag=True, help='show only unique requests per image/pid/method/uri combination') @click.option('--black-list/--white-list', default=True, is_flag=True) -def cli_preference(service_provider: LockdownServiceProvider, out, pids, process_names, images, request, response, - color, unique, black_list): +def mobile_preference(service_provider: LockdownServiceProvider, out, pids, process_names, images, request, response, + color, unique, black_list): """ Sniff using the secret com.apple.CFNetwork.plist configuration. @@ -54,5 +60,25 @@ def cli_preference(service_provider: LockdownServiceProvider, out, pids, process unique=unique).sniff() +@cli.command('profile') +@click.option('pids', '-p', '--pid', type=click.INT, multiple=True, help='filter pid list') +@click.option('--color/--no-color', default=True) +@click.option('process_names', '-pn', '--process-name', multiple=True, help='filter process name list') +@click.option('images', '-i', '--image', multiple=True, help='filter image list') +@click.option('--request/--no-request', is_flag=True, default=True, help='show requests') +@click.option('--response/--no-response', is_flag=True, default=True, help='show responses') +@click.option('-u', '--unique', is_flag=True, help='show only unique requests per image/pid/method/uri combination') +@click.option('--black-list/--white-list', default=True, is_flag=True) +def host_profile(pids, process_names, color, request, response, images, unique, black_list): + """ + Sniff using CFNetworkDiagnostics.mobileconfig profile. + + This requires the specific Apple profile to be installed for the sniff to work. + """ + filters = Filters(pids, process_names, images, black_list) + HostSnifferProfile(filters=filters, request=request, response=response, color=color, + unique=unique).sniff() + + if __name__ == '__main__': cli() diff --git a/harlogger/http_transaction.py b/harlogger/http_transaction.py index 8b5771b..8123ff3 100644 --- a/harlogger/http_transaction.py +++ b/harlogger/http_transaction.py @@ -17,7 +17,12 @@ def parse_transaction(message: str) -> 'HTTPTransaction': parsed_transaction = HTTPTransaction._parse_fields(message=message) if 'Protocol Enqueue' in parsed_transaction: - method, url, http_version = parsed_transaction.pop('Protocol Enqueue').split()[1:] + info = parsed_transaction.pop('Protocol Enqueue').split()[1:] + if len(info) == 2: + method, url = info + http_version = 'unknown' + else: + method, url, http_version = info parsed_transaction.pop('Message') parsed_transaction.pop('Request') res = HTTPRequest(url, method, http_version, parsed_transaction) diff --git a/harlogger/sniffers.py b/harlogger/sniffers.py index c8ab6ee..6ffbfee 100644 --- a/harlogger/sniffers.py +++ b/harlogger/sniffers.py @@ -3,9 +3,10 @@ import posixpath from abc import ABC, abstractmethod from dataclasses import dataclass -from typing import IO, Tuple +from typing import IO, Optional, Tuple from haralyzer import HarEntry +from maclog.log import get_logger from pygments import highlight from pygments.formatters.terminal256 import TerminalTrueColorFormatter from pygments.lexers.textfmts import HttpLexer @@ -36,18 +37,16 @@ class Filters: def should_keep(self, entry_hash: EntryHash) -> bool: """ Filter out entry if one of the criteria specified (pid,image,process_name) """ - in_filters = self.pids is not None and entry_hash.pid in self.pids or \ - self.process_names is not None and entry_hash.process_name in self.process_names or \ - self.images is not None and entry_hash.image in self.images + in_filters = (self.pids is not None and entry_hash.pid in self.pids or + self.process_names is not None and entry_hash.process_name in self.process_names or + self.images is not None and entry_hash.image in self.images) return self.black_list and not in_filters or not self.black_list and in_filters class SnifferBase(ABC): - def __init__(self, lockdown: LockdownClient, filters: Filters = None, unique: bool = False, request: bool = True, + def __init__(self, filters: Filters = None, unique: bool = False, request: bool = True, response: bool = True, color: bool = True, style: str = 'autumn'): - self._lockdown = lockdown - self._os_trace_service = OsTraceService(self._lockdown) self._filters = filters self._request = request self._response = response @@ -74,6 +73,14 @@ def sniff(self) -> None: pass +class MobileSnifferBase(SnifferBase, ABC): + def __init__(self, lockdown: LockdownClient, filters: Filters = None, unique: bool = False, request: bool = True, + response: bool = True, color: bool = True, style: str = 'autumn'): + super().__init__(filters, unique, request, response, color, style) + self._lockdown = lockdown + self._os_trace_service = OsTraceService(self._lockdown) + + class SnifferPreference(SnifferBase): """ Sniff using the secret com.apple.CFNetwork.plist configuration. @@ -140,7 +147,37 @@ def _sniff(self) -> None: continue -class SnifferProfile(SnifferBase): +class SnifferProfileBase(SnifferBase, ABC): + def _handle_entry(self, pid: int, message: str, filename: str, image_name: str, subsystem: Optional[str] = None, + category: Optional[str] = None): + if subsystem != 'com.apple.CFNetwork' or category != 'Diagnostics': + return + + if 'Protocol Received' not in message and 'Protocol Enqueue' not in message: + return + + lines = message.split('\n') + if len(lines) < 2: + return + + http_transaction = HTTPTransaction.parse_transaction(message) + if not http_transaction: + raise HTTPParseError() + entry_hash = EntryHash(pid, + posixpath.basename(filename), + os.path.basename(image_name), + http_transaction.url) + + if not self._filters.should_keep(entry_hash): + return + + if self._request and isinstance(http_transaction, HTTPRequest): + self.show(entry_hash, http_transaction.formatted, '➡️') + if self._response and isinstance(http_transaction, HTTPResponse): + self.show(entry_hash, http_transaction.formatted, '⬅️', f'({http_transaction.url})') + + +class MobileSnifferProfile(MobileSnifferBase, SnifferProfileBase): """ Sniff using CFNetworkDiagnostics.mobileconfig profile. @@ -151,32 +188,29 @@ def __init__(self, lockdown: LockdownClient, filters: Filters = None, unique: bo response: bool = True, color: bool = True, style: str = 'autumn'): super().__init__(lockdown, filters, unique, request, response, color, style) - def sniff(self): + def sniff(self) -> None: for entry in self._os_trace_service.syslog(): - if entry.label is None or entry.label.subsystem != 'com.apple.CFNetwork' or \ - entry.label.category != 'Diagnostics': - continue + subsystem = None + category = None + if entry.label is not None: + subsystem = entry.label.subsystem + category = entry.label.category + self._handle_entry(entry.pid, entry.message, entry.filename, entry.image_name, subsystem=subsystem, + category=category) - if 'Protocol Received' not in entry.message and \ - 'Protocol Enqueue' not in entry.message: - continue - lines = entry.message.split('\n') - if len(lines) < 2: - continue +class HostSnifferProfile(SnifferProfileBase): + """ + Sniff using CFNetworkDiagnostics.mobileconfig profile. - http_transaction = HTTPTransaction.parse_transaction(entry.message) - if not http_transaction: - raise HTTPParseError() - entry_hash = EntryHash(entry.pid, - posixpath.basename(entry.filename), - os.path.basename(entry.image_name), - http_transaction.url) + This requires the specific Apple profile to be installed for the sniff to work. + """ - if not self._filters.should_keep(entry_hash): - continue + def __init__(self, filters: Filters = None, unique: bool = False, request: bool = True, + response: bool = True, color: bool = True, style: str = 'autumn'): + super().__init__(filters, unique, request, response, color, style) - if self._request and isinstance(http_transaction, HTTPRequest): - self.show(entry_hash, http_transaction.formatted, '➡️') - if self._response and isinstance(http_transaction, HTTPResponse): - self.show(entry_hash, http_transaction.formatted, '⬅️', f'({http_transaction.url})') + def sniff(self) -> None: + for entry in get_logger(): + self._handle_entry(entry.process_id, entry.event_message, entry.process_image_path, entry.sender_image_path, + subsystem=entry.subsystem, category=entry.category) diff --git a/pyproject.toml b/pyproject.toml index bc271ff..d2e8911 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ description = "Simple utlity for sniffing decrypted HTTP/HTTPS traffic on an iOS readme = "README.md" requires-python = ">=3.8" license = { file = "LICENSE" } -keywords = ["ios", "http", "https", "har", "sniffer", "jailbroken"] +keywords = ["ios", "osx", "mac", "macos", "http", "https", "har", "sniffer", "jailbroken"] authors = [ { name = "doronz88", email = "doron88@gmail.com" }, { name = "netanelc305", email = "netanelc305@protonmail.com" } diff --git a/requirements.txt b/requirements.txt index 38a1fc8..53ef800 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ click pymobiledevice3>=2.0.2 pygments -haralyzer>=2.4.0 \ No newline at end of file +haralyzer>=2.4.0 +maclog \ No newline at end of file