Add/devicemetrics enhancedproxy (#197)

This PR adds a few missing classes to unarchiver and initial support for LZ4 decompression.
With this change, dproxy is now able to fully dump instruments metrics sessions. Paves the way for adding the feature to go-ios finally.

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/google/gopacket v1.1.19
 	github.com/google/uuid v1.1.2
 	github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40
+	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/sirupsen/logrus v1.6.0
 	github.com/stretchr/testify v1.6.1
 	golang.org/x/crypto v0.0.0-20210812204632-0ba0e8f03122

go.sum

@@ -21,6 +21,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40 h1:EnfXoSqDfSNJv0VBNqY/88RNnhSGYkrHaO0mmFGbVsc=
 github.com/lunixbochs/struc v0.0.0-20200707160740-784aaebc1d40/go.mod h1:vy1vK6wD6j7xX6O6hXe621WabdtNkou2h7uRtTfRMyg=
+github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM=
+github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=

ios/debugproxy/decoders.go

@@ -133,7 +133,11 @@ func logDtxMessageNice(log *log.Entry, msg dtx.Message) {
 		return
 	}
 	if msg.PayloadHeader.MessageType == dtx.UnknownTypeOne {
-		log.Infof("type1: %x", msg.Payload[0])
+		if len(msg.Payload) > 0 {
+			log.Infof("type1 with payload: %x", msg.Payload[0])
+			return
+		}
+		log.Infof("type1 without payload: %+v", msg)
 		return
 	}
 	if msg.PayloadHeader.MessageType == dtx.ResponseWithReturnValueInPayload {

ios/dtx_codec/decoder.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	log "github.com/sirupsen/logrus"
 	"io"
 
 	"github.com/danielpaulus/go-ios/ios/nskeyedarchiver"
@@ -156,6 +157,7 @@ func DecodeNonBlocking(messageBytes []byte) (Message, []byte, error) {
 		return Message{}, make([]byte, 0), NewIncomplete("Payload missing")
 	}
 	result.RawBytes = messageBytes[:totalMessageLength]
+
 	if result.HasPayload() {
 		payload, err := result.parsePayloadBytes(result.RawBytes)
 		if err != nil {
@@ -212,6 +214,15 @@ func (d Message) parsePayloadBytes(messageBytes []byte) ([]interface{}, error) {
 	if d.PayloadHeader.MessageType == UnknownTypeOne {
 		return []interface{}{messageBytes[offset:]}, nil
 	}
+	if d.PayloadHeader.MessageType == LZ4CompressedMessage {
+		uncompressed, err := Decompress(messageBytes[offset:])
+		if err == nil {
+			log.Infof("lz4 compressed %d bytes/ %d uncompressed ", len(messageBytes[offset:]), len(uncompressed))
+		} else {
+			log.Infof("skipping lz4 compressed msg with %d bytes, decompression error %v", len(messageBytes[offset:]), err)
+		}
+		return []interface{}{messageBytes[offset:]}, nil
+	}
 	return nskeyedarchiver.Unarchive(messageBytes[offset:])
 }
 

ios/dtx_codec/decoder_test.go

@@ -32,6 +32,22 @@ func TestErrors(t *testing.T) {
 
 }
 
+func TestLZ4CompressedDtxMessage(t *testing.T) {
+	dat, err := ioutil.ReadFile("fixtures/instruments-metrics-dtx.bin")
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fixtureMsg, _, err := dtx.DecodeNonBlocking(dat)
+	if err != nil {
+		t.Fatal(err)
+	}
+	log.Infof("%v", fixtureMsg)
+	assert.NoError(t, err)
+
+}
+
 func TestCodec2(t *testing.T) {
 	dat, err := ioutil.ReadFile("fixtures/requestChannelWithCodeIdentifier.bin")
 

ios/dtx_codec/dtxmessage.go

@@ -68,21 +68,25 @@ type AuxiliaryHeader struct {
 const (
 	//Ack is the messagetype for a 16 byte long acknowleding DtxMessage.
 	Ack = 0x0
-	//Uknown
+	//Unknown
 	UnknownTypeOne = 0x1
 	//Methodinvocation is the messagetype for a remote procedure call style DtxMessage.
 	Methodinvocation = 0x2
 	//ResponseWithReturnValueInPayload is the response for a method call that has a return value
 	ResponseWithReturnValueInPayload = 0x3
 	//DtxTypeError is the messagetype for a DtxMessage containing an error
-	DtxTypeError = 0x4
+	DtxTypeError         = 0x4
+	LZ4CompressedMessage = 0x0707
 )
 
 //This is only used for creating nice String() output
 var messageTypeLookup = map[int]string{
 	ResponseWithReturnValueInPayload: `ResponseWithReturnValueInPayload`,
 	Methodinvocation:                 `Methodinvocation`,
 	Ack:                              `Ack`,
+	LZ4CompressedMessage:             `LZ4Compressed`,
+	UnknownTypeOne:                   `UnknownType1`,
+	DtxTypeError:                     `Error`,
 }
 
 func (d Message) String() string {

ios/dtx_codec/lz4decompress.go

@@ -0,0 +1,36 @@
+package dtx
+
+import (
+	"encoding/binary"
+	"github.com/pierrec/lz4"
+)
+
+const bv41 = 0x62763431
+
+//https://discuss.appium.io/t/how-to-parse-trace-file-to-get-cpu-performance-usage-data-for-ios-apps/35334/2
+func Decompress(data []byte) ([]byte, error) {
+	//no idea what the first four bytes mean
+	totalUncompressedSize := binary.LittleEndian.Uint32(data)
+	data = data[4:]
+
+	var magic uint32
+	magic = binary.BigEndian.Uint32(data)
+	compressedAgg := make([]byte, 0)
+	for magic == bv41 {
+		//uncompressedSize := binary.LittleEndian.Uint32(data[4:])
+		compressedSize := binary.LittleEndian.Uint32(data[8:])
+		chunk := data[12 : 12+compressedSize]
+		//log.Infof("chunk: %x", chunk)
+		data = data[12+compressedSize:]
+
+		compressedAgg = append(compressedAgg, chunk...)
+		magic = binary.BigEndian.Uint32(data)
+	}
+	uncompressedData := make([]byte, totalUncompressedSize+100)
+	n, err := lz4.UncompressBlock(compressedAgg, uncompressedData)
+	if err != nil {
+		return []byte{}, err
+	}
+	//log.Infof("uncompressed lz4 data of %d bytes", len(uncompressedData[:n]))
+	return uncompressedData[:n], nil
+}

ios/instruments/instruments_metrics.go

@@ -0,0 +1,40 @@
+package instruments
+
+import (
+	"github.com/danielpaulus/go-ios/ios"
+	dtx "github.com/danielpaulus/go-ios/ios/dtx_codec"
+	log "github.com/sirupsen/logrus"
+)
+
+type metricsDispatcher struct {
+	messageChannel chan dtx.Message
+	closeChannel   chan struct{}
+}
+
+func (dispatcher metricsDispatcher) Dispatch(msg dtx.Message) {
+	log.Infof("%+v", msg)
+}
+
+func GetMetrics(device ios.DeviceEntry) (func() (map[string]interface{}, error), func() error, error) {
+	conn, err := connectInstruments(device)
+	if err != nil {
+		return nil, nil, err
+	}
+	dispatcher := metricsDispatcher{messageChannel: make(chan dtx.Message), closeChannel: make(chan struct{})}
+	conn.AddDefaultChannelReceiver(dispatcher)
+	channel := conn.RequestChannelIdentifier(mobileNotificationsChannel, channelDispatcher{})
+	resp, err := channel.MethodCall("setApplicationStateNotificationsEnabled:", true)
+	if err != nil {
+		log.Errorf("resp:%+v, %+v", resp, resp.Payload[0])
+		return nil, nil, err
+	}
+	log.Debugf("appstatenotifications enabled successfully: %+v", resp)
+	resp, err = channel.MethodCall("setMemoryNotificationsEnabled:", true)
+	if err != nil {
+		log.Errorf("resp:%+v, %+v", resp, resp.Payload[0])
+		return nil, nil, err
+	}
+	log.Debugf("memory notifications enabled: %+v", resp)
+
+	return nil, nil, nil
+}

ios/nskeyedarchiver/archiver_test.go

@@ -76,6 +76,42 @@ func TestXCTCaps(t *testing.T) {
 	log.Info(unarchivedObject)
 }
 
+func TestDTCPUClusterInfo(t *testing.T) {
+	nskeyedBytes, err := ioutil.ReadFile("fixtures/dtcpuclusterinfo.bin")
+	if err != nil {
+		log.Error(err)
+		t.Fatal()
+	}
+	unarchivedObject, err := archiver.Unarchive(nskeyedBytes)
+	assert.NoError(t, err)
+	log.Info(unarchivedObject)
+}
+
+func TestDTTapMessage(t *testing.T) {
+	nskeyedBytes, err := ioutil.ReadFile("fixtures/dttapmessage.bin")
+	if err != nil {
+		log.Error(err)
+		t.Fatal()
+	}
+	unarchivedObject, err := archiver.Unarchive(nskeyedBytes)
+	assert.NoError(t, err)
+	log.Info(unarchivedObject)
+}
+
+func TestDTSysmonTap(t *testing.T) {
+	nskeyedBytes, err := ioutil.ReadFile("fixtures/DTSysmonTapMessage.bin")
+
+	if err != nil {
+
+		log.Error(err)
+		t.Fatal()
+	}
+
+	unarchivedObject, err := archiver.Unarchive(nskeyedBytes)
+	assert.NoError(t, err)
+	log.Info(unarchivedObject)
+}
+
 func TestNSUUID(t *testing.T) {
 	nskeyedBytes, err := ioutil.ReadFile("fixtures/nsuuid.bin")
 

ios/nskeyedarchiver/objectivec_classes.go

@@ -17,6 +17,7 @@ func SetupDecoders() {
 	if decodableClasses == nil {
 		decodableClasses = map[string]func(map[string]interface{}, []interface{}) interface{}{
 			"DTActivityTraceTapMessage": NewDTActivityTraceTapMessage,
+			"DTSysmonTapMessage":        NewDTActivityTraceTapMessage,
 			"NSError":                   NewNSError,
 			"NSNull":                    NewNSNullFromArchived,
 			"NSDate":                    NewNSDate,
@@ -29,6 +30,8 @@ func SetupDecoders() {
 			"NSValue":                   NewNSValue,
 			"XCTTestIdentifier":         NewXCTTestIdentifier,
 			"DTTapStatusMessage":        NewDTTapStatusMessage,
+			"DTTapMessage":              NewDTTapMessage,
+			"DTCPUClusterInfo":          NewDTCPUClusterInfo,
 		}
 	}
 }
@@ -347,6 +350,10 @@ type DTTapHeartbeatMessage struct {
 	DTTapMessagePlist map[string]interface{}
 }
 
+type DTTapMessage struct {
+	DTTapMessagePlist map[string]interface{}
+}
+
 type XCTCapabilities struct {
 	CapabilitiesDictionary map[string]interface{}
 }
@@ -363,6 +370,12 @@ func NewDTTapHeartbeatMessage(object map[string]interface{}, objects []interface
 	return DTTapHeartbeatMessage{DTTapMessagePlist: plist}
 }
 
+func NewDTTapMessage(object map[string]interface{}, objects []interface{}) interface{} {
+	ref := object["DTTapMessagePlist"].(plist.UID)
+	plist, _ := extractDictionary(objects[ref].(map[string]interface{}), objects)
+	return DTTapMessage{DTTapMessagePlist: plist}
+}
+
 type DTTapStatusMessage struct {
 	DTTapMessagePlist map[string]interface{}
 }
@@ -384,6 +397,15 @@ func (n NSDate) String() string {
 	return fmt.Sprintf("%s", n.Timestamp)
 }
 
+type DTCPUClusterInfo struct {
+	ClusterID    uint64
+	ClusterFlags uint64
+}
+
+func NewDTCPUClusterInfo(object map[string]interface{}, objects []interface{}) interface{} {
+	return DTCPUClusterInfo{ClusterID: object["_clusterID"].(uint64), ClusterFlags: object["_clusterFlags"].(uint64)}
+}
+
 type NSNull struct {
 	class string
 }