From a5f3343a8e6d34b8dd1cfc209b5862d6af7d8bdb Mon Sep 17 00:00:00 2001 From: Rod Hynes Date: Thu, 30 Jan 2025 20:15:42 -0500 Subject: [PATCH] Add in-proxy WebRTC media stream mode --- psiphon/common/errors/errors.go | 18 + psiphon/common/inproxy/api.go | 226 +- psiphon/common/inproxy/broker.go | 71 +- psiphon/common/inproxy/client.go | 31 +- psiphon/common/inproxy/coordinator.go | 14 +- psiphon/common/inproxy/coordinator_test.go | 84 +- psiphon/common/inproxy/discovery.go | 1 - psiphon/common/inproxy/discoverySTUN.go | 9 +- psiphon/common/inproxy/inproxy_disabled.go | 13 +- psiphon/common/inproxy/inproxy_test.go | 180 +- psiphon/common/inproxy/matcher.go | 52 +- psiphon/common/inproxy/matcher_test.go | 131 +- psiphon/common/inproxy/portmapper_other.go | 2 +- psiphon/common/inproxy/proxy.go | 44 +- psiphon/common/inproxy/session_test.go | 10 +- psiphon/common/inproxy/webrtc.go | 1925 ++++++++++++++---- psiphon/common/parameters/inproxy.go | 8 +- psiphon/common/parameters/parameters.go | 40 +- psiphon/common/parameters/parameters_test.go | 4 +- psiphon/common/protocol/packed.go | 3 +- psiphon/common/quic/obfuscator_test.go | 1 + psiphon/common/quic/quic.go | 61 +- psiphon/common/quic/quic_disabled.go | 4 + psiphon/common/quic/quic_test.go | 2 + psiphon/config.go | 70 +- psiphon/controller.go | 13 +- psiphon/dialParameters.go | 26 + psiphon/inproxy.go | 63 +- psiphon/net.go | 2 +- psiphon/net_darwin.go | 6 +- psiphon/server/api.go | 1 + psiphon/server/meek.go | 14 +- psiphon/server/server_test.go | 54 +- psiphon/server/tunnelServer.go | 31 +- psiphon/tunnel.go | 1 + 35 files changed, 2429 insertions(+), 786 deletions(-) diff --git a/psiphon/common/errors/errors.go b/psiphon/common/errors/errors.go index dfe06f139..7a640fe81 100644 --- a/psiphon/common/errors/errors.go +++ b/psiphon/common/errors/errors.go @@ -25,6 +25,7 @@ package errors import ( "fmt" + "io" "runtime" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/stacktrace" @@ -72,6 +73,23 @@ func Trace(err error) error { return fmt.Errorf("%s#%d: %w", stacktrace.GetFunctionName(pc), line, err) } +// TraceReader wraps the given error with the caller stack frame information, +// except in the case of io.EOF, which is returned unwrapped. This is used to +// preserve io.Reader.Read io.EOF error returns. +func TraceReader(err error) error { + if err == nil { + return nil + } + if err == io.EOF { + return io.EOF + } + pc, _, line, ok := runtime.Caller(1) + if !ok { + return fmt.Errorf("[unknown]: %w", err) + } + return fmt.Errorf("%s#%d: %w", stacktrace.GetFunctionName(pc), line, err) +} + // TraceMsg wraps the given error with the caller stack frame information // and the given message. func TraceMsg(err error, message string) error { diff --git a/psiphon/common/inproxy/api.go b/psiphon/common/inproxy/api.go index 0f9059f7f..9c83ba116 100644 --- a/psiphon/common/inproxy/api.go +++ b/psiphon/common/inproxy/api.go @@ -31,19 +31,81 @@ import ( const ( - // ProxyProtocolVersion1 represents protocol version 1. - ProxyProtocolVersion1 = int32(1) + // ProtocolVersion1 represents protocol version 1, the initial in-proxy + // protocol version number. + ProtocolVersion1 = int32(1) - // MinimumProxyProtocolVersion is the minimum supported version number. - MinimumProxyProtocolVersion = ProxyProtocolVersion1 + // ProtocolVersion2 represents protocol version 2, which adds support for + // proxying over WebRTC media streams. + ProtocolVersion2 = int32(2) + + // LatestProtocolVersion is the current, default protocol version number. + LatestProtocolVersion = ProtocolVersion2 + + // MinimumProxyProtocolVersion is the minimum required protocol version + // number for proxies. + MinimumProxyProtocolVersion = ProtocolVersion1 + + // MinimumClientProtocolVersion is the minimum supported protocol version + // number for clients. + MinimumClientProtocolVersion = ProtocolVersion1 MaxCompartmentIDs = 10 ) -// proxyProtocolVersion is the current protocol version number. -// proxyProtocolVersion is variable, to enable overriding the value in tests. -// This value should not be overridden outside of test cases. -var proxyProtocolVersion = ProxyProtocolVersion1 +// minimumProxyProtocolVersion and minimumClientProtocolVersion are variable +// to enable overriding the values in tests. These value should not be +// overridden outside of test cases. +var ( + minimumProxyProtocolVersion = MinimumProxyProtocolVersion + minimumClientProtocolVersion = MinimumClientProtocolVersion +) + +// negotiateProtocolVersion selects the in-proxy protocol version for a new +// proxy/client match, based on the client's and proxy's reported protocol +// versions, and the client's selected protocol options. Returns false if no +// protocol version selection is possible. +// +// The broker performs the negotiation on behalf of the proxy and client. Both +// the proxy and client initially specify the latest protocol version they +// support. The client specifies the protocol options to use, based on +// tactics and replay. +// +// negotiateProtocolVersion is used by the matcher when searching for +// potential matches; for this reason, the failure case is expected and +// returns a simple boolean intead of formating an error message. +// +// Existing, legacy proxies have the equivalent of an "if +// announceResponse.SelectedProtocolVersion != ProtocolVersion1" check, so +// the SelectedProtocolVersion must be downgraded in that case, if a match is +// possible. +func negotiateProtocolVersion( + proxyProtocolVersion int32, + clientProtocolVersion int32, + useMediaStreams bool) (int32, bool) { + + // When not using WebRTC media streams, introduced in ProtocolVersion2, + // potentially downgrade if either the proxy or client supports only + // ProtocolVersion1. + + if (proxyProtocolVersion == ProtocolVersion1 || + clientProtocolVersion == ProtocolVersion1) && + !useMediaStreams { + return ProtocolVersion1, true + } + + // Select the client's protocol version. + + if proxyProtocolVersion >= clientProtocolVersion { + return clientProtocolVersion, true + } + + // No selection is possible. This includes the case where the proxy + // supports up to ProtocolVersion1 and the client has specified media + // streams. + + return 0, false +} // ID is a unique identifier used to identify inproxy connections and actors. type ID [32]byte @@ -192,7 +254,7 @@ func (p NetworkProtocol) IsStream() bool { // and clients. type ProxyMetrics struct { BaseAPIParameters protocol.PackedAPIParameters `cbor:"1,keyasint,omitempty"` - ProxyProtocolVersion int32 `cbor:"2,keyasint,omitempty"` + ProtocolVersion int32 `cbor:"2,keyasint,omitempty"` NATType NATType `cbor:"3,keyasint,omitempty"` PortMappingTypes PortMappingTypes `cbor:"4,keyasint,omitempty"` MaxClients int32 `cbor:"6,keyasint,omitempty"` @@ -208,10 +270,10 @@ type ProxyMetrics struct { // broker. The broker uses this information when matching proxies and // clients. type ClientMetrics struct { - BaseAPIParameters protocol.PackedAPIParameters `cbor:"1,keyasint,omitempty"` - ProxyProtocolVersion int32 `cbor:"2,keyasint,omitempty"` - NATType NATType `cbor:"3,keyasint,omitempty"` - PortMappingTypes PortMappingTypes `cbor:"4,keyasint,omitempty"` + BaseAPIParameters protocol.PackedAPIParameters `cbor:"1,keyasint,omitempty"` + ProtocolVersion int32 `cbor:"2,keyasint,omitempty"` + NATType NATType `cbor:"3,keyasint,omitempty"` + PortMappingTypes PortMappingTypes `cbor:"4,keyasint,omitempty"` } // ProxyAnnounceRequest is an API request sent from a proxy to a broker, @@ -264,24 +326,26 @@ type WebRTCSessionDescription struct { // corresponds to a valid Psiphon server. // // MustUpgrade is an optional flag that is set by the broker, based on the -// submitted ProxyProtocolVersion, when the proxy app must be upgraded in -// order to function properly. Potential must-upgrade scenarios include -// changes to the personal pairing broker rendezvous algorithm, where no -// protocol backwards compatibility accommodations can ensure a rendezvous -// and match. When MustUpgrade is set, NoMatch is implied. +// submitted ProtocolVersion, when the proxy app must be upgraded in order to +// function properly. Potential must-upgrade scenarios include changes to the +// personal pairing broker rendezvous algorithm, where no protocol backwards +// compatibility accommodations can ensure a rendezvous and match. When +// MustUpgrade is set, NoMatch is implied. + type ProxyAnnounceResponse struct { - TacticsPayload []byte `cbor:"2,keyasint,omitempty"` - Limited bool `cbor:"3,keyasint,omitempty"` - NoMatch bool `cbor:"4,keyasint,omitempty"` - MustUpgrade bool `cbor:"13,keyasint,omitempty"` - ConnectionID ID `cbor:"5,keyasint,omitempty"` - ClientProxyProtocolVersion int32 `cbor:"6,keyasint,omitempty"` - ClientOfferSDP WebRTCSessionDescription `cbor:"7,keyasint,omitempty"` - ClientRootObfuscationSecret ObfuscationSecret `cbor:"8,keyasint,omitempty"` - DoDTLSRandomization bool `cbor:"9,keyasint,omitempty"` - TrafficShapingParameters *DataChannelTrafficShapingParameters `cbor:"10,keyasint,omitempty"` - NetworkProtocol NetworkProtocol `cbor:"11,keyasint,omitempty"` - DestinationAddress string `cbor:"12,keyasint,omitempty"` + TacticsPayload []byte `cbor:"2,keyasint,omitempty"` + Limited bool `cbor:"3,keyasint,omitempty"` + NoMatch bool `cbor:"4,keyasint,omitempty"` + MustUpgrade bool `cbor:"13,keyasint,omitempty"` + ConnectionID ID `cbor:"5,keyasint,omitempty"` + SelectedProtocolVersion int32 `cbor:"6,keyasint,omitempty"` + ClientOfferSDP WebRTCSessionDescription `cbor:"7,keyasint,omitempty"` + ClientRootObfuscationSecret ObfuscationSecret `cbor:"8,keyasint,omitempty"` + DoDTLSRandomization bool `cbor:"9,keyasint,omitempty"` + UseMediaStreams bool `cbor:"14,keyasint,omitempty"` + TrafficShapingParameters *TrafficShapingParameters `cbor:"10,keyasint,omitempty"` + NetworkProtocol NetworkProtocol `cbor:"11,keyasint,omitempty"` + DestinationAddress string `cbor:"12,keyasint,omitempty"` } // ClientOfferRequest is an API request sent from a client to a broker, @@ -305,24 +369,25 @@ type ProxyAnnounceResponse struct { // domain, and destination port for a valid Psiphon tunnel protocol run by // the specified server entry. type ClientOfferRequest struct { - Metrics *ClientMetrics `cbor:"1,keyasint,omitempty"` - CommonCompartmentIDs []ID `cbor:"2,keyasint,omitempty"` - PersonalCompartmentIDs []ID `cbor:"3,keyasint,omitempty"` - ClientOfferSDP WebRTCSessionDescription `cbor:"4,keyasint,omitempty"` - ICECandidateTypes ICECandidateTypes `cbor:"5,keyasint,omitempty"` - ClientRootObfuscationSecret ObfuscationSecret `cbor:"6,keyasint,omitempty"` - DoDTLSRandomization bool `cbor:"7,keyasint,omitempty"` - TrafficShapingParameters *DataChannelTrafficShapingParameters `cbor:"8,keyasint,omitempty"` - PackedDestinationServerEntry []byte `cbor:"9,keyasint,omitempty"` - NetworkProtocol NetworkProtocol `cbor:"10,keyasint,omitempty"` - DestinationAddress string `cbor:"11,keyasint,omitempty"` -} - -// DataChannelTrafficShapingParameters specifies a data channel traffic -// shaping configuration, including random padding and decoy messages. -// Clients determine their own traffic shaping configuration, and generate -// and send a configuration for the peer proxy to use. -type DataChannelTrafficShapingParameters struct { + Metrics *ClientMetrics `cbor:"1,keyasint,omitempty"` + CommonCompartmentIDs []ID `cbor:"2,keyasint,omitempty"` + PersonalCompartmentIDs []ID `cbor:"3,keyasint,omitempty"` + ClientOfferSDP WebRTCSessionDescription `cbor:"4,keyasint,omitempty"` + ICECandidateTypes ICECandidateTypes `cbor:"5,keyasint,omitempty"` + ClientRootObfuscationSecret ObfuscationSecret `cbor:"6,keyasint,omitempty"` + DoDTLSRandomization bool `cbor:"7,keyasint,omitempty"` + UseMediaStreams bool `cbor:"12,keyasint,omitempty"` + TrafficShapingParameters *TrafficShapingParameters `cbor:"8,keyasint,omitempty"` + PackedDestinationServerEntry []byte `cbor:"9,keyasint,omitempty"` + NetworkProtocol NetworkProtocol `cbor:"10,keyasint,omitempty"` + DestinationAddress string `cbor:"11,keyasint,omitempty"` +} + +// TrafficShapingParameters specifies data channel or media stream traffic +// shaping configuration, including random padding and decoy messages (or +// packets). Clients determine their own traffic shaping configuration, and +// generate and send a configuration for the peer proxy to use. +type TrafficShapingParameters struct { MinPaddedMessages int `cbor:"1,keyasint,omitempty"` MaxPaddedMessages int `cbor:"2,keyasint,omitempty"` MinPaddingSize int `cbor:"3,keyasint,omitempty"` @@ -347,19 +412,19 @@ type DataChannelTrafficShapingParameters struct { // connection and its relayed BrokerServerReport. // // MustUpgrade is an optional flag that is set by the broker, based on the -// submitted ProxyProtocolVersion, when the client app must be upgraded in -// order to function properly. Potential must-upgrade scenarios include -// changes to the personal pairing broker rendezvous algorithm, where no -// protocol backwards compatibility accommodations can ensure a rendezvous -// and match. When MustUpgrade is set, NoMatch is implied. +// submitted ProtocolVersion, when the client app must be upgraded in order +// to function properly. Potential must-upgrade scenarios include changes to +// the personal pairing broker rendezvous algorithm, where no protocol +// backwards compatibility accommodations can ensure a rendezvous and match. +// When MustUpgrade is set, NoMatch is implied. type ClientOfferResponse struct { - Limited bool `cbor:"1,keyasint,omitempty"` - NoMatch bool `cbor:"2,keyasint,omitempty"` - MustUpgrade bool `cbor:"7,keyasint,omitempty"` - ConnectionID ID `cbor:"3,keyasint,omitempty"` - SelectedProxyProtocolVersion int32 `cbor:"4,keyasint,omitempty"` - ProxyAnswerSDP WebRTCSessionDescription `cbor:"5,keyasint,omitempty"` - RelayPacketToServer []byte `cbor:"6,keyasint,omitempty"` + Limited bool `cbor:"1,keyasint,omitempty"` + NoMatch bool `cbor:"2,keyasint,omitempty"` + MustUpgrade bool `cbor:"7,keyasint,omitempty"` + ConnectionID ID `cbor:"3,keyasint,omitempty"` + SelectedProtocolVersion int32 `cbor:"4,keyasint,omitempty"` + ProxyAnswerSDP WebRTCSessionDescription `cbor:"5,keyasint,omitempty"` + RelayPacketToServer []byte `cbor:"6,keyasint,omitempty"` } // TODO: Encode SDPs using CBOR without field names, simliar to packed metrics? @@ -373,11 +438,14 @@ type ClientOfferResponse struct { // reason, it should still send ProxyAnswerRequest with AnswerError // populated; the broker will signal the client to abort this connection. type ProxyAnswerRequest struct { - ConnectionID ID `cbor:"1,keyasint,omitempty"` - SelectedProxyProtocolVersion int32 `cbor:"2,keyasint,omitempty"` - ProxyAnswerSDP WebRTCSessionDescription `cbor:"3,keyasint,omitempty"` - ICECandidateTypes ICECandidateTypes `cbor:"4,keyasint,omitempty"` - AnswerError string `cbor:"5,keyasint,omitempty"` + ConnectionID ID `cbor:"1,keyasint,omitempty"` + ProxyAnswerSDP WebRTCSessionDescription `cbor:"3,keyasint,omitempty"` + ICECandidateTypes ICECandidateTypes `cbor:"4,keyasint,omitempty"` + AnswerError string `cbor:"5,keyasint,omitempty"` + + // These fields are no longer used. + // + // SelectedProtocolVersion int32 `cbor:"2,keyasint,omitempty"` } // ProxyAnswerResponse is the acknowledgement for a ProxyAnswerRequest. @@ -496,8 +564,8 @@ func (metrics *ProxyMetrics) ValidateAndGetParametersAndLogFields( return nil, nil, errors.Trace(err) } - if metrics.ProxyProtocolVersion < 0 || metrics.ProxyProtocolVersion > proxyProtocolVersion { - return nil, nil, errors.Tracef("invalid proxy protocol version: %v", metrics.ProxyProtocolVersion) + if metrics.ProtocolVersion < ProtocolVersion1 || metrics.ProtocolVersion > LatestProtocolVersion { + return nil, nil, errors.Tracef("invalid protocol version: %v", metrics.ProtocolVersion) } if !metrics.NATType.IsValid() { @@ -514,7 +582,7 @@ func (metrics *ProxyMetrics) ValidateAndGetParametersAndLogFields( logFields := formatter(logFieldPrefix, geoIPData, baseParams) - logFields[logFieldPrefix+"proxy_protocol_version"] = metrics.ProxyProtocolVersion + logFields[logFieldPrefix+"protocol_version"] = metrics.ProtocolVersion logFields[logFieldPrefix+"nat_type"] = metrics.NATType logFields[logFieldPrefix+"port_mapping_types"] = metrics.PortMappingTypes logFields[logFieldPrefix+"max_clients"] = metrics.MaxClients @@ -549,8 +617,8 @@ func (metrics *ClientMetrics) ValidateAndGetLogFields( return nil, errors.Trace(err) } - if metrics.ProxyProtocolVersion < 0 || metrics.ProxyProtocolVersion > proxyProtocolVersion { - return nil, errors.Tracef("invalid proxy protocol version: %v", metrics.ProxyProtocolVersion) + if metrics.ProtocolVersion < ProtocolVersion1 || metrics.ProtocolVersion > LatestProtocolVersion { + return nil, errors.Tracef("invalid protocol version: %v", metrics.ProtocolVersion) } if !metrics.NATType.IsValid() { @@ -567,7 +635,7 @@ func (metrics *ClientMetrics) ValidateAndGetLogFields( logFields := formatter("", geoIPData, baseParams) - logFields["proxy_protocol_version"] = metrics.ProxyProtocolVersion + logFields["protocol_version"] = metrics.ProtocolVersion logFields["nat_type"] = metrics.NATType logFields["port_mapping_types"] = metrics.PortMappingTypes @@ -620,6 +688,14 @@ func (request *ClientOfferRequest) ValidateAndGetLogFields( formatter common.APIParameterLogFieldFormatter, geoIPData common.GeoIPData) ([]byte, common.LogFields, error) { + // UseMediaStreams requires at least ProtocolVersion2. + if request.UseMediaStreams && + request.Metrics.ProtocolVersion < ProtocolVersion2 { + + return nil, nil, errors.Tracef( + "invalid protocol version: %d", request.Metrics.ProtocolVersion) + } + if len(request.CommonCompartmentIDs) > maxCompartmentIDs { return nil, nil, errors.Tracef( "invalid compartment IDs length: %d", len(request.CommonCompartmentIDs)) @@ -700,13 +776,14 @@ func (request *ClientOfferRequest) ValidateAndGetLogFields( logFields["has_IPv6"] = sdpMetrics.hasIPv6 logFields["has_private_IP"] = sdpMetrics.hasPrivateIP logFields["filtered_ice_candidates"] = sdpMetrics.filteredICECandidates + logFields["use_media_streams"] = request.UseMediaStreams return filteredSDP, logFields, nil } // Validate validates the that client has not specified excess traffic shaping // padding or decoy traffic. -func (params *DataChannelTrafficShapingParameters) Validate() error { +func (params *TrafficShapingParameters) Validate() error { if params.MinPaddedMessages < 0 || params.MinPaddedMessages > params.MaxPaddedMessages || @@ -777,11 +854,6 @@ func (request *ProxyAnswerRequest) ValidateAndGetLogFields( "invalid ICE candidate types: %v", request.ICECandidateTypes) } - if request.SelectedProxyProtocolVersion != ProxyProtocolVersion1 { - return nil, nil, errors.Tracef( - "invalid select proxy protocol version: %v", request.SelectedProxyProtocolVersion) - } - logFields := formatter("", geoIPData, common.APIParameters{}) logFields["connection_id"] = request.ConnectionID diff --git a/psiphon/common/inproxy/broker.go b/psiphon/common/inproxy/broker.go index 18563a55b..80002c942 100644 --- a/psiphon/common/inproxy/broker.go +++ b/psiphon/common/inproxy/broker.go @@ -123,13 +123,13 @@ type BrokerConfig struct { AllowProxy func(common.GeoIPData) bool // PrioritizeProxy is a callback which can indicate whether proxy - // announcements from proxies with the specified GeoIPData and - // APIParameters should be prioritized in the matcher queue. Priority - // proxy announcements match ahead of other proxy announcements, - // regardless of announcement age/deadline. Priority status takes - // precedence over preferred NAT matching. Prioritization applies only to - // common compartment IDs and not personal pairing mode. - PrioritizeProxy func(common.GeoIPData, common.APIParameters) bool + // announcements from proxies with the specified in-proxy protocol + // version, GeoIPData, and APIParameters should be prioritized in the + // matcher queue. Priority proxy announcements match ahead of other proxy + // announcements, regardless of announcement age/deadline. Priority + // status takes precedence over preferred NAT matching. Prioritization + // applies only to common compartment IDs and not personal pairing mode. + PrioritizeProxy func(int, common.GeoIPData, common.APIParameters) bool // AllowClient is a callback which can indicate whether a client with the // given GeoIP data is allowed to match with common compartment ID @@ -483,6 +483,7 @@ func (b *Broker) handleProxyAnnounce( startTime := time.Now() var logFields common.LogFields + var isPriority bool var newTacticsTag string var clientOffer *MatchOffer var matchMetrics *MatchMetrics @@ -529,6 +530,7 @@ func (b *Broker) handleProxyAnnounce( logFields["broker_event"] = "proxy-announce" logFields["broker_id"] = b.brokerID logFields["proxy_id"] = proxyID + logFields["is_priority"] = isPriority logFields["elapsed_time"] = time.Since(startTime) / time.Millisecond logFields["connection_id"] = connectionID if newTacticsTag != "" { @@ -538,6 +540,7 @@ func (b *Broker) handleProxyAnnounce( // Log the target Psiphon server ID (diagnostic ID). The presence // of this field indicates that a match was made. logFields["destination_server_id"] = clientOffer.DestinationServerID + logFields["use_media_streams"] = clientOffer.UseMediaStreams } if timedOut { logFields["timed_out"] = true @@ -571,7 +574,7 @@ func (b *Broker) handleProxyAnnounce( // Return MustUpgrade when the proxy's protocol version is less than the // minimum required. - if announceRequest.Metrics.ProxyProtocolVersion < MinimumProxyProtocolVersion { + if announceRequest.Metrics.ProtocolVersion < minimumProxyProtocolVersion { responsePayload, err := MarshalProxyAnnounceResponse( &ProxyAnnounceResponse{ NoMatch: true, @@ -643,7 +646,6 @@ func (b *Broker) handleProxyAnnounce( // In the common compartment ID case, invoke the callback to check if the // announcement should be prioritized. - isPriority := false if b.config.PrioritizeProxy != nil && !hasPersonalCompartmentIDs { // Limitation: Of the two return values from @@ -659,7 +661,8 @@ func (b *Broker) handleProxyAnnounce( // calls for range filtering, which is not yet supported in the // psiphon/server.MeekServer PrioritizeProxy provider. - isPriority = b.config.PrioritizeProxy(geoIPData, apiParams) + isPriority = b.config.PrioritizeProxy( + int(announceRequest.Metrics.ProtocolVersion), geoIPData, apiParams) } // Await client offer. @@ -685,6 +688,7 @@ func (b *Broker) handleProxyAnnounce( &MatchAnnouncement{ Properties: MatchProperties{ IsPriority: isPriority, + ProtocolVersion: announceRequest.Metrics.ProtocolVersion, CommonCompartmentIDs: commonCompartmentIDs, PersonalCompartmentIDs: announceRequest.PersonalCompartmentIDs, GeoIPData: geoIPData, @@ -746,6 +750,17 @@ func (b *Broker) handleProxyAnnounce( return responsePayload, nil } + // Select the protocol version. The matcher has already checked + // negotiateProtocolVersion, so failure is not expected. + + negotiatedProtocolVersion, ok := negotiateProtocolVersion( + announceRequest.Metrics.ProtocolVersion, + clientOffer.Properties.ProtocolVersion, + clientOffer.UseMediaStreams) + if !ok { + return nil, errors.TraceNew("unexpected negotiateProtocolVersion failure") + } + // Respond with the client offer. The proxy will follow up with an answer // request, which is relayed to the client, and then the WebRTC dial begins. @@ -760,10 +775,11 @@ func (b *Broker) handleProxyAnnounce( &ProxyAnnounceResponse{ TacticsPayload: tacticsPayload, ConnectionID: connectionID, - ClientProxyProtocolVersion: clientOffer.ClientProxyProtocolVersion, + SelectedProtocolVersion: negotiatedProtocolVersion, ClientOfferSDP: clientOffer.ClientOfferSDP, ClientRootObfuscationSecret: clientOffer.ClientRootObfuscationSecret, DoDTLSRandomization: clientOffer.DoDTLSRandomization, + UseMediaStreams: clientOffer.UseMediaStreams, TrafficShapingParameters: clientOffer.TrafficShapingParameters, NetworkProtocol: clientOffer.NetworkProtocol, DestinationAddress: clientOffer.DestinationAddress, @@ -907,7 +923,7 @@ func (b *Broker) handleClientOffer( // Return MustUpgrade when the client's protocol version is less than the // minimum required. - if offerRequest.Metrics.ProxyProtocolVersion < MinimumProxyProtocolVersion { + if offerRequest.Metrics.ProtocolVersion < minimumClientProtocolVersion { responsePayload, err := MarshalClientOfferResponse( &ClientOfferResponse{ NoMatch: true, @@ -944,6 +960,7 @@ func (b *Broker) handleClientOffer( clientMatchOffer = &MatchOffer{ Properties: MatchProperties{ + ProtocolVersion: offerRequest.Metrics.ProtocolVersion, CommonCompartmentIDs: offerRequest.CommonCompartmentIDs, PersonalCompartmentIDs: offerRequest.PersonalCompartmentIDs, GeoIPData: geoIPData, @@ -951,10 +968,10 @@ func (b *Broker) handleClientOffer( NATType: offerRequest.Metrics.NATType, PortMappingTypes: offerRequest.Metrics.PortMappingTypes, }, - ClientProxyProtocolVersion: offerRequest.Metrics.ProxyProtocolVersion, ClientOfferSDP: offerSDP, ClientRootObfuscationSecret: offerRequest.ClientRootObfuscationSecret, DoDTLSRandomization: offerRequest.DoDTLSRandomization, + UseMediaStreams: offerRequest.UseMediaStreams, TrafficShapingParameters: offerRequest.TrafficShapingParameters, NetworkProtocol: offerRequest.NetworkProtocol, DestinationAddress: offerRequest.DestinationAddress, @@ -1069,14 +1086,25 @@ func (b *Broker) handleClientOffer( return nil, errors.Trace(err) } + // Select the protocol version. The matcher has already checked + // negotiateProtocolVersion, so failure is not expected. + + negotiatedProtocolVersion, ok := negotiateProtocolVersion( + proxyMatchAnnouncement.Properties.ProtocolVersion, + offerRequest.Metrics.ProtocolVersion, + offerRequest.UseMediaStreams) + if !ok { + return nil, errors.TraceNew("unexpected negotiateProtocolVersion failure") + } + // Respond with the proxy answer and initial broker/server session packet. responsePayload, err := MarshalClientOfferResponse( &ClientOfferResponse{ - ConnectionID: proxyAnswer.ConnectionID, - SelectedProxyProtocolVersion: proxyAnswer.SelectedProxyProtocolVersion, - ProxyAnswerSDP: proxyAnswer.ProxyAnswerSDP, - RelayPacketToServer: relayPacket, + ConnectionID: proxyAnswer.ConnectionID, + SelectedProtocolVersion: negotiatedProtocolVersion, + ProxyAnswerSDP: proxyAnswer.ProxyAnswerSDP, + RelayPacketToServer: relayPacket, }) if err != nil { return nil, errors.Trace(err) @@ -1180,11 +1208,10 @@ func (b *Broker) handleProxyAnswer( // These fields are used internally in the matcher. proxyAnswer = &MatchAnswer{ - ProxyIP: proxyIP, - ProxyID: initiatorID, - ConnectionID: answerRequest.ConnectionID, - SelectedProxyProtocolVersion: answerRequest.SelectedProxyProtocolVersion, - ProxyAnswerSDP: answerSDP, + ProxyIP: proxyIP, + ProxyID: initiatorID, + ConnectionID: answerRequest.ConnectionID, + ProxyAnswerSDP: answerSDP, } err = b.matcher.Answer(proxyAnswer) diff --git a/psiphon/common/inproxy/client.go b/psiphon/common/inproxy/client.go index b95de367c..154639055 100644 --- a/psiphon/common/inproxy/client.go +++ b/psiphon/common/inproxy/client.go @@ -359,7 +359,8 @@ func dialClientWebRTCConn( // Initialize the WebRTC offer doTLSRandomization := config.WebRTCDialCoordinator.DoDTLSRandomization() - trafficShapingParameters := config.WebRTCDialCoordinator.DataChannelTrafficShapingParameters() + useMediaStreams := config.WebRTCDialCoordinator.UseMediaStreams() + trafficShapingParameters := config.WebRTCDialCoordinator.TrafficShapingParameters() clientRootObfuscationSecret := config.WebRTCDialCoordinator.ClientRootObfuscationSecret() webRTCConn, SDP, SDPMetrics, err := newWebRTCConnForOffer( @@ -369,6 +370,7 @@ func dialClientWebRTCConn( WebRTCDialCoordinator: config.WebRTCDialCoordinator, ClientRootObfuscationSecret: clientRootObfuscationSecret, DoDTLSRandomization: doTLSRandomization, + UseMediaStreams: useMediaStreams, TrafficShapingParameters: trafficShapingParameters, ReliableTransport: config.ReliableTransport, }, @@ -411,10 +413,10 @@ func dialClientWebRTCConn( ctx, &ClientOfferRequest{ Metrics: &ClientMetrics{ - BaseAPIParameters: packedParams, - ProxyProtocolVersion: proxyProtocolVersion, - NATType: config.WebRTCDialCoordinator.NATType(), - PortMappingTypes: config.WebRTCDialCoordinator.PortMappingTypes(), + BaseAPIParameters: packedParams, + ProtocolVersion: LatestProtocolVersion, + NATType: config.WebRTCDialCoordinator.NATType(), + PortMappingTypes: config.WebRTCDialCoordinator.PortMappingTypes(), }, CommonCompartmentIDs: brokerCoordinator.CommonCompartmentIDs(), PersonalCompartmentIDs: personalCompartmentIDs, @@ -422,6 +424,7 @@ func dialClientWebRTCConn( ICECandidateTypes: SDPMetrics.iceCandidateTypes, ClientRootObfuscationSecret: clientRootObfuscationSecret, DoDTLSRandomization: doTLSRandomization, + UseMediaStreams: useMediaStreams, TrafficShapingParameters: trafficShapingParameters, PackedDestinationServerEntry: config.PackedDestinationServerEntry, NetworkProtocol: config.DialNetworkProtocol, @@ -458,11 +461,13 @@ func dialClientWebRTCConn( return nil, true, errors.TraceNew("no match") } - if offerResponse.SelectedProxyProtocolVersion < MinimumProxyProtocolVersion || - offerResponse.SelectedProxyProtocolVersion > proxyProtocolVersion { + if offerResponse.SelectedProtocolVersion < ProtocolVersion1 || + (useMediaStreams && + offerResponse.SelectedProtocolVersion < ProtocolVersion2) || + offerResponse.SelectedProtocolVersion > LatestProtocolVersion { return nil, false, errors.Tracef( - "Unsupported proxy protocol version: %d", - offerResponse.SelectedProxyProtocolVersion) + "Unsupported protocol version: %d", + offerResponse.SelectedProtocolVersion) } // Establish the WebRTC DataChannel connection @@ -473,13 +478,13 @@ func dialClientWebRTCConn( return nil, true, errors.Trace(err) } - awaitDataChannelCtx, awaitDataChannelCancelFunc := context.WithTimeout( + awaitReadyToProxyCtx, awaitReadyToProxyCancelFunc := context.WithTimeout( ctx, common.ValueOrDefault( - config.WebRTCDialCoordinator.WebRTCAwaitDataChannelTimeout(), dataChannelAwaitTimeout)) - defer awaitDataChannelCancelFunc() + config.WebRTCDialCoordinator.WebRTCAwaitReadyToProxyTimeout(), readyToProxyAwaitTimeout)) + defer awaitReadyToProxyCancelFunc() - err = webRTCConn.AwaitInitialDataChannel(awaitDataChannelCtx) + err = webRTCConn.AwaitReadyToProxy(awaitReadyToProxyCtx, offerResponse.ConnectionID) if err != nil { return nil, true, errors.Trace(err) } diff --git a/psiphon/common/inproxy/coordinator.go b/psiphon/common/inproxy/coordinator.go index 257ddea0d..bc1e7bf03 100644 --- a/psiphon/common/inproxy/coordinator.go +++ b/psiphon/common/inproxy/coordinator.go @@ -235,10 +235,14 @@ type WebRTCDialCoordinator interface { // the value. DoDTLSRandomization() bool - // DataChannelTrafficShapingParameters returns parameters specifying how - // to perform data channel traffic shapping -- random padding and decoy - // message. Returns nil when no traffic shaping is to be performed. - DataChannelTrafficShapingParameters() *DataChannelTrafficShapingParameters + // UseMediaStreams indicates whether to use WebRTC media streams to tunnel + // traffic. When false, a WebRTC data channel is used to tunnel traffic. + UseMediaStreams() bool + + // TrafficShapingParameters returns parameters specifying how to perform + // data channel or media stream traffic shapping -- random padding and + // decoy messages. Returns nil when no traffic shaping is to be performed. + TrafficShapingParameters() *TrafficShapingParameters // STUNServerAddress selects a STUN server to use for this dial. When // RFC5780 is true, the STUN server must support RFC5780 NAT discovery; @@ -363,7 +367,7 @@ type WebRTCDialCoordinator interface { DiscoverNATTimeout() time.Duration WebRTCAnswerTimeout() time.Duration WebRTCAwaitPortMappingTimeout() time.Duration - WebRTCAwaitDataChannelTimeout() time.Duration + WebRTCAwaitReadyToProxyTimeout() time.Duration ProxyDestinationDialTimeout() time.Duration ProxyRelayInactivityTimeout() time.Duration } diff --git a/psiphon/common/inproxy/coordinator_test.go b/psiphon/common/inproxy/coordinator_test.go index 14eccc6c1..3e703a249 100644 --- a/psiphon/common/inproxy/coordinator_test.go +++ b/psiphon/common/inproxy/coordinator_test.go @@ -30,6 +30,7 @@ import ( "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/stacktrace" ) @@ -204,7 +205,8 @@ type testWebRTCDialCoordinator struct { networkType NetworkType clientRootObfuscationSecret ObfuscationSecret doDTLSRandomization bool - trafficShapingParameters *DataChannelTrafficShapingParameters + useMediaStreams bool + trafficShapingParameters *TrafficShapingParameters stunServerAddress string stunServerAddressRFC5780 string stunServerAddressSucceeded func(RFC5780 bool, address string) @@ -223,7 +225,7 @@ type testWebRTCDialCoordinator struct { discoverNATTimeout time.Duration webRTCAnswerTimeout time.Duration webRTCAwaitPortMappingTimeout time.Duration - webRTCAwaitDataChannelTimeout time.Duration + webRTCAwaitReadyToProxyTimeout time.Duration proxyDestinationDialTimeout time.Duration proxyRelayInactivityTimeout time.Duration } @@ -252,7 +254,13 @@ func (t *testWebRTCDialCoordinator) DoDTLSRandomization() bool { return t.doDTLSRandomization } -func (t *testWebRTCDialCoordinator) DataChannelTrafficShapingParameters() *DataChannelTrafficShapingParameters { +func (t *testWebRTCDialCoordinator) UseMediaStreams() bool { + t.mutex.Lock() + defer t.mutex.Unlock() + return t.useMediaStreams +} + +func (t *testWebRTCDialCoordinator) TrafficShapingParameters() *TrafficShapingParameters { t.mutex.Lock() defer t.mutex.Unlock() return t.trafficShapingParameters @@ -365,6 +373,35 @@ func (t *testWebRTCDialCoordinator) ResolveAddress(ctx context.Context, network, return net.JoinHostPort(IPs[0].String(), port), nil } +// lossyConn randomly drops 1% of packets sent or received. +type lossyConn struct { + net.PacketConn +} + +func (conn *lossyConn) ReadFrom(p []byte) (int, net.Addr, error) { + for { + n, addr, err := conn.PacketConn.ReadFrom(p) + if err != nil { + return n, addr, err + } + if prng.FlipWeightedCoin(0.01) { + // Drop packet + continue + } + return n, addr, err + } +} + +func (conn *lossyConn) WriteTo(p []byte, addr net.Addr) (int, error) { + if prng.FlipWeightedCoin(0.01) { + // Drop packet + return len(p), nil + } + return conn.PacketConn.WriteTo(p, addr) +} + +// UDPListen wraps the returned net.PacketConn in lossyConn to simulate packet +// loss. func (t *testWebRTCDialCoordinator) UDPListen(_ context.Context) (net.PacketConn, error) { t.mutex.Lock() defer t.mutex.Unlock() @@ -372,9 +409,11 @@ func (t *testWebRTCDialCoordinator) UDPListen(_ context.Context) (net.PacketConn if err != nil { return nil, errors.Trace(err) } - return conn, nil + return &lossyConn{conn}, nil } +// UDPConn wraps the returned net.PacketConn in lossyConn to simulate packet +// loss. func (t *testWebRTCDialCoordinator) UDPConn(_ context.Context, network, remoteAddress string) (net.PacketConn, error) { t.mutex.Lock() defer t.mutex.Unlock() @@ -387,7 +426,7 @@ func (t *testWebRTCDialCoordinator) UDPConn(_ context.Context, network, remoteAd if err != nil { return nil, errors.Trace(err) } - return conn.(*net.UDPConn), nil + return &lossyConn{conn.(*net.UDPConn)}, nil } func (t *testWebRTCDialCoordinator) BindToDevice(fileDescriptor int) error { @@ -423,10 +462,10 @@ func (t *testWebRTCDialCoordinator) WebRTCAwaitPortMappingTimeout() time.Duratio return t.webRTCAwaitPortMappingTimeout } -func (t *testWebRTCDialCoordinator) WebRTCAwaitDataChannelTimeout() time.Duration { +func (t *testWebRTCDialCoordinator) WebRTCAwaitReadyToProxyTimeout() time.Duration { t.mutex.Lock() defer t.mutex.Unlock() - return t.webRTCAwaitDataChannelTimeout + return t.webRTCAwaitReadyToProxyTimeout } func (t *testWebRTCDialCoordinator) ProxyDestinationDialTimeout() time.Duration { @@ -442,11 +481,21 @@ func (t *testWebRTCDialCoordinator) ProxyRelayInactivityTimeout() time.Duration } type testLogger struct { + component string logLevelDebug int32 } func newTestLogger() *testLogger { - return &testLogger{logLevelDebug: 1} + return &testLogger{ + logLevelDebug: 0, + } +} + +func newTestLoggerWithComponent(component string) *testLogger { + return &testLogger{ + component: component, + logLevelDebug: 0, + } } func (logger *testLogger) WithTrace() common.LogTrace { @@ -466,9 +515,14 @@ func (logger *testLogger) WithTraceFields(fields common.LogFields) common.LogTra func (logger *testLogger) LogMetric(metric string, fields common.LogFields) { jsonFields, _ := json.Marshal(fields) + var component string + if len(logger.component) > 0 { + component = fmt.Sprintf("[%s]", logger.component) + } fmt.Printf( - "[%s] METRIC: %s: %s\n", + "[%s]%s METRIC: %s: %s\n", time.Now().UTC().Format(time.RFC3339), + component, metric, string(jsonFields)) } @@ -493,10 +547,14 @@ type testLoggerTrace struct { func (logger *testLoggerTrace) log(priority, message string) { now := time.Now().UTC().Format(time.RFC3339) + var component string + if len(logger.logger.component) > 0 { + component = fmt.Sprintf("[%s]", logger.logger.component) + } if len(logger.fields) == 0 { fmt.Printf( - "[%s] %s: %s: %s\n", - now, priority, logger.trace, message) + "[%s]%s %s: %s: %s\n", + now, component, priority, logger.trace, message) } else { fields := common.LogFields{} for k, v := range logger.fields { @@ -510,8 +568,8 @@ func (logger *testLoggerTrace) log(priority, message string) { } jsonFields, _ := json.Marshal(fields) fmt.Printf( - "[%s] %s: %s: %s %s\n", - now, priority, logger.trace, message, string(jsonFields)) + "[%s]%s %s: %s: %s %s\n", + now, component, priority, logger.trace, message, string(jsonFields)) } } diff --git a/psiphon/common/inproxy/discovery.go b/psiphon/common/inproxy/discovery.go index 95e77e853..db976fa90 100644 --- a/psiphon/common/inproxy/discovery.go +++ b/psiphon/common/inproxy/discovery.go @@ -225,7 +225,6 @@ func discoverNATType( } resultChannel <- result{NATType: MakeNATType(mapping, filtering)} - return }() var r result diff --git a/psiphon/common/inproxy/discoverySTUN.go b/psiphon/common/inproxy/discoverySTUN.go index be8a9f404..773650c26 100644 --- a/psiphon/common/inproxy/discoverySTUN.go +++ b/psiphon/common/inproxy/discoverySTUN.go @@ -142,7 +142,7 @@ func discoverNATFiltering( request = stun.MustBuild(stun.TransactionID, stun.BindingRequest) request.Add(stun.AttrChangeRequest, []byte{0x00, 0x00, 0x00, 0x06}) - response, responseTimeout, err := doSTUNRoundTrip(request, conn, serverAddress) + _, responseTimeout, err := doSTUNRoundTrip(request, conn, serverAddress) if err == nil { return NATFilteringEndpointIndependent, nil } else if !responseTimeout { @@ -154,7 +154,7 @@ func discoverNATFiltering( request = stun.MustBuild(stun.TransactionID, stun.BindingRequest) request.Add(stun.AttrChangeRequest, []byte{0x00, 0x00, 0x00, 0x02}) - response, responseTimeout, err = doSTUNRoundTrip(request, conn, serverAddress) + _, responseTimeout, err = doSTUNRoundTrip(request, conn, serverAddress) if err == nil { return NATFilteringAddressDependent, nil } else if !responseTimeout { @@ -212,7 +212,10 @@ func doSTUNRoundTrip( return nil, false, errors.Trace(err) } - conn.SetReadDeadline(time.Now().Add(discoverNATRoundTripTimeout)) + err = conn.SetReadDeadline(time.Now().Add(discoverNATRoundTripTimeout)) + if err != nil { + return nil, false, errors.Trace(err) + } var buffer [1500]byte n, _, err := conn.ReadFrom(buffer[:]) diff --git a/psiphon/common/inproxy/inproxy_disabled.go b/psiphon/common/inproxy/inproxy_disabled.go index a28af760f..d501bef6e 100644 --- a/psiphon/common/inproxy/inproxy_disabled.go +++ b/psiphon/common/inproxy/inproxy_disabled.go @@ -52,7 +52,7 @@ func Enabled() bool { var errNotEnabled = std_errors.New("operation not enabled") const ( - dataChannelAwaitTimeout = time.Duration(0) + readyToProxyAwaitTimeout = time.Duration(0) ) type webRTCConn struct { @@ -64,7 +64,8 @@ type webRTCConfig struct { WebRTCDialCoordinator WebRTCDialCoordinator ClientRootObfuscationSecret ObfuscationSecret DoDTLSRandomization bool - TrafficShapingParameters *DataChannelTrafficShapingParameters + UseMediaStreams bool + TrafficShapingParameters *TrafficShapingParameters ReliableTransport bool } @@ -75,9 +76,7 @@ func (conn *webRTCConn) SetRemoteSDP( return errors.Trace(errNotEnabled) } -// AwaitInitialDataChannel returns when the data channel is established, or -// when an error has occured. -func (conn *webRTCConn) AwaitInitialDataChannel(ctx context.Context) error { +func (conn *webRTCConn) AwaitReadyToProxy(ctx context.Context, connectionID ID) error { return errors.Trace(errNotEnabled) } @@ -121,6 +120,10 @@ func (conn *webRTCConn) GetMetrics() common.LogFields { return nil } +func GetQUICMaxPacketSizeAdjustment(isIPv6 bool) int { + return 0 +} + type webRTCSDPMetrics struct { iceCandidateTypes []ICECandidateType hasIPv6 bool diff --git a/psiphon/common/inproxy/inproxy_test.go b/psiphon/common/inproxy/inproxy_test.go index 86ea1721b..4a6f202ad 100644 --- a/psiphon/common/inproxy/inproxy_test.go +++ b/psiphon/common/inproxy/inproxy_test.go @@ -86,6 +86,7 @@ func runTestInproxy(doMustUpgrade bool) error { testNATType := NATTypeUnknown testSTUNServerAddress := "stun.nextcloud.com:443" testDisableSTUN := false + testDisablePortMapping := false testNewTacticsPayload := []byte(prng.HexString(100)) testNewTacticsTag := "new-tactics-tag" @@ -115,13 +116,15 @@ func runTestInproxy(doMustUpgrade bool) error { receivedClientMustUpgrade = make(chan struct{}) // trigger MustUpgrade - proxyProtocolVersion = 0 + minimumProxyProtocolVersion = LatestProtocolVersion + 1 + minimumClientProtocolVersion = LatestProtocolVersion + 1 // Minimize test parameters for MustUpgrade case numProxies = 1 proxyMaxClients = 1 numClients = 1 testDisableSTUN = true + testDisablePortMapping = true } testCtx, stopTest := context.WithCancel(context.Background()) @@ -225,7 +228,9 @@ func runTestInproxy(doMustUpgrade bool) error { apiParameterLogFieldFormatter := func( _ string, _ common.GeoIPData, params common.APIParameters) common.LogFields { - return common.LogFields(params) + logFields := common.LogFields{} + logFields.Add(common.LogFields(params)) + return logFields } // Start broker @@ -406,6 +411,12 @@ func runTestInproxy(doMustUpgrade bool) error { brokerListener.Addr().String(), "proxy"), brokerClientRoundTripperSucceeded: roundTripperSucceded, brokerClientRoundTripperFailed: roundTripperFailed, + + // Minimize the delay before proxies reannounce after dial + // failures, which may occur. + announceDelay: 0, + announceMaxBackoffDelay: 0, + announceDelayJitter: 0.0, } webRTCCoordinator := &testWebRTCDialCoordinator{ @@ -413,6 +424,7 @@ func runTestInproxy(doMustUpgrade bool) error { networkType: testNetworkType, natType: testNATType, disableSTUN: testDisableSTUN, + disablePortMapping: testDisablePortMapping, stunServerAddress: testSTUNServerAddress, stunServerAddressRFC5780: testSTUNServerAddress, stunServerAddressSucceeded: stunServerAddressSucceeded, @@ -420,6 +432,11 @@ func runTestInproxy(doMustUpgrade bool) error { setNATType: func(NATType) {}, setPortMappingTypes: func(PortMappingTypes) {}, bindToDevice: func(int) error { return nil }, + + // Minimize the delay before proxies reannounce after failed + // connections, which may occur. + webRTCAwaitReadyToProxyTimeout: 5 * time.Second, + proxyRelayInactivityTimeout: 5 * time.Second, } // Each proxy has its own broker client @@ -433,9 +450,11 @@ func runTestInproxy(doMustUpgrade bool) error { runCtx, cancelRun := context.WithCancel(testCtx) // No deferred cancelRun due to testGroup.Go below + name := fmt.Sprintf("proxy-%d", i) + proxy, err := NewProxy(&ProxyConfig{ - Logger: logger, + Logger: newTestLoggerWithComponent(name), WaitForNetworkConnectivity: func() bool { return true @@ -466,8 +485,8 @@ func runTestInproxy(doMustUpgrade bool) error { ActivityUpdater: func(connectingClients int32, connectedClients int32, bytesUp int64, bytesDown int64, bytesDuration time.Duration) { - fmt.Printf("[%s] ACTIVITY: %d connecting, %d connected, %d up, %d down\n", - time.Now().UTC().Format(time.RFC3339), + fmt.Printf("[%s][%s] ACTIVITY: %d connecting, %d connected, %d up, %d down\n", + time.Now().UTC().Format(time.RFC3339), name, connectingClients, connectedClients, bytesUp, bytesDown) }, @@ -510,13 +529,15 @@ func runTestInproxy(doMustUpgrade bool) error { // Start clients + var completedClientCount atomic.Int64 + logger.WithTrace().Info("START CLIENTS") clientsGroup := new(errgroup.Group) makeClientFunc := func( + clientNum int, isTCP bool, - isMobile bool, brokerClient *BrokerClient, webRTCCoordinator WebRTCDialCoordinator) func() error { @@ -534,13 +555,15 @@ func runTestInproxy(doMustUpgrade bool) error { return func() error { + name := fmt.Sprintf("client-%d", clientNum) + dialCtx, cancelDial := context.WithTimeout(testCtx, 60*time.Second) defer cancelDial() conn, err := DialClient( dialCtx, &ClientConfig{ - Logger: logger, + Logger: newTestLoggerWithComponent(name), BaseAPIParameters: baseAPIParameters, BrokerClient: brokerClient, WebRTCDialCoordinator: webRTCCoordinator, @@ -561,12 +584,22 @@ func runTestInproxy(doMustUpgrade bool) error { relayConn = conn if wrapWithQUIC { + disablePathMTUDiscovery := true quicConn, err := quic.Dial( dialCtx, conn, &net.UDPAddr{Port: 1}, // This address is ignored, but the zero value is not allowed - "test", "QUICv1", nil, quicEchoServer.ObfuscationKey(), nil, nil, true, - false, false, common.WrapClientSessionCache(tls.NewLRUClientSessionCache(0), ""), + "test", + "QUICv1", + nil, + quicEchoServer.ObfuscationKey(), + nil, + nil, + disablePathMTUDiscovery, + GetQUICMaxPacketSizeAdjustment(false), + false, + false, + common.WrapClientSessionCache(tls.NewLRUClientSessionCache(0), ""), ) if err != nil { return errors.Trace(err) @@ -620,7 +653,8 @@ func runTestInproxy(doMustUpgrade bool) error { } n += m } - fmt.Printf("%d bytes sent\n", bytesToSend) + fmt.Printf("[%s][%s] %d bytes sent\n", + time.Now().UTC().Format(time.RFC3339), name, bytesToSend) return nil }) @@ -639,13 +673,21 @@ func runTestInproxy(doMustUpgrade bool) error { } n += m } - fmt.Printf("%d bytes received\n", bytesToSend) + + completed := completedClientCount.Add(1) + + fmt.Printf("[%s][%s] %d bytes received; relay complete (%d/%d)\n", + time.Now().UTC().Format(time.RFC3339), name, + bytesToSend, completed, numClients) select { case <-signalRelayComplete: case <-testCtx.Done(): } + fmt.Printf("[%s][%s] closing\n", + time.Now().UTC().Format(time.RFC3339), name) + relayConn.Close() conn.Close() @@ -656,16 +698,11 @@ func runTestInproxy(doMustUpgrade bool) error { } } - newClientParams := func(isMobile bool) (*BrokerClient, *testWebRTCDialCoordinator, error) { + newClientBrokerClient := func() (*BrokerClient, error) { clientPrivateKey, err := GenerateSessionPrivateKey() if err != nil { - return nil, nil, errors.Trace(err) - } - - clientRootObfuscationSecret, err := GenerateRootObfuscationSecret() - if err != nil { - return nil, nil, errors.Trace(err) + return nil, errors.Trace(err) } brokerCoordinator := &testBrokerDialCoordinator{ @@ -684,6 +721,50 @@ func runTestInproxy(doMustUpgrade bool) error { brokerClientNoMatch: noMatch, } + brokerClient, err := NewBrokerClient(brokerCoordinator) + if err != nil { + return nil, errors.Trace(err) + } + + return brokerClient, nil + } + + newClientWebRTCDialCoordinator := func( + isMobile bool, + useMediaStreams bool) (*testWebRTCDialCoordinator, error) { + + clientRootObfuscationSecret, err := GenerateRootObfuscationSecret() + if err != nil { + return nil, errors.Trace(err) + } + + var trafficShapingParameters *TrafficShapingParameters + if useMediaStreams { + trafficShapingParameters = &TrafficShapingParameters{ + MinPaddedMessages: 0, + MaxPaddedMessages: 10, + MinPaddingSize: 0, + MaxPaddingSize: 254, + MinDecoyMessages: 0, + MaxDecoyMessages: 10, + MinDecoySize: 1, + MaxDecoySize: 1200, + DecoyMessageProbability: 0.5, + } + } else { + trafficShapingParameters = &TrafficShapingParameters{ + MinPaddedMessages: 0, + MaxPaddedMessages: 10, + MinPaddingSize: 0, + MaxPaddingSize: 1500, + MinDecoyMessages: 0, + MaxDecoyMessages: 10, + MinDecoySize: 1, + MaxDecoySize: 1500, + DecoyMessageProbability: 0.5, + } + } + webRTCCoordinator := &testWebRTCDialCoordinator{ networkID: testNetworkID, networkType: testNetworkType, @@ -697,27 +778,18 @@ func runTestInproxy(doMustUpgrade bool) error { clientRootObfuscationSecret: clientRootObfuscationSecret, doDTLSRandomization: prng.FlipCoin(), - trafficShapingParameters: &DataChannelTrafficShapingParameters{ - MinPaddedMessages: 0, - MaxPaddedMessages: 10, - MinPaddingSize: 0, - MaxPaddingSize: 1500, - MinDecoyMessages: 0, - MaxDecoyMessages: 10, - MinDecoySize: 1, - MaxDecoySize: 1500, - DecoyMessageProbability: 0.5, - }, + useMediaStreams: useMediaStreams, + trafficShapingParameters: trafficShapingParameters, setNATType: func(NATType) {}, setPortMappingTypes: func(PortMappingTypes) {}, bindToDevice: func(int) error { return nil }, // With STUN enabled (testDisableSTUN = false), there are cases - // where the WebRTC Data Channel is not successfully established. - // With a short enough timeout here, clients will redial and - // eventually succceed. - webRTCAwaitDataChannelTimeout: 5 * time.Second, + // where the WebRTC peer connection is not successfully + // established. With a short enough timeout here, clients will + // redial and eventually succceed. + webRTCAwaitReadyToProxyTimeout: 5 * time.Second, } if isMobile { @@ -725,20 +797,10 @@ func runTestInproxy(doMustUpgrade bool) error { webRTCCoordinator.disableInboundForMobileNetworks = true } - brokerClient, err := NewBrokerClient(brokerCoordinator) - if err != nil { - return nil, nil, errors.Trace(err) - } - - return brokerClient, webRTCCoordinator, nil + return webRTCCoordinator, nil } - clientBrokerClient, clientWebRTCCoordinator, err := newClientParams(false) - if err != nil { - return errors.Trace(err) - } - - clientMobileBrokerClient, clientMobileWebRTCCoordinator, err := newClientParams(true) + sharedBrokerClient, err := newClientBrokerClient() if err != nil { return errors.Trace(err) } @@ -750,29 +812,30 @@ func runTestInproxy(doMustUpgrade bool) error { isTCP := i%2 == 0 isMobile := i%4 == 0 + useMediaStreams := i%4 < 2 // Exercise BrokerClients shared by multiple clients, but also create // several broker clients. - if i%8 == 0 { - clientBrokerClient, clientWebRTCCoordinator, err = newClientParams(false) - if err != nil { - return errors.Trace(err) - } - - clientMobileBrokerClient, clientMobileWebRTCCoordinator, err = newClientParams(true) + brokerClient := sharedBrokerClient + if i%2 == 0 { + brokerClient, err = newClientBrokerClient() if err != nil { return errors.Trace(err) } } - brokerClient := clientBrokerClient - webRTCCoordinator := clientWebRTCCoordinator - if isMobile { - brokerClient = clientMobileBrokerClient - webRTCCoordinator = clientMobileWebRTCCoordinator + webRTCCoordinator, err := newClientWebRTCDialCoordinator( + isMobile, useMediaStreams) + if err != nil { + return errors.Trace(err) } - clientsGroup.Go(makeClientFunc(isTCP, isMobile, brokerClient, webRTCCoordinator)) + clientsGroup.Go( + makeClientFunc( + i, + isTCP, + brokerClient, + webRTCCoordinator)) } if doMustUpgrade { @@ -1014,6 +1077,7 @@ func newQuicEchoServer() (*quicEchoServer, error) { nil, nil, "127.0.0.1:0", + GetQUICMaxPacketSizeAdjustment(false), obfuscationKey, false) if err != nil { diff --git a/psiphon/common/inproxy/matcher.go b/psiphon/common/inproxy/matcher.go index 8ab307da7..20c92bc7a 100644 --- a/psiphon/common/inproxy/matcher.go +++ b/psiphon/common/inproxy/matcher.go @@ -41,6 +41,7 @@ const ( matcherPendingAnswersTTL = 30 * time.Second matcherPendingAnswersMaxSize = 100000 matcherMaxPreferredNATProbe = 100 + matcherMaxProbe = 1000 matcherRateLimiterReapHistoryFrequencySeconds = 300 matcherRateLimiterMaxCacheEntries = 1000000 @@ -112,9 +113,10 @@ type Matcher struct { } // MatchProperties specifies the compartment, GeoIP, and network topology -// matching roperties of clients and proxies. +// matching properties of clients and proxies. type MatchProperties struct { IsPriority bool + ProtocolVersion int32 CommonCompartmentIDs []ID PersonalCompartmentIDs []ID GeoIPData common.GeoIPData @@ -173,11 +175,11 @@ type MatchAnnouncement struct { // MatchOffer is a client offer to be queued for matching. type MatchOffer struct { Properties MatchProperties - ClientProxyProtocolVersion int32 ClientOfferSDP WebRTCSessionDescription ClientRootObfuscationSecret ObfuscationSecret DoDTLSRandomization bool - TrafficShapingParameters *DataChannelTrafficShapingParameters + UseMediaStreams bool + TrafficShapingParameters *TrafficShapingParameters NetworkProtocol NetworkProtocol DestinationAddress string DestinationServerID string @@ -186,11 +188,10 @@ type MatchOffer struct { // MatchAnswer is a proxy answer, the proxy's follow up to a matched // announcement, to be routed to the awaiting client offer. type MatchAnswer struct { - ProxyIP string - ProxyID ID - ConnectionID ID - SelectedProxyProtocolVersion int32 - ProxyAnswerSDP WebRTCSessionDescription + ProxyIP string + ProxyID ID + ConnectionID ID + ProxyAnswerSDP WebRTCSessionDescription } // MatchMetrics records statistics about the match queue state at the time a @@ -740,7 +741,7 @@ func (m *Matcher) matchAllOffers() { func (m *Matcher) matchOffer(offerEntry *offerEntry) (*announcementEntry, int) { - // Assumes the caller has the queue mutexed locked. + // Assumes the caller has the queue mutexes locked. // Check each candidate announcement in turn, and select a match. There is // an implicit preference for older proxy announcements, sooner to @@ -756,10 +757,6 @@ func (m *Matcher) matchOffer(offerEntry *offerEntry) (*announcementEntry, int) { // rules, such as a configuration encoding knowledge of an ASN's NAT // type, or preferred client/proxy country/ASN matches. - // TODO: match supported protocol versions. Currently, all announces and - // offers must specify ProxyProtocolVersion1, so there's no protocol - // version match logic. - offerProperties := &offerEntry.offer.Properties // Assumes the caller checks that offer specifies either personal @@ -789,13 +786,28 @@ func (m *Matcher) matchOffer(offerEntry *offerEntry) (*announcementEntry, int) { partiallyLimitedNATCount > 0, strictlyLimitedNATCount > 0) + // TODO: add an ExistsCompatibleProtocolVersionMatch check? + // + // Currently, searching for protocol version support that doesn't exist + // may be mitigated by limiting, through tactics, client protocol options + // selection; using the proxy protocol version in PrioritizeProxy; and, + // ultimately, increasing MinimumProxyProtocolVersion. + var bestMatch *announcementEntry bestMatchIndex := -1 bestMatchIsPriority := false bestMatchNAT := false + // matcherMaxProbe limits the linear search through the announcement queue + // to find a match. Currently, the queue implementation provides + // constant-time lookup for matching compartment IDs. Other matching + // aspects may require iterating over the queue items, including the + // strict same-country and ASN constraint and protocol version + // compatibility constraint. Best NAT match is not a strict constraint + // and uses a shorter search limit, matcherMaxPreferredNATProbe. + candidateIndex := -1 - for { + for candidateIndex <= matcherMaxProbe { announcementEntry, isPriority := matchIterator.getNext() if announcementEntry == nil { @@ -825,6 +837,18 @@ func (m *Matcher) matchOffer(offerEntry *offerEntry) (*announcementEntry, int) { announcementProperties := &announcementEntry.announcement.Properties + // Don't match unless the proxy announcement, client offer, and the + // client's selected protocol options are compatible. UseMediaStreams + // requires at least ProtocolVersion2. + + _, ok := negotiateProtocolVersion( + announcementProperties.ProtocolVersion, + offerProperties.ProtocolVersion, + offerEntry.offer.UseMediaStreams) + if !ok { + continue + } + // Disallow matching the same country and ASN, except for personal // compartment ID matches. // diff --git a/psiphon/common/inproxy/matcher_test.go b/psiphon/common/inproxy/matcher_test.go index 3d19c642a..f17f03e8e 100644 --- a/psiphon/common/inproxy/matcher_test.go +++ b/psiphon/common/inproxy/matcher_test.go @@ -83,10 +83,10 @@ func runTestMatcher() error { } } - makeOffer := func(properties *MatchProperties) *MatchOffer { + makeOffer := func(properties *MatchProperties, useMediaStreams bool) *MatchOffer { return &MatchOffer{ - Properties: *properties, - ClientProxyProtocolVersion: ProxyProtocolVersion1, + Properties: *properties, + UseMediaStreams: useMediaStreams, } } @@ -115,12 +115,19 @@ func runTestMatcher() error { if err != nil { resultChan <- errors.Trace(err) return - } else { - err := checkMatchMetrics(matchMetrics) - if err != nil { - resultChan <- errors.Trace(err) - return - } + } + err = checkMatchMetrics(matchMetrics) + if err != nil { + resultChan <- errors.Trace(err) + return + } + _, ok := negotiateProtocolVersion( + matchProperties.ProtocolVersion, + offer.Properties.ProtocolVersion, + offer.UseMediaStreams) + if !ok { + resultChan <- errors.TraceNew("unexpected negotiateProtocolVersion failure") + return } if waitBeforeAnswer != nil { @@ -130,9 +137,8 @@ func runTestMatcher() error { if answerSuccess { err = m.Answer( &MatchAnswer{ - ProxyID: announcement.ProxyID, - ConnectionID: announcement.ConnectionID, - SelectedProxyProtocolVersion: offer.ClientProxyProtocolVersion, + ProxyID: announcement.ProxyID, + ConnectionID: announcement.ConnectionID, }) } else { m.AnswerError(announcement.ProxyID, announcement.ConnectionID) @@ -142,39 +148,55 @@ func runTestMatcher() error { clientIP := randomIPAddress() - clientFunc := func( + baseClientFunc := func( resultChan chan error, clientIP string, matchProperties *MatchProperties, + useMediaStreams bool, timeout time.Duration) { ctx, cancelFunc := context.WithTimeout(context.Background(), timeout) defer cancelFunc() - offer := makeOffer(matchProperties) - answer, _, matchMetrics, err := m.Offer(ctx, clientIP, offer) + offer := makeOffer(matchProperties, useMediaStreams) + _, matchAnnouncement, matchMetrics, err := m.Offer(ctx, clientIP, offer) if err != nil { resultChan <- errors.Trace(err) return } - if answer.SelectedProxyProtocolVersion != offer.ClientProxyProtocolVersion { - resultChan <- errors.TraceNew("unexpected selected proxy protocol version") + err = checkMatchMetrics(matchMetrics) + if err != nil { + resultChan <- errors.Trace(err) return - } else { - err := checkMatchMetrics(matchMetrics) - if err != nil { - resultChan <- errors.Trace(err) - return - } } + _, ok := negotiateProtocolVersion( + matchAnnouncement.Properties.ProtocolVersion, + offer.Properties.ProtocolVersion, + offer.UseMediaStreams) + if !ok { + resultChan <- errors.TraceNew("unexpected negotiateProtocolVersion failure") + return + } + resultChan <- nil } + clientFunc := func(resultChan chan error, clientIP string, + matchProperties *MatchProperties, timeout time.Duration) { + baseClientFunc(resultChan, clientIP, matchProperties, false, timeout) + } + + clientUsingMediaStreamsFunc := func(resultChan chan error, clientIP string, + matchProperties *MatchProperties, timeout time.Duration) { + baseClientFunc(resultChan, clientIP, matchProperties, true, timeout) + } + // Test: announce timeout proxyResultChan := make(chan error) matchProperties := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, CommonCompartmentIDs: []ID{makeID()}, } @@ -373,11 +395,13 @@ func runTestMatcher() error { commonCompartmentIDs := []ID{makeID()} geoIPData1 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C1", ASN: "A1"}, CommonCompartmentIDs: commonCompartmentIDs, } geoIPData2 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C2", ASN: "A2"}, CommonCompartmentIDs: commonCompartmentIDs, } @@ -432,21 +456,25 @@ func runTestMatcher() error { // Test: no compartment match compartment1 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: geoIPData1.GeoIPData, CommonCompartmentIDs: []ID{makeID()}, } compartment2 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: geoIPData2.GeoIPData, PersonalCompartmentIDs: []ID{makeID()}, } compartment3 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: geoIPData2.GeoIPData, CommonCompartmentIDs: []ID{makeID()}, } compartment4 := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: geoIPData2.GeoIPData, PersonalCompartmentIDs: []ID{makeID()}, } @@ -484,7 +512,8 @@ func runTestMatcher() error { // Test: common compartment match compartment1And3 := &MatchProperties{ - GeoIPData: geoIPData2.GeoIPData, + ProtocolVersion: LatestProtocolVersion, + GeoIPData: geoIPData2.GeoIPData, CommonCompartmentIDs: []ID{ compartment1.CommonCompartmentIDs[0], compartment3.CommonCompartmentIDs[0]}, @@ -506,7 +535,8 @@ func runTestMatcher() error { // Test: personal compartment match compartment2And4 := &MatchProperties{ - GeoIPData: geoIPData2.GeoIPData, + ProtocolVersion: LatestProtocolVersion, + GeoIPData: geoIPData2.GeoIPData, PersonalCompartmentIDs: []ID{ compartment2.PersonalCompartmentIDs[0], compartment4.PersonalCompartmentIDs[0]}, @@ -540,27 +570,75 @@ func runTestMatcher() error { return errors.Tracef("unexpected result: %v", err) } + // Test: downgrade-compatible protocol version match + + protocolVersion1 := &MatchProperties{ + ProtocolVersion: ProtocolVersion1, + GeoIPData: common.GeoIPData{Country: "C1", ASN: "A1"}, + CommonCompartmentIDs: commonCompartmentIDs, + } + + protocolVersion2 := &MatchProperties{ + ProtocolVersion: ProtocolVersion2, + GeoIPData: common.GeoIPData{Country: "C2", ASN: "A2"}, + CommonCompartmentIDs: commonCompartmentIDs, + } + + go proxyFunc(proxyResultChan, proxyIP, protocolVersion1, 10*time.Millisecond, nil, true) + go clientFunc(clientResultChan, clientIP, protocolVersion2, 10*time.Millisecond) + + err = <-proxyResultChan + if err != nil { + return errors.Trace(err) + } + + err = <-clientResultChan + if err != nil { + return errors.Trace(err) + } + + // Test: no incompatible protocol version match + + go proxyFunc(proxyResultChan, proxyIP, protocolVersion1, 10*time.Millisecond, nil, true) + go clientUsingMediaStreamsFunc(clientResultChan, clientIP, protocolVersion2, 10*time.Millisecond) + + err = <-proxyResultChan + if err == nil || !strings.HasSuffix(err.Error(), "context deadline exceeded") { + return errors.Tracef("unexpected result: %v", err) + } + + err = <-clientResultChan + if err == nil || !strings.HasSuffix(err.Error(), "context deadline exceeded") { + return errors.Tracef("unexpected result: %v", err) + } + + // Test: downgrade-compatible protocol version match + // Test: proxy preferred NAT match client1Properties := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C1", ASN: "A1"}, NATType: NATTypeFullCone, CommonCompartmentIDs: commonCompartmentIDs, } client2Properties := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C2", ASN: "A2"}, NATType: NATTypeSymmetric, CommonCompartmentIDs: commonCompartmentIDs, } proxy1Properties := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C3", ASN: "A3"}, NATType: NATTypeNone, CommonCompartmentIDs: commonCompartmentIDs, } proxy2Properties := &MatchProperties{ + ProtocolVersion: LatestProtocolVersion, GeoIPData: common.GeoIPData{Country: "C4", ASN: "A4"}, NATType: NATTypeSymmetric, CommonCompartmentIDs: commonCompartmentIDs, @@ -1095,6 +1173,7 @@ func BenchmarkMatcherQueue(b *testing.B) { limitIP: "127.0.0.1", announcement: &MatchAnnouncement{ Properties: MatchProperties{ + ProtocolVersion: LatestProtocolVersion, PersonalCompartmentIDs: []ID{personalCompartmentID}, GeoIPData: common.GeoIPData{}, NetworkType: NetworkTypeWiFi, @@ -1111,13 +1190,13 @@ func BenchmarkMatcherQueue(b *testing.B) { limitIP: "127.0.0.1", offer: &MatchOffer{ Properties: MatchProperties{ + ProtocolVersion: LatestProtocolVersion, PersonalCompartmentIDs: []ID{personalCompartmentID}, GeoIPData: common.GeoIPData{}, NetworkType: NetworkTypeWiFi, NATType: NATTypePortRestrictedCone, PortMappingTypes: []PortMappingType{}, }, - ClientProxyProtocolVersion: ProxyProtocolVersion1, }, answerChan: make(chan *answerInfo, 1), } diff --git a/psiphon/common/inproxy/portmapper_other.go b/psiphon/common/inproxy/portmapper_other.go index e06d23322..2806942d1 100644 --- a/psiphon/common/inproxy/portmapper_other.go +++ b/psiphon/common/inproxy/portmapper_other.go @@ -23,6 +23,6 @@ package inproxy func setPortMapperBindToDevice(_ WebRTCDialCoordinator) { // BindToDevice is not applied on iOS as tailscale.com/net/netns does not - // have an equivilent to SetAndroidProtectFunc for iOS. At this time, + // have an equivalent to SetAndroidProtectFunc for iOS. At this time, // BindToDevice operations on iOS are legacy code and not required. } diff --git a/psiphon/common/inproxy/proxy.go b/psiphon/common/inproxy/proxy.go index 257072c23..06ab8bc39 100644 --- a/psiphon/common/inproxy/proxy.go +++ b/psiphon/common/inproxy/proxy.go @@ -720,13 +720,15 @@ func (p *Proxy) proxyOneClient( } - if announceResponse.ClientProxyProtocolVersion != ProxyProtocolVersion1 { - // This case is currently unexpected, as all clients and proxies use - // ProxyProtocolVersion1. + if announceResponse.SelectedProtocolVersion < ProtocolVersion1 || + (announceResponse.UseMediaStreams && + announceResponse.SelectedProtocolVersion < ProtocolVersion2) || + announceResponse.SelectedProtocolVersion > LatestProtocolVersion { + backOff = true return backOff, errors.Tracef( - "Unsupported proxy protocol version: %d", - announceResponse.ClientProxyProtocolVersion) + "unsupported protocol version: %d", + announceResponse.SelectedProtocolVersion) } // Trigger back-off if the following WebRTC operations fail to establish a @@ -767,7 +769,16 @@ func (p *Proxy) proxyOneClient( WebRTCDialCoordinator: webRTCCoordinator, ClientRootObfuscationSecret: announceResponse.ClientRootObfuscationSecret, DoDTLSRandomization: announceResponse.DoDTLSRandomization, + UseMediaStreams: announceResponse.UseMediaStreams, TrafficShapingParameters: announceResponse.TrafficShapingParameters, + + // In media stream mode, this flag indicates to the proxy that it + // should add the QUIC-based reliability layer wrapping to media + // streams. In data channel mode, this flag is ignored, since the + // client configures the data channel using + // webrtc.DataChannelInit.Ordered, and this configuration is sent + // to the proxy in the client's SDP. + ReliableTransport: announceResponse.NetworkProtocol == NetworkProtocolTCP, }, announceResponse.ClientOfferSDP, hasPersonalCompartmentIDs) @@ -788,11 +799,10 @@ func (p *Proxy) proxyOneClient( _, err = brokerClient.ProxyAnswer( ctx, &ProxyAnswerRequest{ - ConnectionID: announceResponse.ConnectionID, - SelectedProxyProtocolVersion: announceResponse.ClientProxyProtocolVersion, - ProxyAnswerSDP: SDP, - ICECandidateTypes: sdpMetrics.iceCandidateTypes, - AnswerError: webRTCRequestErr, + ConnectionID: announceResponse.ConnectionID, + ProxyAnswerSDP: SDP, + ICECandidateTypes: sdpMetrics.iceCandidateTypes, + AnswerError: webRTCRequestErr, }) if err != nil { if webRTCErr != nil { @@ -818,21 +828,17 @@ func (p *Proxy) proxyOneClient( // create wasted load on destination Psiphon servers, particularly when // WebRTC connections fail. - awaitDataChannelCtx, awaitDataChannelCancelFunc := context.WithTimeout( + awaitReadyToProxyCtx, awaitReadyToProxyCancelFunc := context.WithTimeout( ctx, common.ValueOrDefault( - webRTCCoordinator.WebRTCAwaitDataChannelTimeout(), dataChannelAwaitTimeout)) - defer awaitDataChannelCancelFunc() + webRTCCoordinator.WebRTCAwaitReadyToProxyTimeout(), readyToProxyAwaitTimeout)) + defer awaitReadyToProxyCancelFunc() - err = webRTCConn.AwaitInitialDataChannel(awaitDataChannelCtx) + err = webRTCConn.AwaitReadyToProxy(awaitReadyToProxyCtx, announceResponse.ConnectionID) if err != nil { return backOff, errors.Trace(err) } - p.config.Logger.WithTraceFields(common.LogFields{ - "connectionID": announceResponse.ConnectionID, - }).Info("WebRTC data channel established") - // Dial the destination, a Psiphon server. The broker validates that the // dial destination is a Psiphon server. @@ -1028,7 +1034,7 @@ func (p *Proxy) getMetrics( return &ProxyMetrics{ BaseAPIParameters: packedParams, - ProxyProtocolVersion: proxyProtocolVersion, + ProtocolVersion: LatestProtocolVersion, NATType: webRTCCoordinator.NATType(), PortMappingTypes: webRTCCoordinator.PortMappingTypes(), MaxClients: int32(p.config.MaxClients), diff --git a/psiphon/common/inproxy/session_test.go b/psiphon/common/inproxy/session_test.go index 4ba9f9c72..823267698 100644 --- a/psiphon/common/inproxy/session_test.go +++ b/psiphon/common/inproxy/session_test.go @@ -712,11 +712,11 @@ func runTestNoise() error { if receivedPayload == nil { return errors.TraceNew("missing payload") } - if bytes.Compare(sendPayload, receivedPayload) != 0 { + if !bytes.Equal(sendPayload, receivedPayload) { return errors.TraceNew("incorrect payload") } - if bytes.Compare(responderHandshake.PeerStatic(), initiatorKeys.Public) != 0 { + if !bytes.Equal(responderHandshake.PeerStatic(), initiatorKeys.Public) { return errors.TraceNew("unexpected initiator static public key") } @@ -738,7 +738,7 @@ func runTestNoise() error { if !initiatorReplay.ValidateCounter(nonce, math.MaxUint64) { return errors.TraceNew("replay detected") } - if bytes.Compare(sendPayload, receivedPayload) != 0 { + if !bytes.Equal(sendPayload, receivedPayload) { return errors.TraceNew("incorrect payload") } @@ -764,7 +764,7 @@ func runTestNoise() error { if !responderReplay.ValidateCounter(nonce, math.MaxUint64) { return errors.TraceNew("replay detected") } - if bytes.Compare(sendPayload, receivedPayload) != 0 { + if !bytes.Equal(sendPayload, receivedPayload) { return errors.TraceNew("incorrect payload") } @@ -786,7 +786,7 @@ func runTestNoise() error { if !initiatorReplay.ValidateCounter(nonce, math.MaxUint64) { return errors.TraceNew("replay detected") } - if bytes.Compare(sendPayload, receivedPayload) != 0 { + if !bytes.Equal(sendPayload, receivedPayload) { return errors.TraceNew("incorrect payload") } } diff --git a/psiphon/common/inproxy/webrtc.go b/psiphon/common/inproxy/webrtc.go index c1c77ed8b..c92739fa1 100644 --- a/psiphon/common/inproxy/webrtc.go +++ b/psiphon/common/inproxy/webrtc.go @@ -27,6 +27,8 @@ import ( "encoding/binary" std_errors "errors" "fmt" + "io" + "math" "net" "strconv" "strings" @@ -34,15 +36,21 @@ import ( "sync/atomic" "time" + tls "github.com/Psiphon-Labs/psiphon-tls" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/errors" inproxy_dtls "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/inproxy/dtls" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/prng" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/quic" "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/stacktrace" + "github.com/Psiphon-Labs/psiphon-tunnel-core/psiphon/common/values" + quic_go "github.com/Psiphon-Labs/quic-go" "github.com/pion/datachannel" "github.com/pion/dtls/v2" "github.com/pion/ice/v2" + "github.com/pion/interceptor" pion_logging "github.com/pion/logging" + "github.com/pion/rtp" "github.com/pion/sdp/v3" "github.com/pion/stun" "github.com/pion/transport/v2" @@ -53,11 +61,17 @@ import ( const ( portMappingAwaitTimeout = 2 * time.Second - dataChannelAwaitTimeout = 20 * time.Second + readyToProxyAwaitTimeout = 20 * time.Second + dataChannelBufferedAmountLowThreshold uint64 = 512 * 1024 dataChannelMaxBufferedAmount uint64 = 1024 * 1024 dataChannelMaxMessageSize = 65536 - dataChannelMaxLabelLength = 512 + dataChannelMaxLabelLength = 256 + + mediaTrackMaxUDPPayloadLength = 1200 + mediaTrackRTPPacketOverhead = 12 + 16 + 1 // RTP header, SRTP encryption, and Psiphon padding header + mediaTrackMaxRTPPayloadLength = mediaTrackMaxUDPPayloadLength - mediaTrackRTPPacketOverhead + mediaTrackMaxIDLength = 256 // Psiphon uses a fork of github.com/pion/dtls/v2, selected with go mod // replace, which has an idential API apart from dtls.IsPsiphon. If @@ -78,36 +92,48 @@ const ( // used to relay streams or packets between them. WebRTCConn implements the // net.Conn interface. type webRTCConn struct { - config *webRTCConfig - trafficShapingParameters *DataChannelTrafficShapingParameters - - mutex sync.Mutex - udpConn net.PacketConn - portMapper *portMapper - isClosed bool - closedSignal chan struct{} - peerConnection *webrtc.PeerConnection - dataChannel *webrtc.DataChannel - dataChannelConn datachannel.ReadWriteCloser - dataChannelOpenedSignal chan struct{} - dataChannelOpenedOnce sync.Once - dataChannelWriteBufferSignal chan struct{} - decoyDone bool - iceCandidatePairMetrics common.LogFields - - readMutex sync.Mutex - readBuffer []byte - readOffset int - readLength int - readError error - peerPaddingDone bool - - writeMutex sync.Mutex - trafficShapingPRNG *prng.PRNG - trafficShapingBuffer *bytes.Buffer - paddedMessageCount int - decoyMessageCount int - trafficShapingDone bool + config *webRTCConfig + isOffer bool + + mutex sync.Mutex + udpConn net.PacketConn + portMapper *portMapper + isClosed bool + closedSignal chan struct{} + readyToProxySignal chan struct{} + readyToProxyOnce sync.Once + peerConnection *webrtc.PeerConnection + dataChannel *webrtc.DataChannel + dataChannelConn datachannel.ReadWriteCloser + dataChannelWriteBufferSignal chan struct{} + sendMediaTrack *webrtc.TrackLocalStaticRTP + sendMediaTrackRTP *webrtc.RTPTransceiver + receiveMediaTrack *webrtc.TrackRemote + receiveMediaTrackOpenedSignal chan struct{} + mediaTrackReliabilityLayer *reliableConn + iceCandidatePairMetrics common.LogFields + + readMutex sync.Mutex + readBuffer []byte + readOffset int + readLength int + readError error + peerPaddingDone bool + receiveMediaTrackPacket *rtp.Packet + + writeMutex sync.Mutex + trafficShapingPRNG *prng.PRNG + trafficShapingBuffer *bytes.Buffer + paddedMessageCount int + decoyMessageCount int + trafficShapingDone bool + sendMediaTrackPacket *rtp.Packet + sendMediaTrackSequencer rtp.Sequencer + sendMediaTrackTimestampTick int + sendMediaTrackFrameSizeRange [2]int + sendMediaTrackRemainingFrameSize int + + decoyDone atomic.Bool paddedMessagesSent int32 paddedMessagesReceived int32 @@ -138,8 +164,13 @@ type webRTCConfig struct { // DoDTLSRandomization indicates whether to perform DTLS randomization. DoDTLSRandomization bool - // TrafficShapingParameters indicates whether and how to perform data channel traffic shaping. - TrafficShapingParameters *DataChannelTrafficShapingParameters + // UseMediaStreams indicates whether to use WebRTC media streams to tunnel + // traffic. When false, a WebRTC data channel is used to tunnel traffic. + UseMediaStreams bool + + // TrafficShapingParameters indicates whether and how to perform data + // channel or media track traffic shaping. + TrafficShapingParameters *TrafficShapingParameters // ReliableTransport indicates whether to configure the WebRTC data // channel to use reliable transport. Set ReliableTransport when proxying @@ -372,12 +403,13 @@ func newWebRTCConn( settingEngine.SetICETimeouts(45*time.Second, 0, prng.JitterDuration(15*time.Second, 0.2)) settingEngine.SetICEMaxBindingRequests(10) - // Initialize data channel obfuscation + // Initialize data channel or media streams obfuscation config.Logger.WithTraceFields(common.LogFields{ "dtls_randomization": config.DoDTLSRandomization, "data_channel_traffic_shaping": config.TrafficShapingParameters != nil, - }).Info("webrtc_data_channel_obfuscation") + "use_media_streams": config.UseMediaStreams, + }).Info("webrtc_obfuscation") // Facilitate DTLS Client/ServerHello randomization. The client decides // whether to do DTLS randomization and generates and the proxy receives @@ -420,7 +452,7 @@ func newWebRTCConn( }) // Configure traffic shaping, which adds random padding and decoy messages - // to data channel message flows. + // to data channel message or media track packet flows. var trafficShapingPRNG *prng.PRNG trafficShapingBuffer := new(bytes.Buffer) @@ -431,9 +463,9 @@ func newWebRTCConn( // TODO: also use pion/dtls.Config.PaddingLengthGenerator? - trafficShapingContext := "in-proxy-data-channel-traffic-shaping-offer" + trafficShapingContext := "in-proxy-traffic-shaping-offer" if !isOffer { - trafficShapingContext = "in-proxy-data-channel-traffic-shaping-answer" + trafficShapingContext = "in-proxy-traffic-shaping-answer" } trafficShapingObfuscationSecret, err := deriveObfuscationSecret( @@ -525,12 +557,13 @@ func newWebRTCConn( } conn := &webRTCConn{ - config: config, + config: config, + isOffer: isOffer, udpConn: udpConn, portMapper: portMapper, closedSignal: make(chan struct{}), - dataChannelOpenedSignal: make(chan struct{}), + readyToProxySignal: make(chan struct{}), dataChannelWriteBufferSignal: make(chan struct{}, 1), // A data channel uses SCTP and is message oriented. The maximum @@ -538,7 +571,10 @@ func newWebRTCConn( // https://github.com/pion/webrtc/blob/dce970438344727af9c9965f88d958c55d32e64d/datachannel.go#L19. // This read buffer must be as large as the maximum message size or // else a read may fail with io.ErrShortBuffer. - readBuffer: make([]byte, dataChannelMaxMessageSize), + // + // For media streams, the largest media track RTP packet payload is + // no more than mediaTrackMaxRTPPayloadLength. + readBuffer: make([]byte, max(dataChannelMaxMessageSize, mediaTrackMaxRTPPayloadLength)), trafficShapingPRNG: trafficShapingPRNG, trafficShapingBuffer: trafficShapingBuffer, @@ -565,7 +601,47 @@ func newWebRTCConn( settingEngine.SetICEBindingRequestHandler(conn.onICEBindingRequest) // All settingEngine configuration must be done before calling NewAPI. - webRTCAPI := webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) + + var webRTCAPI *webrtc.API + + if !config.UseMediaStreams { + + webRTCAPI = webrtc.NewAPI(webrtc.WithSettingEngine(settingEngine)) + + } else { + + // Additional webRTCAPI setup for media streams support. + + mediaEngine := &webrtc.MediaEngine{} + err := mediaEngine.RegisterDefaultCodecs() + if err != nil { + return nil, nil, nil, errors.Trace(err) + } + + // pion/webrtc interceptors monitor RTP and send additional traffic + // including NACKs and RTCP. Enable interceptors for the potential + // obfuscation benefit from exhibiting this additional traffic. + // webrtc.RegisterDefaultInterceptors calls ConfigureNack, + // ConfigureRTCPReports, ConfigureTWCCSender. At this time we skip + // ConfigureNack as this appears to generate excess "duplicated + // packet" logs and connection instability. From a connection + // reliability stand point, the underlying QUIC layer provides any + // necessary resends. + interceptors := &interceptor.Registry{} + err = webrtc.ConfigureRTCPReports(interceptors) + if err != nil { + return nil, nil, nil, errors.Trace(err) + } + err = webrtc.ConfigureTWCCSender(mediaEngine, interceptors) + if err != nil { + return nil, nil, nil, errors.Trace(err) + } + + webRTCAPI = webrtc.NewAPI( + webrtc.WithSettingEngine(settingEngine), + webrtc.WithMediaEngine(mediaEngine), + webrtc.WithInterceptorRegistry(interceptors)) + } conn.peerConnection, err = webRTCAPI.NewPeerConnection( webrtc.Configuration{ @@ -583,14 +659,10 @@ func newWebRTCConn( conn.peerConnection.OnSignalingStateChange(conn.onSignalingStateChange) conn.peerConnection.OnDataChannel(conn.onDataChannel) - // As a future enhancement, consider using media channels instead of data - // channels, as media channels may be more common. Proxied QUIC would - // work over an unreliable media channel. Note that a media channel is - // still prefixed with STUN and DTLS exchanges before SRTP begins, so the - // first few packets are the same as a data channel. + if !config.UseMediaStreams && isOffer { - // The offer sets the data channel configuration. - if isOffer { + // Use a data channel to proxy traffic. The client offer sets the data + // channel configuration. dataChannelInit := &webrtc.DataChannelInit{} if !config.ReliableTransport { @@ -601,25 +673,188 @@ func newWebRTCConn( } // Generate a random length label, to vary the DATA_CHANNEL_OPEN - // message length. The label is derived from and replayed via + // message length. This length/value is not replayed. + dataChannelLabel := prng.HexString(prng.Range(1, dataChannelMaxLabelLength)) + + dataChannel, err := conn.peerConnection.CreateDataChannel( + dataChannelLabel, dataChannelInit) + if err != nil { + return nil, nil, nil, errors.Trace(err) + } + + conn.setDataChannel(dataChannel) + } + + if config.UseMediaStreams { + + // Use media streams to proxy traffic. Each peer configures one + // unidirectional media stream track to send its proxied traffic. In + // WebRTC, a media stream consists of a set of tracks. Configure and + // use a single video track. + // + // This implementation is intended to circumvent the WebRTC data + // channel blocking described in "Differential Degradation + // Vulnerabilities in Censorship Circumvention Systems", + // https://arxiv.org/html/2409.06247v1, section 5.2. + + // Select the media track attributes, which are observable, in + // plaintext, in the RTP header. Attributes include the payload + // type/codec and codec timestamp inputs. Attempt to mimic common + // WebRTC media stream traffic by selecting common codecs and video + // frame sizes and timestamp ticks. Each peer's track has its own + // attributes, which is not unusual. This is a basic effort to avoid + // trivial, stateless or minimal state DPI blocking, unlike more + // advanced schemes which replace bytes in actual video streams. The + // client drives attribute selection and replay by specifying // ClientRootObfuscationSecret. - labelObfuscationSecret, err := deriveObfuscationSecret( - config.ClientRootObfuscationSecret, "in-proxy-data-channel-label") + + propertiesContext := "in-proxy-media-track-properties-offer" + if !isOffer { + propertiesContext = "in-proxy-media-track-properties-answer" + } + + propertiesObfuscationSecret, err := deriveObfuscationSecret( + config.ClientRootObfuscationSecret, propertiesContext) if err != nil { return nil, nil, nil, errors.Trace(err) } - seed := prng.Seed(labelObfuscationSecret) - labelPRNG := prng.NewPRNGWithSeed(&seed) - dataChannelLabel := labelPRNG.HexString( - labelPRNG.Range(1, dataChannelMaxLabelLength/2)) - dataChannel, err := conn.peerConnection.CreateDataChannel( - dataChannelLabel, dataChannelInit) + seed := prng.Seed(propertiesObfuscationSecret) + propertiesPRNG := prng.NewPRNGWithSeed(&seed) + + // Omit webrtc.MimeTypeH265, which results in the error: + // "SetRemoteSDP: unable to start track, codec is not supported by remote". + mimeTypes := []string{webrtc.MimeTypeH264, webrtc.MimeTypeVP8, webrtc.MimeTypeVP9, webrtc.MimeTypeAV1} + clockRate := 90000 // Standard 90kHz + frameRates := []int{25, 30, 60} // Common frame rates + + // Select frame sizes from common video modes. Each frame size is + // selected at random from the given range, and the codec timestamp + // is advanced when the resulting "frame size" number of proxied + // bytes is sent. + // + // - Low-resolution video (e.g., QCIF): 1–10 KB per frame. + // - Standard-definition video (480p): 50–200 KB per frame. + // - High-definition video (720p): 100–500 KB per frame. + // - Full HD video (1080p): 300 KB – 1 MB per frame. + // - 4K video: 1–4 MB per frame. + KB := 1024 + MB := 1024 * 1024 + frameSizeRanges := [][2]int{ + {1 * KB, 10 * KB}, + {50 * KB, 200 * KB}, + {100 * KB, 500 * KB}, + {300 * KB, 1 * MB}, + {1 * MB, 4 * MB}} + + mimeType := mimeTypes[propertiesPRNG.Intn(len(mimeTypes))] + frameRate := frameRates[propertiesPRNG.Intn(len(frameRates))] + frameSizeRange := frameSizeRanges[propertiesPRNG.Intn(len(frameSizeRanges))] + + conn.sendMediaTrackTimestampTick = clockRate / frameRate + conn.sendMediaTrackFrameSizeRange = frameSizeRange + + // Initialize the first frame size. The random frame sizes are not + // replayed. + conn.sendMediaTrackRemainingFrameSize = prng.Range( + conn.sendMediaTrackFrameSizeRange[0], conn.sendMediaTrackFrameSizeRange[1]) + + // Generate random IDs, to vary the resulting SDP entry size message + // length. These lengths/values are not replayed. + trackID := prng.HexString(prng.Range(1, mediaTrackMaxIDLength)) + trackStreamID := prng.HexString(prng.Range(1, mediaTrackMaxIDLength)) + + // Initialize a reusable rtp.Packet struct to avoid an allocation per + // write. In SRTP, the packet payload is encrypted while the RTP + // header remains plaintext. + // + // Plaintext RTP header fields: + // + // - Version is always 2. + // + // - Timestamp is initialized here to a random value, as is common, + // and incremented, after writes, for the next video "frame". + // Limitation: in states of low tunnel traffic, the video frame and + // timestamp progression won't look realistic. + // + // - PayloadType is the codec and is auto-populated by pion. + // + // - SequenceNumber is a packet sequence number and populated by + // pion's rtp.NewRandomSequencer, which uses the same logic as + // Chrome's WebRTC implementation. + // + // - SSRC a random stream identifier, distinct from the track/stream + // ID, and is auto-populated by pion. + + conn.sendMediaTrackPacket = &rtp.Packet{ + Header: rtp.Header{ + Version: 2, + Timestamp: uint32(prng.Intn(1 << 32)), + }} + conn.sendMediaTrackSequencer = rtp.NewRandomSequencer() + + // Add the outbound media track to the SDP that is sent to the peer. + + conn.sendMediaTrack, err = webrtc.NewTrackLocalStaticRTP( + webrtc.RTPCodecCapability{ + MimeType: mimeType, + ClockRate: uint32(clockRate), + }, + trackID, + trackStreamID) if err != nil { return nil, nil, nil, errors.Trace(err) } - conn.setDataChannel(dataChannel) + conn.sendMediaTrackRTP, err = conn.peerConnection.AddTransceiverFromTrack( + conn.sendMediaTrack, + webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendrecv}) + if err != nil { + return nil, nil, nil, errors.Trace(err) + } + for _, rtpSender := range conn.peerConnection.GetSenders() { + + // Read incoming packets for this outbound RTP stream. Streams are + // unidirectional for media payload, but there will be incoming + // packets, from the peer, for RTCP, NACK, and other control + // mechanisms. Interceptors are implicitly invoked and the + // packets are then discarded. + go func(rtpSender *webrtc.RTPSender) { + var buffer [1500]byte + for { + _, _, err := conn.sendMediaTrackRTP.Sender().Read(buffer[:]) + if err != nil { + // TODO: log error? + select { + case <-conn.closedSignal: + return + default: + } + } + } + }(rtpSender) + } + + // Initialize the callback that is invoked once we receive an inbound + // packet from the peer's media stream. + // + // Unlike data channels, where webrtc.DataChannel.OnOpen is symmetric + // and invoked on both peers for a single, bidirectional channel, + // webrtc.PeerConnection.OnTrack is unidirectional. And, unlike + // DataChannel.OnOpen, if both peers await OnTrack before proxying, + // the tunnel will hang. One side must start sending data in order + // for OnTrack to be invoked on the other side. + // See: https://github.com/pion/webrtc/issues/989#issuecomment-580424615. + // + // This has implications for AwaitReadyToProxy: in the media stream + // mode, and when not using the media track reliability layer, + // AwaitReadyToProxy returns when the DTLS handshake has completed, + // but before any SRTP packets have been received from the peer. + + conn.receiveMediaTrackOpenedSignal = make(chan struct{}) + conn.receiveMediaTrackPacket = &rtp.Packet{} + + conn.peerConnection.OnTrack(conn.onMediaTrack) } // Prepare to await full ICE completion, including STUN candidates. @@ -897,16 +1132,20 @@ func (conn *webRTCConn) SetRemoteSDP( return nil } -// AwaitInitialDataChannel returns when the data channel is established, or -// when an error has occured. -func (conn *webRTCConn) AwaitInitialDataChannel(ctx context.Context) error { +// AwaitReadyToProxy returns when the data channel is established, or media +// streams are ready to send data, or when an error has occured. +func (conn *webRTCConn) AwaitReadyToProxy(ctx context.Context, connectionID ID) error { // Don't lock the mutex, or else necessary operations will deadlock. select { - case <-conn.dataChannelOpenedSignal: + case <-conn.readyToProxySignal: - // The data channel is connected. + // ICE is complete and DTLS is connected. In data channel mode, the + // data channel is established using SCTP, which involves a further + // handshake. In media stream mode, due to its unidirectional nature, + // there is no equivalent to the the data channel establishment step. + // See OnTrack comment in newWebRTCConn. err := conn.recordSelectedICECandidateStats() if err != nil { @@ -924,6 +1163,23 @@ func (conn *webRTCConn) AwaitInitialDataChannel(ctx context.Context) error { return errors.TraceNew("connection has closed") } + if conn.config.UseMediaStreams && conn.config.ReliableTransport { + + // The SRTP protocol used in media stream mode doesn't offer + // reliable/ordered transport, so when that transport property is + // required, add a reliability layer based on QUIC. This layer is + // fully established here before returning read-to-proxy. + + err := conn.addRTPReliabilityLayer(ctx) + if err != nil { + return errors.Trace(err) + } + } + + conn.config.Logger.WithTraceFields(common.LogFields{ + "connectionID": connectionID, + }).Info("WebRTC tunnel established") + return nil } @@ -1091,14 +1347,23 @@ func (conn *webRTCConn) Close() error { conn.portMapper.close() } + // Neither sendMediaTrack nor receiveMediaTrack have a Close operation. + + if conn.sendMediaTrackRTP != nil { + _ = conn.sendMediaTrackRTP.Stop() + } + if conn.mediaTrackReliabilityLayer != nil { + _ = conn.mediaTrackReliabilityLayer.Close() + } if conn.dataChannelConn != nil { - conn.dataChannelConn.Close() + _ = conn.dataChannelConn.Close() } if conn.dataChannel != nil { - conn.dataChannel.Close() + _ = conn.dataChannel.Close() } if conn.peerConnection != nil { - conn.peerConnection.Close() + // TODO: use PeerConnection.GracefulClose (requires pion/webrtc 3.2.51)? + _ = conn.peerConnection.Close() } // Close the udpConn to interrupt any blocking DTLS handshake: @@ -1108,7 +1373,7 @@ func (conn *webRTCConn) Close() error { // before the UDP socket is closed here. if conn.udpConn != nil { - conn.udpConn.Close() + _ = conn.udpConn.Close() } close(conn.closedSignal) @@ -1127,410 +1392,149 @@ func (conn *webRTCConn) IsClosed() bool { func (conn *webRTCConn) Read(p []byte) (int, error) { - for { - - n, err := conn.readMessage(p) - if err != nil || n > 0 { - return n, err - } + if !conn.config.UseMediaStreams { + // Data channel mode. + n, err := conn.readDataChannel(p) + return n, errors.TraceReader(err) + } - // A decoy message was read; discard and read again. + if conn.mediaTrackReliabilityLayer != nil { + // Media stream mode with reliability layer. + n, err := conn.mediaTrackReliabilityLayer.Read(p) + return n, errors.TraceReader(err) } -} -func (conn *webRTCConn) readMessage(p []byte) (int, error) { + // Media stream mode without reliability layer. + n, err := conn.readMediaTrack(p) + return n, errors.TraceReader(err) +} - // Don't hold this lock, or else concurrent Writes will be blocked. - conn.mutex.Lock() - isClosed := conn.isClosed - dataChannelConn := conn.dataChannelConn - decoyDone := conn.decoyDone - conn.mutex.Unlock() +func (conn *webRTCConn) Write(p []byte) (int, error) { - if isClosed { - return 0, errors.TraceNew("closed") + if !conn.config.UseMediaStreams { + // Data channel mode. + n, err := conn.writeDataChannelMessage(p, false) + return n, errors.Trace(err) } - if dataChannelConn == nil { - return 0, errors.TraceNew("no data channel") + if conn.mediaTrackReliabilityLayer != nil { + // Media stream mode with reliability layer. + n, err := conn.mediaTrackReliabilityLayer.Write(p) + return n, errors.Trace(err) } - // The input read buffer, p, may not be the same length as the message - // read from the data channel. Buffer the read message if another Read - // call is necessary to consume it. As per https://pkg.go.dev/io#Reader, - // dataChannelConn bytes read are processed even when - // dataChannelConn.Read returns an error; the error value is stored and - // returned with the Read call that consumes the end of the message buffer. - - conn.readMutex.Lock() - defer conn.readMutex.Unlock() - - if conn.readOffset == conn.readLength { - n, err := dataChannelConn.Read(conn.readBuffer) - conn.readOffset = 0 - conn.readLength = n - conn.readError = err - - // Skip over padding. + // Media stream mode without reliability layer. + n, err := conn.writeMediaTrackPacket(p, false) + return n, errors.Trace(err) +} - if n > 0 && !conn.peerPaddingDone { +func (conn *webRTCConn) LocalAddr() net.Addr { + conn.mutex.Lock() + defer conn.mutex.Unlock() - paddingSize, n := binary.Varint(conn.readBuffer[0:conn.readLength]) - if (paddingSize == 0 && n <= 0) || paddingSize >= int64(conn.readLength) { - return 0, errors.TraceNew("invalid padding") - } + // This is the local UDP socket address, not the external, public address. + return conn.udpConn.LocalAddr() +} - if paddingSize < 0 { +func (conn *webRTCConn) RemoteAddr() net.Addr { + conn.mutex.Lock() + defer conn.mutex.Unlock() - // When the padding header indicates a padding size of -1, the - // peer is indicating that padding is done. Subsequent - // messages will have no padding header or padding bytes. + // Not supported. + return nil +} - conn.peerPaddingDone = true - conn.readOffset += n +func (conn *webRTCConn) SetDeadline(t time.Time) error { + conn.mutex.Lock() + defer conn.mutex.Unlock() - } else { + return errors.TraceNew("not supported") +} - conn.readOffset += n + int(paddingSize) +func (conn *webRTCConn) SetReadDeadline(t time.Time) error { + conn.mutex.Lock() + defer conn.mutex.Unlock() - atomic.AddInt32(&conn.paddedMessagesReceived, 1) - if conn.readOffset == conn.readLength { - atomic.AddInt32(&conn.decoyMessagesReceived, 1) - } - } - } + if conn.isClosed { + return errors.TraceNew("closed") } - n := copy(p, conn.readBuffer[conn.readOffset:conn.readLength]) - conn.readOffset += n - - var err error - if conn.readOffset == conn.readLength { - err = conn.readError + if conn.config.UseMediaStreams { + // TODO: add support + return errors.TraceNew("not supported") } - // When decoy messages are enabled, periodically response to an incoming - // messages with an immediate outbound decoy message. This is similar to - // the design here: - // https://github.com/Psiphon-Labs/psiphon-tunnel-core/blob/c4f6a593a645db4479a7032a9e97d3c0b905cdfc/psiphon/common/quic/obfuscator.go#L361-L409 - // - // writeMessage handles conn.decoyMessageCount, which is syncronized with - // conn.WriteMutex, as well as other specific logic. Here we just signal - // writeMessage based on the read event. - // - // When the data channel already has buffered writes in excess of a decoy - // message size, the writeMessage skips the decoy message and returns - // without blocking, so Read calls will not block. - - if !decoyDone { - _, _ = conn.writeMessage(nil, true) + readDeadliner, ok := conn.dataChannelConn.(datachannel.ReadDeadliner) + if !ok { + return errors.TraceNew("no data channel") } - return n, errors.Trace(err) -} - -func (conn *webRTCConn) Write(p []byte) (int, error) { - return conn.writeMessage(p, false) + return readDeadliner.SetReadDeadline(t) } -func (conn *webRTCConn) writeMessage(p []byte, decoy bool) (int, error) { +func (conn *webRTCConn) SetWriteDeadline(t time.Time) error { + conn.mutex.Lock() + defer conn.mutex.Unlock() - if p != nil && decoy { - return 0, errors.TraceNew("invalid write parameters") - } + return errors.TraceNew("not supported") +} - // pion/sctp doesn't handle 0-byte writes correctly, so drop/skip at this level. - // - // Testing shows that the SCTP connection stalls after a 0-byte write. In - // the pion/sctp implementation, - // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L254-L278 and - // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L280-L336, it - // appears that a zero-byte write won't send an SCTP messages but does - // increment a sequence number. +// GetMetrics implements the common.MetricsSource interface and returns log +// fields detailing the WebRTC dial parameters. +func (conn *webRTCConn) GetMetrics() common.LogFields { + conn.mutex.Lock() + defer conn.mutex.Unlock() - if len(p) == 0 && !decoy { - return 0, nil - } + logFields := make(common.LogFields) - // Don't hold this lock, or else concurrent Reads will be blocked. - conn.mutex.Lock() - isClosed := conn.isClosed - bufferedAmount := conn.dataChannel.BufferedAmount() - dataChannelConn := conn.dataChannelConn - conn.mutex.Unlock() + logFields.Add(conn.iceCandidatePairMetrics) - if isClosed { - return 0, errors.TraceNew("closed") + randomizeDTLS := "0" + if conn.config.DoDTLSRandomization { + randomizeDTLS = "1" } + logFields["inproxy_webrtc_randomize_dtls"] = randomizeDTLS - if dataChannelConn == nil { - return 0, errors.TraceNew("no data channel") + useMediaStreams := "0" + if conn.config.UseMediaStreams { + useMediaStreams = "1" } + logFields["inproxy_webrtc_use_media_streams"] = useMediaStreams - // Only proceed with a decoy message when no pending writes are buffered. - // - // This check is made before acquiring conn.writeMutex so that, in most - // cases, writeMessage won't block Read calls when a concurrent Write is - // holding conn.writeMutex and potentially blocking on flow control. - // There's still a chance that this test passes, and a concurrent Write - // arrives at the same time. - - if decoy && bufferedAmount > 0 { - return 0, nil - } + logFields["inproxy_webrtc_padded_messages_sent"] = atomic.LoadInt32(&conn.paddedMessagesSent) + logFields["inproxy_webrtc_padded_messages_received"] = atomic.LoadInt32(&conn.paddedMessagesReceived) + logFields["inproxy_webrtc_decoy_messages_sent"] = atomic.LoadInt32(&conn.decoyMessagesSent) + logFields["inproxy_webrtc_decoy_messages_received"] = atomic.LoadInt32(&conn.decoyMessagesReceived) - conn.writeMutex.Lock() - defer conn.writeMutex.Unlock() + return logFields +} - writeSize := len(p) +func (conn *webRTCConn) onConnectionStateChange(state webrtc.PeerConnectionState) { - // Determine padding size and padding header size. + switch state { + case webrtc.PeerConnectionStateConnected: - doPadding := false - paddingSize := 0 - var paddingHeader [binary.MaxVarintLen32]byte - paddingHeaderSize := 0 + if conn.config.UseMediaStreams { - if decoy { + // webrtc.PeerConnectionStateConnected is received once the DTLS + // connection is established. At this point, media track data may + // be sent. In media stream mode, unblock AwaitForReadyToProxy to + // allow peers to start sending data. In data channel mode, wait + // and signal in onDataChannelOpen. - if conn.decoyMessageCount < 1 { - return 0, nil + conn.readyToProxyOnce.Do(func() { close(conn.readyToProxySignal) }) } - if !conn.trafficShapingPRNG.FlipWeightedCoin( - conn.config.TrafficShapingParameters.DecoyMessageProbability) { - return 0, nil - } + case webrtc.PeerConnectionStateDisconnected, + webrtc.PeerConnectionStateFailed, + webrtc.PeerConnectionStateClosed: - conn.decoyMessageCount -= 1 + // Close the WebRTCConn when the connection is no longer connected. Close + // will lock conn.mutex, so do not aquire the lock here. + // + // Currently, ICE Restart is not used, and there is no transition from + // Disconnected back to Connected. - decoySize := conn.trafficShapingPRNG.Range( - conn.config.TrafficShapingParameters.MinDecoySize, - conn.config.TrafficShapingParameters.MaxDecoySize) - - // When sending a decoy message, the entire message is padding. - - doPadding = true - paddingSize = decoySize - - if conn.decoyMessageCount == 0 { - - // Set the shared flag that readMessage uses to stop invoking - // writeMessage for decoy events. - - conn.mutex.Lock() - conn.decoyDone = true - conn.mutex.Unlock() - } - - } else if conn.paddedMessageCount > 0 { - - // Add padding to a normal write. - - conn.paddedMessageCount -= 1 - - doPadding = true - paddingSize = prng.Range( - conn.config.TrafficShapingParameters.MinPaddingSize, - conn.config.TrafficShapingParameters.MaxPaddingSize) - - } else if conn.decoyMessageCount > 0 { - - // Padding normal messages is done, but there are still outstanding - // decoy messages, so add a padding header indicating padding size 0 - // to this normal message. - - doPadding = true - paddingSize = 0 - - } else if !conn.trafficShapingDone { - - // Padding normal messages is done and all decoy messages are sent, so - // send a special padding header with padding size -1, signaling the - // peer that no additional padding will be performed and no - // subsequent messages will contain a padding header. - - doPadding = true - paddingSize = -1 - - } - - if doPadding { - - if paddingSize > 0 { - - // Reduce, if necessary, to stay within the maximum data channel - // message size. This is not expected to happen for the io.Copy use - // case, with 32K message size, plus reasonable padding sizes. - - if writeSize+binary.MaxVarintLen32+paddingSize > dataChannelMaxMessageSize { - paddingSize -= (writeSize + binary.MaxVarintLen32 + paddingSize) - dataChannelMaxMessageSize - if paddingSize < 0 { - paddingSize = 0 - } - } - - // Add padding overhead to total writeSize before the flow control check. - - writeSize += paddingSize - } - - paddingHeaderSize = binary.PutVarint(paddingHeader[:], int64(paddingSize)) - writeSize += paddingHeaderSize - } - - if writeSize > dataChannelMaxMessageSize { - return 0, errors.TraceNew("write too large") - } - - // Flow control is required to ensure that Write calls don't result in - // unbounded buffering in pion/webrtc. Use similar logic and the same - // buffer size thresholds as the pion sample code. - // - // https://github.com/pion/webrtc/tree/master/examples/data-channels-flow-control#when-do-we-need-it: - // > Send or SendText methods are called on DataChannel to send data to - // > the connected peer. The methods return immediately, but it does not - // > mean the data was actually sent onto the wire. Instead, it is - // > queued in a buffer until it actually gets sent out to the wire. - // > - // > When you have a large amount of data to send, it is an application's - // > responsibility to control the buffered amount in order not to - // > indefinitely grow the buffer size to eventually exhaust the memory. - - // If the pion write buffer is too full, wait for a signal that sufficient - // write data has been consumed before writing more. - if !isClosed && bufferedAmount+uint64(writeSize) > dataChannelMaxBufferedAmount { - select { - case <-conn.dataChannelWriteBufferSignal: - case <-conn.closedSignal: - return 0, errors.TraceNew("connection has closed") - } - } - - if conn.trafficShapingDone { - - // When traffic shaping is done, p is written directly without the - // additional trafficShapingBuffer copy. - - // Limitation: if len(p) > 65536, the dataChannelConn.Write will fail. In - // practise, this is not expected to happen with typical use cases such - // as io.Copy, which uses a 32K buffer. - n, err := dataChannelConn.Write(p) - - return n, errors.Trace(err) - } - - conn.trafficShapingBuffer.Reset() - conn.trafficShapingBuffer.Write(paddingHeader[:paddingHeaderSize]) - if paddingSize > 0 { - conn.trafficShapingBuffer.Write(prng.Bytes(paddingSize)) - } - conn.trafficShapingBuffer.Write(p) - - // Limitation: see above; len(p) + padding must be <= 65536. - _, err := dataChannelConn.Write(conn.trafficShapingBuffer.Bytes()) - - if decoy { - atomic.AddInt32(&conn.decoyMessagesSent, 1) - } else if doPadding && paddingSize > 0 { - atomic.AddInt32(&conn.paddedMessagesSent, 1) - } - - if conn.paddedMessageCount == 0 && conn.decoyMessageCount == 0 && paddingSize == -1 { - - // Set flag indicating -1 padding size was sent and release traffic - // shaping resources. - - conn.trafficShapingDone = true - conn.trafficShapingPRNG = nil - conn.trafficShapingBuffer = nil - } - - return len(p), errors.Trace(err) -} - -func (conn *webRTCConn) LocalAddr() net.Addr { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - // This is the local UDP socket address, not the external, public address. - return conn.udpConn.LocalAddr() -} - -func (conn *webRTCConn) RemoteAddr() net.Addr { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - // Not supported. - return nil -} - -func (conn *webRTCConn) SetDeadline(t time.Time) error { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - return errors.TraceNew("not supported") -} - -func (conn *webRTCConn) SetReadDeadline(t time.Time) error { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - if conn.isClosed { - return errors.TraceNew("closed") - } - - readDeadliner, ok := conn.dataChannelConn.(datachannel.ReadDeadliner) - if !ok { - return errors.TraceNew("no data channel") - } - - return readDeadliner.SetReadDeadline(t) -} - -func (conn *webRTCConn) SetWriteDeadline(t time.Time) error { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - return errors.TraceNew("not supported") -} - -// GetMetrics implements the common.MetricsSource interface and returns log -// fields detailing the WebRTC dial parameters. -func (conn *webRTCConn) GetMetrics() common.LogFields { - conn.mutex.Lock() - defer conn.mutex.Unlock() - - logFields := make(common.LogFields) - - logFields.Add(conn.iceCandidatePairMetrics) - - randomizeDTLS := "0" - if conn.config.DoDTLSRandomization { - randomizeDTLS = "1" - } - logFields["inproxy_webrtc_randomize_dtls"] = randomizeDTLS - - logFields["inproxy_webrtc_padded_messages_sent"] = atomic.LoadInt32(&conn.paddedMessagesSent) - logFields["inproxy_webrtc_padded_messages_received"] = atomic.LoadInt32(&conn.paddedMessagesReceived) - logFields["inproxy_webrtc_decoy_messages_sent"] = atomic.LoadInt32(&conn.decoyMessagesSent) - logFields["inproxy_webrtc_decoy_messages_received"] = atomic.LoadInt32(&conn.decoyMessagesReceived) - - return logFields -} - -func (conn *webRTCConn) onConnectionStateChange(state webrtc.PeerConnectionState) { - - // Close the WebRTCConn when the connection is no longer connected. Close - // will lock conn.mutex, so do lot aquire the lock here. - // - // Currently, ICE Restart is not used, and there is no transition from - // Disconnected back to Connected. - - switch state { - case webrtc.PeerConnectionStateDisconnected, - webrtc.PeerConnectionStateFailed, - webrtc.PeerConnectionStateClosed: conn.Close() } @@ -1554,15 +1558,16 @@ func (conn *webRTCConn) onICEBindingRequest(m *stun.Message, local, remote ice.C // SetICEBindingRequestHandler is used to hook onICEBindingRequest into // STUN bind events for logging. The return values is always false as // this callback makes no adjustments to ICE candidate selection. When - // the data channel has already opened, skip logging events, as this - // callback appears to be invoked for keepalive pings. + // the data channel or media track tunnel has already opened, skip + // logging events, as this callback appears to be invoked for keepalive + // pings. if local == nil || remote == nil { return false } select { - case <-conn.dataChannelOpenedSignal: + case <-conn.readyToProxySignal: return false default: } @@ -1577,6 +1582,9 @@ func (conn *webRTCConn) onICEBindingRequest(m *stun.Message, local, remote ice.C func (conn *webRTCConn) onICEConnectionStateChange(state webrtc.ICEConnectionState) { + conn.mutex.Lock() + defer conn.mutex.Unlock() + conn.config.Logger.WithTraceFields(common.LogFields{ "state": state.String(), }).Debug("ICE connection state changed") @@ -1591,7 +1599,7 @@ func (conn *webRTCConn) onICEGatheringStateChange(state webrtc.ICEGathererState) func (conn *webRTCConn) onNegotiationNeeded() { - conn.config.Logger.WithTrace().Info("negotiation needed") + conn.config.Logger.WithTrace().Debug("negotiation needed") } func (conn *webRTCConn) onSignalingStateChange(state webrtc.SignalingState) { @@ -1614,6 +1622,20 @@ func (conn *webRTCConn) onDataChannel(dataChannel *webrtc.DataChannel) { }).Debug("new data channel") } +func (conn *webRTCConn) onMediaTrack(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + + conn.mutex.Lock() + defer conn.mutex.Unlock() + + conn.receiveMediaTrack = track + close(conn.receiveMediaTrackOpenedSignal) + + conn.config.Logger.WithTraceFields(common.LogFields{ + "ID": track.ID(), + "payload_type": track.Kind().String(), + }).Info("media track open") +} + func (conn *webRTCConn) onDataChannelOpen() { conn.mutex.Lock() @@ -1626,7 +1648,7 @@ func (conn *webRTCConn) onDataChannelOpen() { // TODO: can a data channel be connected, disconnected, and then // reestablished in one session? - conn.dataChannelOpenedOnce.Do(func() { close(conn.dataChannelOpenedSignal) }) + conn.readyToProxyOnce.Do(func() { close(conn.readyToProxySignal) }) } conn.config.Logger.WithTraceFields(common.LogFields{ @@ -1637,12 +1659,1043 @@ func (conn *webRTCConn) onDataChannelOpen() { func (conn *webRTCConn) onDataChannelClose() { // Close the WebRTCConn when the data channel is closed. Close will lock - // conn.mutex, so do lot aquire the lock here. + // conn.mutex, so do not aquire the lock here. conn.Close() conn.config.Logger.WithTrace().Info("data channel closed") } +func (conn *webRTCConn) readDataChannel(p []byte) (int, error) { + for { + + n, err := conn.readDataChannelMessage(p) + if err != nil || n > 0 { + return n, errors.TraceReader(err) + } + + // A decoy message was read; discard and read again. + } +} + +func (conn *webRTCConn) readDataChannelMessage(p []byte) (int, error) { + + // Don't hold this lock, or else concurrent Writes will be blocked. + conn.mutex.Lock() + isClosed := conn.isClosed + dataChannelConn := conn.dataChannelConn + conn.mutex.Unlock() + + if isClosed { + return 0, errors.TraceNew("closed") + } + + if dataChannelConn == nil { + return 0, errors.TraceNew("no data channel") + } + + // The input read buffer, p, may not be the same length as the message + // read from the data channel. Buffer the read message if another Read + // call is necessary to consume it. As per https://pkg.go.dev/io#Reader, + // dataChannelConn bytes read are processed even when + // dataChannelConn.Read returns an error; the error value is stored and + // returned with the Read call that consumes the end of the message buffer. + + conn.readMutex.Lock() + defer conn.readMutex.Unlock() + + if conn.readOffset == conn.readLength { + n, err := dataChannelConn.Read(conn.readBuffer) + conn.readOffset = 0 + conn.readLength = n + conn.readError = err + + // Skip over padding. + + if n > 0 && !conn.peerPaddingDone { + + paddingSize, n := binary.Varint(conn.readBuffer[0:conn.readLength]) + if (paddingSize == 0 && n <= 0) || paddingSize >= int64(conn.readLength) { + return 0, errors.TraceNew("invalid padding") + } + + if paddingSize < 0 { + + // When the padding header indicates a padding size of -1, the + // peer is indicating that padding is done. Subsequent + // messages will have no padding header or padding bytes. + + conn.peerPaddingDone = true + conn.readOffset += n + + } else { + + conn.readOffset += n + int(paddingSize) + + atomic.AddInt32(&conn.paddedMessagesReceived, 1) + if conn.readOffset == conn.readLength { + atomic.AddInt32(&conn.decoyMessagesReceived, 1) + } + } + } + } + + n := copy(p, conn.readBuffer[conn.readOffset:conn.readLength]) + conn.readOffset += n + + var err error + if conn.readOffset == conn.readLength { + err = conn.readError + } + + // When decoy messages are enabled, periodically respond to an incoming + // messages with an immediate outbound decoy message. This is similar to + // the design here: + // https://github.com/Psiphon-Labs/psiphon-tunnel-core/blob/c4f6a593a645db4479a7032a9e97d3c0b905cdfc/psiphon/common/quic/obfuscator.go#L361-L409 + // + // writeDataChannelMessage handles conn.decoyMessageCount, which is + // synchronized with conn.WriteMutex, as well as other specific logic. + // Here we just signal writeDataChannelMessage based on the read event. + // + // When the data channel already has buffered writes in excess of a decoy + // message size, the writeDataChannelMessage skips the decoy message and + // returns without blocking, so Read calls will not block. + + if !conn.decoyDone.Load() { + _, _ = conn.writeDataChannelMessage(nil, true) + } + + return n, errors.TraceReader(err) +} + +func (conn *webRTCConn) writeDataChannelMessage(p []byte, decoy bool) (int, error) { + + if p != nil && decoy { + return 0, errors.TraceNew("invalid write parameters") + } + + // pion/sctp doesn't handle 0-byte writes correctly, so drop/skip at this level. + // + // Testing shows that the SCTP connection stalls after a 0-byte write. In + // the pion/sctp implementation, + // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L254-L278 and + // https://github.com/pion/sctp/blob/v1.8.8/stream.go#L280-L336, it + // appears that a zero-byte write won't send an SCTP messages but does + // increment a sequence number. + + if len(p) == 0 && !decoy { + return 0, nil + } + + // Don't hold this lock, or else concurrent Reads will be blocked. + conn.mutex.Lock() + isClosed := conn.isClosed + bufferedAmount := conn.dataChannel.BufferedAmount() + dataChannelConn := conn.dataChannelConn + conn.mutex.Unlock() + + if isClosed { + return 0, errors.TraceNew("closed") + } + + if dataChannelConn == nil { + return 0, errors.TraceNew("no data channel") + } + + // Only proceed with a decoy message when no pending writes are buffered. + // + // This check is made before acquiring conn.writeMutex so that, in most + // cases, writeMessage won't block Read calls when a concurrent Write is + // holding conn.writeMutex and potentially blocking on flow control. + // There's still a chance that this test passes, and a concurrent Write + // arrives at the same time. + + if decoy && bufferedAmount > 0 { + return 0, nil + } + + conn.writeMutex.Lock() + defer conn.writeMutex.Unlock() + + writeSize := len(p) + + // Determine padding size and padding header size. + + doPadding := false + paddingSize := 0 + var paddingHeader [binary.MaxVarintLen32]byte + paddingHeaderSize := 0 + + if decoy { + + if conn.decoyMessageCount < 1 { + return 0, nil + } + + if !conn.trafficShapingPRNG.FlipWeightedCoin( + conn.config.TrafficShapingParameters.DecoyMessageProbability) { + return 0, nil + } + + conn.decoyMessageCount -= 1 + + decoySize := conn.trafficShapingPRNG.Range( + conn.config.TrafficShapingParameters.MinDecoySize, + conn.config.TrafficShapingParameters.MaxDecoySize) + + // When sending a decoy message, the entire message is padding. + + doPadding = true + paddingSize = decoySize + + if conn.decoyMessageCount == 0 { + + // Set the shared flag that readMessage uses to stop invoking + // writeMessage for decoy events. + + conn.decoyDone.Store(true) + } + + } else if conn.paddedMessageCount > 0 { + + // Add padding to a normal write. + + conn.paddedMessageCount -= 1 + + doPadding = true + paddingSize = prng.Range( + conn.config.TrafficShapingParameters.MinPaddingSize, + conn.config.TrafficShapingParameters.MaxPaddingSize) + + } else if conn.decoyMessageCount > 0 { + + // Padding normal messages is done, but there are still outstanding + // decoy messages, so add a padding header indicating padding size 0 + // to this normal message. + + doPadding = true + paddingSize = 0 + + } else if !conn.trafficShapingDone { + + // Padding normal messages is done and all decoy messages are sent, so + // send a special padding header with padding size -1, signaling the + // peer that no additional padding will be performed and no + // subsequent messages will contain a padding header. + + doPadding = true + paddingSize = -1 + + } + + if doPadding { + + if paddingSize > 0 { + + // Reduce, if necessary, to stay within the maximum data channel + // message size. This is not expected to happen for the io.Copy use + // case, with 32K message size, plus reasonable padding sizes. + + if writeSize+binary.MaxVarintLen32+paddingSize > dataChannelMaxMessageSize { + paddingSize -= (writeSize + binary.MaxVarintLen32 + paddingSize) - dataChannelMaxMessageSize + if paddingSize < 0 { + paddingSize = 0 + } + } + + // Add padding overhead to total writeSize before the flow control check. + + writeSize += paddingSize + } + + paddingHeaderSize = binary.PutVarint(paddingHeader[:], int64(paddingSize)) + writeSize += paddingHeaderSize + } + + if writeSize > dataChannelMaxMessageSize { + return 0, errors.TraceNew("write too large") + } + + // Flow control is required to ensure that Write calls don't result in + // unbounded buffering in pion/webrtc. Use similar logic and the same + // buffer size thresholds as the pion sample code. + // + // https://github.com/pion/webrtc/tree/master/examples/data-channels-flow-control#when-do-we-need-it: + // > Send or SendText methods are called on DataChannel to send data to + // > the connected peer. The methods return immediately, but it does not + // > mean the data was actually sent onto the wire. Instead, it is + // > queued in a buffer until it actually gets sent out to the wire. + // > + // > When you have a large amount of data to send, it is an application's + // > responsibility to control the buffered amount in order not to + // > indefinitely grow the buffer size to eventually exhaust the memory. + + // If the pion write buffer is too full, wait for a signal that sufficient + // write data has been consumed before writing more. + if !isClosed && bufferedAmount+uint64(writeSize) > dataChannelMaxBufferedAmount { + select { + case <-conn.dataChannelWriteBufferSignal: + case <-conn.closedSignal: + return 0, errors.TraceNew("connection has closed") + } + } + + if conn.trafficShapingDone { + + // When traffic shaping is done, p is written directly without the + // additional trafficShapingBuffer copy. + + // Limitation: if len(p) > 65536, the dataChannelConn.Write will fail. In + // practise, this is not expected to happen with typical use cases such + // as io.Copy, which uses a 32K buffer. + n, err := dataChannelConn.Write(p) + + return n, errors.Trace(err) + } + + conn.trafficShapingBuffer.Reset() + conn.trafficShapingBuffer.Write(paddingHeader[:paddingHeaderSize]) + if paddingSize > 0 { + conn.trafficShapingBuffer.Write(prng.Bytes(paddingSize)) + } + conn.trafficShapingBuffer.Write(p) + + // Limitation: see above; len(p) + padding must be <= 65536. + _, err := dataChannelConn.Write(conn.trafficShapingBuffer.Bytes()) + + if decoy { + atomic.AddInt32(&conn.decoyMessagesSent, 1) + } else if doPadding && paddingSize > 0 { + atomic.AddInt32(&conn.paddedMessagesSent, 1) + } + + if conn.paddedMessageCount == 0 && conn.decoyMessageCount == 0 && paddingSize == -1 { + + // Set flag indicating -1 padding size was sent and release traffic + // shaping resources. + + conn.trafficShapingDone = true + conn.trafficShapingPRNG = nil + conn.trafficShapingBuffer = nil + } + + return len(p), errors.Trace(err) +} + +// GetQUICMaxPacketSizeAdjustment returns the value to be specified in +// Psiphon's quic-go configuration ClientMaxPacketSizeAdjustment +// ServerMaxPacketSizeAdjustment fields. Psiphon's quic-go max packet size +// adjustment reduces the QUIC payload to accomodate overhead from +// obfuscation, as in Obfuscated QUIC. In the in-proxy case, the same +// mechanism is used to ensure that QUIC packets fit within the space +// available for SRTP packet payloads, allowing for the overhead of the RTP +// packet. Beyond that allowance, the adjustment is tuned to produce SRTP +// packets that match common SRTP traffic with maximum packet sizes of 1200 +// bytes, excluding IP and UDP headers. +// +// INPROXY-QUIC-OSSH must apply GetQUICMaxPacketSizeAdjustment on both the +// client and server side. In addition, the client must disable +// DisablePathMTUDiscovery. +func GetQUICMaxPacketSizeAdjustment(isIPv6 bool) int { + + // Limitations: + // + // - For INPROXY-QUIC-OSSH, the second hop egressing from the proxy is + // identical regardless of whether the 1st hop uses data channel mode + // or media stream mode. Currently, the INPROXY-QUIC-OSSH server won't + // be able to distinguish, early enough, between the modes used by the + // 1st hop. In order to conform with the required adustment for media + // stream mode, the server must always apply the adjustment. This + // reduction in QUIC packet size may impact the performance of data + // channel mode. Furthermore, the lower maximum QUIC packet size is + // directly observable on the 2nd hop. + + // common/quic.MAX_PRE_DISCOVERY_PACKET_SIZE_IPV4 = 1252 + // common/quic.MAX_PRE_DISCOVERY_PACKET_SIZE_IPV6 = 1232 + quicMTU := 1252 + if isIPv6 { + quicMTU = 1232 + } + targetMTUAdjustment := quicMTU - mediaTrackMaxUDPPayloadLength + if targetMTUAdjustment < 0 { + targetMTUAdjustment = 0 + } + + adjustment := targetMTUAdjustment + mediaTrackRTPPacketOverhead + if adjustment < 0 { + adjustment = 0 + } + + return adjustment +} + +func (conn *webRTCConn) readMediaTrack(p []byte) (int, error) { + for { + + n, err := conn.readMediaTrackPacket(p) + if err != nil || n > 0 { + return n, errors.TraceReader(err) + } + + // A decoy message was read; discard and read again. + } +} + +func (conn *webRTCConn) readMediaTrackPacket(p []byte) (int, error) { + + // Await opening the peer's media track, the OnTrack event. This + // synchronization is necessary since AwaitReadyToProxy returns before + // receiving a media track packet from the peer, which triggers OnTrack. + + select { + case <-conn.receiveMediaTrackOpenedSignal: + case <-conn.closedSignal: + return 0, errors.TraceNew("closed") + } + + // Don't hold this lock, or else concurrent Writes will be blocked. + conn.mutex.Lock() + isClosed := conn.isClosed + receiveMediaTrack := conn.receiveMediaTrack + conn.mutex.Unlock() + + if isClosed { + return 0, errors.TraceNew("closed") + } + + if receiveMediaTrack == nil { + return 0, errors.TraceNew("no media track") + } + + conn.readMutex.Lock() + defer conn.readMutex.Unlock() + + // Use the lower-level Read and Unmarshal functions to avoid per-call allocations + // performed by the higher-level ReadRTP. + + n, _, err := receiveMediaTrack.Read(conn.readBuffer) + if err != nil { + return 0, errors.TraceReader(err) + } + err = conn.receiveMediaTrackPacket.Unmarshal(conn.readBuffer[:n]) + if err != nil { + return 0, errors.Trace(err) + } + + payload := conn.receiveMediaTrackPacket.Payload + + if len(payload) < 1 { + return 0, errors.TraceNew("invalid padding") + } + + // Read the padding header byte, which is always present (see comment in + // writeMediaTrackPacket). + + paddingSize := int(payload[0]) + + if paddingSize == 255 { + // When the header is 255, this is a decoy packet with no application + // payload. Discard the entire packet. Return n = 0 bytes read, and + // the caller will read again. + return 0, nil + } + + if len(payload) < 1+paddingSize { + return 0, errors.Tracef("invalid padding: %d < %d", len(payload), 1+paddingSize) + } + + payload = payload[1+paddingSize:] + + // Unlike the data channel case, there is no carry over data left in + // conn.readBuffer between readMediaTrackPacket calls: the entire packet + // payload must be read in this one call. + + if len(p) < len(payload) { + return 0, errors.Tracef("read buffer too short: %d < %d", len(p), len(payload)) + } + + copy(p, payload) + + // When decoy messages are enabled, periodically respond to an incoming + // messages with an immediate outbound decoy message. + // + // writeMediaTrackPacket handles conn.decoyMessageCount, which is + // synchronized with conn.WriteMutex, as well as other specific logic. + // Here we just signal writeDataChannelMessage based on the read event. + + if !conn.decoyDone.Load() { + _, _ = conn.writeMediaTrackPacket(nil, true) + } + + return len(payload), nil +} + +func (conn *webRTCConn) writeMediaTrackPacket(p []byte, decoy bool) (int, error) { + + if p != nil && decoy { + return 0, errors.TraceNew("invalid write parameters") + } + + // Don't hold this lock, or else concurrent Writes will be blocked. + conn.mutex.Lock() + isClosed := conn.isClosed + sendMediaTrack := conn.sendMediaTrack + conn.mutex.Unlock() + + if isClosed { + return 0, errors.TraceNew("closed") + } + + if sendMediaTrack == nil { + return 0, errors.TraceNew("no media track") + } + + conn.writeMutex.Lock() + defer conn.writeMutex.Unlock() + + // Packet writes can't be split. + + maxRTPPayloadLength := mediaTrackMaxRTPPayloadLength + if len(p) > maxRTPPayloadLength { + return 0, errors.Tracef("write too large: %d > %d", len(p), maxRTPPayloadLength) + } + + // Determine padding size and padding header size. + + // Limitation: unlike data channel padding, the header size is fixed, not + // a varint, and is always sent. This is due to the fixed QUIC max packet + // size adjustment. To limit the overhead, and because the maximum SRTP + // payload size is much smaller than the maximum data channel message + // size, the padding is limited to 254 bytes, represented with a 1 byte + // header. The value 255 is reserved to signal that the entire packet is + // a decoy packet. + + conn.trafficShapingBuffer.Reset() + + if decoy { + + if conn.decoyMessageCount < 1 { + return 0, nil + } + + if !conn.trafficShapingPRNG.FlipWeightedCoin( + conn.config.TrafficShapingParameters.DecoyMessageProbability) { + return 0, nil + } + + conn.decoyMessageCount -= 1 + + // When sending a decoy message, the entire message is padding, and + // the padding can be up to the full packet size. + // + // Note that the actual decoy payload size is decoySize+1, including + // the padding header. + + decoySize := conn.trafficShapingPRNG.Range( + conn.config.TrafficShapingParameters.MinDecoySize, + conn.config.TrafficShapingParameters.MaxDecoySize) + + if decoySize > maxRTPPayloadLength-1 { + // Ensure there's space for the 1 byte padding header. + decoySize = maxRTPPayloadLength - 1 + } + + // Set the padding header to 255, which indicates a decoy packet. + conn.trafficShapingBuffer.WriteByte(255) + if decoySize > 0 { + conn.trafficShapingBuffer.Write(prng.Bytes(decoySize)) + } + + if conn.decoyMessageCount == 0 { + // Set the shared flag that readMessage uses to stop invoking + // writeMessage for decoy events. + conn.decoyDone.Store(true) + } + + } else { + + // Add padding to a normal write. + + paddingSize := 0 + + if conn.paddedMessageCount > 0 { + + paddingSize = prng.Range( + conn.config.TrafficShapingParameters.MinPaddingSize, + conn.config.TrafficShapingParameters.MaxPaddingSize) + + if paddingSize > 254 { + // The maximum padding size is 254. + paddingSize = 254 + } + if len(p)+1+paddingSize > maxRTPPayloadLength { + paddingSize -= (len(p) + 1 + paddingSize) - maxRTPPayloadLength + } + if paddingSize < 0 { + paddingSize = 0 + } + + conn.paddedMessageCount -= 1 + } + + conn.trafficShapingBuffer.WriteByte(byte(paddingSize)) + if paddingSize > 0 { + conn.trafficShapingBuffer.Write(prng.Bytes(paddingSize)) + } + conn.trafficShapingBuffer.Write(p) + } + + paddedPayload := conn.trafficShapingBuffer.Bytes() + + // Sanity check, in case there's a bug in the padding logic above; +1 here + // is the padding header. + if len(paddedPayload) > maxRTPPayloadLength+1 { + return 0, errors.Tracef("write too large: %d > %d", len(paddedPayload), maxRTPPayloadLength) + } + + // Send the RTP packet. + + // Dynamic plaintext RTP header values are set here: the sequence number + // is set when sending the packet; the timestamp, initialized in + // newWebRTCConn, is updated once payload equivalent to a complete + // video "frame" has been sent. See the "Plaintext RTP header fields" + // comment in newWebRTCConn. + + conn.sendMediaTrackPacket.SequenceNumber = conn.sendMediaTrackSequencer.NextSequenceNumber() + conn.sendMediaTrackPacket.Payload = paddedPayload + err := sendMediaTrack.WriteRTP(conn.sendMediaTrackPacket) + if err != nil { + return 0, errors.Trace(err) + } + + conn.sendMediaTrackRemainingFrameSize -= len(paddedPayload) + if conn.sendMediaTrackRemainingFrameSize <= 0 { + conn.sendMediaTrackPacket.Timestamp += uint32(conn.sendMediaTrackTimestampTick) + conn.sendMediaTrackRemainingFrameSize = prng.Range(conn.sendMediaTrackFrameSizeRange[0], conn.sendMediaTrackFrameSizeRange[1]) + } + + return len(p), nil +} + +func (conn *webRTCConn) addRTPReliabilityLayer(ctx context.Context) error { + + // Add a QUIC layer over the SRTP packet flow to provide reliable delivery + // and ordering. The proxy runs a QUIC server and the client runs a QUIC + // client that connects to the proxy's server. As all of the QUIC traffic + // is encapsulated in the secure SRTP layer. + + // Wrap the RTP track read and write operations in a mediaTrackPacketConn + // provides the net.PacketConn interface required by quic-go. There is no + // Close-on-error for mediaTrackPacketConn since it doesn't allocate or use + // any resources. + mediaTrackPacketConn := newMediaTrackPacketConn(conn) + + // Use the Psiphon QUIC obfuscated PSK mechanism to facilitate a faster + // QUIC TLS handshake. QUIC client hello randomization is also + // initialized, as it will vary the QUIC handshake traffic shape within + // the SRTP packet flow. + + var obfuscatedPSKKey [32]byte + obfuscationSecret, err := deriveObfuscationSecret( + conn.config.ClientRootObfuscationSecret, "in-proxy-RTP-QUIC-reliability-layer") + if err != nil { + return errors.Trace(err) + } + obfuscationSeed := prng.Seed(obfuscationSecret) + copy(obfuscatedPSKKey[:], prng.NewPRNGWithSeed(&obfuscationSeed).Bytes(len(obfuscatedPSKKey))) + + // To effectively disable them, quic-go's idle timeouts and keep-alives + // are initialized to the maximum possible duration. The higher-level + // WebRTC connection will provide this functionality. + maxDuration := time.Duration(math.MaxInt64) + + // Set the handshake timeout to align with the ctx deadline. Setting + // HandshakeIdleTimeout to maxDuration causes the quic-go dial to fail. + // Assumes ctx has a deadline. + deadline, _ := ctx.Deadline() + handshakeIdleTimeout := time.Until(deadline) / 2 + + if conn.isOffer { + + // The client is a QUIC client. + + // Initialize the obfuscated PSK. + sessionCache := common.WrapClientSessionCache(tls.NewLRUClientSessionCache(1), "") + obfuscatedSessionState, err := tls.NewObfuscatedClientSessionState( + obfuscatedPSKKey, true, false) + if err != nil { + return errors.Trace(err) + } + sessionCache.Put( + "", tls.MakeClientSessionState( + obfuscatedSessionState.SessionTicket, + obfuscatedSessionState.Vers, + obfuscatedSessionState.CipherSuite, + obfuscatedSessionState.MasterSecret, + obfuscatedSessionState.CreatedAt, + obfuscatedSessionState.AgeAdd, + obfuscatedSessionState.UseBy)) + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + InsecureSkipTimeVerify: true, + NextProtos: []string{"h3"}, + ServerName: values.GetHostName(), + ClientSessionCache: sessionCache, + } + + isIPv6 := true // remote addr is synthetic uniqueIPv6Address + maxPacketSizeAdjustment := GetQUICMaxPacketSizeAdjustment(isIPv6) + + // Set ClientMaxPacketSizeAdjustment to so that quic-go will produce + // packets with a small enough max size to produce the overall target + // packet MTU. + quicConfig := &quic_go.Config{ + HandshakeIdleTimeout: handshakeIdleTimeout, + MaxIdleTimeout: maxDuration, + KeepAlivePeriod: maxDuration, + Versions: []quic_go.VersionNumber{0x1}, + ClientHelloSeed: &obfuscationSeed, + ClientMaxPacketSizeAdjustment: maxPacketSizeAdjustment, + DisablePathMTUDiscovery: true, + } + + deadline, ok := ctx.Deadline() + if ok { + quicConfig.HandshakeIdleTimeout = time.Until(deadline) + } + + // Establish the QUIC connection with the server and open a single + // data stream for relaying traffic. + // + // Use DialEarly, in combination with the "established" PSK, for + // 0-RTT, which potentially allows data to be sent with the + // handshake; this could include the open stream message from the + // following OpenStreamSync call. There is no replay concern with + // 0-RTT here, as the QUIC traffic is encapsualted in the secure SRTP + // flow. + + quicConn, err := quic_go.DialEarly( + ctx, + mediaTrackPacketConn, + mediaTrackPacketConn.remoteAddr, + tlsConfig, + quicConfig) + if err != nil { + return errors.Trace(err) + } + + quicStream, err := quicConn.OpenStreamSync(ctx) + if err != nil { + // Ensure any background quic-go goroutines are stopped. + _ = quicConn.CloseWithError(0, "") + return errors.Trace(err) + } + + conn.mediaTrackReliabilityLayer = &reliableConn{ + mediaTrackConn: mediaTrackPacketConn, + quicConn: quicConn, + quicStream: quicStream, + } + + return nil + + } else { + + // The proxy is a QUIC server. + + // Use an ephemeral, self-signed certificate. + certificate, privateKey, _, err := common.GenerateWebServerCertificate( + values.GetHostName()) + if err != nil { + return errors.Trace(err) + } + tlsCertificate, err := tls.X509KeyPair([]byte(certificate), []byte(privateKey)) + if err != nil { + return errors.Trace(err) + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{tlsCertificate}, + NextProtos: []string{"h3"}, + } + tlsConfig.SetSessionTicketKeys([][32]byte{ + obfuscatedPSKKey, + }) + + // Anti-probing via VerifyClientHelloRandom, for passthrough, is not + // necessary here and is not initialized. + quicConfig := &quic_go.Config{ + Allow0RTT: true, + HandshakeIdleTimeout: handshakeIdleTimeout, + MaxIdleTimeout: maxDuration, + KeepAlivePeriod: maxDuration, + MaxIncomingStreams: 1, + MaxIncomingUniStreams: -1, + VerifyClientHelloRandom: nil, + ServerMaxPacketSizeAdjustment: func(addr net.Addr) int { + isIPv6 := true // remote addr is synthetic uniqueIPv6Address + return GetQUICMaxPacketSizeAdjustment(isIPv6) + }, + } + + quicTransport := &quic_go.Transport{ + Conn: mediaTrackPacketConn, + DisableVersionNegotiationPackets: true, + } + + quicListener, err := quicTransport.ListenEarly(tlsConfig, quicConfig) + if err != nil { + return errors.Trace(err) + } + + // Accept the single expected QUIC client and its QUIC data stream. + + quicConn, err := quicListener.Accept(ctx) + if err != nil { + _ = quicTransport.Close() + return errors.Trace(err) + } + + quicStream, err := quicConn.AcceptStream(ctx) + if err != nil { + _ = quicConn.CloseWithError(0, "") + _ = quicTransport.Close() + return errors.Trace(err) + } + + // Closing the quic-go Transport/Listener closes all client + // connections, so retain the Transport for the duration of the + // overall connection. + conn.mediaTrackReliabilityLayer = &reliableConn{ + mediaTrackConn: mediaTrackPacketConn, + quicTransport: quicTransport, + quicConn: quicConn, + quicStream: quicStream, + } + + return nil + } +} + +// incrementingIPv6Address provides successive, distinct IPv6 addresses from +// the 2001:db8::/32 range, reserved for documentation purposes as defined in +// RFC 3849. It will wrap after 2^96 calls. +type incrementingIPv6Address struct { + mutex sync.Mutex + ip [12]byte +} + +var uniqueIPv6Address incrementingIPv6Address + +func (inc *incrementingIPv6Address) next() net.IP { + inc.mutex.Lock() + defer inc.mutex.Unlock() + for i := 11; i >= 0; i-- { + inc.ip[i]++ + if inc.ip[i] != 0 { + break + } + } + ip := make([]byte, 16) + copy(ip[0:4], []byte{0x20, 0x01, 0x0d, 0xb8}) + copy(ip[4:16], inc.ip[:]) + return net.IP(ip) +} + +// mediaTrackPacketConn provides the required net.PacketConn interface for +// quic-go to use to read and write packets to the RTP media track conn. +type mediaTrackPacketConn struct { + webRTCConn *webRTCConn + localAddr net.Addr + remoteAddr net.Addr + isClosed int32 +} + +func newMediaTrackPacketConn(conn *webRTCConn) *mediaTrackPacketConn { + + // Create distinct, artificial local/remote addrs for the synthetic + // net.PacketConn. + // + // For its local operations, quic-go references local/remote addrs for the + // net.PacketConns it uses. Furthermore, the quic-go server listener + // currently uses a singleton multiplexer, connMultiplexer, which panics + // if multiple conns with the same local addr are added. Since this is a + // singleton, this panic occurs even when using distinct quic-go + // listeners per conn. + // + // No actual network traffic is sent to these artificial addresses. + + ip := uniqueIPv6Address.next() + localAddr := &net.UDPAddr{IP: ip, Port: 1} + remoteAddr := &net.UDPAddr{IP: ip, Port: 2} + + return &mediaTrackPacketConn{ + webRTCConn: conn, + localAddr: localAddr, + remoteAddr: remoteAddr, + } +} + +func (conn *mediaTrackPacketConn) ReadFrom(p []byte) (int, net.Addr, error) { + + if atomic.LoadInt32(&conn.isClosed) == 1 { + return 0, conn.remoteAddr, errors.TraceNew("closed") + } + + n, err := conn.webRTCConn.readMediaTrack(p) + return n, conn.remoteAddr, errors.TraceReader(err) +} + +func (conn *mediaTrackPacketConn) WriteTo(p []byte, addr net.Addr) (int, error) { + + if atomic.LoadInt32(&conn.isClosed) == 1 { + return 0, errors.TraceNew("closed") + } + + n, err := conn.webRTCConn.writeMediaTrackPacket(p, false) + return n, errors.Trace(err) +} + +func (conn *mediaTrackPacketConn) Close() error { + if !atomic.CompareAndSwapInt32(&conn.isClosed, 0, 1) { + return nil + } + return nil +} + +func (conn *mediaTrackPacketConn) LocalAddr() net.Addr { + return conn.localAddr +} + +func (conn *mediaTrackPacketConn) SetDeadline(t time.Time) error { + return errors.TraceNew("not supported") +} + +func (conn *mediaTrackPacketConn) SetReadDeadline(t time.Time) error { + + // When a quic-go DialEarly fails, it invokes Transport.Close. In turn, + // Transport.Close calls this SetReadDeadline in order to interrupt any + // blocked read. The underlying pion/webrtc.TrackRemote has a + // SetReadDeadline. However, at this time webRTCConn.receiveMediaTrack + // may be nil, and readMediaTrack may be blocking on + // receiveMediaTrackOpenedSignal. + // + // Simply calling webRTCConn.Close unblocks both that case and the case + // where receiveMediaTrack exists and is blocked on read. + // + // Invoke in a goroutine to avoid a deadlock that would otherwise occur + // when webRTCConn.Close is invoked directly, as it will call down to + // mediaTrackPacketConn.SetReadDeadline via reliableConn.Close. The + // webRTCConn.Close isClosed check ensures there isn't an endless loop of + // calls. + // + // Assumes that mediaTrackPacketConn.SetReadDeadline is called only in + // this terminating quic-go case. + + go func() { + _ = conn.webRTCConn.Close() + }() + + return nil +} + +func (conn *mediaTrackPacketConn) SetWriteDeadline(t time.Time) error { + return errors.TraceNew("not supported") +} + +// reliableConn provides a reliable/ordered delivery layer on top of the media +// track RTP conn. This is implemented as a QUIC connection. +type reliableConn struct { + mediaTrackConn *mediaTrackPacketConn + quicTransport *quic_go.Transport + quicConn quic_go.EarlyConnection + quicStream quic_go.Stream + + readMutex sync.Mutex + writeMutex sync.Mutex + + isClosed int32 +} + +func (conn *reliableConn) Read(b []byte) (int, error) { + + if atomic.LoadInt32(&conn.isClosed) == 1 { + return 0, errors.TraceNew("closed") + } + + // Add mutex to provide full net.Conn concurrency semantics. + // https://github.com/lucas-clemente/quic-go/blob/9cc23135d0477baf83aa4715de39ae7070039cb2/stream.go#L64 + // "Read() and Write() may be called concurrently, but multiple calls to + // "Read() or Write() individually must be synchronized manually." + conn.readMutex.Lock() + defer conn.readMutex.Unlock() + + n, err := conn.quicStream.Read(b) + if quic.IsIETFErrorIndicatingClosed(err) { + _ = conn.Close() + err = io.EOF + } + return n, errors.TraceReader(err) +} + +func (conn *reliableConn) Write(b []byte) (int, error) { + + if atomic.LoadInt32(&conn.isClosed) == 1 { + return 0, errors.TraceNew("closed") + } + + conn.writeMutex.Lock() + defer conn.writeMutex.Unlock() + + n, err := conn.quicStream.Write(b) + if quic.IsIETFErrorIndicatingClosed(err) { + _ = conn.Close() + if n == len(b) { + err = nil + } + } + return n, errors.Trace(err) +} + +func (conn *reliableConn) Close() error { + if !atomic.CompareAndSwapInt32(&conn.isClosed, 0, 1) { + return nil + } + + // Close mediaTrackConn first, or else quic-go's Close will attempt to + // Write, which leads to deadlock between webRTCConn.writeMediaTrack and + // webRTCConn.Close. The graceful QUIC close write will fails, but that's + // not an issue. + + _ = conn.mediaTrackConn.Close() + + err := conn.quicConn.CloseWithError(0, "") + if conn.quicTransport != nil { + conn.quicTransport.Close() + } + return errors.Trace(err) +} + +func (conn *reliableConn) LocalAddr() net.Addr { + return conn.quicConn.LocalAddr() +} + +func (conn *reliableConn) RemoteAddr() net.Addr { + return conn.quicConn.RemoteAddr() +} + +func (conn *reliableConn) SetDeadline(t time.Time) error { + return conn.quicStream.SetDeadline(t) +} + +func (conn *reliableConn) SetReadDeadline(t time.Time) error { + return conn.quicStream.SetReadDeadline(t) +} + +func (conn *reliableConn) SetWriteDeadline(t time.Time) error { + return conn.quicStream.SetWriteDeadline(t) +} + // prepareSDPAddresses adjusts the SDP, pruning local network addresses and // adding any port mapping as a host candidate. func prepareSDPAddresses( @@ -2107,7 +3160,7 @@ func hasInterfaceForPrivateIPAddress(IP net.IP) bool { // Android at this time: https://github.com/golang/go/issues/40569. // // Any errors are silently dropped; the caller will proceed without using - // the input private IP; and equivilent anet calls are made in + // the input private IP; and equivalent anet calls are made in // pionNetwork.Interfaces, with errors logged. netInterfaces, err := anet.Interfaces() diff --git a/psiphon/common/parameters/inproxy.go b/psiphon/common/parameters/inproxy.go index c66b470d5..edea42702 100755 --- a/psiphon/common/parameters/inproxy.go +++ b/psiphon/common/parameters/inproxy.go @@ -82,9 +82,9 @@ func (IDs InproxyCompartmentIDsValue) Validate(checkCompartmentIDList *[]string) return nil } -// InproxyDataChannelTrafficShapingParameters is type-compatible with -// common/inproxy.DataChannelTrafficShapingParameters. -type InproxyDataChannelTrafficShapingParametersValue struct { +// InproxyTrafficShapingParametersValue is type-compatible with +// common/inproxy.TrafficShapingParameters. +type InproxyTrafficShapingParametersValue struct { MinPaddedMessages int MaxPaddedMessages int MinPaddingSize int @@ -96,7 +96,7 @@ type InproxyDataChannelTrafficShapingParametersValue struct { DecoyMessageProbability float64 } -func (p *InproxyDataChannelTrafficShapingParametersValue) Validate() error { +func (p *InproxyTrafficShapingParametersValue) Validate() error { if p.MinPaddedMessages < 0 || p.MaxPaddedMessages < 0 || p.MinPaddingSize < 0 || diff --git a/psiphon/common/parameters/parameters.go b/psiphon/common/parameters/parameters.go index 060038ad7..de3bf7816 100644 --- a/psiphon/common/parameters/parameters.go +++ b/psiphon/common/parameters/parameters.go @@ -421,6 +421,7 @@ const ( InproxyBrokerMatcherOfferRateLimitInterval = "InproxyBrokerMatcherOfferRateLimitInterval" InproxyBrokerMatcherPrioritizeProxiesProbability = "InproxyBrokerMatcherPrioritizeProxiesProbability" InproxyBrokerMatcherPrioritizeProxiesFilter = "InproxyBrokerMatcherPrioritizeProxiesFilter" + InproxyBrokerMatcherPrioritizeProxiesMinVersion = "InproxyBrokerMatcherPrioritizeProxiesMinVersion" InproxyBrokerProxyAnnounceTimeout = "InproxyBrokerProxyAnnounceTimeout" InproxyBrokerClientOfferTimeout = "InproxyBrokerClientOfferTimeout" InproxyBrokerClientOfferPersonalTimeout = "InproxyBrokerClientOfferPersonalTimeout" @@ -438,8 +439,11 @@ const ( InproxyClientRelayedPacketRequestTimeout = "InproxyCloientRelayedPacketRequestTimeout" InproxyBrokerRoundTripStatusCodeFailureThreshold = "InproxyBrokerRoundTripStatusCodeFailureThreshold" InproxyDTLSRandomizationProbability = "InproxyDTLSRandomizationProbability" - InproxyDataChannelTrafficShapingProbability = "InproxyDataChannelTrafficShapingProbability" - InproxyDataChannelTrafficShapingParameters = "InproxyDataChannelTrafficShapingParameters" + InproxyWebRTCMediaStreamsProbability = "InproxyWebRTCMediaStreamsProbability" + InproxyWebRTCDataChannelTrafficShapingProbability = "InproxyWebRTCDataChannelTrafficShapingProbability" + InproxyWebRTCDataChannelTrafficShapingParameters = "InproxyWebRTCDataChannelTrafficShapingParameters" + InproxyWebRTCMediaStreamsTrafficShapingProbability = "InproxyWebRTCMediaStreamsTrafficShapingProbability" + InproxyWebRTCMediaStreamsTrafficShapingParameters = "InproxyWebRTCMediaStreamsTrafficShapingParameters" InproxySTUNServerAddresses = "InproxySTUNServerAddresses" InproxySTUNServerAddressesRFC5780 = "InproxySTUNServerAddressesRFC5780" InproxyProxySTUNServerAddresses = "InproxyProxySTUNServerAddresses" @@ -463,8 +467,8 @@ const ( InproxyClientDiscoverNATTimeout = "InproxyClientDiscoverNATTimeout" InproxyWebRTCAnswerTimeout = "InproxyWebRTCAnswerTimeout" InproxyWebRTCAwaitPortMappingTimeout = "InproxyWebRTCAwaitPortMappingTimeout" - InproxyProxyWebRTCAwaitDataChannelTimeout = "InproxyProxyWebRTCAwaitDataChannelTimeout" - InproxyClientWebRTCAwaitDataChannelTimeout = "InproxyClientWebRTCAwaitDataChannelTimeout" + InproxyProxyWebRTCAwaitReadyToProxyTimeout = "InproxyProxyWebRTCAwaitReadyToProxyTimeout" + InproxyClientWebRTCAwaitReadyToProxyTimeout = "InproxyClientWebRTCAwaitReadyToProxyTimeout" InproxyProxyDestinationDialTimeout = "InproxyProxyDestinationDialTimeout" InproxyProxyRelayInactivityTimeout = "InproxyProxyRelayInactivityTimeout" InproxyPsiphonAPIRequestTimeout = "InproxyPsiphonAPIRequestTimeout" @@ -962,8 +966,9 @@ var defaultParameters = map[string]struct { InproxyBrokerMatcherOfferLimitEntryCount: {value: 10, minimum: 0, flags: serverSideOnly}, InproxyBrokerMatcherOfferRateLimitQuantity: {value: 50, minimum: 0, flags: serverSideOnly}, InproxyBrokerMatcherOfferRateLimitInterval: {value: 1 * time.Minute, minimum: time.Duration(0), flags: serverSideOnly}, - InproxyBrokerMatcherPrioritizeProxiesProbability: {value: 1.0, minimum: 0.0}, - InproxyBrokerMatcherPrioritizeProxiesFilter: {value: KeyStrings{}}, + InproxyBrokerMatcherPrioritizeProxiesProbability: {value: 1.0, minimum: 0.0, flags: serverSideOnly}, + InproxyBrokerMatcherPrioritizeProxiesFilter: {value: KeyStrings{}, flags: serverSideOnly}, + InproxyBrokerMatcherPrioritizeProxiesMinVersion: {value: 0, minimum: 0, flags: serverSideOnly}, InproxyBrokerProxyAnnounceTimeout: {value: 2 * time.Minute, minimum: time.Duration(0), flags: serverSideOnly}, InproxyBrokerClientOfferTimeout: {value: 10 * time.Second, minimum: time.Duration(0), flags: serverSideOnly}, InproxyBrokerClientOfferPersonalTimeout: {value: 5 * time.Second, minimum: time.Duration(0), flags: serverSideOnly}, @@ -981,8 +986,11 @@ var defaultParameters = map[string]struct { InproxyClientRelayedPacketRequestTimeout: {value: 10 * time.Second, minimum: time.Duration(0)}, InproxyBrokerRoundTripStatusCodeFailureThreshold: {value: 2 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyDTLSRandomizationProbability: {value: 0.5, minimum: 0.0}, - InproxyDataChannelTrafficShapingProbability: {value: 0.5, minimum: 0.0}, - InproxyDataChannelTrafficShapingParameters: {value: InproxyDataChannelTrafficShapingParametersValue{0, 10, 0, 1500, 0, 10, 1, 1500, 0.5}}, + InproxyWebRTCMediaStreamsProbability: {value: 0.0, minimum: 0.0}, + InproxyWebRTCDataChannelTrafficShapingProbability: {value: 0.5, minimum: 0.0}, + InproxyWebRTCDataChannelTrafficShapingParameters: {value: InproxyTrafficShapingParametersValue{0, 10, 0, 1500, 0, 10, 1, 1500, 0.5}}, + InproxyWebRTCMediaStreamsTrafficShapingProbability: {value: 0.5, minimum: 0.0}, + InproxyWebRTCMediaStreamsTrafficShapingParameters: {value: InproxyTrafficShapingParametersValue{0, 10, 0, 254, 0, 10, 1, 1200, 0.5}}, InproxySTUNServerAddresses: {value: []string{}}, InproxySTUNServerAddressesRFC5780: {value: []string{}}, InproxyProxySTUNServerAddresses: {value: []string{}}, @@ -1006,8 +1014,8 @@ var defaultParameters = map[string]struct { InproxyClientDiscoverNATTimeout: {value: 10 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyWebRTCAnswerTimeout: {value: 20 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyWebRTCAwaitPortMappingTimeout: {value: 2 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, - InproxyProxyWebRTCAwaitDataChannelTimeout: {value: 30 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, - InproxyClientWebRTCAwaitDataChannelTimeout: {value: 20 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, + InproxyProxyWebRTCAwaitReadyToProxyTimeout: {value: 30 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, + InproxyClientWebRTCAwaitReadyToProxyTimeout: {value: 20 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyProxyDestinationDialTimeout: {value: 20 * time.Second, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyProxyRelayInactivityTimeout: {value: 5 * time.Minute, minimum: time.Duration(0), flags: useNetworkLatencyMultiplier}, InproxyPsiphonAPIRequestTimeout: {value: 10 * time.Second, minimum: 1 * time.Second, flags: useNetworkLatencyMultiplier}, @@ -1566,7 +1574,7 @@ func (p *Parameters) Set( } return nil, errors.Trace(err) } - case InproxyDataChannelTrafficShapingParametersValue: + case InproxyTrafficShapingParametersValue: err := v.Validate() if err != nil { if skipOnError { @@ -2184,12 +2192,12 @@ func (p ParametersAccessor) InproxyCompartmentIDs(name string) InproxyCompartmen return value } -// InproxyDataChannelTrafficShapingParameters returns a -// InproxyDataChannelTrafficShapingParameters parameter value. -func (p ParametersAccessor) InproxyDataChannelTrafficShapingParameters( - name string) InproxyDataChannelTrafficShapingParametersValue { +// InproxyTrafficShapingParameters returns a InproxyTrafficShapingParameters +// parameter value. +func (p ParametersAccessor) InproxyTrafficShapingParameters( + name string) InproxyTrafficShapingParametersValue { - value := InproxyDataChannelTrafficShapingParametersValue{} + value := InproxyTrafficShapingParametersValue{} p.snapshot.getValue(name, &value) return value } diff --git a/psiphon/common/parameters/parameters_test.go b/psiphon/common/parameters/parameters_test.go index 07eb43710..f7da0e6e8 100644 --- a/psiphon/common/parameters/parameters_test.go +++ b/psiphon/common/parameters/parameters_test.go @@ -220,8 +220,8 @@ func TestGetDefaultParameters(t *testing.T) { if !reflect.DeepEqual(v, g) { t.Fatalf("ConjureTransports returned %+v expected %+v", g, v) } - case InproxyDataChannelTrafficShapingParametersValue: - g := p.Get().InproxyDataChannelTrafficShapingParameters(name) + case InproxyTrafficShapingParametersValue: + g := p.Get().InproxyTrafficShapingParameters(name) if !reflect.DeepEqual(v, g) { t.Fatalf("ConjureTransports returned %+v expected %+v", g, v) } diff --git a/psiphon/common/protocol/packed.go b/psiphon/common/protocol/packed.go index 7187dcd93..3cd1b3089 100644 --- a/psiphon/common/protocol/packed.go +++ b/psiphon/common/protocol/packed.go @@ -825,8 +825,9 @@ func init() { {163, "inproxy_dial_broker_offer_duration", intConverter}, {164, "inproxy_dial_webrtc_connection_duration", intConverter}, {165, "inproxy_broker_is_reuse", intConverter}, + {166, "inproxy_webrtc_use_media_streams", intConverter}, - // Next key value = 166 + // Next key value = 167 } for _, spec := range packedAPIParameterSpecs { diff --git a/psiphon/common/quic/obfuscator_test.go b/psiphon/common/quic/obfuscator_test.go index c1bf61725..b53f7ddee 100644 --- a/psiphon/common/quic/obfuscator_test.go +++ b/psiphon/common/quic/obfuscator_test.go @@ -135,6 +135,7 @@ func runNonceTransformer(t *testing.T, quicVersion string) { TransformSpec: transforms.Spec{{"^.{24}", "ffff00000000000000000000"}}, }, false, + 0, false, false, // Disable obfuscated PSK common.WrapClientSessionCache(tls.NewLRUClientSessionCache(0), "test"), diff --git a/psiphon/common/quic/quic.go b/psiphon/common/quic/quic.go index 7ddf92fc5..b9f5e1503 100644 --- a/psiphon/common/quic/quic.go +++ b/psiphon/common/quic/quic.go @@ -144,6 +144,7 @@ func Listen( logger common.Logger, irregularTunnelLogger func(string, error, common.LogFields), address string, + additionalMaxPacketSizeAdjustment int, obfuscationKey string, enableGQUIC bool) (net.Listener, error) { @@ -249,7 +250,11 @@ func Listen( // pumping read packets though mux channels. tlsConfig, ietfQUICConfig, err := makeServerIETFConfig( - obfuscatedPacketConn, verifyClientHelloRandom, tlsCertificate, obfuscationKey) + obfuscatedPacketConn, + additionalMaxPacketSizeAdjustment, + verifyClientHelloRandom, + tlsCertificate, + obfuscationKey) if err != nil { obfuscatedPacketConn.Close() @@ -275,7 +280,12 @@ func Listen( // return and caller calls Accept. muxListener, err := newMuxListener( - logger, verifyClientHelloRandom, obfuscatedPacketConn, tlsCertificate, obfuscationKey) + logger, + obfuscatedPacketConn, + additionalMaxPacketSizeAdjustment, + verifyClientHelloRandom, + tlsCertificate, + obfuscationKey) if err != nil { obfuscatedPacketConn.Close() return nil, errors.Trace(err) @@ -293,6 +303,7 @@ func Listen( func makeServerIETFConfig( conn *ObfuscatedPacketConn, + additionalMaxPacketSizeAdjustment int, verifyClientHelloRandom func(net.Addr, []byte) bool, tlsCertificate tls.Certificate, sharedSecret string) (*tls.Config, *ietf_quic.Config, error) { @@ -325,6 +336,14 @@ func makeServerIETFConfig( }) } + serverMaxPacketSizeAdjustment := conn.serverMaxPacketSizeAdjustment + if additionalMaxPacketSizeAdjustment != 0 { + serverMaxPacketSizeAdjustment = func(addr net.Addr) int { + return conn.serverMaxPacketSizeAdjustment(addr) + + additionalMaxPacketSizeAdjustment + } + } + ietfQUICConfig := &ietf_quic.Config{ Allow0RTT: true, HandshakeIdleTimeout: SERVER_HANDSHAKE_TIMEOUT, @@ -335,7 +354,7 @@ func makeServerIETFConfig( KeepAlivePeriod: CLIENT_IDLE_TIMEOUT / 2, VerifyClientHelloRandom: verifyClientHelloRandom, - ServerMaxPacketSizeAdjustment: conn.serverMaxPacketSizeAdjustment, + ServerMaxPacketSizeAdjustment: serverMaxPacketSizeAdjustment, } return tlsConfig, ietfQUICConfig, nil @@ -405,6 +424,7 @@ func Dial( obfuscationPaddingSeed *prng.Seed, obfuscationNonceTransformerParameters *transforms.ObfuscatorSeedTransformerParameters, disablePathMTUDiscovery bool, + additionalMaxPacketSizeAdjustment int, dialEarly bool, useObfuscatedPSK bool, tlsClientSessionCache *common.TLSClientSessionCacheWrapper) (net.Conn, error) { @@ -472,7 +492,7 @@ func Dial( } } - maxPacketSizeAdjustment := 0 + maxPacketSizeAdjustment := additionalMaxPacketSizeAdjustment if isObfuscated(quicVersion) { obfuscatedPacketConn, err := NewClientObfuscatedPacketConn( @@ -488,9 +508,9 @@ func Dial( } packetConn = obfuscatedPacketConn - // Reserve space for packet obfuscation overhead so that quic-go will - // continue to produce packets of max size 1280. - maxPacketSizeAdjustment = OBFUSCATED_MAX_PACKET_SIZE_ADJUSTMENT + // Reserve additional space for packet obfuscation overhead so that + // quic-go will continue to produce packets of max size 1280. + maxPacketSizeAdjustment += OBFUSCATED_MAX_PACKET_SIZE_ADJUSTMENT } // As an anti-probing measure, QUIC clients must prove knowledge of the @@ -525,7 +545,6 @@ func Dial( connection, err := dialQUIC( ctx, packetConn, - false, remoteAddr, quicSNIAddress, versionNumber, @@ -971,7 +990,6 @@ func (t *QUICTransporter) dialQUIC() (retConnection quicConnection, retErr error connection, err := dialQUIC( ctx, packetConn, - true, remoteAddr, t.quicSNIAddress, versionNumber, @@ -1111,6 +1129,13 @@ func (c *ietfQUICConnection) Close() error { } func (c *ietfQUICConnection) isErrorIndicatingClosed(err error) bool { + if err == nil { + return false + } + return IsIETFErrorIndicatingClosed(err) +} + +func IsIETFErrorIndicatingClosed(err error) bool { if err == nil { return false } @@ -1138,13 +1163,12 @@ func (c *ietfQUICConnection) getClientConnMetrics() quicClientConnMetrics { func dialQUIC( ctx context.Context, packetConn net.PacketConn, - expectNetUDPConn bool, remoteAddr *net.UDPAddr, quicSNIAddress string, versionNumber uint32, clientHelloSeed *prng.Seed, getClientHelloRandom func() ([]byte, error), - clientMaxPacketSizeAdjustment int, + maxPacketSizeAdjustment int, disablePathMTUDiscovery bool, dialEarly bool, obfuscatedPSKKey string, @@ -1164,7 +1188,7 @@ func dialQUIC( ietf_quic.VersionNumber(versionNumber)}, ClientHelloSeed: clientHelloSeed, GetClientHelloRandom: getClientHelloRandom, - ClientMaxPacketSizeAdjustment: clientMaxPacketSizeAdjustment, + ClientMaxPacketSizeAdjustment: maxPacketSizeAdjustment, DisablePathMTUDiscovery: disablePathMTUDiscovery, } @@ -1207,7 +1231,7 @@ func dialQUIC( if err != nil { return nil, errors.Trace(err) } - ss := tls.MakeClientSessionState( + sessionState := tls.MakeClientSessionState( obfuscatedSessionState.SessionTicket, obfuscatedSessionState.Vers, obfuscatedSessionState.CipherSuite, @@ -1216,7 +1240,7 @@ func dialQUIC( obfuscatedSessionState.AgeAdd, obfuscatedSessionState.UseBy, ) - tlsClientSessionCache.Put("", ss) + tlsClientSessionCache.Put("", sessionState) } if dialEarly { @@ -1400,8 +1424,9 @@ type muxListener struct { func newMuxListener( logger common.Logger, - verifyClientHelloRandom func(net.Addr, []byte) bool, conn *ObfuscatedPacketConn, + additionalMaxPacketSizeAdjustment int, + verifyClientHelloRandom func(net.Addr, []byte) bool, tlsCertificate tls.Certificate, sharedSecret string) (*muxListener, error) { @@ -1422,7 +1447,11 @@ func newMuxListener( listener.ietfQUICConn = newMuxPacketConn(conn.LocalAddr(), listener) tlsConfig, ietfQUICConfig, err := makeServerIETFConfig( - conn, verifyClientHelloRandom, tlsCertificate, sharedSecret) + conn, + additionalMaxPacketSizeAdjustment, + verifyClientHelloRandom, + tlsCertificate, + sharedSecret) if err != nil { return nil, errors.Trace(err) } diff --git a/psiphon/common/quic/quic_disabled.go b/psiphon/common/quic/quic_disabled.go index b1796763b..267f5b0e3 100644 --- a/psiphon/common/quic/quic_disabled.go +++ b/psiphon/common/quic/quic_disabled.go @@ -96,3 +96,7 @@ func NewQUICTransporter( return nil, errors.TraceNew("operation is not enabled") } + +func IsIETFErrorIndicatingClosed(_ error) bool { + return false +} diff --git a/psiphon/common/quic/quic_test.go b/psiphon/common/quic/quic_test.go index bad8d181b..364afddff 100644 --- a/psiphon/common/quic/quic_test.go +++ b/psiphon/common/quic/quic_test.go @@ -106,6 +106,7 @@ func runQUIC( nil, irregularTunnelLogger, "127.0.0.1:0", + 0, obfuscationKey, enableGQUIC) if err != nil { @@ -216,6 +217,7 @@ func runQUIC( obfuscationPaddingSeed, nil, disablePathMTUDiscovery, + 0, true, useObfuscatedPSK, clientSessionCache) diff --git a/psiphon/config.go b/psiphon/config.go index 1743004ac..781345ec3 100755 --- a/psiphon/config.go +++ b/psiphon/config.go @@ -1028,8 +1028,11 @@ type Config struct { InproxyClientOfferRetryJitter *float64 InproxyClientRelayedPacketRequestTimeoutMilliseconds *int InproxyDTLSRandomizationProbability *float64 - InproxyDataChannelTrafficShapingProbability *float64 - InproxyDataChannelTrafficShapingParameters *parameters.InproxyDataChannelTrafficShapingParametersValue + InproxyWebRTCMediaStreamsProbability *float64 + InproxyWebRTCDataChannelTrafficShapingProbability *float64 + InproxyWebRTCDataChannelTrafficShapingParameters *parameters.InproxyTrafficShapingParametersValue + InproxyWebRTCMediaStreamsTrafficShapingProbability *float64 + InproxyWebRTCMediaStreamsTrafficShapingParameters *parameters.InproxyTrafficShapingParametersValue InproxySTUNServerAddresses []string InproxySTUNServerAddressesRFC5780 []string InproxyProxySTUNServerAddresses []string @@ -1052,8 +1055,8 @@ type Config struct { InproxyProxyDiscoverNATTimeoutMilliseconds *int InproxyClientDiscoverNATTimeoutMilliseconds *int InproxyWebRTCAnswerTimeoutMilliseconds *int - InproxyProxyWebRTCAwaitDataChannelTimeoutMilliseconds *int - InproxyClientWebRTCAwaitDataChannelTimeoutMilliseconds *int + InproxyProxyWebRTCAwaitReadyToProxyTimeoutMilliseconds *int + InproxyClientWebRTCAwaitReadyToProxyTimeoutMilliseconds *int InproxyProxyDestinationDialTimeoutMilliseconds *int InproxyPsiphonAPIRequestTimeoutMilliseconds *int InproxyProxyTotalActivityNoticePeriodMilliseconds *int @@ -1274,11 +1277,14 @@ func (config *Config) Commit(migrateFromLegacyFields bool) error { } if config.UseNoticeFiles != nil { - setNoticeFiles( + err := setNoticeFiles( homepageFilePath, noticesFilePath, config.UseNoticeFiles.RotatingFileSize, config.UseNoticeFiles.RotatingSyncFrequency) + if err != nil { + return errors.Trace(err) + } } // Emit notices now that notice files are set if configured @@ -1713,7 +1719,7 @@ func (config *Config) SetParameters(tag string, skipOnError bool, applyParameter for _, receiver := range config.GetTacticsAppliedReceivers() { err := receiver.TacticsApplied() if err != nil { - NoticeError("TacticsApplied failed: %v", err) + NoticeError("TacticsApplied failed: %v", errors.Trace(err)) // Log and continue running. } } @@ -2650,12 +2656,24 @@ func (config *Config) makeConfigParameters() map[string]interface{} { applyParameters[parameters.InproxyDTLSRandomizationProbability] = *config.InproxyDTLSRandomizationProbability } - if config.InproxyDataChannelTrafficShapingProbability != nil { - applyParameters[parameters.InproxyDataChannelTrafficShapingProbability] = *config.InproxyDataChannelTrafficShapingProbability + if config.InproxyWebRTCMediaStreamsProbability != nil { + applyParameters[parameters.InproxyWebRTCMediaStreamsProbability] = *config.InproxyWebRTCMediaStreamsProbability + } + + if config.InproxyWebRTCDataChannelTrafficShapingProbability != nil { + applyParameters[parameters.InproxyWebRTCDataChannelTrafficShapingProbability] = *config.InproxyWebRTCDataChannelTrafficShapingProbability + } + + if config.InproxyWebRTCDataChannelTrafficShapingParameters != nil { + applyParameters[parameters.InproxyWebRTCDataChannelTrafficShapingParameters] = *config.InproxyWebRTCDataChannelTrafficShapingParameters + } + + if config.InproxyWebRTCMediaStreamsTrafficShapingProbability != nil { + applyParameters[parameters.InproxyWebRTCMediaStreamsTrafficShapingProbability] = *config.InproxyWebRTCMediaStreamsTrafficShapingProbability } - if config.InproxyDataChannelTrafficShapingParameters != nil { - applyParameters[parameters.InproxyDataChannelTrafficShapingParameters] = *config.InproxyDataChannelTrafficShapingParameters + if config.InproxyWebRTCMediaStreamsTrafficShapingParameters != nil { + applyParameters[parameters.InproxyWebRTCMediaStreamsTrafficShapingParameters] = *config.InproxyWebRTCMediaStreamsTrafficShapingParameters } if len(config.InproxySTUNServerAddresses) > 0 { @@ -2746,12 +2764,12 @@ func (config *Config) makeConfigParameters() map[string]interface{} { applyParameters[parameters.InproxyWebRTCAnswerTimeout] = fmt.Sprintf("%dms", *config.InproxyWebRTCAnswerTimeoutMilliseconds) } - if config.InproxyProxyWebRTCAwaitDataChannelTimeoutMilliseconds != nil { - applyParameters[parameters.InproxyProxyWebRTCAwaitDataChannelTimeout] = fmt.Sprintf("%dms", *config.InproxyProxyWebRTCAwaitDataChannelTimeoutMilliseconds) + if config.InproxyProxyWebRTCAwaitReadyToProxyTimeoutMilliseconds != nil { + applyParameters[parameters.InproxyProxyWebRTCAwaitReadyToProxyTimeout] = fmt.Sprintf("%dms", *config.InproxyProxyWebRTCAwaitReadyToProxyTimeoutMilliseconds) } - if config.InproxyClientWebRTCAwaitDataChannelTimeoutMilliseconds != nil { - applyParameters[parameters.InproxyClientWebRTCAwaitDataChannelTimeout] = fmt.Sprintf("%dms", *config.InproxyClientWebRTCAwaitDataChannelTimeoutMilliseconds) + if config.InproxyClientWebRTCAwaitReadyToProxyTimeoutMilliseconds != nil { + applyParameters[parameters.InproxyClientWebRTCAwaitReadyToProxyTimeout] = fmt.Sprintf("%dms", *config.InproxyClientWebRTCAwaitReadyToProxyTimeoutMilliseconds) } if config.InproxyProxyDestinationDialTimeoutMilliseconds != nil { @@ -3538,13 +3556,25 @@ func (config *Config) setDialParametersHash() { hash.Write([]byte("InproxyDTLSRandomizationProbability")) binary.Write(hash, binary.LittleEndian, *config.InproxyDTLSRandomizationProbability) } - if config.InproxyDataChannelTrafficShapingProbability != nil { - hash.Write([]byte("InproxyDataChannelTrafficShapingProbability")) - binary.Write(hash, binary.LittleEndian, *config.InproxyDataChannelTrafficShapingProbability) + if config.InproxyWebRTCMediaStreamsProbability != nil { + hash.Write([]byte("InproxyWebRTCMediaStreamsProbability")) + binary.Write(hash, binary.LittleEndian, *config.InproxyWebRTCMediaStreamsProbability) + } + if config.InproxyWebRTCDataChannelTrafficShapingProbability != nil { + hash.Write([]byte("InproxyWebRTCDataChannelTrafficShapingProbability")) + binary.Write(hash, binary.LittleEndian, *config.InproxyWebRTCDataChannelTrafficShapingProbability) + } + if config.InproxyWebRTCDataChannelTrafficShapingParameters != nil { + hash.Write([]byte("InproxyWebRTCDataChannelTrafficShapingParameters")) + hash.Write([]byte(fmt.Sprintf("%+v", config.InproxyWebRTCDataChannelTrafficShapingParameters))) + } + if config.InproxyWebRTCMediaStreamsTrafficShapingProbability != nil { + hash.Write([]byte("InproxyWebRTCMediaStreamsTrafficShapingProbability")) + binary.Write(hash, binary.LittleEndian, *config.InproxyWebRTCMediaStreamsTrafficShapingProbability) } - if config.InproxyDataChannelTrafficShapingParameters != nil { - hash.Write([]byte("InproxyDataChannelTrafficShapingParameters")) - hash.Write([]byte(fmt.Sprintf("%+v", config.InproxyDataChannelTrafficShapingParameters))) + if config.InproxyWebRTCMediaStreamsTrafficShapingParameters != nil { + hash.Write([]byte("InproxyWebRTCMediaStreamsTrafficShapingParameters")) + hash.Write([]byte(fmt.Sprintf("%+v", config.InproxyWebRTCMediaStreamsTrafficShapingParameters))) } if config.InproxySTUNServerAddresses != nil { hash.Write([]byte("InproxySTUNServerAddresses")) diff --git a/psiphon/controller.go b/psiphon/controller.go index 8b2c126c1..880c3ade8 100644 --- a/psiphon/controller.go +++ b/psiphon/controller.go @@ -466,9 +466,18 @@ func (controller *Controller) NetworkChanged() { controller.TerminateNextActiveTunnel() if controller.inproxyProxyBrokerClientManager != nil { - controller.inproxyProxyBrokerClientManager.NetworkChanged() + err := controller.inproxyProxyBrokerClientManager.NetworkChanged() + if err != nil { + NoticeError("NetworkChanged failed: %v", errors.Trace(err)) + // Log and continue running. + } + + } + err := controller.inproxyClientBrokerClientManager.NetworkChanged() + if err != nil { + NoticeError("NetworkChanged failed: %v", errors.Trace(err)) + // Log and continue running. } - controller.inproxyClientBrokerClientManager.NetworkChanged() controller.config.networkIDGetter.FlushCache() diff --git a/psiphon/dialParameters.go b/psiphon/dialParameters.go index 3a41de80a..c4298e7e3 100644 --- a/psiphon/dialParameters.go +++ b/psiphon/dialParameters.go @@ -135,6 +135,7 @@ type DialParameters struct { QUICDialEarly bool QUICUseObfuscatedPSK bool QUICDisablePathMTUDiscovery bool + QUICMaxPacketSizeAdjustment int ConjureCachedRegistrationTTL time.Duration ConjureAPIRegistration bool @@ -1267,6 +1268,31 @@ func MakeDialParameters( } } + if protocol.TunnelProtocolUsesQUIC(dialParams.TunnelProtocol) && + dialParams.InproxyWebRTCDialParameters.UseMediaStreams { + + // In the in-proxy WebRTC media stream mode, QUIC packets are + // encapsulated in SRTP packet payloads, and the maximum QUIC + // packet size must be adjusted to fit. In addition, QUIC path + // MTU discovery is disabled, to avoid sending oversized packets. + + // isIPv6 indicates whether quic-go will use a max initial packet + // size appropriate for IPv6 or IPv4; + // GetQUICMaxPacketSizeAdjustment modifies the adjustment + // accordingly. quic-go selects based on the RemoteAddr of the + // net.PacketConn passed to quic.Dial. In the in-proxy case, that + // RemoteAddr, inproxy.ClientConn.RemoteAddr, is synthetic and + // can reflect inproxy.ClientConfig.RemoteAddrOverride, which, in + // turn, is currently based on serverEntry.IpAddress; see + // dialInproxy. Limitation: not compatible with FRONTED-QUIC. + + IPAddress := net.ParseIP(serverEntry.IpAddress) + isIPv6 := IPAddress != nil && IPAddress.To4() == nil + + dialParams.QUICMaxPacketSizeAdjustment = inproxy.GetQUICMaxPacketSizeAdjustment(isIPv6) + dialParams.QUICDisablePathMTUDiscovery = true + } + // dialParams.inproxyConn is left uninitialized until after the dial, // and until then Load will return nil. } diff --git a/psiphon/inproxy.go b/psiphon/inproxy.go index 341ba99fb..cb4cad495 100644 --- a/psiphon/inproxy.go +++ b/psiphon/inproxy.go @@ -1548,7 +1548,7 @@ type InproxyWebRTCDialInstance struct { discoverNATTimeout time.Duration webRTCAnswerTimeout time.Duration webRTCAwaitPortMappingTimeout time.Duration - awaitDataChannelTimeout time.Duration + awaitReadyToProxyTimeout time.Duration proxyDestinationDialTimeout time.Duration proxyRelayInactivityTimeout time.Duration } @@ -1593,7 +1593,7 @@ func NewInproxyWebRTCDialInstance( disableInboundForMobileNetworks := p.Bool(parameters.InproxyDisableInboundForMobileNetworks) disableIPv6ICECandidates := p.Bool(parameters.InproxyDisableIPv6ICECandidates) - var discoverNATTimeout, awaitDataChannelTimeout time.Duration + var discoverNATTimeout, awaitReadyToProxyTimeout time.Duration if isProxy { @@ -1609,7 +1609,7 @@ func NewInproxyWebRTCDialInstance( discoverNATTimeout = p.Duration(parameters.InproxyProxyDiscoverNATTimeout) - awaitDataChannelTimeout = p.Duration(parameters.InproxyProxyWebRTCAwaitDataChannelTimeout) + awaitReadyToProxyTimeout = p.Duration(parameters.InproxyProxyWebRTCAwaitReadyToProxyTimeout) } else { @@ -1625,7 +1625,7 @@ func NewInproxyWebRTCDialInstance( discoverNATTimeout = p.Duration(parameters.InproxyClientDiscoverNATTimeout) - awaitDataChannelTimeout = p.Duration(parameters.InproxyClientWebRTCAwaitDataChannelTimeout) + awaitReadyToProxyTimeout = p.Duration(parameters.InproxyClientWebRTCAwaitReadyToProxyTimeout) } // Parameters such as disabling certain operations and operation timeouts @@ -1652,7 +1652,7 @@ func NewInproxyWebRTCDialInstance( discoverNATTimeout: discoverNATTimeout, webRTCAnswerTimeout: p.Duration(parameters.InproxyWebRTCAnswerTimeout), webRTCAwaitPortMappingTimeout: p.Duration(parameters.InproxyWebRTCAwaitPortMappingTimeout), - awaitDataChannelTimeout: awaitDataChannelTimeout, + awaitReadyToProxyTimeout: awaitReadyToProxyTimeout, proxyDestinationDialTimeout: p.Duration(parameters.InproxyProxyDestinationDialTimeout), proxyRelayInactivityTimeout: p.Duration(parameters.InproxyProxyRelayInactivityTimeout), }, nil @@ -1679,8 +1679,13 @@ func (w *InproxyWebRTCDialInstance) DoDTLSRandomization() bool { } // Implements the inproxy.WebRTCDialCoordinator interface. -func (w *InproxyWebRTCDialInstance) DataChannelTrafficShapingParameters() *inproxy.DataChannelTrafficShapingParameters { - return w.webRTCDialParameters.DataChannelTrafficShapingParameters +func (w *InproxyWebRTCDialInstance) UseMediaStreams() bool { + return w.webRTCDialParameters.UseMediaStreams +} + +// Implements the inproxy.WebRTCDialCoordinator interface. +func (w *InproxyWebRTCDialInstance) TrafficShapingParameters() *inproxy.TrafficShapingParameters { + return w.webRTCDialParameters.TrafficShapingParameters } // Implements the inproxy.WebRTCDialCoordinator interface. @@ -1961,8 +1966,8 @@ func (w *InproxyWebRTCDialInstance) WebRTCAwaitPortMappingTimeout() time.Duratio } // Implements the inproxy.WebRTCDialCoordinator interface. -func (w *InproxyWebRTCDialInstance) WebRTCAwaitDataChannelTimeout() time.Duration { - return w.awaitDataChannelTimeout +func (w *InproxyWebRTCDialInstance) WebRTCAwaitReadyToProxyTimeout() time.Duration { + return w.awaitReadyToProxyTimeout } // Implements the inproxy.WebRTCDialCoordinator interface. @@ -2156,9 +2161,10 @@ func (dialParams *InproxySTUNDialParameters) GetMetrics() common.LogFields { // marshaling. For client in-proxy tunnel dials, DialParameters will manage // WebRTC dial parameter selection and replay. type InproxyWebRTCDialParameters struct { - RootObfuscationSecret inproxy.ObfuscationSecret - DataChannelTrafficShapingParameters *inproxy.DataChannelTrafficShapingParameters - DoDTLSRandomization bool + RootObfuscationSecret inproxy.ObfuscationSecret + UseMediaStreams bool + TrafficShapingParameters *inproxy.TrafficShapingParameters + DoDTLSRandomization bool } // MakeInproxyWebRTCDialParameters generates new InproxyWebRTCDialParameters. @@ -2170,19 +2176,36 @@ func MakeInproxyWebRTCDialParameters( return nil, errors.Trace(err) } - var trafficSharingParams inproxy.DataChannelTrafficShapingParameters - if p.WeightedCoinFlip(parameters.InproxyDataChannelTrafficShapingProbability) { - trafficSharingParams = inproxy.DataChannelTrafficShapingParameters( - p.InproxyDataChannelTrafficShapingParameters( - parameters.InproxyDataChannelTrafficShapingParameters)) + useMediaStreams := p.WeightedCoinFlip(parameters.InproxyWebRTCMediaStreamsProbability) + + var trafficSharingParams *inproxy.TrafficShapingParameters + + if useMediaStreams { + + if p.WeightedCoinFlip(parameters.InproxyWebRTCMediaStreamsTrafficShapingProbability) { + t := inproxy.TrafficShapingParameters( + p.InproxyTrafficShapingParameters( + parameters.InproxyWebRTCMediaStreamsTrafficShapingParameters)) + trafficSharingParams = &t + } + + } else { + + if p.WeightedCoinFlip(parameters.InproxyWebRTCDataChannelTrafficShapingProbability) { + t := inproxy.TrafficShapingParameters( + p.InproxyTrafficShapingParameters( + parameters.InproxyWebRTCDataChannelTrafficShapingParameters)) + trafficSharingParams = &t + } } doDTLSRandomization := p.WeightedCoinFlip(parameters.InproxyDTLSRandomizationProbability) return &InproxyWebRTCDialParameters{ - RootObfuscationSecret: rootObfuscationSecret, - DataChannelTrafficShapingParameters: &trafficSharingParams, - DoDTLSRandomization: doDTLSRandomization, + RootObfuscationSecret: rootObfuscationSecret, + UseMediaStreams: useMediaStreams, + TrafficShapingParameters: trafficSharingParams, + DoDTLSRandomization: doDTLSRandomization, }, nil } diff --git a/psiphon/net.go b/psiphon/net.go index 933eef333..180955db5 100644 --- a/psiphon/net.go +++ b/psiphon/net.go @@ -835,7 +835,7 @@ func ResumeDownload( // Not making failure to write ETag file fatal, in case the entire download // succeeds in this one request. - ioutil.WriteFile(partialETagFilename, []byte(responseETag), 0600) + _ = ioutil.WriteFile(partialETagFilename, []byte(responseETag), 0600) // A partial download occurs when this copy is interrupted. The io.Copy // will fail, leaving a partial download in place (.part and .part.etag). diff --git a/psiphon/net_darwin.go b/psiphon/net_darwin.go index afceb1214..7c06a46a6 100644 --- a/psiphon/net_darwin.go +++ b/psiphon/net_darwin.go @@ -37,7 +37,11 @@ func setSocketBPF(_ []bpf.RawInstruction, _ int) error { } func setAdditionalSocketOptions(socketFd int) { - syscall.SetsockoptInt(socketFd, syscall.SOL_SOCKET, syscall.SO_NOSIGPIPE, 1) + // TODO: return error + err := syscall.SetsockoptInt(socketFd, syscall.SOL_SOCKET, syscall.SO_NOSIGPIPE, 1) + if err != nil { + NoticeError("SetsockoptInt failed: %v", errors.Trace(err)) + } } func makeLocalProxyListener(listenIP string, port int) (net.Listener, bool, error) { diff --git a/psiphon/server/api.go b/psiphon/server/api.go index 79bc91ce5..16128195e 100644 --- a/psiphon/server/api.go +++ b/psiphon/server/api.go @@ -1198,6 +1198,7 @@ var inproxyDialParams = []requestParamSpec{ {"inproxy_dial_broker_offer_duration", isIntString, requestParamOptional | requestParamLogStringAsInt}, {"inproxy_dial_webrtc_connection_duration", isIntString, requestParamOptional | requestParamLogStringAsInt}, {"inproxy_broker_is_reuse", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool}, + {"inproxy_webrtc_use_media_streams", isBooleanFlag, requestParamOptional | requestParamLogFlagAsBool}, } // baseAndDialParams adds baseDialParams and inproxyDialParams to baseParams. diff --git a/psiphon/server/meek.go b/psiphon/server/meek.go index 82ce12477..0dae04bf6 100644 --- a/psiphon/server/meek.go +++ b/psiphon/server/meek.go @@ -1896,7 +1896,9 @@ func (server *MeekServer) inproxyBrokerAllowDomainFrontedDestinations(clientGeoI } func (server *MeekServer) inproxyBrokerPrioritizeProxy( - proxyGeoIPData common.GeoIPData, proxyAPIParams common.APIParameters) bool { + proxyInproxyProtocolVersion int, + proxyGeoIPData common.GeoIPData, + proxyAPIParams common.APIParameters) bool { // Fallback to not-prioritized on failure or nil tactics. p, err := server.support.ServerTacticsParametersCache.Get(GeoIPData(proxyGeoIPData)) @@ -1908,6 +1910,14 @@ func (server *MeekServer) inproxyBrokerPrioritizeProxy( if p.IsNil() { return false } + + // As API parameter filtering currently does not support range matching, the minimum version + // constraint is specified in a seperate parameter. + minProtocolVersion := p.Int(parameters.InproxyBrokerMatcherPrioritizeProxiesMinVersion) + if proxyInproxyProtocolVersion < minProtocolVersion { + return false + } + filter := p.KeyStringsValue(parameters.InproxyBrokerMatcherPrioritizeProxiesFilter) if len(filter) == 0 { return false @@ -1918,9 +1928,11 @@ func (server *MeekServer) inproxyBrokerPrioritizeProxy( return false } } + if !p.WeightedCoinFlip(parameters.InproxyBrokerMatcherPrioritizeProxiesProbability) { return false } + return true } diff --git a/psiphon/server/server_test.go b/psiphon/server/server_test.go index 8a76ab2ca..0e7b276c8 100644 --- a/psiphon/server/server_test.go +++ b/psiphon/server/server_test.go @@ -444,6 +444,39 @@ func TestInproxyPersonalPairing(t *testing.T) { }) } +func TestInproxyOSSHMediaStreams(t *testing.T) { + if !inproxy.Enabled() { + t.Skip("inproxy is not enabled") + } + runServer(t, + &runServerConfig{ + tunnelProtocol: "INPROXY-WEBRTC-OSSH", + requireAuthorization: true, + doTunneledWebRequest: true, + doTunneledNTPRequest: true, + doDanglingTCPConn: true, + doLogHostProvider: true, + doTargetBrokerSpecs: true, + useInproxyMediaStreams: true, + }) +} + +func TestInproxyQUICOSSHMediaStreams(t *testing.T) { + if !inproxy.Enabled() { + t.Skip("inproxy is not enabled") + } + runServer(t, + &runServerConfig{ + tunnelProtocol: "INPROXY-WEBRTC-QUIC-OSSH", + requireAuthorization: true, + doTunneledWebRequest: true, + doTunneledNTPRequest: true, + doLogHostProvider: true, + doTargetBrokerSpecs: true, + useInproxyMediaStreams: true, + }) +} + func TestHotReload(t *testing.T) { runServer(t, &runServerConfig{ @@ -702,6 +735,7 @@ type runServerConfig struct { useLegacyAPIEncoding bool doPersonalPairing bool doRestrictInproxy bool + useInproxyMediaStreams bool } var ( @@ -758,7 +792,8 @@ func runServer(t *testing.T, runConfig *runServerConfig) { runConfig.doTargetBrokerSpecs, brokerIPAddress, brokerPort, - serverEntrySignaturePublicKey) + serverEntrySignaturePublicKey, + runConfig.useInproxyMediaStreams) if err != nil { t.Fatalf("error generating inproxy test config: %s", err) } @@ -2521,7 +2556,7 @@ func checkExpectedServerTunnelLogFields( "inproxy_proxy_device_region", "inproxy_proxy_device_location", "inproxy_proxy_network_type", - "inproxy_proxy_proxy_protocol_version", + "inproxy_proxy_protocol_version", "inproxy_proxy_nat_type", "inproxy_proxy_max_clients", "inproxy_proxy_connecting_clients", @@ -2544,6 +2579,7 @@ func checkExpectedServerTunnelLogFields( "inproxy_broker_dial_address", "inproxy_broker_resolved_ip_address", "inproxy_webrtc_randomize_dtls", + "inproxy_webrtc_use_media_streams", "inproxy_webrtc_padded_messages_sent", "inproxy_webrtc_padded_messages_received", "inproxy_webrtc_decoy_messages_sent", @@ -2619,6 +2655,10 @@ func checkExpectedServerTunnelLogFields( if fields["inproxy_proxy_network_type"].(string) != testNetworkType { return fmt.Errorf("unexpected inproxy_proxy_network_type '%s'", fields["inproxy_proxy_network_type"]) } + + if fields["inproxy_webrtc_use_media_streams"].(bool) != runConfig.useInproxyMediaStreams { + return fmt.Errorf("unexpected inproxy_webrtc_use_media_streams '%v'", fields["inproxy_webrtc_use_media_streams"]) + } } if runConfig.applyPrefix { @@ -3690,7 +3730,8 @@ func generateInproxyTestConfig( doTargetBrokerSpecs bool, brokerIPAddress string, brokerPort int, - serverEntrySignaturePublicKey string) (*inproxyTestConfig, error) { + serverEntrySignaturePublicKey string, + useInproxyMediaStreams bool) (*inproxyTestConfig, error) { // Generate in-proxy configuration. // @@ -3869,9 +3910,15 @@ func generateInproxyTestConfig( "InproxyDisableSTUN": true, "InproxyDisablePortMapping": true, "InproxyDisableIPv6ICECandidates": true, + "InproxyWebRTCMediaStreamsProbability": %s, %s ` + mediaStreamsProbability := "0.0" + if useInproxyMediaStreams { + mediaStreamsProbability = "1.0" + } + tacticsParametersJSON := fmt.Sprintf( tacticsParametersJSONFormat, brokerSessionPublicKeyStr, @@ -3881,6 +3928,7 @@ func generateInproxyTestConfig( clientBrokerSpecsJSON, commonCompartmentIDStr, commonCompartmentIDStr, + mediaStreamsProbability, maxRequestTimeoutsJSON) config := &inproxyTestConfig{ diff --git a/psiphon/server/tunnelServer.go b/psiphon/server/tunnelServer.go index cf943b9f7..f3cecd424 100644 --- a/psiphon/server/tunnelServer.go +++ b/psiphon/server/tunnelServer.go @@ -168,8 +168,36 @@ func (server *TunnelServer) Run() error { } else if protocol.TunnelProtocolUsesQUIC(tunnelProtocol) { + usesInproxy := protocol.TunnelProtocolUsesInproxy(tunnelProtocol) + // in-proxy QUIC tunnel protocols don't support gQUIC. - enableGQUIC := support.Config.EnableGQUIC && !protocol.TunnelProtocolUsesInproxy(tunnelProtocol) + enableGQUIC := support.Config.EnableGQUIC && !usesInproxy + + maxPacketSizeAdjustment := 0 + if usesInproxy { + + // In the in-proxy WebRTC media stream mode, QUIC packets sent + // back to the client, via the proxy, are encapsulated in + // SRTP packet payloads, and the maximum QUIC packet size + // must be adjusted to fit. + // + // Limitation: the WebRTC data channel mode does not have the + // same QUIC packet size constraint, since data channel + // messages can be far larger (up to 65536 bytes). However, + // the server, at this point, does not know whether + // individual connections are using WebRTC media streams or + // data channels on the first hop, and will no know until API + // handshake information is delivered after the QUIC, OSSH, + // and SSH handshakes are completed. Currently the max packet + // size adjustment is set unconditionally. For data channels, + // this will result in suboptimal packet sizes (10s of bytes) + // and a corresponding different traffic shape on the 2nd hop. + + IPAddress := net.ParseIP(support.Config.ServerIPAddress) + isIPv6 := IPAddress != nil && IPAddress.To4() == nil + + maxPacketSizeAdjustment = inproxy.GetQUICMaxPacketSizeAdjustment(isIPv6) + } logTunnelProtocol := tunnelProtocol listener, err = quic.Listen( @@ -180,6 +208,7 @@ func (server *TunnelServer) Run() error { errors.Trace(err), LogFields(logFields)) }, localAddress, + maxPacketSizeAdjustment, support.Config.ObfuscatedSSHKey, enableGQUIC) diff --git a/psiphon/tunnel.go b/psiphon/tunnel.go index 6dc27b11e..961560975 100644 --- a/psiphon/tunnel.go +++ b/psiphon/tunnel.go @@ -945,6 +945,7 @@ func dialTunnel( dialParams.ObfuscatedQUICPaddingSeed, dialParams.ObfuscatedQUICNonceTransformerParameters, dialParams.QUICDisablePathMTUDiscovery, + dialParams.QUICMaxPacketSizeAdjustment, dialParams.QUICDialEarly, dialParams.QUICUseObfuscatedPSK, dialParams.quicTLSClientSessionCache)