diff --git a/internal/useragent/useragent.go b/internal/useragent/useragent.go index 78fc93d..b022ad0 100644 --- a/internal/useragent/useragent.go +++ b/internal/useragent/useragent.go @@ -5,10 +5,10 @@ // // Basic usage: // -// client := &http.Client{ -// Transport: useragent.Set(http.DefaultTransport, "MyRobot/1.0"), -// } -// resp, err := client.Get("https://...") +// client := &http.Client{ +// Transport: useragent.Set(http.DefaultTransport, "MyRobot/1.0"), +// } +// resp, err := client.Get("https://...") package useragent import ( @@ -49,7 +49,8 @@ func (t uaT) RoundTrip(r *http.Request) (*http.Response, error) { r2.Header[k] = v } r2.Header.Set("User-Agent", t.userAgent) - if r.URL.Host == "twitter.com" || strings.HasSuffix(r.URL.Host, ".twitter.com") { + if r.URL.Host == "twitter.com" || strings.HasSuffix(r.URL.Host, ".twitter.com") || + r.URL.Host == "x.com" || strings.HasSuffix(r.URL.Host, ".x.com") { r2.Header.Set("User-Agent", "DiscourseBot/1.0") } return t.Transport.RoundTrip(r2) diff --git a/unfurlist.go b/unfurlist.go index 5b5a0d6..f2546bf 100644 --- a/unfurlist.go +++ b/unfurlist.go @@ -472,8 +472,9 @@ func (h *unfurlHandler) fetchData(ctx context.Context, URL string) (*pageChunk, return &pageChunk{url: resp.Request.URL}, errors.New("bad status: " + resp.Status) } if resp.Header.Get("Content-Encoding") == "deflate" && - strings.HasSuffix(resp.Request.Host, "twitter.com") { - // twitter sends unsolicited deflate-encoded responses + (strings.HasSuffix(resp.Request.Host, "twitter.com") || + strings.HasSuffix(resp.Request.Host, "x.com")) { + // twitter/X sends unsolicited deflate-encoded responses // violating RFC; workaround this. // See https://golang.org/issues/18779 for background var err error