diff --git a/snscrape/base.py b/snscrape/base.py index 0c34000..a121005 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -274,6 +274,9 @@ def _request(self, method, url, params = None, data = None, headers = None, time def _get(self, *args, **kwargs): return self._request('GET', *args, **kwargs) + def _head(self, *args, **kwargs): + return requests.head(*args, allow_redirects=False, timeout=10) + def _post(self, *args, **kwargs): return self._request('POST', *args, **kwargs) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index d1719a8..195cbd1 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -1081,7 +1081,12 @@ def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, ca card.url = u.url break else: - _logger.warning(f'Could not translate t.co card URL on tweet {tweetId}') + try: + u = self._head(card.url) + assert u.status_code >= 300 and u.status_code < 400 + card.url = u.headers["location"] + except: + _logger.warning(f'Could not translate t.co card URL on tweet {tweetId}') if 'bookmark_count' in tweet: kwargs['bookmarkCount'] = tweet['bookmark_count'] kwargs['conversationControlPolicy'] = ConversationControlPolicy._from_policy(tweet.get('conversation_control', {'policy': None})['policy'])