From 72b26f2373f3fecf53bdf9c62d7408df3d15a329 Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Thu, 15 Oct 2020 08:14:14 -0700 Subject: [PATCH 01/22] Scrape images, video, and post forwarding information for Telegram channel posts --- snscrape/modules/telegram.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 11cc19bb..8b0ece75 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -9,7 +9,7 @@ import snscrape.base import typing import urllib.parse - +import base64 _logger = logging.getLogger(__name__) _SINGLE_MEDIA_LINK_PATTERN = re.compile(r'^https://t\.me/[^/]+/\d+\?single$') @@ -30,6 +30,9 @@ class TelegramPost(snscrape.base.Item): date: datetime.datetime content: str outlinks: list + images: list + video: str + forwarded: str linkPreview: typing.Optional[LinkPreview] = None outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') @@ -90,17 +93,38 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): _logger.warning(f'Possibly incorrect URL: {rawUrl!r}') url = rawUrl.replace('//t.me/', '//t.me/s/') date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') + images = [] + video = None + forwarded = None if (message := post.find('div', class_ = 'tgme_widget_message_text')): - content = message.text + content = message.get_text(separator="\n") + + for video_tag in post.find_all('video'): + video = video_tag['src'] + + if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): + forwarded = forward_tag['href'].split('t.me/')[1].split('/')[0] + outlinks = [] for link in post.find_all('a'): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): # Author links at the top (avatar and name) continue if link['href'] == rawUrl or link['href'] == url: + style = link.attrs.get('style', '') # Generic filter of links to the post itself, catches videos, photos, and the date link - continue + if style != '': + image = re.findall('url\(\'(.*?)\'\)', style) + if len(image) == 1: + images.append(image[0]) + continue if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): + style = link.attrs.get('style', '') + image = re.findall('url\(\'(.*?)\'\)', style) + if len(image) == 1: + images.append(image[0]) + # resp = self._get(image[0]) + # encoded_string = base64.b64encode(resp.content) # Individual photo or video link continue href = urllib.parse.urljoin(pageUrl, link['href']) @@ -109,6 +133,8 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: content = None outlinks = [] + images = [] + video = None linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} @@ -125,7 +151,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: _logger.warning(f'Could not process link preview image on {url}') linkPreview = LinkPreview(**kwargs) - yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview) + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, video = video, forwarded = forwarded) def get_items(self): r, soup = self._initial_page() From de4ebed81f3f6a4bb4c65630daab6ec63784959b Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Thu, 24 Feb 2022 18:08:12 +0100 Subject: [PATCH 02/22] Fix KeyError caused by retweets without URLs in TwitterProfileScraper --- snscrape/modules/twitter.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 7c78f69f..337843f3 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -605,6 +605,9 @@ def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, ca i = kwargs['tcooutlinks'].index(card.url) except ValueError: _logger.warning('Could not find card URL in tcooutlinks') + except KeyError: + # retweets are missing this attribute + pass else: card.url = kwargs['outlinks'][i] return Tweet(**kwargs) From b8efce2a127af07f03d83000cb08f4cbc4a45029 Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Tue, 8 Mar 2022 15:10:15 +0100 Subject: [PATCH 03/22] Clean up unnecessary imports --- snscrape/modules/telegram.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 8b0ece75..ed812220 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -9,7 +9,6 @@ import snscrape.base import typing import urllib.parse -import base64 _logger = logging.getLogger(__name__) _SINGLE_MEDIA_LINK_PATTERN = re.compile(r'^https://t\.me/[^/]+/\d+\?single$') @@ -123,9 +122,6 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): image = re.findall('url\(\'(.*?)\'\)', style) if len(image) == 1: images.append(image[0]) - # resp = self._get(image[0]) - # encoded_string = base64.b64encode(resp.content) - # Individual photo or video link continue href = urllib.parse.urljoin(pageUrl, link['href']) if href not in outlinks: From ed829163a09d5153a6489e2a01ce6b1421356fce Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Tue, 29 Mar 2022 01:12:07 -0500 Subject: [PATCH 04/22] added capability to extract the number of channel members when the the string in membersDiv has the word 'subscribers' rather than 'members'. --- snscrape/modules/telegram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 8b0ece75..a65671c3 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -177,8 +177,8 @@ def _get_entity(self): raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') membersDiv = soup.find('div', class_ = 'tgme_page_extra') - if membersDiv.text.endswith(' members'): - kwargs['members'] = int(membersDiv.text[:-8].replace(' ', '')) + if membersDiv.text.endswith((' members', ' subscribers')): + kwargs['members'] = int(''.join(membersDiv.text.split(' ')[:-1])) kwargs['photo'] = soup.find('img', class_ = 'tgme_page_photo_image').attrs['src'] r, soup = self._initial_page() From fb8d73ac95011b7ad848a6048d3eed1880e80f21 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Tue, 29 Mar 2022 13:15:53 -0500 Subject: [PATCH 05/22] handled case where channel has no profile image --- snscrape/modules/telegram.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index a65671c3..399ce95b 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -179,7 +179,11 @@ def _get_entity(self): membersDiv = soup.find('div', class_ = 'tgme_page_extra') if membersDiv.text.endswith((' members', ' subscribers')): kwargs['members'] = int(''.join(membersDiv.text.split(' ')[:-1])) - kwargs['photo'] = soup.find('img', class_ = 'tgme_page_photo_image').attrs['src'] + photoImg = soup.find('img', class_ = 'tgme_page_photo_image') + if photoImg is not None: + kwargs['photo'] = photoImg.attrs['src'] + else: + kwargs['photo'] = None r, soup = self._initial_page() if '/s/' not in r.url: # Redirect on channels without public posts From d32c9add8a3691c81c9091dc1a7d079e9871379f Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 30 Mar 2022 18:13:15 -0500 Subject: [PATCH 06/22] added capability to scrape multiple videos from a single post --- snscrape/modules/telegram.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 399ce95b..0f093eea 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -31,7 +31,7 @@ class TelegramPost(snscrape.base.Item): content: str outlinks: list images: list - video: str + videos: list forwarded: str linkPreview: typing.Optional[LinkPreview] = None @@ -94,13 +94,13 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): url = rawUrl.replace('//t.me/', '//t.me/s/') date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') images = [] - video = None + videos = [] forwarded = None if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.get_text(separator="\n") for video_tag in post.find_all('video'): - video = video_tag['src'] + videos.append(video_tag['src']) if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): forwarded = forward_tag['href'].split('t.me/')[1].split('/')[0] @@ -134,7 +134,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): content = None outlinks = [] images = [] - video = None + videos = [] linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} @@ -151,7 +151,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: _logger.warning(f'Could not process link preview image on {url}') linkPreview = LinkPreview(**kwargs) - yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, video = video, forwarded = forwarded) + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, videos = videos, forwarded = forwarded) def get_items(self): r, soup = self._initial_page() From a7eb54d226b5397a7a3c1cd502f22a8a31f719fd Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 30 Mar 2022 21:07:17 -0500 Subject: [PATCH 07/22] implemented Media dataclasses for Telegram, and added variable for extracting a post's view count --- snscrape/modules/telegram.py | 94 ++++++++++++++++++++++++++---------- 1 file changed, 69 insertions(+), 25 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 0f093eea..0484c9c1 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -30,9 +30,9 @@ class TelegramPost(snscrape.base.Item): date: datetime.datetime content: str outlinks: list - images: list - videos: list + media: typing.Optional[typing.List['Medium']] forwarded: str + views: int = None linkPreview: typing.Optional[LinkPreview] = None outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') @@ -62,6 +62,29 @@ class Channel(snscrape.base.Entity): def __str__(self): return f'https://t.me/s/{self.username}' +class Medium: + pass + + +@dataclasses.dataclass +class Photo(Medium): + previewUrl: str + fullUrl: str + +@dataclasses.dataclass +class Image(Medium): + url: str + +@dataclasses.dataclass +class Video(Medium): + thumbnailUrl: str + duration: float + url: typing.Optional[str] = None + +@dataclasses.dataclass +class Gif(Medium): + thumbnailUrl: str + url: typing.Optional[str] = None class TelegramChannelScraper(snscrape.base.Scraper): name = 'telegram-channel' @@ -93,18 +116,34 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): _logger.warning(f'Possibly incorrect URL: {rawUrl!r}') url = rawUrl.replace('//t.me/', '//t.me/s/') date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') - images = [] - videos = [] + media = [] forwarded = None if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.get_text(separator="\n") - for video_tag in post.find_all('video'): - videos.append(video_tag['src']) + for video_player in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): + style = video_player.find('i')['style'] + videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style) + videoTag = video_player.find('video') + if videoTag is None: + videoUrl = None + else: + videoUrl = videoTag['src'] + mKwargs = { + 'thumbnailUrl': videoThumbnailUrl, + 'url': videoUrl, + } + timeTag = video_player.find('time') + if timeTag is None: + cls = Gif + else: + cls = Video + durationStr = video_player.find('time').text.split(':') + mKwargs['duration'] = sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationStr))]) + media.append(cls(**mKwargs)) if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): forwarded = forward_tag['href'].split('t.me/')[1].split('/')[0] - outlinks = [] for link in post.find_all('a'): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): @@ -114,15 +153,15 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): style = link.attrs.get('style', '') # Generic filter of links to the post itself, catches videos, photos, and the date link if style != '': - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) + imageUrls = re.findall('url\(\'(.*?)\'\)', style) + if len(imageUrls) == 1: + media.append(Image(url = imageUrls[0])) continue if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): style = link.attrs.get('style', '') - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) + imageUrls = re.findall('url\(\'(.*?)\'\)', style) + if len(imageUrls) == 1: + media.append(Image(url = imageUrls[0])) # resp = self._get(image[0]) # encoded_string = base64.b64encode(resp.content) # Individual photo or video link @@ -133,8 +172,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: content = None outlinks = [] - images = [] - videos = [] + media = [] linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} @@ -151,7 +189,13 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: _logger.warning(f'Could not process link preview image on {url}') linkPreview = LinkPreview(**kwargs) - yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, images = images, videos = videos, forwarded = forwarded) + viewsSpan = post.find('span', class_ = 'tgme_widget_message_views') + if viewsSpan is None: + views = None + else: + views = parse_num(viewsSpan.text) + + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, media = media, forwarded = forwarded, views = views) def get_items(self): r, soup = self._initial_page() @@ -204,15 +248,6 @@ def _get_entity(self): if (descriptionDiv := channelInfoDiv.find('div', class_ = 'tgme_channel_info_description')): kwargs['description'] = descriptionDiv.text - def parse_num(s): - s = s.replace(' ', '') - if s.endswith('M'): - return int(float(s[:-1]) * 1e6), 10 ** (6 if '.' not in s else 6 - len(s[:-1].split('.')[1])) - elif s.endswith('K'): - return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) - else: - return int(s), 1 - for div in channelInfoDiv.find_all('div', class_ = 'tgme_channel_info_counter'): value, granularity = parse_num(div.find('span', class_ = 'counter_value').text) type_ = div.find('span', class_ = 'counter_type').text @@ -231,3 +266,12 @@ def _cli_setup_parser(cls, subparser): @classmethod def _cli_from_args(cls, args): return cls._cli_construct(args, args.channel) + +def parse_num(s): + s = s.replace(' ', '') + if s.endswith('M'): + return int(float(s[:-1]) * 1e6), 10 ** (6 if '.' not in s else 6 - len(s[:-1].split('.')[1])) + elif s.endswith('K'): + return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) + else: + return int(s), 1 \ No newline at end of file From 4e59638e7c00b57d63ae06a84c6a4a90d9e7ee49 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 30 Mar 2022 21:33:03 -0500 Subject: [PATCH 08/22] added a forwardedUrl attribute to TelegramPost and made forwarded attribute type Channel. --- snscrape/modules/telegram.py | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 0484c9c1..19aa22ec 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -24,23 +24,6 @@ class LinkPreview: image: typing.Optional[str] = None -@dataclasses.dataclass -class TelegramPost(snscrape.base.Item): - url: str - date: datetime.datetime - content: str - outlinks: list - media: typing.Optional[typing.List['Medium']] - forwarded: str - views: int = None - linkPreview: typing.Optional[LinkPreview] = None - - outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') - - def __str__(self): - return self.url - - @dataclasses.dataclass class Channel(snscrape.base.Entity): username: str @@ -62,6 +45,23 @@ class Channel(snscrape.base.Entity): def __str__(self): return f'https://t.me/s/{self.username}' +@dataclasses.dataclass +class TelegramPost(snscrape.base.Item): + url: str + date: datetime.datetime + content: str + outlinks: list + forwarded: typing.Optional['Channel'] = None + forwardedUrl: typing.Optional[str] = None + media: typing.Optional[typing.List['Medium']] = None + views: typing.Optional[int] = None + linkPreview: typing.Optional[LinkPreview] = None + + outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') + + def __str__(self): + return self.url + class Medium: pass @@ -118,6 +118,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') media = [] forwarded = None + forwardedUrl = None if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.get_text(separator="\n") @@ -143,7 +144,11 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): mKwargs['duration'] = sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationStr))]) media.append(cls(**mKwargs)) if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): - forwarded = forward_tag['href'].split('t.me/')[1].split('/')[0] + forwardedUrl = forward_tag['href'] + forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] + forwardedChannelScraper = TelegramChannelScraper(name = forwardedName) + forwarded = forwardedChannelScraper._get_entity() + outlinks = [] for link in post.find_all('a'): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): @@ -195,7 +200,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: views = parse_num(viewsSpan.text) - yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, media = media, forwarded = forwarded, views = views) + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) def get_items(self): r, soup = self._initial_page() From 2ce014ade4104a748b7a6a0ada19fcab1460e832 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Sun, 3 Apr 2022 01:45:25 -0500 Subject: [PATCH 09/22] fixed edge case for videos that have data-link-attr but no href attribute --- snscrape/modules/vkontakte.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index ea33b444..3193abe0 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -177,11 +177,15 @@ def _post_div_to_item(self, post, isCopy = False): continue if 'data-video' in a.attrs: # Video + if 'data-link-attr' in a.attrs: + hrefUrl = urllib.parse.unquote(a.attrs['data-link-attr'].split('to=')[1].split('&')[0]) + else: + hrefUrl = f'https://vk.com{a["href"]}' video = Video( id = a['data-video'], list = a['data-list'], duration = int(a['data-duration']), - url = f'https://vk.com{a["href"]}', + url = hrefUrl, thumbUrl = a['style'][(begin := a['style'].find('background-image: url(') + 22) : a['style'].find(')', begin)], ) continue From babcddda19a0eca7d7413464666cbeb145d76f2d Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Sun, 17 Apr 2022 03:55:37 -0500 Subject: [PATCH 10/22] made Telegram scraper not return full channel info for forwarded_from attribute; fixed video edge cases. --- snscrape/modules/telegram.py | 39 +++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 19aa22ec..565322ce 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -27,9 +27,9 @@ class LinkPreview: @dataclasses.dataclass class Channel(snscrape.base.Entity): username: str - title: str - verified: bool - photo: str + title: typing.Optional[str] = None + verified: typing.Optional[bool] = None + photo: typing.Optional[str] = None description: typing.Optional[str] = None members: typing.Optional[int] = None photos: typing.Optional[snscrape.base.IntWithGranularity] = None @@ -123,14 +123,18 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): content = message.get_text(separator="\n") for video_player in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): - - style = video_player.find('i')['style'] - videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style) - videoTag = video_player.find('video') - if videoTag is None: - videoUrl = None + iTag = video_player.find('i') + if iTag is None: + videoUrl = None + videoThumbnailUrl = None else: - videoUrl = videoTag['src'] + style = iTag['style'] + videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style)[0] + videoTag = video_player.find('video') + if videoTag is None: + videoUrl = None + else: + videoUrl = videoTag['src'] mKwargs = { 'thumbnailUrl': videoThumbnailUrl, 'url': videoUrl, @@ -146,8 +150,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): forwardedUrl = forward_tag['href'] forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] - forwardedChannelScraper = TelegramChannelScraper(name = forwardedName) - forwarded = forwardedChannelScraper._get_entity() + forwarded = Channel(username = forwardedName) outlinks = [] for link in post.find_all('a'): @@ -213,7 +216,7 @@ def get_items(self): if not pageLink: break nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) - r = self._get(nextPageUrl, headers = self._headers) + r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) if r.status_code != 200: raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') @@ -279,4 +282,12 @@ def parse_num(s): elif s.endswith('K'): return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) else: - return int(s), 1 \ No newline at end of file + return int(s), 1 + +def telegramResponseOkCallback(r): + if r.status_code == 200: + return (True, None) + elif r.status_code // 100 == 5: + return (False, f'status code: {r.status_code}') + else: + return (False, None) \ No newline at end of file From 1e4e0c278dce468fa98626234d522fa8e51af5e6 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Sun, 17 Apr 2022 04:33:22 -0500 Subject: [PATCH 11/22] fixed issue where Telegram scraper terminated early because some pages didn't have a next page link (added reasonable default) --- snscrape/modules/telegram.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 565322ce..ad49066b 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -214,8 +214,13 @@ def get_items(self): yield from self._soup_to_items(soup, r.url) pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True}) if not pageLink: - break + nextPostIndex = int(nextPageUrl.split('=')[-1]) - 20 + if nextPostIndex > 20: + pageLink = {'href': nextPageUrl.split('=')[0] + f'={nextPostIndex}'} + else: + break nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) + print(f'nextPageUrl: {nextPageUrl}') r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) if r.status_code != 200: raise snscrape.base.ScraperException(f'Got status code {r.status_code}') From b276c3cc27f35e57f287fb0b815e19b94b31487f Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Sun, 17 Apr 2022 06:50:43 -0500 Subject: [PATCH 12/22] fixed issue where some videos and photos weren't being scraped (because they weren't in a post containing a 'tgme_widget_message_text' div --- snscrape/modules/telegram.py | 132 ++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 58 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index ad49066b..c3d055d7 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -65,14 +65,8 @@ def __str__(self): class Medium: pass - @dataclasses.dataclass class Photo(Medium): - previewUrl: str - fullUrl: str - -@dataclasses.dataclass -class Image(Medium): url: str @dataclasses.dataclass @@ -81,6 +75,12 @@ class Video(Medium): duration: float url: typing.Optional[str] = None +@dataclasses.dataclass +class VoiceMessage(Medium): + url: str + duration: str + bars:typing.List[float] + @dataclasses.dataclass class Gif(Medium): thumbnailUrl: str @@ -117,70 +117,81 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): url = rawUrl.replace('//t.me/', '//t.me/s/') date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') media = [] + outlinks = [] forwarded = None forwardedUrl = None + if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.get_text(separator="\n") - for video_player in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): - iTag = video_player.find('i') - if iTag is None: - videoUrl = None - videoThumbnailUrl = None - else: - style = iTag['style'] - videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style)[0] - videoTag = video_player.find('video') - if videoTag is None: - videoUrl = None - else: - videoUrl = videoTag['src'] - mKwargs = { - 'thumbnailUrl': videoThumbnailUrl, - 'url': videoUrl, - } - timeTag = video_player.find('time') - if timeTag is None: - cls = Gif - else: - cls = Video - durationStr = video_player.find('time').text.split(':') - mKwargs['duration'] = sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationStr))]) - media.append(cls(**mKwargs)) if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): forwardedUrl = forward_tag['href'] forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] forwarded = Channel(username = forwardedName) - outlinks = [] - for link in post.find_all('a'): - if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): - # Author links at the top (avatar and name) - continue - if link['href'] == rawUrl or link['href'] == url: - style = link.attrs.get('style', '') - # Generic filter of links to the post itself, catches videos, photos, and the date link - if style != '': - imageUrls = re.findall('url\(\'(.*?)\'\)', style) - if len(imageUrls) == 1: - media.append(Image(url = imageUrls[0])) - continue - if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): - style = link.attrs.get('style', '') + else: + content = None + + outlinks = [] + for link in post.find_all('a'): + if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): + # Author links at the top (avatar and name) + continue + if link['href'] == rawUrl or link['href'] == url: + style = link.attrs.get('style', '') + # Generic filter of links to the post itself, catches videos, photos, and the date link + if style != '': imageUrls = re.findall('url\(\'(.*?)\'\)', style) if len(imageUrls) == 1: - media.append(Image(url = imageUrls[0])) - # resp = self._get(image[0]) - # encoded_string = base64.b64encode(resp.content) - # Individual photo or video link + media.append(Photo(url = imageUrls[0])) continue - href = urllib.parse.urljoin(pageUrl, link['href']) - if href not in outlinks: - outlinks.append(href) - else: - content = None - outlinks = [] - media = [] + if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): + style = link.attrs.get('style', '') + imageUrls = re.findall('url\(\'(.*?)\'\)', style) + if len(imageUrls) == 1: + media.append(Photo(url = imageUrls[0])) + # resp = self._get(image[0]) + # encoded_string = base64.b64encode(resp.content) + # Individual photo or video link + continue + href = urllib.parse.urljoin(pageUrl, link['href']) + if (href not in outlinks) and (href != rawUrl): + outlinks.append(href) + + for voice_player in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): + audioUrl = voice_player.find('audio')['src'] + durationStr = voice_player.find('time').text.split(':') + duration = durationStrToSeconds(durationStr) + barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voice_player.find('div', {'class': 'bar'}).find_all('s')] + + media.append(VoiceMessage(url = audioUrl, duration = duration, bars = barHeights)) + + for video_player in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): + iTag = video_player.find('i') + if iTag is None: + videoUrl = None + videoThumbnailUrl = None + else: + style = iTag['style'] + videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style)[0] + videoTag = video_player.find('video') + if videoTag is None: + videoUrl = None + else: + videoUrl = videoTag['src'] + mKwargs = { + 'thumbnailUrl': videoThumbnailUrl, + 'url': videoUrl, + } + timeTag = video_player.find('time') + if timeTag is None: + cls = Gif + else: + cls = Video + durationStr = video_player.find('time').text.split(':') + mKwargs['duration'] = durationStrToSeconds(durationStr) + media.append(cls(**mKwargs)) + linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} @@ -197,6 +208,9 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: _logger.warning(f'Could not process link preview image on {url}') linkPreview = LinkPreview(**kwargs) + if kwargs['href'] in outlinks: + outlinks.remove(kwargs['href']) + viewsSpan = post.find('span', class_ = 'tgme_widget_message_views') if viewsSpan is None: views = None @@ -220,7 +234,6 @@ def get_items(self): else: break nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) - print(f'nextPageUrl: {nextPageUrl}') r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) if r.status_code != 200: raise snscrape.base.ScraperException(f'Got status code {r.status_code}') @@ -289,6 +302,9 @@ def parse_num(s): else: return int(s), 1 +def durationStrToSeconds(durationStr): + return sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationStr))]) + def telegramResponseOkCallback(r): if r.status_code == 200: return (True, None) From 97d38e5cde20219dfff814f0b84d85d416fef86c Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 21 Apr 2022 09:41:53 -0500 Subject: [PATCH 13/22] added additional termination criteria to Telegram scraper --- snscrape/modules/telegram.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index c3d055d7..89245f47 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -224,10 +224,20 @@ def get_items(self): if '/s/' not in r.url: _logger.warning('No public post list for this user') return + nextPageUrl = '' while True: yield from self._soup_to_items(soup, r.url) + try: + if soup.find('a', attrs = {'class': 'tgme_widget_message_date'}, href = True)['href'].split('/')[-1] == '1': + # if message 1 is the first message in the page, terminate scraping + break + except: + pass pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True}) if not pageLink: + # some pages are missing a "tme_messages_more" tag, causing early termination + if '=' not in nextPageUrl: + nextPageUrl = soup.find('link', attrs = {'rel': 'canonical'}, href = True)['href'] nextPostIndex = int(nextPageUrl.split('=')[-1]) - 20 if nextPostIndex > 20: pageLink = {'href': nextPageUrl.split('=')[0] + f'={nextPostIndex}'} From 9b3faec9803cebfc5606f7c36eb74ebe8f2e6973 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 21 Apr 2022 18:06:43 -0500 Subject: [PATCH 14/22] added additional attributes for hashtags and user mentions, removed redundant outlinks --- snscrape/modules/telegram.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 89245f47..bed72cf8 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -50,7 +50,9 @@ class TelegramPost(snscrape.base.Item): url: str date: datetime.datetime content: str - outlinks: list + outlinks: typing.List[str] = None + mentions: typing.List[str] = None + hashtags: typing.List[str] = None forwarded: typing.Optional['Channel'] = None forwardedUrl: typing.Optional[str] = None media: typing.Optional[typing.List['Medium']] = None @@ -133,6 +135,8 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): content = None outlinks = [] + mentions = [] + hashtags = [] for link in post.find_all('a'): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): # Author links at the top (avatar and name) @@ -154,8 +158,14 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): # encoded_string = base64.b64encode(resp.content) # Individual photo or video link continue + if link.text.startswith('@'): + mentions.append(link.text.strip('@')) + continue + if link.text.startswith('#'): + hashtags.append(link.text.strip('#')) + continue href = urllib.parse.urljoin(pageUrl, link['href']) - if (href not in outlinks) and (href != rawUrl): + if (href not in outlinks) and (href != rawUrl) and (href != forwardedUrl): outlinks.append(href) for voice_player in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): @@ -217,7 +227,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: views = parse_num(viewsSpan.text) - yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) + yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, mentions = mentions, hashtags = hashtags, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) def get_items(self): r, soup = self._initial_page() From 21f7b620ec4d89102912700be6fa41a8001a8692 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 21 Apr 2022 18:26:31 -0500 Subject: [PATCH 15/22] moved forward finding out of tgme_widget_message_text clause, since it wasn't correctly getting the forwarding information in forwarded posts that contained attachments but no text --- snscrape/modules/telegram.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index bed72cf8..9cd7573e 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -123,14 +123,13 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): forwarded = None forwardedUrl = None + if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): + forwardedUrl = forward_tag['href'] + forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] + forwarded = Channel(username = forwardedName) + if (message := post.find('div', class_ = 'tgme_widget_message_text')): content = message.get_text(separator="\n") - - if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): - forwardedUrl = forward_tag['href'] - forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] - forwarded = Channel(username = forwardedName) - else: content = None From 5648e957d0d1f759f68a2dfe21a137bf7c0ca608 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 27 Apr 2022 16:41:24 -0500 Subject: [PATCH 16/22] improved consistency of code formatting and added _STYLE_MEDIA_URL_PATTERN as variable --- snscrape/modules/telegram.py | 64 +++++++++++++++++------------------- 1 file changed, 31 insertions(+), 33 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 9cd7573e..ab445613 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -13,7 +13,7 @@ _logger = logging.getLogger(__name__) _SINGLE_MEDIA_LINK_PATTERN = re.compile(r'^https://t\.me/[^/]+/\d+\?single$') - +_STYLE_MEDIA_URL_PATTERN = re.compile(r'url\(\'(.*?)\'\)') @dataclasses.dataclass class LinkPreview: @@ -45,6 +45,7 @@ class Channel(snscrape.base.Entity): def __str__(self): return f'https://t.me/s/{self.username}' + @dataclasses.dataclass class TelegramPost(snscrape.base.Item): url: str @@ -64,30 +65,36 @@ class TelegramPost(snscrape.base.Item): def __str__(self): return self.url + class Medium: pass + @dataclasses.dataclass class Photo(Medium): url: str + @dataclasses.dataclass class Video(Medium): thumbnailUrl: str duration: float url: typing.Optional[str] = None + @dataclasses.dataclass class VoiceMessage(Medium): url: str duration: str bars:typing.List[float] + @dataclasses.dataclass class Gif(Medium): thumbnailUrl: str url: typing.Optional[str] = None + class TelegramChannelScraper(snscrape.base.Scraper): name = 'telegram-channel' @@ -120,11 +127,13 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): date = datetime.datetime.strptime(dateDiv.find('time', datetime = True)['datetime'].replace('-', '', 2).replace(':', ''), '%Y%m%dT%H%M%S%z') media = [] outlinks = [] + mentions = [] + hashtags = [] forwarded = None forwardedUrl = None - if (forward_tag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): - forwardedUrl = forward_tag['href'] + if (forwardTag := post.find('a', class_ = 'tgme_widget_message_forwarded_from_name')): + forwardedUrl = forwardTag['href'] forwardedName = forwardedUrl.split('t.me/')[1].split('/')[0] forwarded = Channel(username = forwardedName) @@ -133,9 +142,6 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: content = None - outlinks = [] - mentions = [] - hashtags = [] for link in post.find_all('a'): if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): # Author links at the top (avatar and name) @@ -144,13 +150,13 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): style = link.attrs.get('style', '') # Generic filter of links to the post itself, catches videos, photos, and the date link if style != '': - imageUrls = re.findall('url\(\'(.*?)\'\)', style) + imageUrls = _STYLE_MEDIA_URL_PATTERN.findall(style) if len(imageUrls) == 1: media.append(Photo(url = imageUrls[0])) continue if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): style = link.attrs.get('style', '') - imageUrls = re.findall('url\(\'(.*?)\'\)', style) + imageUrls = _STYLE_MEDIA_URL_PATTERN.findall(style) if len(imageUrls) == 1: media.append(Photo(url = imageUrls[0])) # resp = self._get(image[0]) @@ -167,37 +173,34 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): if (href not in outlinks) and (href != rawUrl) and (href != forwardedUrl): outlinks.append(href) - for voice_player in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): - audioUrl = voice_player.find('audio')['src'] - durationStr = voice_player.find('time').text.split(':') + for voicePlayer in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): + audioUrl = voicePlayer.find('audio')['src'] + durationStr = voicePlayer.find('time').text duration = durationStrToSeconds(durationStr) - barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voice_player.find('div', {'class': 'bar'}).find_all('s')] + barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voicePlayer.find('div', {'class': 'bar'}).find_all('s')] media.append(VoiceMessage(url = audioUrl, duration = duration, bars = barHeights)) - for video_player in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): - iTag = video_player.find('i') + for videoPlayer in post.find_all('a', {'class': 'tgme_widget_message_video_player'}): + iTag = videoPlayer.find('i') if iTag is None: videoUrl = None videoThumbnailUrl = None else: style = iTag['style'] - videoThumbnailUrl = re.findall('url\(\'(.*?)\'\)', style)[0] - videoTag = video_player.find('video') - if videoTag is None: - videoUrl = None - else: - videoUrl = videoTag['src'] + videoThumbnailUrl = _STYLE_MEDIA_URL_PATTERN.findall(style)[0] + videoTag = videoPlayer.find('video') + videoUrl = None if videoTag is None else videoTag['src'] mKwargs = { 'thumbnailUrl': videoThumbnailUrl, 'url': videoUrl, } - timeTag = video_player.find('time') + timeTag = videoPlayer.find('time') if timeTag is None: cls = Gif else: cls = Video - durationStr = video_player.find('time').text.split(':') + durationStr = videoPlayer.find('time').text mKwargs['duration'] = durationStrToSeconds(durationStr) media.append(cls(**mKwargs)) @@ -221,10 +224,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): outlinks.remove(kwargs['href']) viewsSpan = post.find('span', class_ = 'tgme_widget_message_views') - if viewsSpan is None: - views = None - else: - views = parse_num(viewsSpan.text) + views = None if viewsSpan is None else parse_num(viewsSpan.text) yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, mentions = mentions, hashtags = hashtags, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) @@ -318,16 +318,14 @@ def parse_num(s): return int(float(s[:-1]) * 1e6), 10 ** (6 if '.' not in s else 6 - len(s[:-1].split('.')[1])) elif s.endswith('K'): return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) - else: - return int(s), 1 + return int(s), 1 def durationStrToSeconds(durationStr): - return sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationStr))]) + durationList = durationStr.split(':') + return sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationList))]) def telegramResponseOkCallback(r): if r.status_code == 200: return (True, None) - elif r.status_code // 100 == 5: - return (False, f'status code: {r.status_code}') - else: - return (False, None) \ No newline at end of file + return (False, f'{r.status_code=}') + \ No newline at end of file From e2d922301e114c0fe9956715bc304cbed68d2610 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Mon, 9 May 2022 09:37:36 -0500 Subject: [PATCH 17/22] forgot to save modified twitter.py module --- snscrape/modules/twitter.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 9a845b07..048df342 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -874,18 +874,6 @@ def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, ca kwargs['cashtags'] = [o['text'] for o in tweet['entities']['symbols']] if card: kwargs['card'] = card -<<<<<<< HEAD - # Try to convert the URL to the non-shortened/t.co one - try: - i = kwargs['tcooutlinks'].index(card.url) - except ValueError: - _logger.warning('Could not find card URL in tcooutlinks') - except KeyError: - # retweets are missing this attribute - pass - else: - card.url = kwargs['outlinks'][i] -======= if hasattr(card, 'url') and '//t.co/' in card.url: # Try to convert the URL to the non-shortened/t.co one # Retweets inherit the card but not the outlinks; try to get them from the retweeted tweet instead in that case. @@ -895,7 +883,6 @@ def _make_tweet(self, tweet, user, retweetedTweet = None, quotedTweet = None, ca card.url = retweetedTweet.outlinks[retweetedTweet.tcooutlinks.index(card.url)] else: _logger.warning(f'Could not translate t.co card URL on tweet {tweetId}') ->>>>>>> ed3ea944d177157d688786470c8369198b0ce8ce return Tweet(**kwargs) def _make_medium(self, medium, tweetId): From 65723f10ff7afbba075e8077e65c8836bceab2ef Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 25 May 2022 06:47:47 -0500 Subject: [PATCH 18/22] fixed merge --- snscrape/modules/telegram.py | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 1008328e..395e9a30 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -203,36 +203,6 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): mKwargs['duration'] = durationStrToSeconds(durationStr) media.append(cls(**mKwargs)) -<<<<<<< HEAD - outlinks = [] - for link in post.find_all('a'): - if any(x in link.parent.attrs.get('class', []) for x in ('tgme_widget_message_user', 'tgme_widget_message_author')): - # Author links at the top (avatar and name) - continue - if link['href'] == rawUrl or link['href'] == url: - style = link.attrs.get('style', '') - # Generic filter of links to the post itself, catches videos, photos, and the date link - if style != '': - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) - continue - if _SINGLE_MEDIA_LINK_PATTERN.match(link['href']): - style = link.attrs.get('style', '') - image = re.findall('url\(\'(.*?)\'\)', style) - if len(image) == 1: - images.append(image[0]) - continue - href = urllib.parse.urljoin(pageUrl, link['href']) - if href not in outlinks: - outlinks.append(href) - else: - content = None - outlinks = [] - images = [] - video = None -======= ->>>>>>> master linkPreview = None if (linkPreviewA := post.find('a', class_ = 'tgme_widget_message_link_preview')): kwargs = {} From 56e4232083879c624954a9d8c8953112f25b7bff Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 23 Jun 2022 11:51:13 -0500 Subject: [PATCH 19/22] fixed typo --- snscrape/modules/telegram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 395e9a30..3e3a512c 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -321,7 +321,7 @@ def parse_num(s): def durationStrToSeconds(durationStr): durationList = durationStr.split(':') - return sum([int(s) * int(g) for s, g in zip([1, 60, 360], reversed(durationList))]) + return sum([int(s) * int(g) for s, g in zip([1, 60, 3600], reversed(durationList))]) def telegramResponseOkCallback(r): if r.status_code == 200: From 056cd6215cf96af517711bb07c46f46078233d39 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Thu, 23 Jun 2022 15:47:18 -0500 Subject: [PATCH 20/22] incorporated requested changes from maintainer, removed modifications to VK module --- snscrape/modules/telegram.py | 23 ++++++++++++++--------- snscrape/modules/vkontakte.py | 8 ++------ 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 3e3a512c..16f2dc51 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -56,7 +56,7 @@ class TelegramPost(snscrape.base.Item): forwarded: typing.Optional['Channel'] = None forwardedUrl: typing.Optional[str] = None media: typing.Optional[typing.List['Medium']] = None - views: typing.Optional[int] = None + views: typing.Optional[snscrape.base.IntWithGranularity] = None linkPreview: typing.Optional[LinkPreview] = None outlinksss = snscrape.base._DeprecatedProperty('outlinksss', lambda self: ' '.join(self.outlinks), 'outlinks') @@ -175,7 +175,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): for voicePlayer in post.find_all('a', {'class': 'tgme_widget_message_voice_player'}): audioUrl = voicePlayer.find('audio')['src'] durationStr = voicePlayer.find('time').text - duration = durationStrToSeconds(durationStr) + duration = _durationStrToSeconds(durationStr) barHeights = [float(s['style'].split(':')[-1].strip(';%')) for s in voicePlayer.find('div', {'class': 'bar'}).find_all('s')] media.append(VoiceMessage(url = audioUrl, duration = duration, bars = barHeights)) @@ -200,7 +200,7 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): else: cls = Video durationStr = videoPlayer.find('time').text - mKwargs['duration'] = durationStrToSeconds(durationStr) + mKwargs['duration'] = _durationStrToSeconds(durationStr) media.append(cls(**mKwargs)) linkPreview = None @@ -223,7 +223,12 @@ def _soup_to_items(self, soup, pageUrl, onlyUsername = False): outlinks.remove(kwargs['href']) viewsSpan = post.find('span', class_ = 'tgme_widget_message_views') - views = None if viewsSpan is None else parse_num(viewsSpan.text) + views = None if viewsSpan is None else _parse_num(viewsSpan.text) + + outlinks = outlinks if outlinks else None + media = media if media else None + mentions = mentions if mentions else None + hashtags = hashtags if hashtags else None yield TelegramPost(url = url, date = date, content = content, outlinks = outlinks, mentions = mentions, hashtags = hashtags, linkPreview = linkPreview, media = media, forwarded = forwarded, forwardedUrl = forwardedUrl, views = views) @@ -252,7 +257,7 @@ def get_items(self): else: break nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) - r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) + r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = _telegramResponseOkCallback) if r.status_code != 200: raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') @@ -293,7 +298,7 @@ def _get_entity(self): kwargs['description'] = descriptionDiv.text for div in channelInfoDiv.find_all('div', class_ = 'tgme_channel_info_counter'): - value, granularity = parse_num(div.find('span', class_ = 'counter_value').text) + value, granularity = _parse_num(div.find('span', class_ = 'counter_value').text) type_ = div.find('span', class_ = 'counter_type').text if type_ == 'members': # Already extracted more accurately from /channel, skip @@ -311,7 +316,7 @@ def _cli_setup_parser(cls, subparser): def _cli_from_args(cls, args): return cls._cli_construct(args, args.channel) -def parse_num(s): +def _parse_num(s): s = s.replace(' ', '') if s.endswith('M'): return int(float(s[:-1]) * 1e6), 10 ** (6 if '.' not in s else 6 - len(s[:-1].split('.')[1])) @@ -319,11 +324,11 @@ def parse_num(s): return int(float(s[:-1]) * 1000), 10 ** (3 if '.' not in s else 3 - len(s[:-1].split('.')[1])) return int(s), 1 -def durationStrToSeconds(durationStr): +def _durationStrToSeconds(durationStr): durationList = durationStr.split(':') return sum([int(s) * int(g) for s, g in zip([1, 60, 3600], reversed(durationList))]) -def telegramResponseOkCallback(r): +def _telegramResponseOkCallback(r): if r.status_code == 200: return (True, None) return (False, f'{r.status_code=}') diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index 3193abe0..9353cd5f 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -177,15 +177,11 @@ def _post_div_to_item(self, post, isCopy = False): continue if 'data-video' in a.attrs: # Video - if 'data-link-attr' in a.attrs: - hrefUrl = urllib.parse.unquote(a.attrs['data-link-attr'].split('to=')[1].split('&')[0]) - else: - hrefUrl = f'https://vk.com{a["href"]}' video = Video( id = a['data-video'], list = a['data-list'], duration = int(a['data-duration']), - url = hrefUrl, + url = f'https://vk.com{a["href"]}', thumbUrl = a['style'][(begin := a['style'].find('background-image: url(') + 22) : a['style'].find(')', begin)], ) continue @@ -389,4 +385,4 @@ def _cli_setup_parser(cls, subparser): @classmethod def _cli_from_args(cls, args): - return cls._cli_construct(args, args.username) + return cls._cli_construct(args, args.username) \ No newline at end of file From 73f10a4f24a42e9642bcea9dc955ed91c92de41e Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Tue, 5 Jul 2022 10:23:26 -0500 Subject: [PATCH 21/22] fixed edge case where channel with no members fails _get_entity --- snscrape/modules/telegram.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 16f2dc51..2e1322f9 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -270,8 +270,12 @@ def _get_entity(self): raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') membersDiv = soup.find('div', class_ = 'tgme_page_extra') - if membersDiv.text.endswith((' members', ' subscribers')): - kwargs['members'] = int(''.join(membersDiv.text.split(' ')[:-1])) + if membersDiv.text.split(',')[0].endswith((' members', ' subscribers')): + membersStr = ''.join(membersDiv.text.split(',')[0].split(' ')[:-1]) + if membersStr == 'no': + kwargs['members'] = 0 + else: + kwargs['members'] = int(membersStr) photoImg = soup.find('img', class_ = 'tgme_page_photo_image') if photoImg is not None: kwargs['photo'] = photoImg.attrs['src'] From cbdfeed8124ecf85e143b172343499d709fcbbf1 Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 30 Nov 2022 12:59:38 -0600 Subject: [PATCH 22/22] fixed edge case where members information wasnt included --- snscrape/modules/telegram.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index 2e1322f9..64b8ca41 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -270,12 +270,13 @@ def _get_entity(self): raise snscrape.base.ScraperException(f'Got status code {r.status_code}') soup = bs4.BeautifulSoup(r.text, 'lxml') membersDiv = soup.find('div', class_ = 'tgme_page_extra') - if membersDiv.text.split(',')[0].endswith((' members', ' subscribers')): - membersStr = ''.join(membersDiv.text.split(',')[0].split(' ')[:-1]) - if membersStr == 'no': - kwargs['members'] = 0 - else: - kwargs['members'] = int(membersStr) + if membersDiv is not None: + if membersDiv.text.split(',')[0].endswith((' members', ' subscribers')): + membersStr = ''.join(membersDiv.text.split(',')[0].split(' ')[:-1]) + if membersStr == 'no': + kwargs['members'] = 0 + else: + kwargs['members'] = int(membersStr) photoImg = soup.find('img', class_ = 'tgme_page_photo_image') if photoImg is not None: kwargs['photo'] = photoImg.attrs['src']