youtube-dl/youtube_dl/extractor/weibo.py
2017-12-26 16:24:56 +08:00

152 lines
5.7 KiB
Python

# coding: utf-8
from __future__ import unicode_literals
from .common import InfoExtractor
from urllib.request import Request
from urllib.parse import urlencode
from urllib import parse
import json
import random as rnd
from os import path
import re
from ..utils import (
js_to_json,
)
class WeiboIE(InfoExtractor):
_VALID_URL = r'https?://weibo\.com/[0-9]+/(?P<id>[a-zA-Z0-9]+)'
_TEST = {
'url': 'https://weibo.com/6275294458/Fp6RGfbff?type=comment',
'info_dict': {
'id': 'Fp6RGfbff',
'ext': 'mp4',
'title': 'You should have servants to massage you,... 来自Hosico_猫 - 微博',
}
}
def _real_extract(self, url):
video_id = self._match_id(url)
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Safari/537.36',
'Upgrade-Insecure-Requests': '1',
}
# to get Referer url for genvisitor
webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="first visit the page")
visitor_url = urlh.geturl()
data = urlencode({
"cb": "gen_callback",
"fp": '{"os":"2","browser":"Gecko57,0,0,0","fonts":"undefined","screenInfo":"1440*900*24","plugins":""}',
}).encode()
headers = {
'Accept-Encoding': 'gzip, deflate, br',
'Accept': '*/*',
'Referer': visitor_url,
}
r_genvisitor = Request(
'https://passport.weibo.com/visitor/genvisitor',
data = data,
headers = headers,
method = 'POST'
)
webpage,urlh = self._download_webpage_handle(r_genvisitor, video_id, note="gen visitor")
p = webpage.split("&&")[1] # split "gen_callback && gen_callback(...)"
i1 = p.find('{')
i2 = p.rfind('}')
j = p[i1:i2+1] # get JSON object
d = json.loads(j)
tid = d["data"]["tid"]
cnfd = "%03d" % d["data"]["confidence"]
param = urlencode({
'a': 'incarnate',
't': tid,
'w': 2,
'c': cnfd,
'cb': 'cross_domain',
'from': 'weibo',
'_rand': rnd.random()
})
gencallback_url = "https://passport.weibo.com/visitor/visitor?" + param
webpage,urlh = self._download_webpage_handle(gencallback_url, video_id, note="gen callback")
webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="retry to visit the page")
# TODO more code goes here, for example ...
title = self._html_search_regex(r'<title>(.+?)</title>', webpage, 'title')
video_sources_text = self._search_regex("video-sources=\\\\\"(.+?)\"", webpage, 'video_sources')
video_formats = parse.parse_qs(video_sources_text)
formats = []
supported_resolutions = ['720', '480']
for res in supported_resolutions:
f = video_formats.get(res)
if isinstance(f, list):
if len(f) > 0:
vid_url = f[0]
formats.append({
'url': vid_url,
'format': 'mp4',
'height': int(res),
})
self._sort_formats(formats)
uploader = self._og_search_property('nick-name', webpage, 'uploader', default = None)
return {
'id': video_id,
'title': title,
'uploader': uploader,
'formats': formats
# TODO more properties (see youtube_dl/extractor/common.py)
}
class WeiboMobileIE(InfoExtractor):
_VALID_URL = r'https?://m.weibo.cn/status/(?P<id>[0-9]+)(\?.+)?'
_TEST = {
'url': 'https://m.weibo.cn/status/4189191225395228?wm=3333_2001&sourcetype=weixin&featurecode=newtitle&from=singlemessage&isappinstalled=0',
'info_dict': {
'id': '4189191225395228',
'ext': 'mp4',
'title': '午睡当然是要甜甜蜜蜜的啦',
'uploader': '柴犬柴犬'
}
}
def _real_extract(self, url):
video_id = self._match_id(url)
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en,zh-CN;q=0.9,zh;q=0.8',
'Upgrade-Insecure-Requests': '1',
}
# to get Referer url for genvisitor
webpage,urlh = self._download_webpage_handle(url, video_id, headers=headers, note="visit the page")
js_code = self._search_regex(r'var\s+\$render_data\s*=\s*\[({.*})\]\[0\] \|\| {};', webpage, 'js_code', flags = re.DOTALL)
weibo_info = self._parse_json(js_code, video_id, transform_source=js_to_json)
page_info = weibo_info['status']['page_info']
title = weibo_info['status']['status_title']
format = {
'url': page_info['media_info']['stream_url'],
'format': 'mp4',
}
formats = [format]
uploader = weibo_info['status']['user']['screen_name']
return {
'id': video_id,
'title': title,
'uploader': uploader,
'formats': formats
# TODO more properties (see youtube_dl/extractor/common.py)
}