Skip to content

Feature/pyppeteer merge better error handling #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Feature/pyppeteer merge better error handling #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8f9c3d8
merge better error handling
vdemay Nov 30, 2020
a26c71c
Ooops add missing coonflict fixes
vdemay Nov 30, 2020
0e2cffb
Continue to fix merge
vdemay Nov 30, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ api = TikTokApi()

results = 10

trending = api.trending(count=results)
# Since TikTok changed their API you need to use the custom_verifyFp option.
# In your web browser you will need to go to TikTok, Log in and get the s_v_web_id value.
trending = api.trending(count=results, custom_verifyFp="")

for tiktok in trending:
# Prints the id of the tiktok
Expand Down Expand Up @@ -132,7 +134,7 @@ executablePath - The path to your chromedriver if you don't want global install
##### The trending Method

```
api.trending(self, count=30, referrer="https://www.tiktok.com/@ondymikula/video/6756762109670477061", language='en', proxy=None)
api.trending(self, count=30, referrer="https://www.tiktok.com/@ondymikula/video/6756762109670477061", language='en', proxy=None, , custom_verifyFp="")
```

count - this is how many trending Tiktoks you want to be returned.
Expand All @@ -142,7 +144,7 @@ Trending returns an array of dictionaries. Example structure [here](https://www.
##### The get_Video_By_TikTok Method

```
api.get_Video_By_TikTok(data, language='en', proxy=None)
api.get_Video_By_TikTok(data, language='en', proxy=None, , custom_verifyFp="")
```

data - The tiktok dictionary returned from the API. Will return bytes.
Expand Down Expand Up @@ -301,7 +303,7 @@ getSuggestedMusicIDCrawler(self, count=30, startingId='6745191554350760966', lan
##### The get_Video_By_DownloadURL Method

```
api.get_Video_By_DownloadURL(url, language='en', proxy=None)
api.get_Video_By_DownloadURL(url, language='en', proxy=None, custom_verifyFp="")
```

url - The download url that's found in the TikTok dictionary. TikTok['video']['downloadAddr']
Expand All @@ -310,7 +312,7 @@ url - The download url that's found in the TikTok dictionary. TikTok['video']['d
##### The get_Video_By_Url Method

```
api.get_Video_By_Url(video_url, return_bytes=0)
api.get_Video_By_Url(video_url, return_bytes=0, custom_verifyFp="")
```

video_url - The video you want to get url.
Expand All @@ -320,7 +322,7 @@ return_bytes - The default value is 0, when it is set to 1 the function instead
##### The get_Video_No_Watermark_Faster Method

```
api.get_Video_No_Watermark(video_url, return_bytes=0, language='en', proxy=None)
api.get_Video_No_Watermark(video_url, return_bytes=0, language='en', proxy=None, custom_verifyFp="")
```

video_url - The video you want to get url.
Expand All @@ -329,7 +331,7 @@ return_bytes - The default value is 0, when it is set to 1 the function instead

If you request without bytes you will need to make a call to the URL it responds yourself to get bytes.
```
url = api.get_Video_No_Watermark_ID('6829267836783971589', return_bytes=0)
url = api.get_Video_No_Watermark_ID('6829267836783971589', return_bytes=0, custom_verifyFp="")

import requests
video_bytes = requests.get(url, headers={"User-Agent": "okhttp"}).content
Expand Down Expand Up @@ -370,7 +372,7 @@ You can use this method if you really want, but just use the 3 above it.
##### The get_Video_No_Watermark_ID Method

```
api.get_Video_No_Watermark_ID(self, video_id, return_bytes=1, proxy=None)
api.get_Video_No_Watermark_ID(self, video_id, return_bytes=1, proxy=None, custom_verifyFp="")
```

video_id - The video id you want to get.
Expand All @@ -388,7 +390,7 @@ video_bytes = requests.get(url, headers={"User-Agent": "okhttp"}).content

##### The get_Video_No_Watermark Method
```
api.get_Video_No_Watermark(self, video_url, return_bytes=0, proxy=None)
api.get_Video_No_Watermark(self, video_url, return_bytes=0, proxy=None, custom_verifyFp="")
```

This endpoint returns a url that is able to be opened in any browser, but sacrifices speed for this convenience. Any old request library can return the bytes if you decide to return a url.
Expand Down
11 changes: 6 additions & 5 deletions TikTokApi/browser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def __init__(
self.args = args
self.args.append("--user-agent=" + self.userAgent)

if self.proxy != None:
if self.proxy is not None:
if "@" in self.proxy:
self.args.append(
"--proxy-server="
Expand All @@ -96,7 +96,7 @@ def __init__(

self.options.update(options)

if self.executablePath != None:
if self.executablePath is not None:
self.options["executablePath"] = self.executablePath

if async_support:
Expand Down Expand Up @@ -133,7 +133,8 @@ async def newParams(self) -> None:

# self.browser_language = await self.page.evaluate("""() => { return navigator.language || navigator.userLanguage; }""")
self.browser_language = ""
# self.timezone_name = await self.page.evaluate("""() => { return Intl.DateTimeFormat().resolvedOptions().timeZone; }""")
# self.timezone_name = await self.page.evaluate("""() => { return
# Intl.DateTimeFormat().resolvedOptions().timeZone; }""")
self.timezone_name = ""
# self.browser_platform = await self.page.evaluate("""() => { return window.navigator.platform; }""")
self.browser_platform = ""
Expand Down Expand Up @@ -263,7 +264,7 @@ async def find_redirect(self):
self.browser.process.communicate()

def __format_proxy(self, proxy):
if proxy != None:
if proxy is not None:
return {"http": proxy, "https": proxy}
else:
return None
Expand All @@ -272,4 +273,4 @@ def __get_js(self):
return requests.get(
"https://sf16-muse-va.ibytedtos.com/obj/rc-web-sdk-gcs/acrawler.js",
proxies=self.__format_proxy(self.proxy),
).text
).text
178 changes: 126 additions & 52 deletions TikTokApi/tiktok.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,20 @@
import os
from .utilities import update_messager
from .browser import browser
from simplejson import JSONDecodeError

BASE_URL = "https://m.tiktok.com/"

class TikTokCaptchaError(Exception):
def __init__(self, message="TikTok blocks this request displaying a Captcha \nTip: Consider using a proxy or a custom_verifyFp as method parameters"):
self.message = message
super().__init__(self.message )

class TikTokNotFoundError(Exception):
def __init__(self, message="The requested object does not exists"):
self.message = message
super().__init__(self.message )

class TikTokApi:
def __init__(self, **kwargs):
"""The TikTokApi class. Used to interact with TikTok.
Expand Down Expand Up @@ -59,14 +70,14 @@ def __init__(self, **kwargs):
self.request_delay = kwargs.get("request_delay", None)

def external_signer(self, url, custom_did=None):
if custom_did != None:
if custom_did is not None:
query = {
"url": url,
"custom_did": custom_did
"url": url,
"custom_did": custom_did
}
else:
query = {
"url": url,
"url": url,
}
data = requests.get(self.signer_url + "?{}".format(urlencode(query)))
parsed_data = data.json()
Expand Down Expand Up @@ -96,7 +107,7 @@ def getData(self, b, **kwargs) -> dict:
if self.request_delay is not None:
time.sleep(self.request_delay)

if self.proxy != None:
if self.proxy is not None:
proxy = self.proxy

if self.signer_url == None:
Expand All @@ -121,19 +132,32 @@ def getData(self, b, **kwargs) -> dict:
"sec-fetch-mode": "cors",
"sec-fetch-site": "same-site",
"user-agent": userAgent,
"cookie": "tt_webid_v2=" + did + ';s_v_web_id=' + kwargs.get("custom_verifyFp", "verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
"cookie": "tt_webid_v2=" +
did +
';s_v_web_id=' +
kwargs.get(
"custom_verifyFp",
"verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
},
proxies=self.__format_proxy(proxy),
)
try:
json = r.json()
if json.get('type') == 'verify' :
logging.error("Tiktok wants to display a catcha. Response is:\n" + r.text)
raise TikTokCaptchaError()
return r.json()
except Exception as e:
logging.error(e)
logging.error(
"Converting response to JSON failed response is below (probably empty)"
)
logging.info(r.text)
raise Exception("Invalid Response")
except JSONDecodeError as e:
text = r.text
logging.error("TikTok response: " + text)
if len(text) == 0 :
raise Exception("Empty response from Tiktok to " + url) from None
else :
logging.error(
"Converting response to JSON failed response is below (probably empty)"
)
logging.error(e)
raise Exception("Invalid Response") from e

def getBytes(self, b, **kwargs) -> bytes:
"""Returns bytes of a response from TikTok.
Expand Down Expand Up @@ -678,17 +702,27 @@ def getMusicObjectFull(self, id, **kwargs):
maxCount,
did,
) = self.__process_kwargs__(kwargs)
r = requests.get("https://www.tiktok.com/music/-{}".format(id), headers={
r = requests.get(
"https://www.tiktok.com/music/-{}".format(id),
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" + kwargs.get("custom_verifyFp", "verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
}, proxies=self.__format_proxy(kwargs.get("proxy", None)))
"Cookie": "s_v_web_id=" +
kwargs.get(
"custom_verifyFp",
"verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
},
proxies=self.__format_proxy(
kwargs.get(
"proxy",
None)))
t = r.text
j_raw = t.split('<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]
j_raw = t.split(
'<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]
return json.loads(j_raw)['props']['pageProps']['musicInfo']

def byHashtag(self, hashtag, count=30, offset=0, **kwargs) -> dict:
Expand Down Expand Up @@ -765,8 +799,12 @@ def getHashtagObject(self, hashtag, **kwargs) -> dict:
api_url = "{}node/share/tag/{}?{}&{}".format(
BASE_URL, quote(hashtag), self.__add_new_params__(), urlencode(query)
)

b = browser(api_url, **kwargs)
return self.getData(b, **kwargs)
data = self.getData(b, **kwargs)
if data['challengeInfo'].get('challenge') is None:
raise TikTokNotFoundError("Challenge {} does not exist".format(hashtag))
return data

def getHashtagDetails(self, hashtag, **kwargs) -> dict:
"""Returns a hashtag object.
Expand Down Expand Up @@ -991,21 +1029,37 @@ def getUser(self, username, **kwargs) -> dict:
maxCount,
did,
) = self.__process_kwargs__(kwargs)
r = requests.get("https://tiktok.com/@{}?lang=en".format(username), headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"path": "/{}".format(username),
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" + kwargs.get("custom_verifyFp", "verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
}, proxies=self.__format_proxy(kwargs.get("proxy", None)))
r = requests.get(
"https://tiktok.com/@{}?lang=en".format(username),
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"path": "/{}".format(username),
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" +
kwargs.get(
"custom_verifyFp",
"verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
},
proxies=self.__format_proxy(
kwargs.get(
"proxy",
None)))

t = r.text

j_raw = t.split('<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]

try:
j_raw = t.split('<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]
except IndexError:
if not t:
logging.error("Tiktok response is empty")
else :
logging.error("Tiktok response: \n " + t)
raise TikTokCaptchaError() from None

return json.loads(j_raw)['props']['pageProps']

def getSuggestedUsersbyID(
Expand Down Expand Up @@ -1339,36 +1393,56 @@ def get_Video_No_Watermark(self, video_url, return_bytes=0, **kwargs) -> bytes:
return r.content

def get_music_title(self, id, **kwargs):
r = requests.get("https://www.tiktok.com/music/-{}".format(id), headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" + kwargs.get("custom_verifyFp", "verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
}, proxies=self.__format_proxy(kwargs.get("proxy", None)))
r = requests.get(
"https://www.tiktok.com/music/-{}".format(id),
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" +
kwargs.get(
"custom_verifyFp",
"verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
},
proxies=self.__format_proxy(
kwargs.get(
"proxy",
None)))
t = r.text
j_raw = t.split('<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]
j_raw = t.split(
'<script id="__NEXT_DATA__" type="application/json" crossorigin="anonymous">')[1].split("</script>")[0]
return json.loads(j_raw)['props']['pageProps']['musicInfo']['title']

def get_secUid(self, username, **kwargs):
r = requests.get("https://tiktok.com/@{}?lang=en".format(username), headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"path": "/{}".format(username),
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" + kwargs.get("custom_verifyFp", "verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
}, proxies=self.__format_proxy(kwargs.get("proxy", None)))
r = requests.get(
"https://tiktok.com/@{}?lang=en".format(username),
headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
"authority": "www.tiktok.com",
"path": "/{}".format(username),
"Accept-Encoding": "gzip, deflate",
"Connection": "keep-alive",
"Host": "www.tiktok.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.111 Safari/537.36",
"Cookie": "s_v_web_id=" +
kwargs.get(
"custom_verifyFp",
"verify_khgp4f49_V12d4mRX_MdCO_4Wzt_Ar0k_z4RCQC9pUDpX"),
},
proxies=self.__format_proxy(
kwargs.get(
"proxy",
None)))
try:
return r.text.split('"secUid":"')[1].split('","secret":')[0]
except IndexError as e:
logging.info(r.text)
logging.error(e)
raise Exception("Retrieving the user secUid failed. Likely due to TikTok wanting captcha validation. Try to use a proxy.")
raise Exception(
"Retrieving the user secUid failed. Likely due to TikTok wanting captcha validation. Try to use a proxy.")
#
# PRIVATE METHODS
#
Expand All @@ -1377,7 +1451,7 @@ def __format_proxy(self, proxy) -> dict:
"""
Formats the proxy object
"""
if proxy == None and self.proxy != None:
if proxy is None and self.proxy is not None:
proxy = self.proxy
if proxy is not None:
return {"http": proxy, "https": proxy}
Expand Down
Loading