From 6367ab7ffe8886969cbd6a99c3e828c8c4a7d64a Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sun, 16 May 2021 15:19:03 +0300 Subject: [PATCH 01/10] fleast: Generate oauth token in runtime --- .gitignore | 1 - main.py | 6 +++--- twitch.py | 58 ++++++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 51 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 1f87ab3..e62cb77 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ __pycache__ *.swp .token -.oauth .secret server.conf .*.log diff --git a/main.py b/main.py index 327c283..39c53d3 100755 --- a/main.py +++ b/main.py @@ -12,8 +12,8 @@ class FleastServer(object): try: with open('.token', 'r') as reader: self.twitch_token = reader.read().strip() - with open('.oauth', 'r') as reader: - self.oauth_token = reader.read().strip() + with open('.secret', 'r') as reader: + self.secret = reader.read().strip() with open('./web/fl.html', 'r') as reader: self.index_page = reader.read() with open('./web/fl_template_main.html', 'r') as reader: @@ -22,7 +22,7 @@ class FleastServer(object): self.templ_stream = reader.read() with open('./web/fl_template_lang.html', 'r') as reader: self.templ_lang = reader.read().splitlines() - self.client = TwitchClient(self.twitch_token, self.oauth_token, freq=1) + self.client = TwitchClient(self.twitch_token, self.secret, freq=1) except: print("Cannot read token for twitch app or templates, abort.") exit(1) diff --git a/twitch.py b/twitch.py index aa12b2a..d1e27aa 100644 --- a/twitch.py +++ b/twitch.py @@ -2,21 +2,54 @@ import cherrypy import requests import threading import time + +from oauthlib.oauth2 import BackendApplicationClient +from requests_oauthlib import OAuth2Session from urllib.parse import quote class TwitchClient: - def __init__(self, token, oauth, freq=2): + def __init__(self, token, secret, freq=2): self.token = token - self.oauth = oauth self.lock = threading.Lock() - self.header_v6 = {'Client-ID': self.token, 'Authorization': 'Bearer ' + self.oauth} + self.header_v6 = {'Client-ID': self.token} self.urlbase_v6 = 'https://api.twitch.tv/helix' self.last_q = time.time() self.delay = 1 / freq + # authentication + self.oauth = '' + self.oauth_token_url = 'https://id.twitch.tv/oauth2/token' + self.auth_secret = secret + oauth_clint = BackendApplicationClient(client_id=self.token) + self.oauth_session = OAuth2Session(client=oauth_clint) + self.update_oauth() + + def update_oauth(self): + """ + Update self.oauth token based on client id and secret. + :return: nothing + """ + token = self.oauth_session.fetch_token(token_url=self.oauth_token_url, + client_secret=self.auth_secret, + include_client_id=True) + self.oauth = token['access_token'] + + def do_q_auth_v6(self, base, header): + """ + Do query with v6 authentication header and single retry. + :param base: string with requesting URL + :param header: dictionary of http headers + :return: string with response or None + """ + result = self.do_q(base, header | {'Authorization': 'Bearer ' + self.oauth}) + if result is not None: + return result + self.update_oauth() + return self.do_q(base, header | {'Authorization': 'Bearer ' + self.oauth}) + def do_q(self, base, header): """ Do query for twitch server @@ -31,8 +64,13 @@ class TwitchClient: if delta < self.delay: time.sleep(delta) # Sleep remaining time r = requests.get(base, headers=header).json() + error_message = r.get("error", "") + if len(error_message) > 0: + cherrypy.log(f'Request: fail with error "{error_message}"') + r = None + else: + cherrypy.log('Request: OK') self.last_q = time.time() - cherrypy.log('Request: OK') except requests.exceptions.RequestException as e: cherrypy.log('Request: FAIL') cherrypy.log('Error: {}'.format(e)) @@ -62,7 +100,7 @@ class TwitchClient: :return: string with get query result or None """ header, base = self.get_base('v6') - return self.do_q(base + q, header) + return self.do_q_auth_v6(base + q, header) def get_game_id_v6(self, name): """ @@ -72,7 +110,7 @@ class TwitchClient: """ header, base = self.get_base('v6') - r = self.do_q('{}/games?name={}'.format(base, name), header) + r = self.do_q_auth_v6('{}/games?name={}'.format(base, name), header) if r and r.get('data'): return r['data'][0]['id'], r['data'][0]['name'] @@ -92,11 +130,11 @@ class TwitchClient: header, base = self.get_base('v6') init_q_template = "{}/streams?language={}&first={}&game_id={}" q_template = "{}/streams?language={}&first={}&after={}&game_id={}" - data = self.do_q(init_q_template.format(base, lang, 100, game_id[0]), header) + data = self.do_q_auth_v6(init_q_template.format(base, lang, 100, game_id[0]), header) result['streams'].extend(data['data']) while len(data.get('data', [])) > 0: # there must be non zero value, but search is kinda broken now result['streams'].extend(data['data']) - data = self.do_q(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id[0]), header) + data = self.do_q_auth_v6(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id[0]), header) return self.unique_streams_v6(result) def get_irl_live_streams_v6(self, lang): @@ -111,11 +149,11 @@ class TwitchClient: game_id += '&game_id={}'.format(irl_id) result = {'_total': 0, 'streams': []} - data = self.do_q(init_q_template.format(base, lang, 100, game_id), header) + data = self.do_q_auth_v6(init_q_template.format(base, lang, 100, game_id), header) result['streams'].extend(data['data']) while len(data.get('data', [])) > 0: # there must be non zero value, but search is kinda broken now result['streams'].extend(data['data']) - data = self.do_q(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id), header) + data = self.do_q_auth_v6(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id), header) return self.unique_streams_v6(result) def unique_streams_v6(self, result): From 6ca8f3cd3c3c5f4eab500cfa4ad803073e44d35c Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sun, 16 May 2021 15:27:15 +0300 Subject: [PATCH 02/10] Bump version number --- README.md | 19 +++++++++++++++++++ main.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97a2859..8049a11 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,25 @@ Least-favourite twitch streamers here ## Change Log +### 1.9 +#### Fixed +- Char escaping and various typos. +- Unique filter for streams. + +#### Added +- Runtime ouath token generation. + +#### Changed +- Use threshold on amount of returned streams to resend requests. + +#### Removed +- v5 (kraken) API support. +- Daemonizer component. + +### 1.8 +#### Fixed +- Search queries for v5 and v6 API + ### 1.7 - Fixed bug with displaying special symbols in stream name ### 1.6 diff --git a/main.py b/main.py index 39c53d3..485fa37 100755 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ import cherrypy from twitch import TwitchClient -ver = '1.9.0-pre-3' +ver = '1.9.0' class FleastServer(object): From be894106a355e98a390306f24840806bfae547b0 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sun, 16 May 2021 15:33:01 +0300 Subject: [PATCH 03/10] Fix string formatting for older python3 versions --- twitch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/twitch.py b/twitch.py index d1e27aa..8d54eb8 100644 --- a/twitch.py +++ b/twitch.py @@ -66,7 +66,7 @@ class TwitchClient: r = requests.get(base, headers=header).json() error_message = r.get("error", "") if len(error_message) > 0: - cherrypy.log(f'Request: fail with error "{error_message}"') + cherrypy.log('Request: fail with error "%s"' % error_message) r = None else: cherrypy.log('Request: OK') From ca2b4f4d59d5ef453826053bc561a0e1a6245c1e Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sat, 3 Jul 2021 11:14:24 +0300 Subject: [PATCH 04/10] twitch: Do not add initial result search twice --- twitch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/twitch.py b/twitch.py index 8d54eb8..ceec7ac 100644 --- a/twitch.py +++ b/twitch.py @@ -150,7 +150,6 @@ class TwitchClient: result = {'_total': 0, 'streams': []} data = self.do_q_auth_v6(init_q_template.format(base, lang, 100, game_id), header) - result['streams'].extend(data['data']) while len(data.get('data', [])) > 0: # there must be non zero value, but search is kinda broken now result['streams'].extend(data['data']) data = self.do_q_auth_v6(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id), header) From 821389605a9a43d99bc2d039d30b69f759de6e95 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sat, 3 Jul 2021 11:15:05 +0300 Subject: [PATCH 05/10] twitch: Handle responses without cursor value --- twitch.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/twitch.py b/twitch.py index ceec7ac..2c5e060 100644 --- a/twitch.py +++ b/twitch.py @@ -152,6 +152,8 @@ class TwitchClient: data = self.do_q_auth_v6(init_q_template.format(base, lang, 100, game_id), header) while len(data.get('data', [])) > 0: # there must be non zero value, but search is kinda broken now result['streams'].extend(data['data']) + if data['pagination'].get("cursor", None) is None: # sometimes server return results without cursor + break data = self.do_q_auth_v6(q_template.format(base, lang, 100, data['pagination']['cursor'], game_id), header) return self.unique_streams_v6(result) From 538f50847e27d3235e3d50202ec0184694f29c8b Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sat, 3 Jul 2021 11:19:17 +0300 Subject: [PATCH 06/10] Bump version number to v1.9.1 --- README.md | 5 +++++ main.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8049a11..ad1e802 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,11 @@ Least-favourite twitch streamers here ## Change Log +### 1.9.1 +#### Fixed +- Server does not fail when processing API response withour `cursor` field. +- String formatting for older python3 versions. + ### 1.9 #### Fixed - Char escaping and various typos. diff --git a/main.py b/main.py index 485fa37..2270d96 100755 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ import cherrypy from twitch import TwitchClient -ver = '1.9.0' +ver = '1.9.1' class FleastServer(object): From 8d8226a5fa756bccddea9d73a9f90e7ae645cdbe Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 23 Dec 2021 18:11:13 +0300 Subject: [PATCH 07/10] Refresh website design - adaptive 4-wide grid for stream, - bootstrap elements - better wording --- web/fl.html | 140 ++++++++++++++++++++++-------------- web/fl_template_lang.html | 25 ++++--- web/fl_template_main.html | 136 +++++++++++++++++++++-------------- web/fl_template_stream.html | 4 +- web/style.css | 99 ------------------------- 5 files changed, 186 insertions(+), 218 deletions(-) delete mode 100644 web/style.css diff --git a/web/fl.html b/web/fl.html index e554367..0f1026b 100644 --- a/web/fl.html +++ b/web/fl.html @@ -3,64 +3,100 @@ FLeast - + - - -
-
-

Usage:

-

- Write the name of the game and select the language.
- The name must _exactly_ match the name of the game on twitch.tv - i.e. csgo must be "Counter-Strike: Global Offensive".

- IRL is back! Sort of.
- You can type IRL in game field and it will output streams from - Just Chatting, Travel & Outdoors, etc... -

-
- - - - - - - - - - - - -
- - - -
- - - - - -
-
-
+ + + +
+
+

Fleast — find new favourite small streamer

+

- Developed by Alex Vanin | twitter | - github
- Version: {_version_} + Search streams by specifying category from twitch.tv and the + language of the stream.
Streams will be sorted by ascending order.

- +

+ The category must be specified by exact name.
For example, + if you are looking for Counter-Strike, then you should write + Counter-Strike: Global Offensive. +

+

+ You can use IRL like in good old days.
The output will + combine Just Chatting, Travel & Outdoors, and other categories. +

+
+ + + + + + + + + + +
+ Category + + Language +
+ + + + + +
+
+
+
+
+ Developed by Alexey Vanin | twitter | + github
+ Version: {_version_} +

+
+
+
diff --git a/web/fl_template_lang.html b/web/fl_template_lang.html index 609810a..9142703 100644 --- a/web/fl_template_lang.html +++ b/web/fl_template_lang.html @@ -1,13 +1,12 @@ - - - - - - - - - - - - - + + + + + + + + + + + + \ No newline at end of file diff --git a/web/fl_template_main.html b/web/fl_template_main.html index df7fd10..a72eb86 100644 --- a/web/fl_template_main.html +++ b/web/fl_template_main.html @@ -3,66 +3,98 @@ FLeast - + + + - -
-
-

Usage:

-

- Write the name of the game and select the language.
- The name must _exactly_ match the name of the game on twitch.tv - i.e. csgo must be "Counter-Strike: Global Offensive".

- You can type IRL in game field and it will output streams from - Just Chatting, Travel & Outdoors, etc... -

-
- - - - - - - - -
- - - -
- - - + + + + + + + - - - + + + +
+ Category + + Language +
+ + + - - -
+ +
- - -
-

Found {_stream_num_} streams:


-
+ +
+

Found {_stream_num_} streams:

+
{_stream_list_} +
+
+
+
+ Developed by Alexey Vanin | twitter | + github
+ Version: {_version_} +

-
- -
-

- Developed by Alex Vanin | twitter | - github
- Version: {_version_} -

-
+ + + diff --git a/web/fl_template_stream.html b/web/fl_template_stream.html index 6712182..dca3a08 100644 --- a/web/fl_template_stream.html +++ b/web/fl_template_stream.html @@ -1,5 +1,5 @@ -
-
+
+
{2}
{3} :{4} diff --git a/web/style.css b/web/style.css deleted file mode 100644 index 203b276..0000000 --- a/web/style.css +++ /dev/null @@ -1,99 +0,0 @@ -body{background: #333333; line-height:1; font-family: arial;} -h1{font-size: 25px;}h2{font-size: 21px;}h3{font-size: 18px;}h4{font-size: 16px;} -table{border-collapse:collapse;border-spacing:0} -hr{display:block;height:1px;border:0;border-top:1px solid #ccc;margin:1em 0;padding:0} - -#pageContent { - margin:0;padding:0;border:0;outline:0 - max-width: 1000px; - margin: auto; - border: none; -} - -#header { - padding:10px; - -} - -main { - float: left; - width: 60%; -} -aside { - float: right; - width: 30%; -} -article { - border-bottom: 2px dotted #999; - padding-bottom: 20px; - margin-bottom: 20px; -} -article h2 { - font-weight: normal; - margin-bottom: 12px; -} -article p { - -} -main section { - -} -footer { - background: #AEC6CF; - max-width: 1000px; - margin: auto; - clear: both; - text-align: right; -} -footer p { - padding: 20px; -} - -aside > div { - margin: 10px auto; - background: #AEC6CF; - min-height: 100px; -} - -body > section { - max-width: 1000px; - margin: auto; - padding: 30px 0px; - border-bottom: 1px solid #999; - color: #fff; -} - -table.Input td, table.Input th { - border: 0px solid #AAAAAA; - padding: 2px 2px; -} - -.container { - display: flex; - flex-wrap: wrap; - padding: 0px 0px 0px 10px -} - -.inner { - overflow: auto; - word-wrap: break-word; - background-color: transparent; - width: 330px; - padding: 0px 0px 30px 0px -} - -.strname { - font-weight: bold; -} - -.struser { - font-size: 15px; - font-weight: bold; - color: #0f0 -} -.strviews{ - font-size: 15px; - font-weight: bold; - color: #f00 -} - From 5daa97af3d2a595970df3e63a00dc7000b0f5c78 Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 23 Dec 2021 18:14:10 +0300 Subject: [PATCH 08/10] Release v1.10.0 --- README.md | 4 ++++ main.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index ad1e802..2d1f3a1 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,10 @@ Least-favourite twitch streamers here ## Change Log +### 1.10.0 +#### Changed +- Refresh website design + ### 1.9.1 #### Fixed - Server does not fail when processing API response withour `cursor` field. diff --git a/main.py b/main.py index 2270d96..07a12a7 100755 --- a/main.py +++ b/main.py @@ -4,7 +4,7 @@ import cherrypy from twitch import TwitchClient -ver = '1.9.1' +ver = '1.10.0' class FleastServer(object): From e3229948426f41484e9b4bc19c5a67dd811aa6ae Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Thu, 23 Dec 2021 23:07:46 +0300 Subject: [PATCH 09/10] Update list of IRL category --- twitch.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/twitch.py b/twitch.py index 2c5e060..194b2fb 100644 --- a/twitch.py +++ b/twitch.py @@ -143,8 +143,28 @@ class TwitchClient: q_template = "{}/streams?language={}&first={}&after={}{}" game_id = '' - irl_ids = ["509660", "509673", "509667", "509669", "509670", "509658", - "509672", "509671", "509664", "509663", "417752", "509659"] + """ + 417752: Talk Shows & Podcasts + 509658: Just Chatting + 509659: ASMR + 509660: Art + 509663: Special Events + 509664: Tabletop RPGs + 509667: Food & Drink + 509669: Beauty & Body Art + 509670: Science & Technology + 509671: Fitness & Health [exclude] + 509672: Travel & Outdoors + 509673: Makers & Crafting + 515214: Politics [exclude] + 518203: Sports + 116747788: Pools, Hot Tubs, and Beaches + 272263131: Animals, Aquariums, and Zoos [exclude] + 1469308723: Software and Game Development + """ + irl_ids = ["417752", "509658", "509659", "509660", "509663", "509664", + "509667", "509669", "509670", "509672", "509673", "518203", + "116747788", "1469308723"] for irl_id in irl_ids: game_id += '&game_id={}'.format(irl_id) From a2c4582a0c4a5977c998a89011cb1b0f7c0518db Mon Sep 17 00:00:00 2001 From: Alex Vanin Date: Sun, 6 Feb 2022 14:31:05 +0300 Subject: [PATCH 10/10] web: Break long stream names to fin in div --- main.py | 2 +- web/fl.html | 1 + web/fl_template_main.html | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 07a12a7..bee436d 100755 --- a/main.py +++ b/main.py @@ -90,7 +90,7 @@ class FleastServer(object): s['thumbnail_url'].format(width=320, height=180), self.to_html(s['title']), s['user_name'], - s['viewer_count']) + '\n' + s['viewer_count']) return self.templ_main.format(_stream_num_=len(data['streams']), _game_name_=game, diff --git a/web/fl.html b/web/fl.html index 0f1026b..2a93ab7 100644 --- a/web/fl.html +++ b/web/fl.html @@ -22,6 +22,7 @@ .strname {{ font-weight: bold; + word-break: break-word; }} .struser {{ diff --git a/web/fl_template_main.html b/web/fl_template_main.html index a72eb86..d205725 100644 --- a/web/fl_template_main.html +++ b/web/fl_template_main.html @@ -22,6 +22,7 @@ .strname {{ font-weight: bold; + word-break: break-word; }} .struser {{