twint Twitter爬虫报asyncio awaitable错误且拉取推文受限求助

基于twint框架开发的Twitter爬虫此前已稳定运行数月，2023年9月起在Python 3.8、Spyder 5.0.5环境中运行时出现两类异常：

• 触发TypeError，错误提示为「An asyncio.Future, a coroutine or an awaitable is required」
• 指定日期范围内仅能拉取到有限数量的推文

错误日志 #

[SpyderKernelApp] ERROR | Exception in message handler:
Traceback (most recent call last):
  File "C:\Users\james.coldman\Anaconda3\lib\site-packages\spyder_kernels\comms\frontendcomm.py", line 164, in poll_one
    asyncio.run(handler(out_stream, ident, msg))
  File "C:\Users\james.coldman\AppData\Roaming\Python\Python3.8\site-packages\nest_asyncio.py", line 32, in run
    return loop.run_until_complete(future)
  File "C:\Users\james.coldman\AppData\Roaming\Python\Python3.8\site-packages\nest_asyncio.py", line 60, in run_until_complete
    f = asyncio.ensure_future(future, loop=self)
  File "C:\Users\james.coldman\Anaconda3\lib\asyncio\tasks.py", line 673, in ensure_future
    raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
TypeError: An asyncio.Future, a coroutine or an awaitable is required
[SpyderKernelApp] ERROR | Exception in message handler:
Traceback (most recent call last):
  File "C:\Users\james.coldman\Anaconda3\lib\site-packages\spyder_kernels\comms\frontendcomm.py", line 164, in poll_one
    asyncio.run(handler(out_stream, ident, msg))
  File "C:\Users\james.coldman\AppData\Roaming\Python\Python3.8\site-packages\nest_asyncio.py", line 32, in run
    return loop.run_until_complete(future)
  File "C:\Users\james.coldman\AppData\Roaming\Python\Python3.8\site-packages\nest_asyncio.py", line 60, in run_until_complete
    f = asyncio.ensure_future(future, loop=self)
  File "C:\Users\james.coldman\Anaconda3\lib\asyncio\tasks.py", line 673, in ensure_future
    raise TypeError('An asyncio.Future, a coroutine or an awaitable is '
TypeError: An asyncio.Future, a coroutine or an awaitable is required

运行脚本 #

import nest_asyncio
import twint
import pandas as pd
import re
import os
import random
from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
from wordcloud import WordCloud, STOPWORDS
nest_asyncio.apply()


RE_EMOJI = re.compile('[\U00010000-\U0010ffff]', flags=re.UNICODE)
def strip_emoji(text):
    return RE_EMOJI.sub(r'', text)

def sentiment_score(tweet):
    score = analyser.polarity_scores(tweet)
    return score['neg'], score['neu'], score['pos'], score['compound']

def scrape(search_term, start_date, end_date):
    random_code = '%030x' % random.randrange(16**30)
    #configuration
    config = twint.Config()
    config.Search = search_term
    config.Lang = "en"
    config.Limit = 1000000
    config.Since = start_date
    config.Until = end_date
    config.Store_csv = True
    config.Output = "twintTerms"+search_term+str(random_code)+".csv"
    #running search
    twint.run.Search(config)
    #convert csv to excel file and append to df_list
    df = pd.read_csv("twintTerms"+search_term+str(random_code)+".csv")
    df['Search Term'] = search_term
    df['Start Date'] = start_date
    df['End Date'] = end_date
    en = df[df['language'] == "en"]
    #delete csv ready for next search term
    if os.path.exists("twintTerms"+search_term+str(random_code)+".csv"):
      os.remove("twintTerms"+search_term+str(random_code)+".csv")
    else:
      pass
    return en
      
def fetch_scrapes(search_dict):
    search_terms = search_dict['Search Term']
    start_dates = search_dict['Start Date']
    end_dates = search_dict['End Date']
    responses = []
    for key in search_terms.keys(): #build a list of futures for async to run
        results = scrape(str(search_terms[key]), str(start_dates[key]), str(end_dates[key]))
        responses.append(results)
    return responses


if __name__ == "__main__":
    print("Twitter Scrape Starting...")
    #delete any leftover files
    if os.path.exists("twitterResults.xlsx"):
        os.remove("twitterResults.xlsx")
    else:
        pass
    
    #create results directory
    if not os.path.exists('data/results'):
        os.makedirs('data/results')
    
    #pull in search terms
    search_df = pd.read_excel(r"data/TwitterTerms.xlsx")
    search_dict = search_df.to_dict()
    
    #run scrape
    dataframe_list = fetch_scrapes(search_dict)
    
    tweet_df = pd.concat(dataframe_list)
    tweet_df = tweet_df[['date','time','tweet','replies_count','retweets_count','likes_count','Search Term','Start Date','End Date']]
    tweet_df = tweet_df.drop_duplicates(['tweet','Search Term'])
    
    #remove emojis
    tweet_df['tweet'] = tweet_df['tweet'].apply(strip_emoji)
    
    #sentiment analysis
    analyser = SentimentIntensityAnalyzer()
    tweet_df[['Negative','Neutral','Positive','Compound']] = tweet_df['tweet'].apply(lambda x : pd.Series(sentiment_score(x)))
    tweet_df.to_excel(r"data/results/rawData.xlsx")
    grouped = tweet_df.groupby(['Search Term','Start Date','End Date']).agg(['mean','count'])
    grouped.to_csv(r'data/results/twitterResults.csv')
    
    #Most frequent words across topics
    extra_stopwords_list = [term.split(" ") for term in list(set(tweet_df['Search Term'].tolist()))]
    extra_stopwords_list = [i for j in extra_stopwords_list for i in j]
    words_list = []
    for team in list(set(tweet_df['Search Term'].tolist())):
        print(team)
        team_df = tweet_df[(tweet_df['Search Term'] == team)]
        raw_string = ' '.join(team_df['tweet'])
        no_links = re.sub(r'http\S+', '', raw_string)
        STOPWORDS = set(list(STOPWORDS) + extra_stopwords_list + ['amp'])
        wordcloud = WordCloud(stopwords = STOPWORDS).generate(no_links)
        most_popular = list(wordcloud.words_.keys())
        most_popular = [word for word in most_popular if len(word) > 2]
        perc_list = [[word,len(team_df[team_df['tweet'].str.contains(word)])] for word in most_popular]
        words_list.append(perc_list)
        df = pd.DataFrame()
        df['Word']= [i[0] for i in perc_list]
        df['Tweets Featured In'] = [i[1] for i in perc_list]
    test = []
    for lst in words_list:
        for lst2 in lst:
            word = lst2[0]
            counter = []
            for lst3 in words_list:
                for lst4 in lst3:
                    if word == lst4[0]:
                        counter.append(word)
            if len(counter) > 1:
                test.append(word)
    test = list(set(test))
    df_list = []
    for team in list(set(tweet_df['Search Term'].tolist())):
        team_df = tweet_df[tweet_df['Search Term'] == team]
        perc_list = [[word, (len(team_df[team_df['tweet'].str.contains(word)])/len(team_df))*100] for word in test]
        df = pd.DataFrame()
        df['Word'] = [i[0] for i in perc_list]
        df[team] = [i[1] for i in perc_list]
        df.set_index(team)
        df_list.append(df)
    words_df = pd.concat(df_list, axis = 1)
    words_df.to_excel(r"data/results/wordFrequencies.xlsx")
    print("Scrape Completed...")

TypeError报错修复 #

报错核心原因是Spyder 5.0.5内核自带的异步事件循环和手动调用的nest_asyncio补丁冲突，导致asyncio.run接收的处理对象不是合法可等待类型：

• 优先使用命令行终端直接执行py脚本，绕过Spyder交互式内核的异步兼容问题，90%以上同类报错可直接解决
• 若必须使用Spyder运行，注释代码中的nest_asyncio.apply()行，同时在twint配置中新增config.Force_tor = False关闭强制异步调用
• 升级twint到最新维护版本，不要使用PyPI源默认的2019年停止更新的旧版本，旧版本异步逻辑和Python 3.8+存在兼容缺陷

爬取推文数量不足修复 #

2023年9月X平台（原Twitter）大幅收紧未登录用户的接口访问权限，旧版twint默认的未登录爬取逻辑被限制，单次最多返回3200条历史推文，部分时间范围数据直接不返回：

• 在twint配置中添加X平台登录凭证，新增配置项：

config.Username = "你的X平台登录用户名"
config.Password = "你的X平台登录密码"
config.Retries = 10

• 将大的日期范围拆分为按天/按周的小时间切片，每爬完一个切片随机休眠30-60秒，避免触发平台限流机制
• 若仍存在数据缺失，可开启配置项config.Debug = True查看具体接口返回报错，排查是否触发账号临时限制

内容的提问来源于stack exchange，提问作者James Coldman