将中断置于循环中以获取 reddit 抓取代码

如何解决将中断置于循环中以获取 reddit 抓取代码

我希望通过 python 获取特定日期之间的所有 reddit 评论。下面的代码是我正在尝试的，它从 start_time (= int(dt.datetime(2020,5,1,0).timestamp())) 开始，但是当它到达 end_time (= end_time = int(dt.datetime(2020,4,20,0).timestamp())) 四月一日。

有谁知道如何更新它以在到达此 end_time 时导致循环中断？

submission_df = pd.DataFrame(None,columns=["id","title","score","author","author_flair_text","body","total_awards_received","awarders","created_utc","full_link","num_comments"
                                           ])
values = []
def downloadFromUrl(df,object_type):
    count = 0
    previous_epoch = int(start_time) #change this to just start_time?
    if len(values) > 0:
        previous_epoch = values[-1]['created_utc'] - 1
    print(f"Saving {object_type}s")


    while True:
        new_url = url.format(object_type,subreddit) + str(previous_epoch)
        json = requests.get(new_url)
        try:
            json_data = json.json()
        except:
            print(json)
        else:
            if 'data' not in json_data:
                break
            objects = json_data['data']
            if len(objects) == 0:
                break

            for submission in objects:
                previous_epoch = submission['created_utc'] - 1
                count += 1
                if object_type == 'comment':
                    try:
                        handle.write(str(submission['score']))
                        handle.write(" : ")
                        handle.write(datetime.fromtimestamp(submission['created_utc']).strftime("%Y-%m-%d"))
                        handle.write("\n")
                        text = submission['body']
                        textASCII = text.encode(encoding='ascii',errors='ignore').decode()
                        handle.write(textASCII)
                        handle.write("\n-------------------------------\n")
                    except Exception as err:
                        print(f"Couldn't print comment: https://www.reddit.com{submission['permalink']}")
                        print(traceback.format_exc())

                elif object_type == 'submission':
                    if 'awarders' in submission:
                        awarders = submission['awarders']
                    else:
                        awarders = None
                    if 'total_awards_received' in submission:
                        total_awards = submission['total_awards_received']
                    else:
                        total_awards = None
                    if 'body' in submission:
                        body = submission['body']
                    else:
                        body = None

                    try:
                        values.append(dict(zip(submission_df.columns,[submission['id'],submission['title'],submission['score'],submission['author'],submission['author_flair_text'],body,total_awards,awarders,submission['created_utc'],submission['full_link'],submission['num_comments'],submission['over_18']])))
                    except Exception as err:
                        print(f"Couldn't print post: {submission['url']}")
                        print(traceback.format_exc())
                    
        print(f"Saved {len(values)} of {count} {object_type}s through {datetime.fromtimestamp(previous_epoch).strftime('%Y-%m-%d')}")

    print(f"Saved {count} {object_type}s")
    return values
    # handle.close()")
    
new_values = downloadFromUrl(submission_df,"submission")

解决方法

创建对象数组后永远不会修改它，因此它的长度永远不会改变。即使你这样做了，你也是在循环内创建对象，所以它总是满的。