ChatGPT Helped Python Tweepy my Tweets Onto My Website
by Mike Levin
Friday, January 27, 2023It’s amazing to me that I don’t have it documented here in this journal yet, but a few days ago, I hit the Twitter API using the Python tweepy package and it went well. I only got about 3K of my 11K tweets since 2007, but it’s a start. Here’s the page https://mikelev.in/tweets/ and here’s the code. It’s going to need some cleanup, but it’s a start.
ChatGPT Assisted Coding Session, Hitting Twitter API
import tweepy
import pandas as pd
from sqlitedict import SqliteDict as sqldict
with open("creds/twitter.txt", "rt") as fh:
api_key, api_key_secret, bearer_token = fh.read().split("\n")
auth = tweepy.OAuth2BearerHandler(bearer_token)
api = tweepy.API(auth)
username = "miklevin"
# Technique 1
with sqldict("tweets/raw_backfill.db") as db:
max_id = None
while True:
tweets = api.user_timeline(
screen_name=username, tweet_mode="extended", count=200, max_id=max_id
)
if len(tweets) == 0:
break
db[str(max_id)] = tweets
db.commit()
max_id = tweets[-1].id - 1
# Technique 2
with sqldict("tweets/raw2_backfill.db") as db:
max_id = 7567832972707979263
for page in tweepy.Cursor(api.user_timeline, screen_name=username, max_id=max_id).pages():
if len(tweets) == 0:
break
db[str(max_id)] = tweets
db.commit()
max_id = tweets[-1].id - 1
with sqldict("tweets/raw2_backfill.db") as db:
for key in db:
print(key)
with sqldict("tweets/raw_backfill.db") as db:
with sqldict("tweets/tweets.db") as db2:
for max_id in db:
tweets = db[max_id]
for tweet in tweets:
db2[tweet.id_str] = tweet
db2.commit()
table = []
with sqldict("tweets/tweets.db") as db:
for i, tweet_id in enumerate(db):
tweet = db[tweet_id]
id = tweet.id
created_at = tweet.created_at
full_text = tweet.full_text
url = f"https://twitter.com/miklevin/status/{id}"
row = (id, created_at, full_text, url)
table.append(row)
columns = ["id", "created_at", "full_text", "url"]
df = pd.DataFrame(table, columns=columns)
df.to_csv("tweets/tweets.csv", index=False)
front_matter = """---
title: Tweets
group: homepage
permalink: /tweets/
description: A page of all my tweets.
---
"""
with open("../hide/MikeLev.in/tweets.md", "wt") as fh:
fh.write(front_matter)
with open("../hide/MikeLev.in/tweets.md", "at") as fh:
for i, row in enumerate(table):
id, created_at, full_text, url = row
fh.write(f"\n\n## {created_at}")
fh.write(f"\n\n{full_text}")
fh.write(f"\n{url}")
print("Done")