Twitter DM downloader
This commit is contained in:
14
.env.example
Normal file
14
.env.example
Normal file
@@ -0,0 +1,14 @@
|
||||
# Since the ".env" file is gitignored, you can use the ".env.example" file to
|
||||
# build a new ".env" file when you clone the repo. Keep this file up-to-date
|
||||
# when you add new variables to `.env`.
|
||||
|
||||
# This file will be committed to version control, so make sure not to have any
|
||||
# secrets in it. If you are cloning this repo, create a copy of this file named
|
||||
# ".env" and populate it with your secrets.
|
||||
|
||||
# When adding additional environment variables, the schema in "/src/env.mjs"
|
||||
# should be updated accordingly.
|
||||
|
||||
# Example:
|
||||
# SERVERVAR="foo"
|
||||
# NEXT_PUBLIC_CLIENTVAR="bar"
|
||||
35
.eslintrc.cjs
Normal file
35
.eslintrc.cjs
Normal file
@@ -0,0 +1,35 @@
|
||||
// eslint-disable-next-line @typescript-eslint/no-var-requires
|
||||
const path = require("path");
|
||||
|
||||
/** @type {import("eslint").Linter.Config} */
|
||||
const config = {
|
||||
overrides: [
|
||||
{
|
||||
extends: [
|
||||
"plugin:@typescript-eslint/recommended-requiring-type-checking",
|
||||
],
|
||||
files: ["*.ts", "*.tsx"],
|
||||
parserOptions: {
|
||||
project: path.join(__dirname, "tsconfig.json"),
|
||||
},
|
||||
},
|
||||
],
|
||||
parser: "@typescript-eslint/parser",
|
||||
parserOptions: {
|
||||
project: path.join(__dirname, "tsconfig.json"),
|
||||
},
|
||||
plugins: ["@typescript-eslint"],
|
||||
extends: ["next/core-web-vitals", "plugin:@typescript-eslint/recommended"],
|
||||
rules: {
|
||||
"@typescript-eslint/consistent-type-imports": [
|
||||
"warn",
|
||||
{
|
||||
prefer: "type-imports",
|
||||
fixStyle: "inline-type-imports",
|
||||
},
|
||||
],
|
||||
"@typescript-eslint/no-unused-vars": ["warn", { argsIgnorePattern: "^_" }],
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
42
.gitignore
vendored
Normal file
42
.gitignore
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
|
||||
|
||||
# dependencies
|
||||
/node_modules
|
||||
/.pnp
|
||||
.pnp.js
|
||||
|
||||
# testing
|
||||
/coverage
|
||||
|
||||
# database
|
||||
/prisma/db.sqlite
|
||||
/prisma/db.sqlite-journal
|
||||
|
||||
# next.js
|
||||
/.next/
|
||||
/out/
|
||||
next-env.d.ts
|
||||
|
||||
# production
|
||||
/build
|
||||
|
||||
# misc
|
||||
.DS_Store
|
||||
*.pem
|
||||
|
||||
# debug
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# local env files
|
||||
# do not commit any .env files to git, except for the .env.example file. https://create.t3.gg/en/usage/env-variables#using-environment-variables
|
||||
.env
|
||||
.env*.local
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
||||
# typescript
|
||||
*.tsbuildinfo
|
||||
28
README.md
Normal file
28
README.md
Normal file
@@ -0,0 +1,28 @@
|
||||
# Create T3 App
|
||||
|
||||
This is a [T3 Stack](https://create.t3.gg/) project bootstrapped with `create-t3-app`.
|
||||
|
||||
## What's next? How do I make an app with this?
|
||||
|
||||
We try to keep this project as simple as possible, so you can start with just the scaffolding we set up for you, and add additional things later when they become necessary.
|
||||
|
||||
If you are not familiar with the different technologies used in this project, please refer to the respective docs. If you still are in the wind, please join our [Discord](https://t3.gg/discord) and ask for help.
|
||||
|
||||
- [Next.js](https://nextjs.org)
|
||||
- [NextAuth.js](https://next-auth.js.org)
|
||||
- [Prisma](https://prisma.io)
|
||||
- [Tailwind CSS](https://tailwindcss.com)
|
||||
- [tRPC](https://trpc.io)
|
||||
|
||||
## Learn More
|
||||
|
||||
To learn more about the [T3 Stack](https://create.t3.gg/), take a look at the following resources:
|
||||
|
||||
- [Documentation](https://create.t3.gg/)
|
||||
- [Learn the T3 Stack](https://create.t3.gg/en/faq#what-learning-resources-are-currently-available) — Check out these awesome tutorials
|
||||
|
||||
You can check out the [create-t3-app GitHub repository](https://github.com/t3-oss/create-t3-app) — your feedback and contributions are welcome!
|
||||
|
||||
## How do I deploy this?
|
||||
|
||||
Follow our deployment guides for [Vercel](https://create.t3.gg/en/deployment/vercel), [Netlify](https://create.t3.gg/en/deployment/netlify) and [Docker](https://create.t3.gg/en/deployment/docker) for more information.
|
||||
89
download_dms.py
Normal file
89
download_dms.py
Normal file
@@ -0,0 +1,89 @@
|
||||
import requests
|
||||
import csv
|
||||
import time
|
||||
from datetime import datetime
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from requests_oauthlib import OAuth1
|
||||
load_dotenv()
|
||||
|
||||
def fetch_dms(auth):
|
||||
all_dms = []
|
||||
next_token = None
|
||||
|
||||
while True:
|
||||
# Construct URL with pagination
|
||||
url = 'https://api.twitter.com/2/dm_events'
|
||||
params = {
|
||||
'dm_event.fields': 'id,text,created_at,sender_id,recipient_id',
|
||||
'max_results': 100 # Max allowed per request
|
||||
}
|
||||
|
||||
if next_token:
|
||||
params['pagination_token'] = next_token
|
||||
|
||||
response = requests.get(
|
||||
url,
|
||||
auth=auth,
|
||||
params=params
|
||||
)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.json())
|
||||
break
|
||||
|
||||
data = response.json()
|
||||
|
||||
if 'data' in data:
|
||||
all_dms.extend(data['data'])
|
||||
|
||||
# Check if there are more DMs to fetch
|
||||
if 'meta' in data and 'next_token' in data['meta']:
|
||||
next_token = data['meta']['next_token']
|
||||
# Add a small delay to avoid rate limits
|
||||
time.sleep(1)
|
||||
else:
|
||||
break
|
||||
|
||||
return all_dms
|
||||
|
||||
def save_to_csv(dms, filename='twitter_dms.csv'):
|
||||
if not dms:
|
||||
print("No DMs to save")
|
||||
return
|
||||
|
||||
# Get all possible keys from the DMs
|
||||
fieldnames = set()
|
||||
for dm in dms:
|
||||
fieldnames.update(dm.keys())
|
||||
|
||||
with open(filename, 'w', newline='', encoding='utf-8') as csvfile:
|
||||
writer = csv.DictWriter(csvfile, fieldnames=list(fieldnames))
|
||||
writer.writeheader()
|
||||
writer.writerows(dms)
|
||||
|
||||
print(f"Saved {len(dms)} DMs to {filename}")
|
||||
|
||||
def main():
|
||||
# Create OAuth1 auth object
|
||||
auth = OAuth1(
|
||||
os.getenv("TWITTER_CLIENT_ID"),
|
||||
os.getenv("TWITTER_CLIENT_SECRET"),
|
||||
os.getenv("TWITTER_ACCESS_TOKEN"),
|
||||
os.getenv("TWITTER_ACCESS_TOKEN_SECRET")
|
||||
)
|
||||
|
||||
print("Fetching DMs...")
|
||||
dms = fetch_dms(auth)
|
||||
|
||||
if dms:
|
||||
print(f"Found {len(dms)} DMs")
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
filename = f'twitter_dms_{timestamp}.csv'
|
||||
save_to_csv(dms, filename)
|
||||
else:
|
||||
print("No DMs found")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
22
next.config.mjs
Normal file
22
next.config.mjs
Normal file
@@ -0,0 +1,22 @@
|
||||
/**
|
||||
* Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation. This is especially useful
|
||||
* for Docker builds.
|
||||
*/
|
||||
await import("./src/env.mjs");
|
||||
|
||||
/** @type {import("next").NextConfig} */
|
||||
const config = {
|
||||
reactStrictMode: true,
|
||||
|
||||
/**
|
||||
* If you have `experimental: { appDir: true }` set, then you must comment the below `i18n` config
|
||||
* out.
|
||||
*
|
||||
* @see https://github.com/vercel/next.js/issues/41980
|
||||
*/
|
||||
i18n: {
|
||||
locales: ["en"],
|
||||
defaultLocale: "en",
|
||||
},
|
||||
};
|
||||
export default config;
|
||||
5660
package-lock.json
generated
Normal file
5660
package-lock.json
generated
Normal file
File diff suppressed because it is too large
Load Diff
45
package.json
Normal file
45
package.json
Normal file
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"name": "dm-dump",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "next build",
|
||||
"dev": "next dev",
|
||||
"lint": "next lint",
|
||||
"start": "next start"
|
||||
},
|
||||
"dependencies": {
|
||||
"@t3-oss/env-nextjs": "^0.3.1",
|
||||
"@tanstack/react-query": "^4.29.7",
|
||||
"@trpc/client": "^10.26.0",
|
||||
"@trpc/next": "^10.26.0",
|
||||
"@trpc/react-query": "^10.26.0",
|
||||
"@trpc/server": "^10.26.0",
|
||||
"next": "^13.4.2",
|
||||
"oauth-1.0a": "^2.2.6",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"superjson": "1.12.2",
|
||||
"zod": "^3.21.4"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/eslint": "^8.37.0",
|
||||
"@types/node": "^18.16.0",
|
||||
"@types/prettier": "^2.7.2",
|
||||
"@types/react": "^18.2.6",
|
||||
"@types/react-dom": "^18.2.4",
|
||||
"@typescript-eslint/eslint-plugin": "^5.59.6",
|
||||
"@typescript-eslint/parser": "^5.59.6",
|
||||
"autoprefixer": "^10.4.14",
|
||||
"eslint": "^8.40.0",
|
||||
"eslint-config-next": "^13.4.2",
|
||||
"postcss": "^8.4.21",
|
||||
"prettier": "^2.8.8",
|
||||
"prettier-plugin-tailwindcss": "^0.2.8",
|
||||
"tailwindcss": "^3.3.0",
|
||||
"typescript": "^5.0.4"
|
||||
},
|
||||
"ct3aMetadata": {
|
||||
"initVersion": "7.13.1"
|
||||
}
|
||||
}
|
||||
8
postcss.config.cjs
Normal file
8
postcss.config.cjs
Normal file
@@ -0,0 +1,8 @@
|
||||
const config = {
|
||||
plugins: {
|
||||
tailwindcss: {},
|
||||
autoprefixer: {},
|
||||
},
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
6
prettier.config.cjs
Normal file
6
prettier.config.cjs
Normal file
@@ -0,0 +1,6 @@
|
||||
/** @type {import("prettier").Config} */
|
||||
const config = {
|
||||
plugins: [require.resolve("prettier-plugin-tailwindcss")],
|
||||
};
|
||||
|
||||
module.exports = config;
|
||||
BIN
public/favicon.ico
Normal file
BIN
public/favicon.ico
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
65
src/components/TwitterDMDownloader.tsx
Normal file
65
src/components/TwitterDMDownloader.tsx
Normal file
@@ -0,0 +1,65 @@
|
||||
import { useState } from "react";
|
||||
import { api } from "~/utils/api";
|
||||
|
||||
function downloadCSV(data: string, filename: string) {
|
||||
const blob = new Blob([data], { type: "text/csv" });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement("a");
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
a.click();
|
||||
}
|
||||
|
||||
export function TwitterDMDownloader() {
|
||||
const [isLoading, setIsLoading] = useState(false);
|
||||
|
||||
const { mutateAsync: getDMs } = api.twitter.getDMs.useMutation();
|
||||
const { data: authUrlData } = api.twitter.getAuthUrl.useQuery();
|
||||
|
||||
const handleAuth = async () => {
|
||||
if (!authUrlData?.authUrl) return;
|
||||
|
||||
const popup = window.open(
|
||||
authUrlData.authUrl,
|
||||
"Twitter Auth",
|
||||
"width=600,height=600"
|
||||
);
|
||||
|
||||
// Create the event listener function
|
||||
const handleMessage = async (event: MessageEvent) => {
|
||||
if (event.data.type === "TWITTER_OAUTH_CALLBACK") {
|
||||
// Remove the event listener first to prevent multiple calls
|
||||
window.removeEventListener("message", handleMessage);
|
||||
|
||||
setIsLoading(true);
|
||||
try {
|
||||
const { csvData } = await getDMs({
|
||||
code: event.data.code,
|
||||
codeVerifier: authUrlData.codeVerifier
|
||||
});
|
||||
downloadCSV(csvData, "twitter-dms.csv");
|
||||
} catch (error) {
|
||||
console.error("Error fetching DMs:", error);
|
||||
alert("Failed to fetch DMs. Please try again.");
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Add the event listener
|
||||
window.addEventListener("message", handleMessage);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="flex flex-col items-center gap-4">
|
||||
<button
|
||||
onClick={handleAuth}
|
||||
disabled={isLoading}
|
||||
className="rounded bg-blue-500 px-4 py-2 text-white hover:bg-blue-600 disabled:opacity-50"
|
||||
>
|
||||
{isLoading ? "Downloading DMs..." : "Download Twitter DMs"}
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
35
src/env.mjs
Normal file
35
src/env.mjs
Normal file
@@ -0,0 +1,35 @@
|
||||
import { createEnv } from "@t3-oss/env-nextjs";
|
||||
import { z } from "zod";
|
||||
|
||||
export const env = createEnv({
|
||||
/**
|
||||
* Specify your server-side environment variables schema here. This way you can ensure the app
|
||||
* isn't built with invalid env vars.
|
||||
*/
|
||||
server: {
|
||||
NODE_ENV: z.enum(["development", "test", "production"]),
|
||||
},
|
||||
|
||||
/**
|
||||
* Specify your client-side environment variables schema here. This way you can ensure the app
|
||||
* isn't built with invalid env vars. To expose them to the client, prefix them with
|
||||
* `NEXT_PUBLIC_`.
|
||||
*/
|
||||
client: {
|
||||
// NEXT_PUBLIC_CLIENTVAR: z.string().min(1),
|
||||
},
|
||||
|
||||
/**
|
||||
* You can't destruct `process.env` as a regular object in the Next.js edge runtimes (e.g.
|
||||
* middlewares) or client-side so we need to destruct manually.
|
||||
*/
|
||||
runtimeEnv: {
|
||||
NODE_ENV: process.env.NODE_ENV,
|
||||
// NEXT_PUBLIC_CLIENTVAR: process.env.NEXT_PUBLIC_CLIENTVAR,
|
||||
},
|
||||
/**
|
||||
* Run `build` or `dev` with `SKIP_ENV_VALIDATION` to skip env validation.
|
||||
* This is especially useful for Docker builds.
|
||||
*/
|
||||
skipValidation: !!process.env.SKIP_ENV_VALIDATION,
|
||||
});
|
||||
9
src/pages/_app.tsx
Normal file
9
src/pages/_app.tsx
Normal file
@@ -0,0 +1,9 @@
|
||||
import { type AppType } from "next/app";
|
||||
import { api } from "~/utils/api";
|
||||
import "~/styles/globals.css";
|
||||
|
||||
const MyApp: AppType = ({ Component, pageProps }) => {
|
||||
return <Component {...pageProps} />;
|
||||
};
|
||||
|
||||
export default api.withTRPC(MyApp);
|
||||
18
src/pages/api/trpc/[trpc].ts
Normal file
18
src/pages/api/trpc/[trpc].ts
Normal file
@@ -0,0 +1,18 @@
|
||||
import { createNextApiHandler } from "@trpc/server/adapters/next";
|
||||
import { env } from "~/env.mjs";
|
||||
import { appRouter } from "~/server/api/root";
|
||||
import { createTRPCContext } from "~/server/api/trpc";
|
||||
|
||||
// export API handler
|
||||
export default createNextApiHandler({
|
||||
router: appRouter,
|
||||
createContext: createTRPCContext,
|
||||
onError:
|
||||
env.NODE_ENV === "development"
|
||||
? ({ path, error }) => {
|
||||
console.error(
|
||||
`❌ tRPC failed on ${path ?? "<no-path>"}: ${error.message}`,
|
||||
);
|
||||
}
|
||||
: undefined,
|
||||
});
|
||||
13
src/pages/callback.tsx
Normal file
13
src/pages/callback.tsx
Normal file
@@ -0,0 +1,13 @@
|
||||
import { useEffect } from "react";
|
||||
|
||||
export default function TwitterCallback() {
|
||||
useEffect(() => {
|
||||
const code = new URLSearchParams(window.location.search).get("code");
|
||||
if (code) {
|
||||
window.opener.postMessage({ type: "TWITTER_OAUTH_CALLBACK", code }, "*");
|
||||
window.close();
|
||||
}
|
||||
}, []);
|
||||
|
||||
return <div>Processing authentication...</div>;
|
||||
}
|
||||
19
src/pages/index.tsx
Normal file
19
src/pages/index.tsx
Normal file
@@ -0,0 +1,19 @@
|
||||
import { type NextPage } from "next";
|
||||
import Head from "next/head";
|
||||
import { TwitterDMDownloader } from "~/components/TwitterDMDownloader";
|
||||
const Home: NextPage = () => {
|
||||
return (
|
||||
<>
|
||||
<Head>
|
||||
<title>Twitter DM Downloader</title>
|
||||
<meta name="description" content="Download your Twitter DMs" />
|
||||
<link rel="icon" href="/favicon.ico" />
|
||||
</Head>
|
||||
<main className="flex min-h-screen flex-col items-center justify-center bg-gradient-to-b from-[#2e026d] to-[#15162c]">
|
||||
<TwitterDMDownloader />
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default Home;
|
||||
15
src/server/api/root.ts
Normal file
15
src/server/api/root.ts
Normal file
@@ -0,0 +1,15 @@
|
||||
import { exampleRouter } from "~/server/api/routers/example";
|
||||
import { createTRPCRouter } from "~/server/api/trpc";
|
||||
import { twitterRouter } from "~/server/api/routers/twitter";
|
||||
/**
|
||||
* This is the primary router for your server.
|
||||
*
|
||||
* All routers added in /api/routers should be manually added here.
|
||||
*/
|
||||
export const appRouter = createTRPCRouter({
|
||||
example: exampleRouter,
|
||||
twitter: twitterRouter,
|
||||
});
|
||||
|
||||
// export type definition of API
|
||||
export type AppRouter = typeof appRouter;
|
||||
12
src/server/api/routers/example.ts
Normal file
12
src/server/api/routers/example.ts
Normal file
@@ -0,0 +1,12 @@
|
||||
import { z } from "zod";
|
||||
import { createTRPCRouter, publicProcedure } from "~/server/api/trpc";
|
||||
|
||||
export const exampleRouter = createTRPCRouter({
|
||||
hello: publicProcedure
|
||||
.input(z.object({ text: z.string() }))
|
||||
.query(({ input }) => {
|
||||
return {
|
||||
greeting: `Hello ${input.text}`,
|
||||
};
|
||||
}),
|
||||
});
|
||||
254
src/server/api/routers/twitter.ts
Normal file
254
src/server/api/routers/twitter.ts
Normal file
@@ -0,0 +1,254 @@
|
||||
import { z } from "zod";
|
||||
import { createTRPCRouter, publicProcedure } from "../trpc";
|
||||
|
||||
function generateRandomState() {
|
||||
return Math.random().toString(36).substring(2, 15);
|
||||
}
|
||||
|
||||
function generateCodeVerifier() {
|
||||
const length = 64;
|
||||
const charset = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~';
|
||||
let result = '';
|
||||
for (let i = 0; i < length; i++) {
|
||||
result += charset.charAt(Math.floor(Math.random() * charset.length));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
async function generateCodeChallenge(verifier: string) {
|
||||
const encoder = new TextEncoder();
|
||||
const data = encoder.encode(verifier);
|
||||
const hash = await crypto.subtle.digest('SHA-256', data);
|
||||
return btoa(String.fromCharCode(...new Uint8Array(hash)))
|
||||
.replace(/\+/g, '-')
|
||||
.replace(/\//g, '_')
|
||||
.replace(/=+$/, '');
|
||||
}
|
||||
|
||||
function convertDMsToCSV(dms: any[], users: any[] = []) {
|
||||
console.log("Processing DMs for CSV conversion");
|
||||
|
||||
// Create a map of user IDs to usernames for quick lookup
|
||||
const userMap = new Map();
|
||||
if (users && Array.isArray(users)) {
|
||||
users.forEach(user => {
|
||||
userMap.set(user.id, {
|
||||
username: user.username || '',
|
||||
name: user.name || ''
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Define the headers for our CSV
|
||||
const headers = [
|
||||
'id', 'created_at', 'sender_id', 'sender_username', 'sender_name',
|
||||
'recipient_id', 'recipient_username', 'recipient_name',
|
||||
'text', 'dm_conversation_id'
|
||||
];
|
||||
|
||||
// Extract the relevant data from each DM
|
||||
const rows = dms.filter(dm => dm.event_type === 'MessageCreate').map(dm => {
|
||||
// For MessageCreate events with 2 participants
|
||||
if (dm.participant_ids && dm.participant_ids.length === 2) {
|
||||
const recipientId = dm.participant_ids.find((id: string) => id !== dm.sender_id) || '';
|
||||
|
||||
// Get user info from the map
|
||||
const senderInfo = userMap.get(dm.sender_id) || { username: '', name: '' };
|
||||
const recipientInfo = userMap.get(recipientId) || { username: '', name: '' };
|
||||
|
||||
return {
|
||||
id: dm.id || '',
|
||||
created_at: dm.created_at || '',
|
||||
sender_id: dm.sender_id || '',
|
||||
sender_username: senderInfo.username,
|
||||
sender_name: senderInfo.name,
|
||||
recipient_id: recipientId,
|
||||
recipient_username: recipientInfo.username,
|
||||
recipient_name: recipientInfo.name,
|
||||
text: dm.text ? dm.text.replace(/"/g, '""') : '', // Escape quotes for CSV
|
||||
dm_conversation_id: dm.dm_conversation_id || ''
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}).filter(Boolean); // Remove null entries
|
||||
|
||||
// Convert to CSV format
|
||||
const csvContent = [
|
||||
// Add the headers
|
||||
headers.join(','),
|
||||
// Add each row
|
||||
...rows.map(row => headers.map(header =>
|
||||
`"${(row as any)[header] || ''}"`
|
||||
).join(','))
|
||||
].join('\n');
|
||||
|
||||
return csvContent;
|
||||
}
|
||||
|
||||
export const twitterRouter = createTRPCRouter({
|
||||
// Initialize OAuth flow
|
||||
getAuthUrl: publicProcedure.query(async () => {
|
||||
const TWITTER_CLIENT_ID = process.env.TWITTER_CLIENT_ID!;
|
||||
const REDIRECT_URI = process.env.TWITTER_REDIRECT_URI!;
|
||||
|
||||
// Use the test values provided by Twitter
|
||||
const codeVerifier = '8KxxO-RPl0bLSxX5AWwgdiFbMnry_VOKzFeIlVA7NoA';
|
||||
const codeChallenge = 'y_SfRG4BmOES02uqWeIkIgLQAlTBggyf_G7uKT51ku8';
|
||||
|
||||
console.log('Using test verifier:', codeVerifier);
|
||||
console.log('Using test challenge:', codeChallenge);
|
||||
|
||||
const authUrl = `https://twitter.com/i/oauth2/authorize?` +
|
||||
`response_type=code&` +
|
||||
`client_id=${TWITTER_CLIENT_ID}&` +
|
||||
`redirect_uri=${REDIRECT_URI}&` +
|
||||
`scope=dm.read%20dm.write%20tweet.read%20users.read%20offline.access&` +
|
||||
`state=${generateRandomState()}&` +
|
||||
`code_challenge=${codeChallenge}&` +
|
||||
`code_challenge_method=S256`;
|
||||
|
||||
return { authUrl, codeVerifier };
|
||||
}),
|
||||
|
||||
// Handle OAuth callback and fetch DMs
|
||||
getDMs: publicProcedure
|
||||
.input(z.object({
|
||||
code: z.string(),
|
||||
codeVerifier: z.string()
|
||||
}))
|
||||
.mutation(async ({ input }) => {
|
||||
try {
|
||||
const token = await exchangeCodeForToken(input.code, input.codeVerifier);
|
||||
const response = await fetch('https://api.twitter.com/2/dm_events?max_results=100&event_types=MessageCreate&dm_event.fields=created_at,dm_conversation_id,participant_ids,sender_id,text&expansions=sender_id,participant_ids&user.fields=username,name,profile_image_url', {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("DM fetch failed:", data);
|
||||
throw new Error(`Failed to fetch DMs: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
// Extract users from the includes section
|
||||
const users = data.includes?.users || [];
|
||||
|
||||
// Fetch all DMs with pagination
|
||||
const allDMs = await fetchAllDMs(token);
|
||||
|
||||
// Convert to CSV with user information
|
||||
const csvData = convertDMsToCSV(allDMs, users);
|
||||
|
||||
return { csvData };
|
||||
} catch (error) {
|
||||
console.error("Error in getDMs:", error);
|
||||
throw error;
|
||||
}
|
||||
}),
|
||||
});
|
||||
|
||||
// Helper function to fetch all DMs using pagination
|
||||
async function fetchAllDMs(token: string) {
|
||||
try {
|
||||
let allDMs: any[] = [];
|
||||
let paginationToken: string | undefined = undefined;
|
||||
let hasMorePages = true;
|
||||
|
||||
while (hasMorePages) {
|
||||
// Build the URL with pagination token if available
|
||||
let url = 'https://api.twitter.com/2/dm_events?max_results=100';
|
||||
url += '&event_types=MessageCreate'; // Only get MessageCreate events
|
||||
url += '&dm_event.fields=created_at,dm_conversation_id,participant_ids,sender_id,text';
|
||||
url += '&expansions=sender_id,participant_ids'; // Add expansions for user info
|
||||
url += '&user.fields=username,name,profile_image_url'; // Request user fields
|
||||
|
||||
if (paginationToken) {
|
||||
url += `&pagination_token=${paginationToken}`;
|
||||
}
|
||||
|
||||
console.log(`Fetching DMs with URL: ${url}`);
|
||||
|
||||
const response = await fetch(url, {
|
||||
headers: {
|
||||
Authorization: `Bearer ${token}`,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text();
|
||||
console.error("DM fetch failed:", errorText);
|
||||
throw new Error(`Failed to fetch DMs: ${errorText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!data.data || !Array.isArray(data.data)) {
|
||||
console.error("Unexpected DM response format:", data);
|
||||
throw new Error('Received invalid DM data format from Twitter');
|
||||
}
|
||||
|
||||
// Add this page of DMs to our collection
|
||||
allDMs = [...allDMs, ...data.data];
|
||||
|
||||
// Check if there are more pages
|
||||
if (data.meta && data.meta.next_token) {
|
||||
paginationToken = data.meta.next_token;
|
||||
console.log(`Found next page token: ${paginationToken}`);
|
||||
} else {
|
||||
hasMorePages = false;
|
||||
console.log("No more pages to fetch");
|
||||
}
|
||||
|
||||
// Safety check - don't fetch too many pages
|
||||
if (allDMs.length > 10000) {
|
||||
console.log("Reached maximum DM count, stopping pagination");
|
||||
hasMorePages = false;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Total DMs fetched: ${allDMs.length}`);
|
||||
return allDMs;
|
||||
} catch (error) {
|
||||
console.error("Error in fetchAllDMs:", error);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function exchangeCodeForToken(code: string, codeVerifier: string): Promise<string> {
|
||||
const TWITTER_CLIENT_ID = process.env.TWITTER_CLIENT_ID!;
|
||||
const TWITTER_CLIENT_SECRET = process.env.TWITTER_CLIENT_SECRET!;
|
||||
const REDIRECT_URI = process.env.TWITTER_REDIRECT_URI!;
|
||||
|
||||
console.log("Exchanging code for token with params:", {
|
||||
code,
|
||||
codeVerifier,
|
||||
redirectUri: REDIRECT_URI
|
||||
});
|
||||
|
||||
const response = await fetch('https://api.twitter.com/2/oauth2/token', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/x-www-form-urlencoded',
|
||||
Authorization: `Basic ${Buffer.from(
|
||||
`${TWITTER_CLIENT_ID}:${TWITTER_CLIENT_SECRET}`
|
||||
).toString('base64')}`,
|
||||
},
|
||||
body: new URLSearchParams({
|
||||
grant_type: 'authorization_code',
|
||||
code,
|
||||
redirect_uri: REDIRECT_URI,
|
||||
code_verifier: codeVerifier,
|
||||
}).toString(),
|
||||
});
|
||||
|
||||
const data = await response.json();
|
||||
|
||||
if (!response.ok) {
|
||||
console.error("Token exchange failed:", data);
|
||||
throw new Error(`Failed to exchange code: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
return data.access_token;
|
||||
}
|
||||
92
src/server/api/trpc.ts
Normal file
92
src/server/api/trpc.ts
Normal file
@@ -0,0 +1,92 @@
|
||||
/**
|
||||
* YOU PROBABLY DON'T NEED TO EDIT THIS FILE, UNLESS:
|
||||
* 1. You want to modify request context (see Part 1).
|
||||
* 2. You want to create a new middleware or type of procedure (see Part 3).
|
||||
*
|
||||
* TL;DR - This is where all the tRPC server stuff is created and plugged in. The pieces you will
|
||||
* need to use are documented accordingly near the end.
|
||||
*/
|
||||
|
||||
import { initTRPC } from "@trpc/server";
|
||||
import { type CreateNextContextOptions } from "@trpc/server/adapters/next";
|
||||
import superjson from "superjson";
|
||||
import { ZodError } from "zod";
|
||||
|
||||
/**
|
||||
* 1. CONTEXT
|
||||
*
|
||||
* This section defines the "contexts" that are available in the backend API.
|
||||
*
|
||||
* These allow you to access things when processing a request, like the database, the session, etc.
|
||||
*/
|
||||
|
||||
type CreateContextOptions = Record<string, never>;
|
||||
|
||||
/**
|
||||
* This helper generates the "internals" for a tRPC context. If you need to use it, you can export
|
||||
* it from here.
|
||||
*
|
||||
* Examples of things you may need it for:
|
||||
* - testing, so we don't have to mock Next.js' req/res
|
||||
* - tRPC's `createSSGHelpers`, where we don't have req/res
|
||||
*
|
||||
* @see https://create.t3.gg/en/usage/trpc#-serverapitrpcts
|
||||
*/
|
||||
const createInnerTRPCContext = (_opts: CreateContextOptions) => {
|
||||
return {};
|
||||
};
|
||||
|
||||
/**
|
||||
* This is the actual context you will use in your router. It will be used to process every request
|
||||
* that goes through your tRPC endpoint.
|
||||
*
|
||||
* @see https://trpc.io/docs/context
|
||||
*/
|
||||
export const createTRPCContext = (_opts: CreateNextContextOptions) => {
|
||||
return createInnerTRPCContext({});
|
||||
};
|
||||
|
||||
/**
|
||||
* 2. INITIALIZATION
|
||||
*
|
||||
* This is where the tRPC API is initialized, connecting the context and transformer. We also parse
|
||||
* ZodErrors so that you get typesafety on the frontend if your procedure fails due to validation
|
||||
* errors on the backend.
|
||||
*/
|
||||
|
||||
const t = initTRPC.context<typeof createTRPCContext>().create({
|
||||
transformer: superjson,
|
||||
errorFormatter({ shape, error }) {
|
||||
return {
|
||||
...shape,
|
||||
data: {
|
||||
...shape.data,
|
||||
zodError:
|
||||
error.cause instanceof ZodError ? error.cause.flatten() : null,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
/**
|
||||
* 3. ROUTER & PROCEDURE (THE IMPORTANT BIT)
|
||||
*
|
||||
* These are the pieces you use to build your tRPC API. You should import these a lot in the
|
||||
* "/src/server/api/routers" directory.
|
||||
*/
|
||||
|
||||
/**
|
||||
* This is how you create new routers and sub-routers in your tRPC API.
|
||||
*
|
||||
* @see https://trpc.io/docs/router
|
||||
*/
|
||||
export const createTRPCRouter = t.router;
|
||||
|
||||
/**
|
||||
* Public (unauthenticated) procedure
|
||||
*
|
||||
* This is the base piece you use to build new queries and mutations on your tRPC API. It does not
|
||||
* guarantee that a user querying is authorized, but you can still access user session data if they
|
||||
* are logged in.
|
||||
*/
|
||||
export const publicProcedure = t.procedure;
|
||||
3
src/styles/globals.css
Normal file
3
src/styles/globals.css
Normal file
@@ -0,0 +1,3 @@
|
||||
@tailwind base;
|
||||
@tailwind components;
|
||||
@tailwind utilities;
|
||||
67
src/utils/api.ts
Normal file
67
src/utils/api.ts
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* This is the client-side entrypoint for your tRPC API. It is used to create the `api` object which
|
||||
* contains the Next.js App-wrapper, as well as your type-safe React Query hooks.
|
||||
*
|
||||
* We also create a few inference helpers for input and output types.
|
||||
*/
|
||||
import { httpBatchLink, loggerLink } from "@trpc/client";
|
||||
import { createTRPCNext } from "@trpc/next";
|
||||
import { type inferRouterInputs, type inferRouterOutputs } from "@trpc/server";
|
||||
import superjson from "superjson";
|
||||
import { type AppRouter } from "~/server/api/root";
|
||||
|
||||
const getBaseUrl = () => {
|
||||
if (typeof window !== "undefined") return ""; // browser should use relative url
|
||||
if (process.env.VERCEL_URL) return `https://${process.env.VERCEL_URL}`; // SSR should use vercel url
|
||||
return `http://localhost:${process.env.PORT ?? 3000}`; // dev SSR should use localhost
|
||||
};
|
||||
|
||||
/** A set of type-safe react-query hooks for your tRPC API. */
|
||||
export const api = createTRPCNext<AppRouter>({
|
||||
config() {
|
||||
return {
|
||||
/**
|
||||
* Transformer used for data de-serialization from the server.
|
||||
*
|
||||
* @see https://trpc.io/docs/data-transformers
|
||||
*/
|
||||
transformer: superjson,
|
||||
|
||||
/**
|
||||
* Links used to determine request flow from client to server.
|
||||
*
|
||||
* @see https://trpc.io/docs/links
|
||||
*/
|
||||
links: [
|
||||
loggerLink({
|
||||
enabled: (opts) =>
|
||||
process.env.NODE_ENV === "development" ||
|
||||
(opts.direction === "down" && opts.result instanceof Error),
|
||||
}),
|
||||
httpBatchLink({
|
||||
url: `${getBaseUrl()}/api/trpc`,
|
||||
}),
|
||||
],
|
||||
};
|
||||
},
|
||||
/**
|
||||
* Whether tRPC should await queries when server rendering pages.
|
||||
*
|
||||
* @see https://trpc.io/docs/nextjs#ssr-boolean-default-false
|
||||
*/
|
||||
ssr: false,
|
||||
});
|
||||
|
||||
/**
|
||||
* Inference helper for inputs.
|
||||
*
|
||||
* @example type HelloInput = RouterInputs['example']['hello']
|
||||
*/
|
||||
export type RouterInputs = inferRouterInputs<AppRouter>;
|
||||
|
||||
/**
|
||||
* Inference helper for outputs.
|
||||
*
|
||||
* @example type HelloOutput = RouterOutputs['example']['hello']
|
||||
*/
|
||||
export type RouterOutputs = inferRouterOutputs<AppRouter>;
|
||||
9
tailwind.config.ts
Normal file
9
tailwind.config.ts
Normal file
@@ -0,0 +1,9 @@
|
||||
import { type Config } from "tailwindcss";
|
||||
|
||||
export default {
|
||||
content: ["./src/**/*.{js,ts,jsx,tsx}"],
|
||||
theme: {
|
||||
extend: {},
|
||||
},
|
||||
plugins: [],
|
||||
} satisfies Config;
|
||||
33
tsconfig.json
Normal file
33
tsconfig.json
Normal file
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "es2017",
|
||||
"lib": ["dom", "dom.iterable", "esnext"],
|
||||
"allowJs": true,
|
||||
"checkJs": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"module": "esnext",
|
||||
"moduleResolution": "node",
|
||||
"resolveJsonModule": true,
|
||||
"isolatedModules": true,
|
||||
"jsx": "preserve",
|
||||
"incremental": true,
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"~/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": [
|
||||
".eslintrc.cjs",
|
||||
"next-env.d.ts",
|
||||
"**/*.ts",
|
||||
"**/*.tsx",
|
||||
"**/*.cjs",
|
||||
"**/*.mjs"
|
||||
],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
Reference in New Issue
Block a user